[doc] build from commit 97814c62a8

This commit is contained in:
inetol
2026-01-16 09:59:45 +00:00
commit 6c01d9b15b
472 changed files with 105711 additions and 0 deletions

4
.buildinfo Normal file
View File

@@ -0,0 +1,4 @@
# Sphinx build info version 1
# This file records the configuration used when building these files. When it is not found, a full rebuild will be done.
config: 094f4f45c33c27474020219ae76e488c
tags: 645f666f9bcd5a90fca523b33c5a78b7

0
.nojekyll Normal file
View File

98
404.html Normal file
View File

@@ -0,0 +1,98 @@
<!DOCTYPE html>
<html lang="en" data-content_root="./">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Page not found &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="/_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="/_static/searxng.css?v=52e4ff28" />
<script src="/_static/documentation_options.js?v=5932b86d"></script>
<script src="/_static/doctools.js?v=9bcbadda"></script>
<script src="/_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="/_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="/genindex.html" />
<link rel="search" title="Search" href="/search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="/genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="/py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="/index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">Page not found</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Page not found</h1>
Unfortunately we couldn't find the content you were looking for.
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="/index.html">
<img class="logo" src="/_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="/index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="/user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="/own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="/admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="/dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="/utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="/src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="/index.html">Overview</a>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="/search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

1
CNAME Normal file
View File

@@ -0,0 +1 @@
docs.searxng.org

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg" version="1.1"
baseProfile="full" width="70px" height="40px"
viewBox="0 0 700 400"
>
<line x1="180" y1="370"
x2="500" y2="50"
stroke="black" stroke-width="15px"
/>
<polygon points="585 0 525 25 585 50"
transform="rotate(135 525 25)"
/>
</svg>

After

Width:  |  Height:  |  Size: 385 B

31
_images/arch_public.dot Normal file
View File

@@ -0,0 +1,31 @@
digraph G {
node [style=filled, shape=box, fillcolor="#ffffcc", fontname=Sans];
edge [fontname="Sans"];
browser [label="browser", shape=tab, fillcolor=aliceblue];
rp [label="reverse proxy"];
static [label="static files", shape=folder, href="url to configure static files", fillcolor=lightgray];
uwsgi [label="uwsgi", shape=parallelogram href="https://docs.searxng.org/utils/searxng.sh.html"]
valkey [label="valkey DB", shape=cylinder];
searxng1 [label="SearXNG #1", fontcolor=blue3];
searxng2 [label="SearXNG #2", fontcolor=blue3];
searxng3 [label="SearXNG #3", fontcolor=blue3];
searxng4 [label="SearXNG #4", fontcolor=blue3];
browser -> rp [label="HTTPS"]
subgraph cluster_searxng {
label = "SearXNG instance" fontname=Sans;
bgcolor="#fafafa";
{ rank=same; static rp };
rp -> static [label="optional: reverse proxy serves static files", fillcolor=slategray, fontcolor=slategray];
rp -> uwsgi [label="http:// (tcp) or unix:// (socket)"];
uwsgi -> searxng1 -> valkey;
uwsgi -> searxng2 -> valkey;
uwsgi -> searxng3 -> valkey;
uwsgi -> searxng4 -> valkey;
}
}

1
_images/browserstack.svg Normal file

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 7.2 KiB

1
_images/docker.svg Normal file

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 5.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 59 KiB

3
_images/hello.dot Normal file
View File

@@ -0,0 +1,3 @@
graph G {
Hello -- World
}

10
_images/svg_image.svg Normal file
View File

@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- originate: https://commons.wikimedia.org/wiki/File:Variable_Resistor.svg -->
<svg xmlns="http://www.w3.org/2000/svg"
version="1.1" baseProfile="full"
width="70px" height="40px" viewBox="0 0 700 400">
<line x1="0" y1="200" x2="700" y2="200" stroke="black" stroke-width="20px"/>
<rect x="100" y="100" width="500" height="200" fill="white" stroke="black" stroke-width="20px"/>
<line x1="180" y1="370" x2="500" y2="50" stroke="black" stroke-width="15px"/>
<polygon points="585 0 525 25 585 50" transform="rotate(135 525 25)"/>
</svg>

After

Width:  |  Height:  |  Size: 580 B

47
_images/translation.svg Normal file

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 18 KiB

1
_images/tuta.svg Normal file
View File

@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 1024 384"><path fill="#410002" d="M479.178 119.294c-.533-.016-.998.357-1.218 1.078l-24.438 78.364.005-.004c-8.59 27.537 4.516 46.485 34.268 46.485 4.336 0 10.006-.357 11.776-.797.885-.264 1.33-.71 1.594-1.506l5.134-17.177c.264-.973-.09-1.77-1.507-1.683-4.517.445-8.588.797-12.308.797-14.964 0-21.075-7.968-16.646-22.224l10.446-33.47h30.373c.797 0 1.506-.445 1.858-1.418l5.401-17.355c.264-.973-.264-1.681-1.417-1.681H492.66l3.895-12.662c.265-.885.089-1.418-.62-2.034l-15.761-14.255c-.332-.3-.676-.45-.996-.459zm173.64 0c-.532-.016-.996.357-1.218 1.078l-24.436 78.364.005-.004c-8.59 27.537 4.517 46.485 34.268 46.485 4.342 0 10.004-.357 11.778-.797.884-.264 1.324-.71 1.593-1.506l5.133-17.177c.26-.973 0-1.77-1.508-1.683-4.517.445-8.59.797-12.307.797-14.966 0-21.077-7.968-16.646-22.224l10.445-33.47H690.3c.795 0 1.504-.445 1.854-1.418l5.402-17.355c.265-.973-.26-1.681-1.414-1.681H666.3l3.896-12.662c.265-.885.087-1.418-.618-2.034l-15.765-14.255c-.332-.3-.676-.45-.996-.459zm-48.32 29.404c-.974 0-1.503.444-1.862 1.417L590.502 188.9c-7.525 23.998-19.478 37.721-31.965 37.721-12.487 0-17.797-9.83-13.105-24.883l16.028-51.178c.351-1.149-.088-1.857-1.328-1.857H539.94c-.97 0-1.505.444-1.86 1.417l-15.497 49.233.008-.005c-8.765 27.982 5.315 46.31 27.452 46.31 12.747 0 22.756-6.111 29.93-16.118l-.176 12.838c0 1.241.621 1.593 1.681 1.593h14.17c1.064 0 1.504-.445 1.859-1.418l28.512-91.997c.35-1.15-.09-1.858-1.33-1.858zm147.96.005c-43.653 0-60.654 37.719-60.654 62.157-.09 21.339 13.282 34.798 31.08 34.798v.004c11.868 0 21.693-5.314 29.133-16.117v12.836c0 1.061.62 1.596 1.594 1.596h14.166c.974 0 1.505-.446 1.86-1.42l28.777-92.086c.265-.973-.266-1.768-1.24-1.768zm-.616 20.54h17.265l-6.197 19.57c-7.35 23.289-18.684 37.896-32.585 37.896-10.094 0-15.585-6.907-15.585-18.15 0-17.976 13.722-39.315 37.102-39.315z"/><path fill="#850122" d="M226.561 106.964c-.558.007-1.043.428-1.043 1.095V251.59c0 1.594 2.04 1.594 2.48 0L261.38 143.3c.445-1.241.446-2.039-.62-3.1l-33.204-32.762c-.299-.332-.66-.478-.996-.474zm55.983 41.739c-1.241 0-1.594.444-2.039 1.417l-43.919 142.203c-.176.797.177 1.594 1.242 1.594h145.747c1.418 0 2.04-.62 2.48-1.858l44.098-141.499c.445-1.417-.18-1.857-1.417-1.857zm-40.022-58.62c-1.418 0-1.594 1.242-.797 2.04l35.065 35.24c.796.798 1.594 1.061 2.836 1.061h149.467c1.065 0 1.68-1.24.62-2.214l-34.63-34.885c-.796-.796-1.592-1.242-3.274-1.242z"/></svg>

After

Width:  |  Height:  |  Size: 2.4 KiB

202
_modules/index.html Normal file
View File

@@ -0,0 +1,202 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Overview: module code &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../_static/searxng.css?v=52e4ff28" />
<script src="../_static/documentation_options.js?v=5932b86d"></script>
<script src="../_static/doctools.js?v=9bcbadda"></script>
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">Overview: module code</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>All modules for which code is available</h1>
<ul><li><a href="builtins.html">builtins</a></li>
<li><a href="searx/answerers/_core.html">searx.answerers._core</a></li>
<li><a href="searx/answerers/random.html">searx.answerers.random</a></li>
<li><a href="searx/answerers/statistics.html">searx.answerers.statistics</a></li>
<li><a href="searx/autocomplete.html">searx.autocomplete</a></li>
<li><a href="searx/babel_extract.html">searx.babel_extract</a></li>
<li><a href="searx/botdetection/_helpers.html">searx.botdetection._helpers</a></li>
<li><a href="searx/botdetection/config.html">searx.botdetection.config</a></li>
<li><a href="searx/botdetection/http_sec_fetch.html">searx.botdetection.http_sec_fetch</a></li>
<li><a href="searx/botdetection/ip_lists.html">searx.botdetection.ip_lists</a></li>
<li><a href="searx/botdetection/link_token.html">searx.botdetection.link_token</a></li>
<li><a href="searx/botdetection/trusted_proxies.html">searx.botdetection.trusted_proxies</a></li>
<li><a href="searx/brand.html">searx.brand</a></li>
<li><a href="searx/cache.html">searx.cache</a></li>
<li><a href="searx/enginelib.html">searx.enginelib</a></li>
<ul><li><a href="searx/enginelib/traits.html">searx.enginelib.traits</a></li>
</ul><li><a href="searx/engines.html">searx.engines</a></li>
<ul><li><a href="searx/engines/annas_archive.html">searx.engines.annas_archive</a></li>
<li><a href="searx/engines/archlinux.html">searx.engines.archlinux</a></li>
<li><a href="searx/engines/astrophysics_data_system.html">searx.engines.astrophysics_data_system</a></li>
<li><a href="searx/engines/azure.html">searx.engines.azure</a></li>
<li><a href="searx/engines/bing.html">searx.engines.bing</a></li>
<li><a href="searx/engines/bing_images.html">searx.engines.bing_images</a></li>
<li><a href="searx/engines/bing_news.html">searx.engines.bing_news</a></li>
<li><a href="searx/engines/bing_videos.html">searx.engines.bing_videos</a></li>
<li><a href="searx/engines/brave.html">searx.engines.brave</a></li>
<li><a href="searx/engines/command.html">searx.engines.command</a></li>
<li><a href="searx/engines/core.html">searx.engines.core</a></li>
<li><a href="searx/engines/dailymotion.html">searx.engines.dailymotion</a></li>
<li><a href="searx/engines/demo_offline.html">searx.engines.demo_offline</a></li>
<li><a href="searx/engines/demo_online.html">searx.engines.demo_online</a></li>
<li><a href="searx/engines/duckduckgo.html">searx.engines.duckduckgo</a></li>
<li><a href="searx/engines/duckduckgo_definitions.html">searx.engines.duckduckgo_definitions</a></li>
<li><a href="searx/engines/github_code.html">searx.engines.github_code</a></li>
<li><a href="searx/engines/google.html">searx.engines.google</a></li>
<li><a href="searx/engines/google_images.html">searx.engines.google_images</a></li>
<li><a href="searx/engines/google_news.html">searx.engines.google_news</a></li>
<li><a href="searx/engines/google_scholar.html">searx.engines.google_scholar</a></li>
<li><a href="searx/engines/google_videos.html">searx.engines.google_videos</a></li>
<li><a href="searx/engines/json_engine.html">searx.engines.json_engine</a></li>
<li><a href="searx/engines/marginalia.html">searx.engines.marginalia</a></li>
<li><a href="searx/engines/mrs.html">searx.engines.mrs</a></li>
<li><a href="searx/engines/odysee.html">searx.engines.odysee</a></li>
<li><a href="searx/engines/peertube.html">searx.engines.peertube</a></li>
<li><a href="searx/engines/qwant.html">searx.engines.qwant</a></li>
<li><a href="searx/engines/radio_browser.html">searx.engines.radio_browser</a></li>
<li><a href="searx/engines/recoll.html">searx.engines.recoll</a></li>
<li><a href="searx/engines/reuters.html">searx.engines.reuters</a></li>
<li><a href="searx/engines/sepiasearch.html">searx.engines.sepiasearch</a></li>
<li><a href="searx/engines/springer.html">searx.engines.springer</a></li>
<li><a href="searx/engines/sqlite.html">searx.engines.sqlite</a></li>
<li><a href="searx/engines/startpage.html">searx.engines.startpage</a></li>
<li><a href="searx/engines/tineye.html">searx.engines.tineye</a></li>
<li><a href="searx/engines/torznab.html">searx.engines.torznab</a></li>
<li><a href="searx/engines/tubearchivist.html">searx.engines.tubearchivist</a></li>
<li><a href="searx/engines/voidlinux.html">searx.engines.voidlinux</a></li>
<li><a href="searx/engines/wikidata.html">searx.engines.wikidata</a></li>
<li><a href="searx/engines/wikipedia.html">searx.engines.wikipedia</a></li>
<li><a href="searx/engines/xpath.html">searx.engines.xpath</a></li>
<li><a href="searx/engines/yahoo.html">searx.engines.yahoo</a></li>
<li><a href="searx/engines/zlibrary.html">searx.engines.zlibrary</a></li>
</ul><li><a href="searx/exceptions.html">searx.exceptions</a></li>
<li><a href="searx/extended_types.html">searx.extended_types</a></li>
<li><a href="searx/favicons/cache.html">searx.favicons.cache</a></li>
<li><a href="searx/favicons/config.html">searx.favicons.config</a></li>
<li><a href="searx/favicons/proxy.html">searx.favicons.proxy</a></li>
<li><a href="searx/favicons/resolvers.html">searx.favicons.resolvers</a></li>
<li><a href="searx/infopage.html">searx.infopage</a></li>
<li><a href="searx/limiter.html">searx.limiter</a></li>
<li><a href="searx/locales.html">searx.locales</a></li>
<li><a href="searx/plugins/_core.html">searx.plugins._core</a></li>
<li><a href="searx/plugins/calculator.html">searx.plugins.calculator</a></li>
<li><a href="searx/plugins/hash_plugin.html">searx.plugins.hash_plugin</a></li>
<li><a href="searx/plugins/hostnames.html">searx.plugins.hostnames</a></li>
<li><a href="searx/plugins/infinite_scroll.html">searx.plugins.infinite_scroll</a></li>
<li><a href="searx/plugins/self_info.html">searx.plugins.self_info</a></li>
<li><a href="searx/plugins/time_zone.html">searx.plugins.time_zone</a></li>
<li><a href="searx/plugins/tor_check.html">searx.plugins.tor_check</a></li>
<li><a href="searx/plugins/unit_converter.html">searx.plugins.unit_converter</a></li>
<li><a href="searx/result_types.html">searx.result_types</a></li>
<ul><li><a href="searx/result_types/_base.html">searx.result_types._base</a></li>
<li><a href="searx/result_types/answer.html">searx.result_types.answer</a></li>
<li><a href="searx/result_types/code.html">searx.result_types.code</a></li>
<li><a href="searx/result_types/file.html">searx.result_types.file</a></li>
<li><a href="searx/result_types/keyvalue.html">searx.result_types.keyvalue</a></li>
<li><a href="searx/result_types/paper.html">searx.result_types.paper</a></li>
</ul><li><a href="searx/search.html">searx.search</a></li>
<ul><li><a href="searx/search/models.html">searx.search.models</a></li>
<li><a href="searx/search/processors/abstract.html">searx.search.processors.abstract</a></li>
<li><a href="searx/search/processors/offline.html">searx.search.processors.offline</a></li>
<li><a href="searx/search/processors/online.html">searx.search.processors.online</a></li>
<li><a href="searx/search/processors/online_currency.html">searx.search.processors.online_currency</a></li>
<li><a href="searx/search/processors/online_dictionary.html">searx.search.processors.online_dictionary</a></li>
<li><a href="searx/search/processors/online_url_search.html">searx.search.processors.online_url_search</a></li>
</ul><li><a href="searx/settings_loader.html">searx.settings_loader</a></li>
<li><a href="searx/sqlitedb.html">searx.sqlitedb</a></li>
<li><a href="searx/utils.html">searx.utils</a></li>
<li><a href="searx/valkeydb.html">searx.valkeydb</a></li>
<li><a href="searx/valkeylib.html">searx.valkeylib</a></li>
<li><a href="searx/weather.html">searx.weather</a></li>
<li><a href="searxng_extra/update/update_engine_descriptions.html">searxng_extra.update.update_engine_descriptions</a></li>
<li><a href="searxng_extra/update/update_engine_traits.html">searxng_extra.update.update_engine_traits</a></li>
<li><a href="searxng_extra/update/update_external_bangs.html">searxng_extra.update.update_external_bangs</a></li>
<li><a href="searxng_extra/update/update_locales.html">searxng_extra.update.update_locales</a></li>
<li><a href="searxng_extra/update/update_pygments.html">searxng_extra.update.update_pygments</a></li>
</ul>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../index.html">
<img class="logo" src="../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../index.html">Overview</a>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,308 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.answerers._core &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.answerers._core</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.answerers._core</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="c1"># pylint: disable=too-few-public-methods, missing-module-docstring</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">abc</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">importlib</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">pathlib</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">warnings</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">dataclasses</span><span class="w"> </span><span class="kn">import</span> <span class="n">dataclass</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">load_module</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types.answer</span><span class="w"> </span><span class="kn">import</span> <span class="n">BaseAnswer</span>
<span class="n">_default</span> <span class="o">=</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)</span><span class="o">.</span><span class="n">parent</span>
<span class="n">log</span><span class="p">:</span> <span class="n">logging</span><span class="o">.</span><span class="n">Logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;searx.answerers&quot;</span><span class="p">)</span>
<div class="viewcode-block" id="AnswererInfo">
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.AnswererInfo">[docs]</a>
<span class="nd">@dataclass</span>
<span class="k">class</span><span class="w"> </span><span class="nc">AnswererInfo</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Object that holds information about an answerer, these infos are shown</span>
<span class="sd"> to the user in the Preferences menu.</span>
<span class="sd"> To be able to translate the information into other languages, the text must</span>
<span class="sd"> be written in English and translated with :py:obj:`flask_babel.gettext`.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Name of the *answerer*.&quot;&quot;&quot;</span>
<span class="n">description</span><span class="p">:</span> <span class="nb">str</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Short description of the *answerer*.&quot;&quot;&quot;</span>
<span class="n">examples</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;List of short examples of the usage / of query terms.&quot;&quot;&quot;</span>
<span class="n">keywords</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;See :py:obj:`Answerer.keywords`&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="Answerer">
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.Answerer">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">Answerer</span><span class="p">(</span><span class="n">abc</span><span class="o">.</span><span class="n">ABC</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Abstract base class of answerers.&quot;&quot;&quot;</span>
<span class="n">keywords</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Keywords to which the answerer has *answers*.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="Answerer.answer">
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.Answerer.answer">[docs]</a>
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">answer</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="n">BaseAnswer</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Function that returns a list of answers to the question/query.&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="Answerer.info">
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.Answerer.info">[docs]</a>
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">AnswererInfo</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Information about the *answerer*, see :py:obj:`AnswererInfo`.&quot;&quot;&quot;</span></div>
</div>
<div class="viewcode-block" id="ModuleAnswerer">
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.ModuleAnswerer">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">ModuleAnswerer</span><span class="p">(</span><span class="n">Answerer</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A wrapper class for legacy *answerers* where the names (keywords, answer,</span>
<span class="sd"> info) are implemented on the module level (not in a class).</span>
<span class="sd"> .. note::</span>
<span class="sd"> For internal use only!</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mod</span><span class="p">):</span>
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;keywords&quot;</span><span class="p">,</span> <span class="s2">&quot;self_info&quot;</span><span class="p">,</span> <span class="s2">&quot;answer&quot;</span><span class="p">]:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">mod</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">SystemExit</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">mod</span><span class="o">.</span><span class="n">keywords</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">SystemExit</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">module</span> <span class="o">=</span> <span class="n">mod</span>
<span class="bp">self</span><span class="o">.</span><span class="n">keywords</span> <span class="o">=</span> <span class="n">mod</span><span class="o">.</span><span class="n">keywords</span> <span class="c1"># type: ignore</span>
<div class="viewcode-block" id="ModuleAnswerer.answer">
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.ModuleAnswerer.answer">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">answer</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="n">BaseAnswer</span><span class="p">]:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="o">.</span><span class="n">answer</span><span class="p">(</span><span class="n">query</span><span class="p">)</span></div>
<div class="viewcode-block" id="ModuleAnswerer.info">
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.ModuleAnswerer.info">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">AnswererInfo</span><span class="p">:</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="o">.</span><span class="n">self_info</span><span class="p">()</span>
<span class="n">kwargs</span><span class="p">[</span><span class="s2">&quot;keywords&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">keywords</span>
<span class="k">return</span> <span class="n">AnswererInfo</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
</div>
<div class="viewcode-block" id="AnswerStorage">
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.AnswerStorage">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">AnswerStorage</span><span class="p">(</span><span class="nb">dict</span><span class="p">):</span> <span class="c1"># type: ignore</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A storage for managing the *answerers* of SearXNG. With the</span>
<span class="sd"> :py:obj:`AnswerStorage.ask`” method, a caller can ask questions to all</span>
<span class="sd"> *answerers* and receives a list of the results.&quot;&quot;&quot;</span>
<span class="n">answerer_list</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="n">Answerer</span><span class="p">]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The list of :py:obj:`Answerer` in this storage.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">answerer_list</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<div class="viewcode-block" id="AnswerStorage.load_builtins">
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.AnswerStorage.load_builtins">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">load_builtins</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Loads ``answerer.py`` modules from the python packages in</span>
<span class="sd"> :origin:`searx/answerers`. The python modules are wrapped by</span>
<span class="sd"> :py:obj:`ModuleAnswerer`.&quot;&quot;&quot;</span>
<span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">_default</span><span class="o">.</span><span class="n">iterdir</span><span class="p">():</span>
<span class="k">if</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;_&quot;</span><span class="p">):</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="n">f</span><span class="o">.</span><span class="n">is_file</span><span class="p">()</span> <span class="ow">and</span> <span class="n">f</span><span class="o">.</span><span class="n">suffix</span> <span class="o">==</span> <span class="s2">&quot;.py&quot;</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">register_by_fqn</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;searx.answerers.</span><span class="si">{</span><span class="n">f</span><span class="o">.</span><span class="n">stem</span><span class="si">}</span><span class="s2">.SXNGAnswerer&quot;</span><span class="p">)</span>
<span class="k">continue</span>
<span class="c1"># for backward compatibility (if a fork has additional answerers)</span>
<span class="k">if</span> <span class="n">f</span><span class="o">.</span><span class="n">is_dir</span><span class="p">()</span> <span class="ow">and</span> <span class="p">(</span><span class="n">f</span> <span class="o">/</span> <span class="s2">&quot;answerer.py&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;answerer module </span><span class="si">{</span><span class="n">f</span><span class="si">}</span><span class="s2"> is deprecated / migrate to searx.answerers.Answerer&quot;</span><span class="p">,</span> <span class="ne">DeprecationWarning</span>
<span class="p">)</span>
<span class="n">mod</span> <span class="o">=</span> <span class="n">load_module</span><span class="p">(</span><span class="s2">&quot;answerer.py&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">f</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">register</span><span class="p">(</span><span class="n">ModuleAnswerer</span><span class="p">(</span><span class="n">mod</span><span class="p">))</span></div>
<div class="viewcode-block" id="AnswerStorage.register_by_fqn">
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.AnswerStorage.register_by_fqn">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">register_by_fqn</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fqn</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Register a :py:obj:`Answerer` via its fully qualified class namen(FQN).&quot;&quot;&quot;</span>
<span class="n">mod_name</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">obj_name</span> <span class="o">=</span> <span class="n">fqn</span><span class="o">.</span><span class="n">rpartition</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
<span class="n">mod</span> <span class="o">=</span> <span class="n">importlib</span><span class="o">.</span><span class="n">import_module</span><span class="p">(</span><span class="n">mod_name</span><span class="p">)</span>
<span class="n">code_obj</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">mod</span><span class="p">,</span> <span class="n">obj_name</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">code_obj</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;answerer </span><span class="si">{</span><span class="n">fqn</span><span class="si">}</span><span class="s2"> is not implemented&quot;</span>
<span class="n">log</span><span class="o">.</span><span class="n">critical</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">register</span><span class="p">(</span><span class="n">code_obj</span><span class="p">())</span></div>
<div class="viewcode-block" id="AnswerStorage.register">
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.AnswerStorage.register">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">register</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">answerer</span><span class="p">:</span> <span class="n">Answerer</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Register a :py:obj:`Answerer`.&quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">answerer_list</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">answerer</span><span class="p">)</span>
<span class="k">for</span> <span class="n">_kw</span> <span class="ow">in</span> <span class="n">answerer</span><span class="o">.</span><span class="n">keywords</span><span class="p">:</span>
<span class="bp">self</span><span class="p">[</span><span class="n">_kw</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">_kw</span><span class="p">,</span> <span class="p">[])</span>
<span class="bp">self</span><span class="p">[</span><span class="n">_kw</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">answerer</span><span class="p">)</span></div>
<div class="viewcode-block" id="AnswerStorage.ask">
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.AnswerStorage.ask">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">ask</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="n">BaseAnswer</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;An answerer is identified via keywords, if there is a keyword at the</span>
<span class="sd"> first position in the ``query`` for which there is one or more</span>
<span class="sd"> answerers, then these are called, whereby the entire ``query`` is passed</span>
<span class="sd"> as argument to the answerer function.&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">keyword</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">for</span> <span class="n">keyword</span> <span class="ow">in</span> <span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">():</span>
<span class="k">if</span> <span class="n">keyword</span><span class="p">:</span>
<span class="k">break</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">keyword</span> <span class="ow">or</span> <span class="n">keyword</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">for</span> <span class="n">answerer</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">[</span><span class="n">keyword</span><span class="p">]:</span>
<span class="k">for</span> <span class="n">answer</span> <span class="ow">in</span> <span class="n">answerer</span><span class="o">.</span><span class="n">answer</span><span class="p">(</span><span class="n">query</span><span class="p">):</span>
<span class="c1"># In case of *answers* prefix ``answerer:`` is set, see searx.result_types.Result</span>
<span class="n">answer</span><span class="o">.</span><span class="n">engine</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;answerer: </span><span class="si">{</span><span class="n">keyword</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">answer</span><span class="p">)</span>
<span class="k">return</span> <span class="n">results</span></div>
<span class="nd">@property</span>
<span class="k">def</span><span class="w"> </span><span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="n">AnswererInfo</span><span class="p">]:</span>
<span class="k">return</span> <span class="p">[</span><span class="n">a</span><span class="o">.</span><span class="n">info</span><span class="p">()</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">answerer_list</span><span class="p">]</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,192 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.answerers.random &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.answerers.random</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.answerers.random</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="c1"># pylint: disable=missing-module-docstring</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">hashlib</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">random</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">string</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">uuid</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">Answer</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types.answer</span><span class="w"> </span><span class="kn">import</span> <span class="n">BaseAnswer</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">Answerer</span><span class="p">,</span> <span class="n">AnswererInfo</span>
<span class="k">def</span><span class="w"> </span><span class="nf">random_characters</span><span class="p">():</span>
<span class="n">random_string_letters</span> <span class="o">=</span> <span class="n">string</span><span class="o">.</span><span class="n">ascii_lowercase</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">digits</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">ascii_uppercase</span>
<span class="k">return</span> <span class="p">[</span><span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">random_string_letters</span><span class="p">)</span> <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">8</span><span class="p">,</span> <span class="mi">32</span><span class="p">))]</span>
<span class="k">def</span><span class="w"> </span><span class="nf">random_string</span><span class="p">():</span>
<span class="k">return</span> <span class="s1">&#39;&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">random_characters</span><span class="p">())</span>
<span class="k">def</span><span class="w"> </span><span class="nf">random_float</span><span class="p">():</span>
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">random</span><span class="p">())</span>
<span class="k">def</span><span class="w"> </span><span class="nf">random_int</span><span class="p">():</span>
<span class="n">random_int_max</span> <span class="o">=</span> <span class="mi">2</span><span class="o">**</span><span class="mi">31</span>
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="o">-</span><span class="n">random_int_max</span><span class="p">,</span> <span class="n">random_int_max</span><span class="p">))</span>
<span class="k">def</span><span class="w"> </span><span class="nf">random_sha256</span><span class="p">():</span>
<span class="n">m</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">()</span>
<span class="n">m</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="s1">&#39;&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">random_characters</span><span class="p">())</span><span class="o">.</span><span class="n">encode</span><span class="p">())</span>
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">m</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">())</span>
<span class="k">def</span><span class="w"> </span><span class="nf">random_uuid</span><span class="p">():</span>
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid4</span><span class="p">())</span>
<span class="k">def</span><span class="w"> </span><span class="nf">random_color</span><span class="p">():</span>
<span class="n">color</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">%06x</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mh">0xFFFFFF</span><span class="p">)</span>
<span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;#</span><span class="si">{</span><span class="n">color</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span>
<div class="viewcode-block" id="SXNGAnswerer">
<a class="viewcode-back" href="../../../dev/answerers/random.html#searx.answerers.random.SXNGAnswerer">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGAnswerer</span><span class="p">(</span><span class="n">Answerer</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Random value generator&quot;&quot;&quot;</span>
<span class="n">keywords</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;random&quot;</span><span class="p">]</span>
<span class="n">random_types</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;string&quot;</span><span class="p">:</span> <span class="n">random_string</span><span class="p">,</span>
<span class="s2">&quot;int&quot;</span><span class="p">:</span> <span class="n">random_int</span><span class="p">,</span>
<span class="s2">&quot;float&quot;</span><span class="p">:</span> <span class="n">random_float</span><span class="p">,</span>
<span class="s2">&quot;sha256&quot;</span><span class="p">:</span> <span class="n">random_sha256</span><span class="p">,</span>
<span class="s2">&quot;uuid&quot;</span><span class="p">:</span> <span class="n">random_uuid</span><span class="p">,</span>
<span class="s2">&quot;color&quot;</span><span class="p">:</span> <span class="n">random_color</span><span class="p">,</span>
<span class="p">}</span>
<div class="viewcode-block" id="SXNGAnswerer.info">
<a class="viewcode-back" href="../../../dev/answerers/random.html#searx.answerers.random.SXNGAnswerer.info">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">AnswererInfo</span><span class="p">(</span>
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__doc__</span><span class="p">),</span>
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Generate different random values&quot;</span><span class="p">),</span>
<span class="n">keywords</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">keywords</span><span class="p">,</span>
<span class="n">examples</span><span class="o">=</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;random </span><span class="si">{</span><span class="n">x</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">random_types</span><span class="p">],</span>
<span class="p">)</span></div>
<div class="viewcode-block" id="SXNGAnswerer.answer">
<a class="viewcode-back" href="../../../dev/answerers/random.html#searx.answerers.random.SXNGAnswerer.answer">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">answer</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="n">BaseAnswer</span><span class="p">]:</span>
<span class="n">parts</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">parts</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">2</span> <span class="ow">or</span> <span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">random_types</span><span class="p">:</span>
<span class="k">return</span> <span class="p">[]</span>
<span class="k">return</span> <span class="p">[</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">random_types</span><span class="p">[</span><span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">]]())]</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,178 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.answerers.statistics &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.answerers.statistics</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.answerers.statistics</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="c1"># pylint: disable=missing-module-docstring</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">functools</span><span class="w"> </span><span class="kn">import</span> <span class="n">reduce</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">operator</span><span class="w"> </span><span class="kn">import</span> <span class="n">mul</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.numbers</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">sxng_request</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">Answer</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types.answer</span><span class="w"> </span><span class="kn">import</span> <span class="n">BaseAnswer</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">Answerer</span><span class="p">,</span> <span class="n">AnswererInfo</span>
<span class="n">kw2func</span> <span class="o">=</span> <span class="p">[</span>
<span class="p">(</span><span class="s2">&quot;min&quot;</span><span class="p">,</span> <span class="nb">min</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;max&quot;</span><span class="p">,</span> <span class="nb">max</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;avg&quot;</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="nb">sum</span><span class="p">(</span><span class="n">args</span><span class="p">)</span> <span class="o">/</span> <span class="nb">len</span><span class="p">(</span><span class="n">args</span><span class="p">)),</span>
<span class="p">(</span><span class="s2">&quot;sum&quot;</span><span class="p">,</span> <span class="nb">sum</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;prod&quot;</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="n">reduce</span><span class="p">(</span><span class="n">mul</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="mi">1</span><span class="p">)),</span>
<span class="p">(</span><span class="s2">&quot;range&quot;</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="nb">max</span><span class="p">(</span><span class="n">args</span><span class="p">)</span> <span class="o">-</span> <span class="nb">min</span><span class="p">(</span><span class="n">args</span><span class="p">)),</span>
<span class="p">]</span>
<div class="viewcode-block" id="SXNGAnswerer">
<a class="viewcode-back" href="../../../dev/answerers/statistics.html#searx.answerers.statistics.SXNGAnswerer">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGAnswerer</span><span class="p">(</span><span class="n">Answerer</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Statistics functions&quot;&quot;&quot;</span>
<span class="n">keywords</span> <span class="o">=</span> <span class="p">[</span><span class="n">kw</span> <span class="k">for</span> <span class="n">kw</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">kw2func</span><span class="p">]</span>
<div class="viewcode-block" id="SXNGAnswerer.info">
<a class="viewcode-back" href="../../../dev/answerers/statistics.html#searx.answerers.statistics.SXNGAnswerer.info">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">AnswererInfo</span><span class="p">(</span>
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__doc__</span><span class="p">),</span>
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Compute </span><span class="si">{func}</span><span class="s2"> of the arguments&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">func</span><span class="o">=</span><span class="s1">&#39;/&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keywords</span><span class="p">))),</span>
<span class="n">keywords</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">keywords</span><span class="p">,</span>
<span class="n">examples</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;avg 123 548 2.04 24.2&quot;</span><span class="p">],</span>
<span class="p">)</span></div>
<div class="viewcode-block" id="SXNGAnswerer.answer">
<a class="viewcode-back" href="../../../dev/answerers/statistics.html#searx.answerers.statistics.SXNGAnswerer.answer">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">answer</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="n">BaseAnswer</span><span class="p">]:</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">parts</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">parts</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">2</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">ui_locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">sxng_request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s1">&#39;locale&#39;</span><span class="p">),</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;-&#39;</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">args</span> <span class="o">=</span> <span class="p">[</span><span class="n">babel</span><span class="o">.</span><span class="n">numbers</span><span class="o">.</span><span class="n">parse_decimal</span><span class="p">(</span><span class="n">num</span><span class="p">,</span> <span class="n">ui_locale</span><span class="p">,</span> <span class="n">numbering_system</span><span class="o">=</span><span class="s2">&quot;latn&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">num</span> <span class="ow">in</span> <span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">:]]</span>
<span class="k">except</span><span class="p">:</span> <span class="c1"># pylint: disable=bare-except</span>
<span class="c1"># seems one of the args is not a float type, can&#39;t be converted to float</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">func</span> <span class="ow">in</span> <span class="n">kw2func</span><span class="p">:</span>
<span class="k">if</span> <span class="n">k</span> <span class="o">==</span> <span class="n">parts</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">func</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">numbers</span><span class="o">.</span><span class="n">format_decimal</span><span class="p">(</span><span class="n">res</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">ui_locale</span><span class="p">)</span>
<span class="n">f_str</span> <span class="o">=</span> <span class="s1">&#39;, &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">numbers</span><span class="o">.</span><span class="n">format_decimal</span><span class="p">(</span><span class="n">arg</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">ui_locale</span><span class="p">)</span> <span class="k">for</span> <span class="n">arg</span> <span class="ow">in</span> <span class="n">args</span><span class="p">)</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;[</span><span class="si">{</span><span class="n">ui_locale</span><span class="si">}</span><span class="s2">] </span><span class="si">{</span><span class="n">k</span><span class="si">}</span><span class="s2">(</span><span class="si">{</span><span class="n">f_str</span><span class="si">}</span><span class="s2">) = </span><span class="si">{</span><span class="n">res</span><span class="si">}</span><span class="s2"> &quot;</span><span class="p">))</span>
<span class="k">break</span>
<span class="k">return</span> <span class="n">results</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,482 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.autocomplete &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.autocomplete</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.autocomplete</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;This module implements functions needed for the autocompleter.&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=use-dict-literal</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">html</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span><span class="p">,</span> <span class="n">quote_plus</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">lxml.etree</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">lxml.html</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">httpx</span><span class="w"> </span><span class="kn">import</span> <span class="n">HTTPError</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">settings</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">engines</span><span class="p">,</span>
<span class="n">google</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="k">as</span> <span class="n">http_get</span><span class="p">,</span> <span class="n">post</span> <span class="k">as</span> <span class="n">http_post</span> <span class="c1"># pyright: ignore[reportUnknownVariableType]</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineResponseException</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extr</span><span class="p">,</span> <span class="n">gen_useragent</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="k">def</span><span class="w"> </span><span class="nf">update_kwargs</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span> <span class="c1"># type: ignore</span>
<span class="k">if</span> <span class="s1">&#39;timeout&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="p">:</span>
<span class="n">kwargs</span><span class="p">[</span><span class="s1">&#39;timeout&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">settings</span><span class="p">[</span><span class="s1">&#39;outgoing&#39;</span><span class="p">][</span><span class="s1">&#39;request_timeout&#39;</span><span class="p">]</span>
<span class="n">kwargs</span><span class="p">[</span><span class="s1">&#39;raise_for_httperror&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">:</span> <span class="c1"># type: ignore</span>
<span class="n">update_kwargs</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
<span class="k">return</span> <span class="n">http_get</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
<span class="k">def</span><span class="w"> </span><span class="nf">post</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">:</span> <span class="c1"># type: ignore</span>
<span class="n">update_kwargs</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
<span class="k">return</span> <span class="n">http_post</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
<span class="k">def</span><span class="w"> </span><span class="nf">baidu</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="c1"># baidu search autocompleter</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;https://www.baidu.com/sugrec?&quot;</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">base_url</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;ie&#39;</span><span class="p">:</span> <span class="s1">&#39;utf-8&#39;</span><span class="p">,</span> <span class="s1">&#39;json&#39;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">&#39;prod&#39;</span><span class="p">:</span> <span class="s1">&#39;pc&#39;</span><span class="p">,</span> <span class="s1">&#39;wd&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">}))</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="n">data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">if</span> <span class="s1">&#39;g&#39;</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;g&#39;</span><span class="p">]:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="s1">&#39;q&#39;</span><span class="p">])</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">brave</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="c1"># brave search autocompleter</span>
<span class="n">url</span> <span class="o">=</span> <span class="s1">&#39;https://search.brave.com/api/suggest?&#39;</span>
<span class="n">url</span> <span class="o">+=</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">})</span>
<span class="n">country</span> <span class="o">=</span> <span class="s1">&#39;all&#39;</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;cookies&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;country&#39;</span><span class="p">:</span> <span class="n">country</span><span class="p">}}</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="n">data</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">[</span><span class="mi">1</span><span class="p">]:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">dbpedia</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="n">autocomplete_url</span> <span class="o">=</span> <span class="s1">&#39;https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?&#39;</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">autocomplete_url</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">QueryString</span><span class="o">=</span><span class="n">query</span><span class="p">)))</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">etree</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;//Result/Label//text()&#39;</span><span class="p">)]</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">duckduckgo</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Autocomplete from DuckDuckGo. Supports DuckDuckGo&#39;s languages&quot;&quot;&quot;</span>
<span class="n">traits</span> <span class="o">=</span> <span class="n">engines</span><span class="p">[</span><span class="s1">&#39;duckduckgo&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">traits</span>
<span class="n">args</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;kl&#39;</span><span class="p">:</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">),</span>
<span class="p">}</span>
<span class="n">url</span> <span class="o">=</span> <span class="s1">&#39;https://duckduckgo.com/ac/?type=list&amp;&#39;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="n">j</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">j</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">j</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="k">return</span> <span class="n">results</span>
<div class="viewcode-block" id="google_complete">
<a class="viewcode-back" href="../../dev/engines/online/google.html#searx.autocomplete.google_complete">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">google_complete</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Autocomplete from Google. Supports Google&#39;s languages and subdomains</span>
<span class="sd"> (:py:obj:`searx.engines.google.get_google_info`) by using the async REST</span>
<span class="sd"> API::</span>
<span class="sd"> https://{subdomain}/complete/search?{args}</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">google_info</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">google</span><span class="o">.</span><span class="n">get_google_info</span><span class="p">({</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">:</span> <span class="n">sxng_locale</span><span class="p">},</span> <span class="n">engines</span><span class="p">[</span><span class="s1">&#39;google&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">traits</span><span class="p">)</span>
<span class="n">url</span> <span class="o">=</span> <span class="s1">&#39;https://</span><span class="si">{subdomain}</span><span class="s1">/complete/search?</span><span class="si">{args}</span><span class="s1">&#39;</span>
<span class="n">args</span> <span class="o">=</span> <span class="n">urlencode</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;client&#39;</span><span class="p">:</span> <span class="s1">&#39;gws-wiz&#39;</span><span class="p">,</span>
<span class="s1">&#39;hl&#39;</span><span class="p">:</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;hl&#39;</span><span class="p">],</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subdomain</span><span class="o">=</span><span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;subdomain&#39;</span><span class="p">],</span> <span class="n">args</span><span class="o">=</span><span class="n">args</span><span class="p">))</span>
<span class="k">if</span> <span class="n">resp</span> <span class="ow">and</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="n">json_txt</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">[</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;[&#39;</span><span class="p">)</span> <span class="p">:</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;]&#39;</span><span class="p">,</span> <span class="o">-</span><span class="mi">3</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">json_txt</span><span class="p">)</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span><span class="o">.</span><span class="n">text_content</span><span class="p">())</span>
<span class="k">return</span> <span class="n">results</span></div>
<div class="viewcode-block" id="mwmbl">
<a class="viewcode-back" href="../../dev/engines/online/mwmbl.html#searx.autocomplete.mwmbl">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">mwmbl</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Autocomplete from Mwmbl_.&quot;&quot;&quot;</span>
<span class="c1"># mwmbl autocompleter</span>
<span class="n">url</span> <span class="o">=</span> <span class="s1">&#39;https://api.mwmbl.org/search/complete?</span><span class="si">{query}</span><span class="s1">&#39;</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">})))</span><span class="o">.</span><span class="n">json</span><span class="p">()[</span><span class="mi">1</span><span class="p">]</span>
<span class="c1"># results starting with `go:` are direct urls and not useful for auto completion</span>
<span class="k">return</span> <span class="p">[</span><span class="n">result</span> <span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">results</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">result</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;go: &quot;</span><span class="p">)</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">result</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;search: &quot;</span><span class="p">)]</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">naver</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="c1"># Naver search autocompleter</span>
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;https://ac.search.naver.com/nx/ac?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;q&#39;</span><span class="p">:</span><span class="w"> </span><span class="n">query</span><span class="p">,</span><span class="w"> </span><span class="s1">&#39;r_format&#39;</span><span class="p">:</span><span class="w"> </span><span class="s1">&#39;json&#39;</span><span class="p">,</span><span class="w"> </span><span class="s1">&#39;st&#39;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">})</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="n">data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">if</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;items&#39;</span><span class="p">):</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;items&#39;</span><span class="p">][</span><span class="mi">0</span><span class="p">]:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">qihu360search</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="c1"># 360Search search autocompleter</span>
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;https://sug.so.360.cn/suggest?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;format&#39;</span><span class="p">:</span><span class="w"> </span><span class="s1">&#39;json&#39;</span><span class="p">,</span><span class="w"> </span><span class="s1">&#39;word&#39;</span><span class="p">:</span><span class="w"> </span><span class="n">query</span><span class="p">})</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="n">data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">if</span> <span class="s1">&#39;result&#39;</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;result&#39;</span><span class="p">]:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="s1">&#39;word&#39;</span><span class="p">])</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">quark</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="c1"># Quark search autocompleter</span>
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;https://sugs.m.sm.cn/web?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;q&#39;</span><span class="p">:</span><span class="w"> </span><span class="n">query</span><span class="p">})</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;r&#39;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="s1">&#39;w&#39;</span><span class="p">])</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">seznam</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="c1"># seznam search autocompleter</span>
<span class="n">url</span> <span class="o">=</span> <span class="s1">&#39;https://suggest.seznam.cz/fulltext/cs?</span><span class="si">{query}</span><span class="s1">&#39;</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span>
<span class="n">url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">query</span><span class="o">=</span><span class="n">urlencode</span><span class="p">(</span>
<span class="p">{</span><span class="s1">&#39;phrase&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span> <span class="s1">&#39;cursorPosition&#39;</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">query</span><span class="p">),</span> <span class="s1">&#39;format&#39;</span><span class="p">:</span> <span class="s1">&#39;json-2&#39;</span><span class="p">,</span> <span class="s1">&#39;highlight&#39;</span><span class="p">:</span> <span class="s1">&#39;1&#39;</span><span class="p">,</span> <span class="s1">&#39;count&#39;</span><span class="p">:</span> <span class="s1">&#39;6&#39;</span><span class="p">}</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[</span>
<span class="s1">&#39;&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">part</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;text&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;text&#39;</span><span class="p">,</span> <span class="p">[])])</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;result&#39;</span><span class="p">,</span> <span class="p">[])</span>
<span class="k">if</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;itemType&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="o">==</span> <span class="s1">&#39;ItemType.TEXT&#39;</span>
<span class="p">]</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">sogou</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="c1"># Sogou search autocompleter</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;https://sor.html5.qq.com/api/getsug?&quot;</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">base_url</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;m&#39;</span><span class="p">:</span> <span class="s1">&#39;searxng&#39;</span><span class="p">,</span> <span class="s1">&#39;key&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">}))</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="n">raw_json</span> <span class="o">=</span> <span class="n">extr</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="s2">&quot;[&quot;</span><span class="p">,</span> <span class="s2">&quot;]&quot;</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;[</span><span class="si">{</span><span class="n">raw_json</span><span class="si">}</span><span class="s2">]]&quot;</span><span class="p">)</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="k">except</span> <span class="n">json</span><span class="o">.</span><span class="n">JSONDecodeError</span><span class="p">:</span>
<span class="k">pass</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">startpage</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Autocomplete from Startpage&#39;s Firefox extension.</span>
<span class="sd"> Supports the languages specified in lang_map.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">lang_map</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;da&#39;</span><span class="p">:</span> <span class="s1">&#39;dansk&#39;</span><span class="p">,</span>
<span class="s1">&#39;de&#39;</span><span class="p">:</span> <span class="s1">&#39;deutsch&#39;</span><span class="p">,</span>
<span class="s1">&#39;en&#39;</span><span class="p">:</span> <span class="s1">&#39;english&#39;</span><span class="p">,</span>
<span class="s1">&#39;es&#39;</span><span class="p">:</span> <span class="s1">&#39;espanol&#39;</span><span class="p">,</span>
<span class="s1">&#39;fr&#39;</span><span class="p">:</span> <span class="s1">&#39;francais&#39;</span><span class="p">,</span>
<span class="s1">&#39;nb&#39;</span><span class="p">:</span> <span class="s1">&#39;norsk&#39;</span><span class="p">,</span>
<span class="s1">&#39;nl&#39;</span><span class="p">:</span> <span class="s1">&#39;nederlands&#39;</span><span class="p">,</span>
<span class="s1">&#39;pl&#39;</span><span class="p">:</span> <span class="s1">&#39;polski&#39;</span><span class="p">,</span>
<span class="s1">&#39;pt&#39;</span><span class="p">:</span> <span class="s1">&#39;portugues&#39;</span><span class="p">,</span>
<span class="s1">&#39;sv&#39;</span><span class="p">:</span> <span class="s1">&#39;svenska&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">base_lang</span> <span class="o">=</span> <span class="n">sxng_locale</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">lui</span> <span class="o">=</span> <span class="n">lang_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">base_lang</span><span class="p">,</span> <span class="s1">&#39;english&#39;</span><span class="p">)</span>
<span class="n">url_params</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;format&#39;</span><span class="p">:</span> <span class="s1">&#39;opensearch&#39;</span><span class="p">,</span>
<span class="s1">&#39;segment&#39;</span><span class="p">:</span> <span class="s1">&#39;startpage.defaultffx&#39;</span><span class="p">,</span>
<span class="s1">&#39;lui&#39;</span><span class="p">:</span> <span class="n">lui</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;https://www.startpage.com/suggestions?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">url_params</span><span class="p">)</span><span class="si">}</span><span class="s1">&#39;</span>
<span class="c1"># Needs user agent, returns a 204 otherwise</span>
<span class="n">h</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;User-Agent&#39;</span><span class="p">:</span> <span class="n">gen_useragent</span><span class="p">()}</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">h</span><span class="p">)</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> <span class="o">&gt;=</span> <span class="mi">2</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="k">except</span> <span class="n">json</span><span class="o">.</span><span class="n">JSONDecodeError</span><span class="p">:</span>
<span class="k">pass</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">stract</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="c1"># stract autocompleter (beta)</span>
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;https://stract.com/beta/api/autosuggest?q=</span><span class="si">{</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">query</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">post</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[</span><span class="n">html</span><span class="o">.</span><span class="n">unescape</span><span class="p">(</span><span class="n">suggestion</span><span class="p">[</span><span class="s1">&#39;raw&#39;</span><span class="p">])</span> <span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()]</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">swisscows</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="c1"># swisscows autocompleter</span>
<span class="n">url</span> <span class="o">=</span> <span class="s1">&#39;https://swisscows.ch/api/suggest?</span><span class="si">{query}</span><span class="s1">&amp;itemsCount=5&#39;</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">})))</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">qwant</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Autocomplete from Qwant. Supports Qwant&#39;s regions.&quot;&quot;&quot;</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">engines</span><span class="p">[</span><span class="s1">&#39;qwant&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="s1">&#39;en_US&#39;</span><span class="p">)</span>
<span class="n">url</span> <span class="o">=</span> <span class="s1">&#39;https://api.qwant.com/v3/suggest?</span><span class="si">{query}</span><span class="s1">&#39;</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span> <span class="s1">&#39;locale&#39;</span><span class="p">:</span> <span class="n">locale</span><span class="p">,</span> <span class="s1">&#39;version&#39;</span><span class="p">:</span> <span class="s1">&#39;2&#39;</span><span class="p">})))</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">if</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;status&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;success&#39;</span><span class="p">:</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;items&#39;</span><span class="p">]:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="s1">&#39;value&#39;</span><span class="p">])</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">wikipedia</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Autocomplete from Wikipedia. Supports Wikipedia&#39;s languages (aka netloc).&quot;&quot;&quot;</span>
<span class="n">eng_traits</span> <span class="o">=</span> <span class="n">engines</span><span class="p">[</span><span class="s1">&#39;wikipedia&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">traits</span>
<span class="n">wiki_lang</span> <span class="o">=</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="s1">&#39;en&#39;</span><span class="p">)</span>
<span class="n">wiki_netloc</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;wiki_netloc&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">wiki_lang</span><span class="p">,</span> <span class="s1">&#39;en.wikipedia.org&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">args</span> <span class="o">=</span> <span class="n">urlencode</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;action&#39;</span><span class="p">:</span> <span class="s1">&#39;opensearch&#39;</span><span class="p">,</span>
<span class="s1">&#39;format&#39;</span><span class="p">:</span> <span class="s1">&#39;json&#39;</span><span class="p">,</span>
<span class="s1">&#39;formatversion&#39;</span><span class="p">:</span> <span class="s1">&#39;2&#39;</span><span class="p">,</span>
<span class="s1">&#39;search&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;namespace&#39;</span><span class="p">:</span> <span class="s1">&#39;0&#39;</span><span class="p">,</span>
<span class="s1">&#39;limit&#39;</span><span class="p">:</span> <span class="s1">&#39;10&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;https://</span><span class="si">{</span><span class="n">wiki_netloc</span><span class="si">}</span><span class="s1">/w/api.php?</span><span class="si">{</span><span class="n">args</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">yandex</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="c1"># yandex autocompleter</span>
<span class="n">url</span> <span class="o">=</span> <span class="s2">&quot;https://suggest.yandex.com/suggest-ff.cgi?</span><span class="si">{0}</span><span class="s2">&quot;</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">urlencode</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">part</span><span class="o">=</span><span class="n">query</span><span class="p">))))</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">resp</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">backends</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Callable</span><span class="p">[[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;360search&#39;</span><span class="p">:</span> <span class="n">qihu360search</span><span class="p">,</span>
<span class="s1">&#39;baidu&#39;</span><span class="p">:</span> <span class="n">baidu</span><span class="p">,</span>
<span class="s1">&#39;brave&#39;</span><span class="p">:</span> <span class="n">brave</span><span class="p">,</span>
<span class="s1">&#39;dbpedia&#39;</span><span class="p">:</span> <span class="n">dbpedia</span><span class="p">,</span>
<span class="s1">&#39;duckduckgo&#39;</span><span class="p">:</span> <span class="n">duckduckgo</span><span class="p">,</span>
<span class="s1">&#39;google&#39;</span><span class="p">:</span> <span class="n">google_complete</span><span class="p">,</span>
<span class="s1">&#39;mwmbl&#39;</span><span class="p">:</span> <span class="n">mwmbl</span><span class="p">,</span>
<span class="s1">&#39;naver&#39;</span><span class="p">:</span> <span class="n">naver</span><span class="p">,</span>
<span class="s1">&#39;quark&#39;</span><span class="p">:</span> <span class="n">quark</span><span class="p">,</span>
<span class="s1">&#39;qwant&#39;</span><span class="p">:</span> <span class="n">qwant</span><span class="p">,</span>
<span class="s1">&#39;seznam&#39;</span><span class="p">:</span> <span class="n">seznam</span><span class="p">,</span>
<span class="s1">&#39;sogou&#39;</span><span class="p">:</span> <span class="n">sogou</span><span class="p">,</span>
<span class="s1">&#39;startpage&#39;</span><span class="p">:</span> <span class="n">startpage</span><span class="p">,</span>
<span class="s1">&#39;stract&#39;</span><span class="p">:</span> <span class="n">stract</span><span class="p">,</span>
<span class="s1">&#39;swisscows&#39;</span><span class="p">:</span> <span class="n">swisscows</span><span class="p">,</span>
<span class="s1">&#39;wikipedia&#39;</span><span class="p">:</span> <span class="n">wikipedia</span><span class="p">,</span>
<span class="s1">&#39;yandex&#39;</span><span class="p">:</span> <span class="n">yandex</span><span class="p">,</span>
<span class="p">}</span>
<span class="k">def</span><span class="w"> </span><span class="nf">search_autocomplete</span><span class="p">(</span><span class="n">backend_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="n">backend</span> <span class="o">=</span> <span class="n">backends</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">backend_name</span><span class="p">)</span>
<span class="k">if</span> <span class="n">backend</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="p">[]</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">return</span> <span class="n">backend</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">)</span>
<span class="k">except</span> <span class="p">(</span><span class="n">HTTPError</span><span class="p">,</span> <span class="n">SearxEngineResponseException</span><span class="p">):</span>
<span class="k">return</span> <span class="p">[]</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../index.html">
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../index.html">Overview</a>
<ul>
<li><a href="../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,165 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.babel_extract &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.babel_extract</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.babel_extract</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;This module implements the :origin:`searxng_msg &lt;babel.cfg&gt;` extractor to</span>
<span class="sd">extract messages from:</span>
<span class="sd">- :origin:`searx/searxng.msg`</span>
<span class="sd">The ``searxng.msg`` files are selected by Babel_, see Babel&#39;s configuration in</span>
<span class="sd">:origin:`babel.cfg`::</span>
<span class="sd"> searxng_msg = searx.babel_extract.extract</span>
<span class="sd"> ...</span>
<span class="sd"> [searxng_msg: **/searxng.msg]</span>
<span class="sd">A ``searxng.msg`` file is a python file that is *executed* by the</span>
<span class="sd">:py:obj:`extract` function. Additional ``searxng.msg`` files can be added by:</span>
<span class="sd">1. Adding a ``searxng.msg`` file in one of the SearXNG python packages and</span>
<span class="sd">2. implement a method in :py:obj:`extract` that yields messages from this file.</span>
<span class="sd">.. _Babel: https://babel.pocoo.org/en/latest/index.html</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">os</span><span class="w"> </span><span class="kn">import</span> <span class="n">path</span>
<span class="n">SEARXNG_MSG_FILE</span> <span class="o">=</span> <span class="s2">&quot;searxng.msg&quot;</span>
<span class="n">_MSG_FILES</span> <span class="o">=</span> <span class="p">[</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="vm">__file__</span><span class="p">),</span> <span class="n">SEARXNG_MSG_FILE</span><span class="p">)]</span>
<div class="viewcode-block" id="extract">
<a class="viewcode-back" href="../../src/searx.babel_extract.html#searx.babel_extract.extract">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">extract</span><span class="p">(</span>
<span class="c1"># pylint: disable=unused-argument</span>
<span class="n">fileobj</span><span class="p">,</span>
<span class="n">keywords</span><span class="p">,</span>
<span class="n">comment_tags</span><span class="p">,</span>
<span class="n">options</span><span class="p">,</span>
<span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Extract messages from ``searxng.msg`` files by a custom extractor_.</span>
<span class="sd"> .. _extractor:</span>
<span class="sd"> https://babel.pocoo.org/en/latest/messages.html#writing-extraction-methods</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">fileobj</span><span class="o">.</span><span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">_MSG_FILES</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;don&#39;t know how to extract messages from </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">fileobj</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="n">namespace</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">exec</span><span class="p">(</span><span class="n">fileobj</span><span class="o">.</span><span class="n">read</span><span class="p">(),</span> <span class="p">{},</span> <span class="n">namespace</span><span class="p">)</span> <span class="c1"># pylint: disable=exec-used</span>
<span class="k">for</span> <span class="n">obj_name</span> <span class="ow">in</span> <span class="n">namespace</span><span class="p">[</span><span class="s1">&#39;__all__&#39;</span><span class="p">]:</span>
<span class="n">obj</span> <span class="o">=</span> <span class="n">namespace</span><span class="p">[</span><span class="n">obj_name</span><span class="p">]</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
<span class="k">for</span> <span class="n">msg</span> <span class="ow">in</span> <span class="n">obj</span><span class="p">:</span>
<span class="c1"># (lineno, funcname, message, comments)</span>
<span class="k">yield</span> <span class="mi">0</span><span class="p">,</span> <span class="s1">&#39;_&#39;</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">obj_name</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">]</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">msg</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">yield</span> <span class="mi">0</span><span class="p">,</span> <span class="s1">&#39;_&#39;</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">obj_name</span><span class="si">}</span><span class="s2">[&#39;</span><span class="si">{</span><span class="n">k</span><span class="si">}</span><span class="s2">&#39;]&quot;</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">obj_name</span><span class="si">}</span><span class="s2"> should be list or dict&quot;</span><span class="p">)</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../index.html">
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../index.html">Overview</a>
<ul>
<li><a href="../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,196 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.botdetection._helpers &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.botdetection._helpers</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.botdetection._helpers</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="c1"># pylint: disable=missing-module-docstring, invalid-name</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;log_error_only_once&quot;</span><span class="p">,</span> <span class="s2">&quot;dump_request&quot;</span><span class="p">,</span> <span class="s2">&quot;get_network&quot;</span><span class="p">,</span> <span class="s2">&quot;logger&quot;</span><span class="p">,</span> <span class="s2">&quot;too_many_requests&quot;</span><span class="p">]</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">IPv4Network</span><span class="p">,</span>
<span class="n">IPv6Network</span><span class="p">,</span>
<span class="n">IPv4Address</span><span class="p">,</span>
<span class="n">IPv6Address</span><span class="p">,</span>
<span class="n">ip_network</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">werkzeug</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">config</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">&#39;botdetection&#39;</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">dump_request</span><span class="p">(</span><span class="n">request</span><span class="p">:</span> <span class="n">flask</span><span class="o">.</span><span class="n">Request</span><span class="p">):</span>
<span class="k">return</span> <span class="p">(</span>
<span class="n">request</span><span class="o">.</span><span class="n">path</span>
<span class="o">+</span> <span class="s2">&quot; || X-Forwarded-For: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;X-Forwarded-For&#39;</span><span class="p">)</span>
<span class="o">+</span> <span class="s2">&quot; || X-Real-IP: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;X-Real-IP&#39;</span><span class="p">)</span>
<span class="o">+</span> <span class="s2">&quot; || form: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">form</span>
<span class="o">+</span> <span class="s2">&quot; || Accept: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Accept&#39;</span><span class="p">)</span>
<span class="o">+</span> <span class="s2">&quot; || Accept-Language: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Accept-Language&#39;</span><span class="p">)</span>
<span class="o">+</span> <span class="s2">&quot; || Accept-Encoding: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Accept-Encoding&#39;</span><span class="p">)</span>
<span class="o">+</span> <span class="s2">&quot; || Content-Type: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Content-Type&#39;</span><span class="p">)</span>
<span class="o">+</span> <span class="s2">&quot; || Content-Length: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Content-Length&#39;</span><span class="p">)</span>
<span class="o">+</span> <span class="s2">&quot; || Connection: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Connection&#39;</span><span class="p">)</span>
<span class="o">+</span> <span class="s2">&quot; || User-Agent: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;User-Agent&#39;</span><span class="p">)</span>
<span class="o">+</span> <span class="s2">&quot; || Sec-Fetch-Site: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Sec-Fetch-Site&#39;</span><span class="p">)</span>
<span class="o">+</span> <span class="s2">&quot; || Sec-Fetch-Mode: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Sec-Fetch-Mode&#39;</span><span class="p">)</span>
<span class="o">+</span> <span class="s2">&quot; || Sec-Fetch-Dest: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Sec-Fetch-Dest&#39;</span><span class="p">)</span>
<span class="p">)</span>
<div class="viewcode-block" id="too_many_requests">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.too_many_requests">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">too_many_requests</span><span class="p">(</span><span class="n">network</span><span class="p">:</span> <span class="n">IPv4Network</span> <span class="o">|</span> <span class="n">IPv6Network</span><span class="p">,</span> <span class="n">log_msg</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">werkzeug</span><span class="o">.</span><span class="n">Response</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns a HTTP 429 response object and writes a ERROR message to the</span>
<span class="sd"> &#39;botdetection&#39; logger. This function is used in part by the filter methods</span>
<span class="sd"> to return the default ``Too Many Requests`` response.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;BLOCK </span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">network</span><span class="o">.</span><span class="n">compressed</span><span class="p">,</span> <span class="n">log_msg</span><span class="p">)</span>
<span class="k">return</span> <span class="n">flask</span><span class="o">.</span><span class="n">make_response</span><span class="p">((</span><span class="s1">&#39;Too Many Requests&#39;</span><span class="p">,</span> <span class="mi">429</span><span class="p">))</span></div>
<div class="viewcode-block" id="get_network">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.get_network">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_network</span><span class="p">(</span><span class="n">real_ip</span><span class="p">:</span> <span class="n">IPv4Address</span> <span class="o">|</span> <span class="n">IPv6Address</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="s2">&quot;config.Config&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">IPv4Network</span> <span class="o">|</span> <span class="n">IPv6Network</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns the (client) network of whether the ``real_ip`` is part of.</span>
<span class="sd"> The ``ipv4_prefix`` and ``ipv6_prefix`` define the number of leading bits in</span>
<span class="sd"> an address that are compared to determine whether or not an address is part</span>
<span class="sd"> of a (client) network.</span>
<span class="sd"> .. code:: toml</span>
<span class="sd"> [botdetection]</span>
<span class="sd"> ipv4_prefix = 32</span>
<span class="sd"> ipv6_prefix = 48</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">prefix</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">cfg</span><span class="p">[</span><span class="s2">&quot;botdetection.ipv4_prefix&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">real_ip</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="mi">6</span><span class="p">:</span>
<span class="n">prefix</span> <span class="o">=</span> <span class="n">cfg</span><span class="p">[</span><span class="s2">&quot;botdetection.ipv6_prefix&quot;</span><span class="p">]</span>
<span class="n">network</span> <span class="o">=</span> <span class="n">ip_network</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">real_ip</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="n">prefix</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">strict</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="c1"># logger.debug(&quot;get_network(): %s&quot;, network.compressed)</span>
<span class="k">return</span> <span class="n">network</span></div>
<span class="n">_logged_errors</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">def</span><span class="w"> </span><span class="nf">log_error_only_once</span><span class="p">(</span><span class="n">err_msg</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="k">if</span> <span class="n">err_msg</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">_logged_errors</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">err_msg</span><span class="p">)</span>
<span class="n">_logged_errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">err_msg</span><span class="p">)</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,528 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.botdetection.config &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.botdetection.config</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.botdetection.config</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Configuration class :py:class:`Config` with deep-update, schema validation</span>
<span class="sd">and deprecated names.</span>
<span class="sd">The :py:class:`Config` class implements a configuration that is based on</span>
<span class="sd">structured dictionaries. The configuration schema is defined in a dictionary</span>
<span class="sd">structure and the configuration data is given in a dictionary structure.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">copy</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">pathlib</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">..compat</span><span class="w"> </span><span class="kn">import</span> <span class="n">tomllib</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;Config&#39;</span><span class="p">,</span> <span class="s1">&#39;UNSET&#39;</span><span class="p">,</span> <span class="s1">&#39;SchemaIssue&#39;</span><span class="p">,</span> <span class="s1">&#39;set_global_cfg&#39;</span><span class="p">,</span> <span class="s1">&#39;get_global_cfg&#39;</span><span class="p">]</span>
<span class="n">log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
<span class="n">CFG</span><span class="p">:</span> <span class="s2">&quot;Config | None&quot;</span> <span class="o">=</span> <span class="kc">None</span>
<span class="sd">&quot;&quot;&quot;Global config of the botdetection.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">set_global_cfg</span><span class="p">(</span><span class="n">cfg</span><span class="p">:</span> <span class="s2">&quot;Config&quot;</span><span class="p">):</span>
<span class="k">global</span> <span class="n">CFG</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="n">CFG</span> <span class="o">=</span> <span class="n">cfg</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_global_cfg</span><span class="p">()</span> <span class="o">-&gt;</span> <span class="s2">&quot;Config&quot;</span><span class="p">:</span>
<span class="k">if</span> <span class="n">CFG</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Botdetection&#39;s config is not yet initialized.&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">CFG</span>
<span class="nd">@typing</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">FALSE</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Class of ``False`` singleton&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=multiple-statements</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">msg</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">msg</span> <span class="o">=</span> <span class="n">msg</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__bool__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">msg</span>
<span class="fm">__repr__</span> <span class="o">=</span> <span class="fm">__str__</span>
<span class="n">UNSET</span> <span class="o">=</span> <span class="n">FALSE</span><span class="p">(</span><span class="s1">&#39;&lt;UNSET&gt;&#39;</span><span class="p">)</span>
<div class="viewcode-block" id="SchemaIssue">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.SchemaIssue">[docs]</a>
<span class="nd">@typing</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">SchemaIssue</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Exception to store and/or raise a message from a schema issue.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">level</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s1">&#39;warn&#39;</span><span class="p">,</span> <span class="s1">&#39;invalid&#39;</span><span class="p">],</span> <span class="n">msg</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">level</span> <span class="o">=</span> <span class="n">level</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;[cfg schema </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">level</span><span class="si">}</span><span class="s2">] </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span></div>
<div class="viewcode-block" id="Config">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">Config</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Base class used for configuration&quot;&quot;&quot;</span>
<span class="n">UNSET</span><span class="p">:</span> <span class="nb">object</span> <span class="o">=</span> <span class="n">UNSET</span>
<span class="nd">@classmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">from_toml</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">schema_file</span><span class="p">:</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">,</span> <span class="n">cfg_file</span><span class="p">:</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">,</span> <span class="n">deprecated</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="s2">&quot;Config&quot;</span><span class="p">:</span>
<span class="c1"># init schema</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;load schema file: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">schema_file</span><span class="p">)</span>
<span class="n">cfg</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">(</span><span class="n">cfg_schema</span><span class="o">=</span><span class="n">toml_load</span><span class="p">(</span><span class="n">schema_file</span><span class="p">),</span> <span class="n">deprecated</span><span class="o">=</span><span class="n">deprecated</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">cfg_file</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
<span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;missing config file: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">cfg_file</span><span class="p">)</span>
<span class="k">return</span> <span class="n">cfg</span>
<span class="c1"># load configuration</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;load config file: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">cfg_file</span><span class="p">)</span>
<span class="n">upd_cfg</span> <span class="o">=</span> <span class="n">toml_load</span><span class="p">(</span><span class="n">cfg_file</span><span class="p">)</span>
<span class="n">is_valid</span><span class="p">,</span> <span class="n">issue_list</span> <span class="o">=</span> <span class="n">cfg</span><span class="o">.</span><span class="n">validate</span><span class="p">(</span><span class="n">upd_cfg</span><span class="p">)</span>
<span class="k">for</span> <span class="n">msg</span> <span class="ow">in</span> <span class="n">issue_list</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">msg</span><span class="p">))</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_valid</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;schema of </span><span class="si">{</span><span class="n">cfg_file</span><span class="si">}</span><span class="s2"> is invalid!&quot;</span><span class="p">)</span>
<span class="n">cfg</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">upd_cfg</span><span class="p">)</span>
<span class="k">return</span> <span class="n">cfg</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg_schema</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">],</span> <span class="n">deprecated</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Constructor of class Config.</span>
<span class="sd"> :param cfg_schema: Schema of the configuration</span>
<span class="sd"> :param deprecated: dictionary that maps deprecated configuration names to a messages</span>
<span class="sd"> These values are needed for validation, see :py:obj:`validate`.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">cfg_schema</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">cfg_schema</span>
<span class="bp">self</span><span class="o">.</span><span class="n">deprecated</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">deprecated</span>
<span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">cfg_schema</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
<div class="viewcode-block" id="Config.validate">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config.validate">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">validate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Validation of dictionary ``cfg`` on :py:obj:`Config.SCHEMA`.</span>
<span class="sd"> Validation is done by :py:obj:`validate`.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">validate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg_schema</span><span class="p">,</span> <span class="n">cfg</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">deprecated</span><span class="p">)</span></div>
<div class="viewcode-block" id="Config.update">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config.update">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">update</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">upd_cfg</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Update this configuration by ``upd_cfg``.&quot;&quot;&quot;</span>
<span class="n">dict_deepupdate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="p">,</span> <span class="n">upd_cfg</span><span class="p">)</span></div>
<div class="viewcode-block" id="Config.default">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config.default">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">default</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns default value of field ``name`` in ``self.cfg_schema``.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">value</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg_schema</span><span class="p">)</span></div>
<div class="viewcode-block" id="Config.get">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config.get">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="n">UNSET</span><span class="p">,</span> <span class="n">replace</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns the value to which ``name`` points in the configuration.</span>
<span class="sd"> If there is no such ``name`` in the config and the ``default`` is</span>
<span class="sd"> :py:obj:`UNSET`, a :py:obj:`KeyError` is raised.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">parent</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_parent_dict</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
<span class="n">val</span> <span class="o">=</span> <span class="n">parent</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="n">UNSET</span><span class="p">)</span>
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
<span class="k">if</span> <span class="n">default</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
<span class="n">val</span> <span class="o">=</span> <span class="n">default</span>
<span class="k">if</span> <span class="n">replace</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">val</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="n">val</span> <span class="o">=</span> <span class="n">val</span> <span class="o">%</span> <span class="bp">self</span>
<span class="k">return</span> <span class="n">val</span></div>
<div class="viewcode-block" id="Config.set">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config.set">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">val</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Set the value to which ``name`` points in the configuration.</span>
<span class="sd"> If there is no such ``name`` in the config, a :py:obj:`KeyError` is</span>
<span class="sd"> raised.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">parent</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_parent_dict</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
<span class="n">parent</span><span class="p">[</span><span class="n">name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]]</span> <span class="o">=</span> <span class="n">val</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">_get_parent_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]:</span>
<span class="n">parent_name</span> <span class="o">=</span> <span class="s1">&#39;.&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
<span class="k">if</span> <span class="n">parent_name</span><span class="p">:</span>
<span class="n">parent</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span><span class="p">(</span><span class="n">parent_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">parent</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span>
<span class="k">if</span> <span class="p">(</span><span class="n">parent</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">)</span> <span class="ow">or</span> <span class="p">(</span><span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">parent</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)):</span>
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="n">parent_name</span><span class="p">)</span>
<span class="k">return</span> <span class="n">parent</span>
<div class="viewcode-block" id="Config.path">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config.path">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">path</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="n">UNSET</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get a :py:class:`pathlib.Path` object from a config string.&quot;&quot;&quot;</span>
<span class="n">val</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">default</span><span class="p">)</span>
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
<span class="k">if</span> <span class="n">default</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
<span class="k">return</span> <span class="n">default</span>
<span class="k">return</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">val</span><span class="p">))</span></div>
<div class="viewcode-block" id="Config.pyobj">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config.pyobj">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">pyobj</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="n">UNSET</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get python object referred by full qualiffied name (FQN) in the config</span>
<span class="sd"> string.&quot;&quot;&quot;</span>
<span class="n">fqn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">default</span><span class="p">)</span>
<span class="k">if</span> <span class="n">fqn</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
<span class="k">if</span> <span class="n">default</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
<span class="k">return</span> <span class="n">default</span>
<span class="p">(</span><span class="n">modulename</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">fqn</span><span class="p">)</span><span class="o">.</span><span class="n">rsplit</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">m</span> <span class="o">=</span> <span class="nb">__import__</span><span class="p">(</span><span class="n">modulename</span><span class="p">,</span> <span class="p">{},</span> <span class="p">{},</span> <span class="p">[</span><span class="n">name</span><span class="p">],</span> <span class="mi">0</span><span class="p">)</span>
<span class="k">return</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">m</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span></div>
</div>
<span class="k">def</span><span class="w"> </span><span class="nf">toml_load</span><span class="p">(</span><span class="n">file_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">):</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">file_name</span><span class="p">,</span> <span class="s2">&quot;rb&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="k">return</span> <span class="n">tomllib</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
<span class="k">except</span> <span class="n">tomllib</span><span class="o">.</span><span class="n">TOMLDecodeError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
<span class="n">msg</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">exc</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\t</span><span class="s1">&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">,</span> <span class="s1">&#39; &#39;</span><span class="p">)</span>
<span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">file_name</span><span class="p">,</span> <span class="n">msg</span><span class="p">)</span>
<span class="k">raise</span>
<span class="c1"># working with dictionaries</span>
<span class="k">def</span><span class="w"> </span><span class="nf">value</span><span class="p">(</span><span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">data_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns the value to which ``name`` points in the ``dat_dict``.</span>
<span class="sd"> .. code: python</span>
<span class="sd"> &gt;&gt;&gt; data_dict = {</span>
<span class="sd"> &quot;foo&quot;: {&quot;bar&quot;: 1 },</span>
<span class="sd"> &quot;bar&quot;: {&quot;foo&quot;: 2 },</span>
<span class="sd"> &quot;foobar&quot;: [1, 2, 3],</span>
<span class="sd"> }</span>
<span class="sd"> &gt;&gt;&gt; value(&#39;foobar&#39;, data_dict)</span>
<span class="sd"> [1, 2, 3]</span>
<span class="sd"> &gt;&gt;&gt; value(&#39;foo.bar&#39;, data_dict)</span>
<span class="sd"> 1</span>
<span class="sd"> &gt;&gt;&gt; value(&#39;foo.bar.xxx&#39;, data_dict)</span>
<span class="sd"> &lt;UNSET&gt;</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">ret_val</span> <span class="o">=</span> <span class="n">data_dict</span>
<span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">ret_val</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="n">ret_val</span> <span class="o">=</span> <span class="n">ret_val</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">part</span><span class="p">,</span> <span class="n">UNSET</span><span class="p">)</span>
<span class="k">if</span> <span class="n">ret_val</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
<span class="k">break</span>
<span class="k">return</span> <span class="n">ret_val</span>
<span class="k">def</span><span class="w"> </span><span class="nf">validate</span><span class="p">(</span>
<span class="n">schema_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">],</span> <span class="n">data_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">],</span> <span class="n">deprecated</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">bool</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">SchemaIssue</span><span class="p">]]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Deep validation of dictionary in ``data_dict`` against dictionary in</span>
<span class="sd"> ``schema_dict``. Argument deprecated is a dictionary that maps deprecated</span>
<span class="sd"> configuration names to a messages::</span>
<span class="sd"> deprecated = {</span>
<span class="sd"> &quot;foo.bar&quot; : &quot;config &#39;foo.bar&#39; is deprecated, use &#39;bar.foo&#39;&quot;,</span>
<span class="sd"> &quot;...&quot; : &quot;...&quot;</span>
<span class="sd"> }</span>
<span class="sd"> The function returns a python tuple ``(is_valid, issue_list)``:</span>
<span class="sd"> ``is_valid``:</span>
<span class="sd"> A bool value indicating ``data_dict`` is valid or not.</span>
<span class="sd"> ``issue_list``:</span>
<span class="sd"> A list of messages (:py:obj:`SchemaIssue`) from the validation::</span>
<span class="sd"> [schema warn] data_dict: deprecated &#39;fontlib.foo&#39;: &lt;DEPRECATED[&#39;foo.bar&#39;]&gt;</span>
<span class="sd"> [schema invalid] data_dict: key unknown &#39;fontlib.foo&#39;</span>
<span class="sd"> [schema invalid] data_dict: type mismatch &#39;fontlib.foo&#39;: expected ..., is ...</span>
<span class="sd"> If ``schema_dict`` or ``data_dict`` is not a dictionary type a</span>
<span class="sd"> :py:obj:`SchemaIssue` is raised.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">names</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">is_valid</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">issue_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">SchemaIssue</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">schema_dict</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">raise</span> <span class="n">SchemaIssue</span><span class="p">(</span><span class="s1">&#39;invalid&#39;</span><span class="p">,</span> <span class="s2">&quot;schema_dict is not a dict type&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data_dict</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">raise</span> <span class="n">SchemaIssue</span><span class="p">(</span><span class="s1">&#39;invalid&#39;</span><span class="p">,</span> <span class="sa">f</span><span class="s2">&quot;data_dict issue</span><span class="si">{</span><span class="s1">&#39;.&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">names</span><span class="p">)</span><span class="si">}</span><span class="s2"> is not a dict type&quot;</span><span class="p">)</span>
<span class="n">is_valid</span><span class="p">,</span> <span class="n">issue_list</span> <span class="o">=</span> <span class="n">_validate</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">issue_list</span><span class="p">,</span> <span class="n">schema_dict</span><span class="p">,</span> <span class="n">data_dict</span><span class="p">,</span> <span class="n">deprecated</span><span class="p">)</span>
<span class="k">return</span> <span class="n">is_valid</span><span class="p">,</span> <span class="n">issue_list</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_validate</span><span class="p">(</span>
<span class="n">names</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
<span class="n">issue_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">SchemaIssue</span><span class="p">],</span>
<span class="n">schema_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">],</span>
<span class="n">data_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">],</span>
<span class="n">deprecated</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">bool</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">SchemaIssue</span><span class="p">]]:</span>
<span class="n">is_valid</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">data_value</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">data_value</span> <span class="ow">in</span> <span class="n">data_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">names</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
<span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;.&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">names</span><span class="p">)</span>
<span class="n">deprecated_msg</span> <span class="o">=</span> <span class="n">deprecated</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
<span class="c1"># print(&quot;XXX %s: key %s // data_value: %s&quot; % (name, key, data_value))</span>
<span class="k">if</span> <span class="n">deprecated_msg</span><span class="p">:</span>
<span class="n">issue_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">SchemaIssue</span><span class="p">(</span><span class="s1">&#39;warn&#39;</span><span class="p">,</span> <span class="sa">f</span><span class="s2">&quot;data_dict &#39;</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">&#39;: deprecated - </span><span class="si">{</span><span class="n">deprecated_msg</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">))</span>
<span class="n">schema_value</span> <span class="o">=</span> <span class="n">value</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">schema_dict</span><span class="p">)</span>
<span class="c1"># print(&quot;YYY %s: key %s // schema_value: %s&quot; % (name, key, schema_value))</span>
<span class="k">if</span> <span class="n">schema_value</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">deprecated_msg</span><span class="p">:</span>
<span class="n">issue_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">SchemaIssue</span><span class="p">(</span><span class="s1">&#39;invalid&#39;</span><span class="p">,</span> <span class="sa">f</span><span class="s2">&quot;data_dict &#39;</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">&#39;: key unknown in schema_dict&quot;</span><span class="p">))</span>
<span class="n">is_valid</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">schema_value</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">type</span><span class="p">(</span><span class="n">data_value</span><span class="p">):</span> <span class="c1"># pylint: disable=unidiomatic-typecheck</span>
<span class="n">issue_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="n">SchemaIssue</span><span class="p">(</span>
<span class="s1">&#39;invalid&#39;</span><span class="p">,</span>
<span class="p">(</span><span class="sa">f</span><span class="s2">&quot;data_dict: type mismatch &#39;</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">&#39;:&quot;</span> <span class="sa">f</span><span class="s2">&quot; expected </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">schema_value</span><span class="p">)</span><span class="si">}</span><span class="s2">, is: </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">data_value</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">),</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="n">is_valid</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data_value</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="n">_valid</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">_validate</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">issue_list</span><span class="p">,</span> <span class="n">schema_dict</span><span class="p">,</span> <span class="n">data_value</span><span class="p">,</span> <span class="n">deprecated</span><span class="p">)</span>
<span class="n">is_valid</span> <span class="o">=</span> <span class="n">is_valid</span> <span class="ow">and</span> <span class="n">_valid</span>
<span class="n">names</span><span class="o">.</span><span class="n">pop</span><span class="p">()</span>
<span class="k">return</span> <span class="n">is_valid</span><span class="p">,</span> <span class="n">issue_list</span>
<span class="k">def</span><span class="w"> </span><span class="nf">dict_deepupdate</span><span class="p">(</span><span class="n">base_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">],</span> <span class="n">upd_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">],</span> <span class="n">names</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Deep-update of dictionary in ``base_dict`` by dictionary in ``upd_dict``.</span>
<span class="sd"> For each ``upd_key`` &amp; ``upd_val`` pair in ``upd_dict``:</span>
<span class="sd"> 0. If types of ``base_dict[upd_key]`` and ``upd_val`` do not match raise a</span>
<span class="sd"> :py:obj:`TypeError`.</span>
<span class="sd"> 1. If ``base_dict[upd_key]`` is a dict: recursively deep-update it by ``upd_val``.</span>
<span class="sd"> 2. If ``base_dict[upd_key]`` not exist: set ``base_dict[upd_key]`` from a</span>
<span class="sd"> (deep-) copy of ``upd_val``.</span>
<span class="sd"> 3. If ``upd_val`` is a list, extend list in ``base_dict[upd_key]`` by the</span>
<span class="sd"> list in ``upd_val``.</span>
<span class="sd"> 4. If ``upd_val`` is a set, update set in ``base_dict[upd_key]`` by set in</span>
<span class="sd"> ``upd_val``.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-many-branches</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">base_dict</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;argument &#39;base_dict&#39; is not a dictionary type&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">upd_dict</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;argument &#39;upd_dict&#39; is not a dictionary type&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">names</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">names</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">upd_key</span><span class="p">,</span> <span class="n">upd_val</span> <span class="ow">in</span> <span class="n">upd_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="c1"># For each upd_key &amp; upd_val pair in upd_dict:</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">upd_val</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">if</span> <span class="n">upd_key</span> <span class="ow">in</span> <span class="n">base_dict</span><span class="p">:</span>
<span class="c1"># if base_dict[upd_key] exists, recursively deep-update it</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">],</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;type mismatch </span><span class="si">{</span><span class="s1">&#39;.&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">names</span><span class="p">)</span><span class="si">}</span><span class="s2">: is not a dict type in base_dict&quot;</span><span class="p">)</span>
<span class="n">dict_deepupdate</span><span class="p">(</span>
<span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">],</span>
<span class="n">upd_val</span><span class="p">,</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
<span class="n">names</span>
<span class="o">+</span> <span class="p">[</span>
<span class="n">upd_key</span><span class="p">,</span>
<span class="p">],</span>
<span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># if base_dict[upd_key] not exist, set base_dict[upd_key] from deepcopy of upd_val</span>
<span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">upd_val</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">upd_val</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
<span class="k">if</span> <span class="n">upd_key</span> <span class="ow">in</span> <span class="n">base_dict</span><span class="p">:</span>
<span class="c1"># if base_dict[upd_key] exists, base_dict[up_key] is extended by</span>
<span class="c1"># the list from upd_val</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;type mismatch </span><span class="si">{</span><span class="s1">&#39;.&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">names</span><span class="p">)</span><span class="si">}</span><span class="s2">: is not a list type in base_dict&quot;</span><span class="p">)</span>
<span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">]</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">upd_val</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># if base_dict[upd_key] doesn&#39;t exists, set base_dict[key] from a deepcopy of the</span>
<span class="c1"># list in upd_val.</span>
<span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">upd_val</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">upd_val</span><span class="p">,</span> <span class="nb">set</span><span class="p">):</span>
<span class="k">if</span> <span class="n">upd_key</span> <span class="ow">in</span> <span class="n">base_dict</span><span class="p">:</span>
<span class="c1"># if base_dict[upd_key] exists, base_dict[up_key] is updated by the set in upd_val</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">],</span> <span class="nb">set</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;type mismatch </span><span class="si">{</span><span class="s1">&#39;.&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">names</span><span class="p">)</span><span class="si">}</span><span class="s2">: is not a set type in base_dict&quot;</span><span class="p">)</span>
<span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">upd_val</span><span class="o">.</span><span class="n">copy</span><span class="p">())</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># if base_dict[upd_key] doesn&#39;t exists, set base_dict[upd_key] from a copy of the</span>
<span class="c1"># set in upd_val</span>
<span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">upd_val</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># for any other type of upd_val replace or add base_dict[upd_key] by a copy</span>
<span class="c1"># of upd_val</span>
<span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">upd_val</span><span class="p">)</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,214 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.botdetection.http_sec_fetch &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.botdetection.http_sec_fetch</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.botdetection.http_sec_fetch</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">Method ``http_sec_fetch``</span>
<span class="sd">-------------------------</span>
<span class="sd">The ``http_sec_fetch`` method protect resources from web attacks with `Fetch</span>
<span class="sd">Metadata`_. A request is filtered out in case of:</span>
<span class="sd">- http header Sec-Fetch-Mode_ is invalid</span>
<span class="sd">- http header Sec-Fetch-Dest_ is invalid</span>
<span class="sd">.. _Fetch Metadata:</span>
<span class="sd"> https://developer.mozilla.org/en-US/docs/Glossary/Fetch_metadata_request_header</span>
<span class="sd">.. _Sec-Fetch-Dest:</span>
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/API/Request/destination</span>
<span class="sd">.. _Sec-Fetch-Mode:</span>
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/API/Request/mode</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=unused-argument</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">IPv4Network</span><span class="p">,</span>
<span class="n">IPv6Network</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">werkzeug</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">config</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">._helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
<div class="viewcode-block" id="is_browser_supported">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.http_sec_fetch.is_browser_supported">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">is_browser_supported</span><span class="p">(</span><span class="n">user_agent</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Check if the browser supports Sec-Fetch headers.</span>
<span class="sd"> https://caniuse.com/mdn-http_headers_sec-fetch-dest</span>
<span class="sd"> https://caniuse.com/mdn-http_headers_sec-fetch-mode</span>
<span class="sd"> https://caniuse.com/mdn-http_headers_sec-fetch-site</span>
<span class="sd"> Supported browsers:</span>
<span class="sd"> - Chrome &gt;= 80</span>
<span class="sd"> - Firefox &gt;= 90</span>
<span class="sd"> - Safari &gt;= 16.4</span>
<span class="sd"> - Edge (mirrors Chrome)</span>
<span class="sd"> - Opera (mirrors Chrome)</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">user_agent</span> <span class="o">=</span> <span class="n">user_agent</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="c1"># Chrome/Chromium/Edge/Opera</span>
<span class="n">chrome_match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;chrome/(\d+)&#39;</span><span class="p">,</span> <span class="n">user_agent</span><span class="p">)</span>
<span class="k">if</span> <span class="n">chrome_match</span><span class="p">:</span>
<span class="n">version</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">chrome_match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
<span class="k">return</span> <span class="n">version</span> <span class="o">&gt;=</span> <span class="mi">80</span>
<span class="c1"># Firefox</span>
<span class="n">firefox_match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;firefox/(\d+)&#39;</span><span class="p">,</span> <span class="n">user_agent</span><span class="p">)</span>
<span class="k">if</span> <span class="n">firefox_match</span><span class="p">:</span>
<span class="n">version</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">firefox_match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
<span class="k">return</span> <span class="n">version</span> <span class="o">&gt;=</span> <span class="mi">90</span>
<span class="c1"># Safari</span>
<span class="n">safari_match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;version/(\d+)\.(\d+)&#39;</span><span class="p">,</span> <span class="n">user_agent</span><span class="p">)</span>
<span class="k">if</span> <span class="n">safari_match</span><span class="p">:</span>
<span class="n">major</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">safari_match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
<span class="n">minor</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">safari_match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span>
<span class="k">return</span> <span class="n">major</span> <span class="o">&gt;</span> <span class="mi">16</span> <span class="ow">or</span> <span class="p">(</span><span class="n">major</span> <span class="o">==</span> <span class="mi">16</span> <span class="ow">and</span> <span class="n">minor</span> <span class="o">&gt;=</span> <span class="mi">4</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">False</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">filter_request</span><span class="p">(</span>
<span class="n">network</span><span class="p">:</span> <span class="n">IPv4Network</span> <span class="o">|</span> <span class="n">IPv6Network</span><span class="p">,</span>
<span class="n">request</span><span class="p">:</span> <span class="n">flask</span><span class="o">.</span><span class="n">Request</span><span class="p">,</span>
<span class="n">cfg</span><span class="p">:</span> <span class="n">config</span><span class="o">.</span><span class="n">Config</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">werkzeug</span><span class="o">.</span><span class="n">Response</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">request</span><span class="o">.</span><span class="n">is_secure</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
<span class="s2">&quot;Sec-Fetch cannot be verified for non-secure requests (HTTP headers are not set/sent by the client).&quot;</span>
<span class="p">)</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="c1"># Only check Sec-Fetch headers for supported browsers</span>
<span class="n">user_agent</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;User-Agent&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">is_browser_supported</span><span class="p">(</span><span class="n">user_agent</span><span class="p">):</span>
<span class="n">val</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;Sec-Fetch-Mode&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">val</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">&#39;navigate&#39;</span><span class="p">,</span> <span class="s1">&#39;cors&#39;</span><span class="p">):</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;invalid Sec-Fetch-Mode &#39;</span><span class="si">%s</span><span class="s2">&#39;&quot;</span><span class="p">,</span> <span class="n">val</span><span class="p">)</span>
<span class="k">return</span> <span class="n">flask</span><span class="o">.</span><span class="n">redirect</span><span class="p">(</span><span class="n">flask</span><span class="o">.</span><span class="n">url_for</span><span class="p">(</span><span class="s1">&#39;index&#39;</span><span class="p">),</span> <span class="n">code</span><span class="o">=</span><span class="mi">302</span><span class="p">)</span>
<span class="n">val</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;Sec-Fetch-Site&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">val</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">&#39;same-origin&#39;</span><span class="p">,</span> <span class="s1">&#39;same-site&#39;</span><span class="p">,</span> <span class="s1">&#39;none&#39;</span><span class="p">):</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;invalid Sec-Fetch-Site &#39;</span><span class="si">%s</span><span class="s2">&#39;&quot;</span><span class="p">,</span> <span class="n">val</span><span class="p">)</span>
<span class="n">flask</span><span class="o">.</span><span class="n">redirect</span><span class="p">(</span><span class="n">flask</span><span class="o">.</span><span class="n">url_for</span><span class="p">(</span><span class="s1">&#39;index&#39;</span><span class="p">),</span> <span class="n">code</span><span class="o">=</span><span class="mi">302</span><span class="p">)</span>
<span class="n">val</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;Sec-Fetch-Dest&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">val</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">&#39;document&#39;</span><span class="p">,</span> <span class="s1">&#39;empty&#39;</span><span class="p">):</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;invalid Sec-Fetch-Dest &#39;</span><span class="si">%s</span><span class="s2">&#39;&quot;</span><span class="p">,</span> <span class="n">val</span><span class="p">)</span>
<span class="n">flask</span><span class="o">.</span><span class="n">redirect</span><span class="p">(</span><span class="n">flask</span><span class="o">.</span><span class="n">url_for</span><span class="p">(</span><span class="s1">&#39;index&#39;</span><span class="p">),</span> <span class="n">code</span><span class="o">=</span><span class="mi">302</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">None</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,194 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.botdetection.ip_lists &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.botdetection.ip_lists</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.botdetection.ip_lists</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;.. _botdetection.ip_lists:</span>
<span class="sd">Method ``ip_lists``</span>
<span class="sd">-------------------</span>
<span class="sd">The ``ip_lists`` method implements :py:obj:`block-list &lt;block_ip&gt;` and</span>
<span class="sd">:py:obj:`pass-list &lt;pass_ip&gt;`.</span>
<span class="sd">.. code:: toml</span>
<span class="sd"> [botdetection.ip_lists]</span>
<span class="sd"> pass_ip = [</span>
<span class="sd"> &#39;167.235.158.251&#39;, # IPv4 of check.searx.space</span>
<span class="sd"> &#39;192.168.0.0/16&#39;, # IPv4 private network</span>
<span class="sd"> &#39;fe80::/10&#39;, # IPv6 linklocal</span>
<span class="sd"> ]</span>
<span class="sd"> block_ip = [</span>
<span class="sd"> &#39;93.184.216.34&#39;, # IPv4 of example.org</span>
<span class="sd"> &#39;257.1.1.1&#39;, # invalid IP --&gt; will be ignored, logged in ERROR class</span>
<span class="sd"> ]</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=unused-argument</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Tuple</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">ip_network</span><span class="p">,</span>
<span class="n">IPv4Address</span><span class="p">,</span>
<span class="n">IPv6Address</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">config</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">._helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">&#39;ip_limit&#39;</span><span class="p">)</span>
<span class="n">SEARXNG_ORG</span> <span class="o">=</span> <span class="p">[</span>
<span class="c1"># https://github.com/searxng/searxng/pull/2484#issuecomment-1576639195</span>
<span class="s1">&#39;167.235.158.251&#39;</span><span class="p">,</span> <span class="c1"># IPv4 check.searx.space</span>
<span class="s1">&#39;2a01:04f8:1c1c:8fc2::/64&#39;</span><span class="p">,</span> <span class="c1"># IPv6 check.searx.space</span>
<span class="p">]</span>
<span class="sd">&quot;&quot;&quot;Passlist of IPs from the SearXNG organization, e.g. `check.searx.space`.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="pass_ip">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.ip_lists.pass_ip">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">pass_ip</span><span class="p">(</span><span class="n">real_ip</span><span class="p">:</span> <span class="n">IPv4Address</span> <span class="o">|</span> <span class="n">IPv6Address</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">config</span><span class="o">.</span><span class="n">Config</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">bool</span><span class="p">,</span> <span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Checks if the IP on the subnet is in one of the members of the</span>
<span class="sd"> ``botdetection.ip_lists.pass_ip`` list.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">cfg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;botdetection.ip_lists.pass_searxng_org&#39;</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="k">for</span> <span class="n">net</span> <span class="ow">in</span> <span class="n">SEARXNG_ORG</span><span class="p">:</span>
<span class="n">net</span> <span class="o">=</span> <span class="n">ip_network</span><span class="p">(</span><span class="n">net</span><span class="p">,</span> <span class="n">strict</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="k">if</span> <span class="n">real_ip</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="n">net</span><span class="o">.</span><span class="n">version</span> <span class="ow">and</span> <span class="n">real_ip</span> <span class="ow">in</span> <span class="n">net</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">True</span><span class="p">,</span> <span class="sa">f</span><span class="s2">&quot;IP matches </span><span class="si">{</span><span class="n">net</span><span class="o">.</span><span class="n">compressed</span><span class="si">}</span><span class="s2"> in SEARXNG_ORG list.&quot;</span>
<span class="k">return</span> <span class="n">ip_is_subnet_of_member_in_list</span><span class="p">(</span><span class="n">real_ip</span><span class="p">,</span> <span class="s1">&#39;botdetection.ip_lists.pass_ip&#39;</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span></div>
<div class="viewcode-block" id="block_ip">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.ip_lists.block_ip">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">block_ip</span><span class="p">(</span><span class="n">real_ip</span><span class="p">:</span> <span class="n">IPv4Address</span> <span class="o">|</span> <span class="n">IPv6Address</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">config</span><span class="o">.</span><span class="n">Config</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">bool</span><span class="p">,</span> <span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Checks if the IP on the subnet is in one of the members of the</span>
<span class="sd"> ``botdetection.ip_lists.block_ip`` list.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">block</span><span class="p">,</span> <span class="n">msg</span> <span class="o">=</span> <span class="n">ip_is_subnet_of_member_in_list</span><span class="p">(</span><span class="n">real_ip</span><span class="p">,</span> <span class="s1">&#39;botdetection.ip_lists.block_ip&#39;</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span>
<span class="k">if</span> <span class="n">block</span><span class="p">:</span>
<span class="n">msg</span> <span class="o">+=</span> <span class="s2">&quot; To remove IP from list, please contact the maintainer of the service.&quot;</span>
<span class="k">return</span> <span class="n">block</span><span class="p">,</span> <span class="n">msg</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">ip_is_subnet_of_member_in_list</span><span class="p">(</span>
<span class="n">real_ip</span><span class="p">:</span> <span class="n">IPv4Address</span> <span class="o">|</span> <span class="n">IPv6Address</span><span class="p">,</span> <span class="n">list_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">config</span><span class="o">.</span><span class="n">Config</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">bool</span><span class="p">,</span> <span class="nb">str</span><span class="p">]:</span>
<span class="k">for</span> <span class="n">net</span> <span class="ow">in</span> <span class="n">cfg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">list_name</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="p">[]):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">net</span> <span class="o">=</span> <span class="n">ip_network</span><span class="p">(</span><span class="n">net</span><span class="p">,</span> <span class="n">strict</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;invalid IP </span><span class="si">%s</span><span class="s2"> in </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">net</span><span class="p">,</span> <span class="n">list_name</span><span class="p">)</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="n">real_ip</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="n">net</span><span class="o">.</span><span class="n">version</span> <span class="ow">and</span> <span class="n">real_ip</span> <span class="ow">in</span> <span class="n">net</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">True</span><span class="p">,</span> <span class="sa">f</span><span class="s2">&quot;IP matches </span><span class="si">{</span><span class="n">net</span><span class="o">.</span><span class="n">compressed</span><span class="si">}</span><span class="s2"> in </span><span class="si">{</span><span class="n">list_name</span><span class="si">}</span><span class="s2">.&quot;</span>
<span class="k">return</span> <span class="kc">False</span><span class="p">,</span> <span class="sa">f</span><span class="s2">&quot;IP is not a member of an item in the f</span><span class="si">{</span><span class="n">list_name</span><span class="si">}</span><span class="s2"> list&quot;</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,272 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.botdetection.link_token &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.botdetection.link_token</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.botdetection.link_token</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">Method ``link_token``</span>
<span class="sd">---------------------</span>
<span class="sd">The ``link_token`` method evaluates a request as :py:obj:`suspicious</span>
<span class="sd">&lt;is_suspicious&gt;` if the URL ``/client&lt;token&gt;.css`` is not requested by the</span>
<span class="sd">client. By adding a random component (the token) in the URL, a bot can not send</span>
<span class="sd">a ping by request a static URL.</span>
<span class="sd">.. note::</span>
<span class="sd"> This method requires a valkey DB and needs a HTTP X-Forwarded-For_ header.</span>
<span class="sd">To get in use of this method a flask URL route needs to be added:</span>
<span class="sd">.. code:: python</span>
<span class="sd"> @app.route(&#39;/client&lt;token&gt;.css&#39;, methods=[&#39;GET&#39;, &#39;POST&#39;])</span>
<span class="sd"> def client_token(token=None):</span>
<span class="sd"> link_token.ping(request, token)</span>
<span class="sd"> return Response(&#39;&#39;, mimetype=&#39;text/css&#39;)</span>
<span class="sd">And in the HTML template from flask a stylesheet link is needed (the value of</span>
<span class="sd">``link_token`` comes from :py:obj:`get_token`):</span>
<span class="sd">.. code:: html</span>
<span class="sd"> &lt;link rel=&quot;stylesheet&quot;</span>
<span class="sd"> href=&quot;{{ url_for(&#39;client_token&#39;, token=link_token) }}&quot;</span>
<span class="sd"> type=&quot;text/css&quot; &gt;</span>
<span class="sd">.. _X-Forwarded-For:</span>
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">IPv4Network</span><span class="p">,</span>
<span class="n">IPv6Network</span><span class="p">,</span>
<span class="n">ip_address</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">string</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">random</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.valkeylib</span><span class="w"> </span><span class="kn">import</span> <span class="n">secret_hash</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">._helpers</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">get_network</span><span class="p">,</span>
<span class="n">logger</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">config</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">valkeydb</span>
<span class="n">TOKEN_LIVE_TIME</span> <span class="o">=</span> <span class="mi">600</span>
<span class="sd">&quot;&quot;&quot;Lifetime (sec) of limiter&#39;s CSS token.&quot;&quot;&quot;</span>
<span class="n">PING_LIVE_TIME</span> <span class="o">=</span> <span class="mi">3600</span>
<span class="sd">&quot;&quot;&quot;Lifetime (sec) of the ping-key from a client (request)&quot;&quot;&quot;</span>
<span class="n">PING_KEY</span> <span class="o">=</span> <span class="s1">&#39;SearXNG_limiter.ping&#39;</span>
<span class="sd">&quot;&quot;&quot;Prefix of all ping-keys generated by :py:obj:`get_ping_key`&quot;&quot;&quot;</span>
<span class="n">TOKEN_KEY</span> <span class="o">=</span> <span class="s1">&#39;SearXNG_limiter.token&#39;</span>
<span class="sd">&quot;&quot;&quot;Key for which the current token is stored in the DB&quot;&quot;&quot;</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">&#39;botdetection.link_token&#39;</span><span class="p">)</span>
<div class="viewcode-block" id="is_suspicious">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.link_token.is_suspicious">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">is_suspicious</span><span class="p">(</span><span class="n">network</span><span class="p">:</span> <span class="n">IPv4Network</span> <span class="o">|</span> <span class="n">IPv6Network</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">flask</span><span class="o">.</span><span class="n">Request</span><span class="p">,</span> <span class="n">renew</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Checks whether a valid ping is exists for this (client) network, if not</span>
<span class="sd"> this request is rated as *suspicious*. If a valid ping exists and argument</span>
<span class="sd"> ``renew`` is ``True`` the expire time of this ping is reset to</span>
<span class="sd"> :py:obj:`PING_LIVE_TIME`.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">valkey_client</span> <span class="o">=</span> <span class="n">valkeydb</span><span class="o">.</span><span class="n">get_valkey_client</span><span class="p">()</span>
<span class="n">ping_key</span> <span class="o">=</span> <span class="n">get_ping_key</span><span class="p">(</span><span class="n">network</span><span class="p">,</span> <span class="n">request</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">valkey_client</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">ping_key</span><span class="p">):</span>
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;missing ping (IP: </span><span class="si">%s</span><span class="s2">) / request: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">network</span><span class="o">.</span><span class="n">compressed</span><span class="p">,</span> <span class="n">ping_key</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="k">if</span> <span class="n">renew</span><span class="p">:</span>
<span class="n">valkey_client</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">ping_key</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">ex</span><span class="o">=</span><span class="n">PING_LIVE_TIME</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;found ping for (client) network </span><span class="si">%s</span><span class="s2"> -&gt; </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">network</span><span class="o">.</span><span class="n">compressed</span><span class="p">,</span> <span class="n">ping_key</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">False</span></div>
<div class="viewcode-block" id="ping">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.link_token.ping">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">ping</span><span class="p">(</span><span class="n">request</span><span class="p">:</span> <span class="n">flask</span><span class="o">.</span><span class="n">Request</span><span class="p">,</span> <span class="n">token</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;This function is called by a request to URL ``/client&lt;token&gt;.css``. If</span>
<span class="sd"> ``token`` is valid a :py:obj:`PING_KEY` for the client is stored in the DB.</span>
<span class="sd"> The expire time of this ping-key is :py:obj:`PING_LIVE_TIME`.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">valkey_client</span> <span class="o">=</span> <span class="n">valkeydb</span><span class="o">.</span><span class="n">get_valkey_client</span><span class="p">()</span>
<span class="n">cfg</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">get_global_cfg</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">token_is_valid</span><span class="p">(</span><span class="n">token</span><span class="p">):</span>
<span class="k">return</span>
<span class="n">real_ip</span> <span class="o">=</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">remote_addr</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">network</span> <span class="o">=</span> <span class="n">get_network</span><span class="p">(</span><span class="n">real_ip</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span>
<span class="n">ping_key</span> <span class="o">=</span> <span class="n">get_ping_key</span><span class="p">(</span><span class="n">network</span><span class="p">,</span> <span class="n">request</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
<span class="s2">&quot;store ping_key for (client) network </span><span class="si">%s</span><span class="s2"> (IP </span><span class="si">%s</span><span class="s2">) -&gt; </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">network</span><span class="o">.</span><span class="n">compressed</span><span class="p">,</span> <span class="n">real_ip</span><span class="o">.</span><span class="n">compressed</span><span class="p">,</span> <span class="n">ping_key</span>
<span class="p">)</span>
<span class="n">valkey_client</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">ping_key</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">ex</span><span class="o">=</span><span class="n">PING_LIVE_TIME</span><span class="p">)</span></div>
<div class="viewcode-block" id="get_ping_key">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.link_token.get_ping_key">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_ping_key</span><span class="p">(</span><span class="n">network</span><span class="p">:</span> <span class="n">IPv4Network</span> <span class="o">|</span> <span class="n">IPv6Network</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">flask</span><span class="o">.</span><span class="n">Request</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Generates a hashed key that fits (more or less) to a *WEB-browser</span>
<span class="sd"> session* in a network.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="p">(</span>
<span class="n">PING_KEY</span>
<span class="o">+</span> <span class="s2">&quot;[&quot;</span>
<span class="o">+</span> <span class="n">secret_hash</span><span class="p">(</span>
<span class="n">network</span><span class="o">.</span><span class="n">compressed</span> <span class="o">+</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Accept-Language&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;User-Agent&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="p">)</span>
<span class="o">+</span> <span class="s2">&quot;]&quot;</span>
<span class="p">)</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">token_is_valid</span><span class="p">(</span><span class="n">token</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="n">valid</span> <span class="o">=</span> <span class="n">token</span> <span class="o">==</span> <span class="n">get_token</span><span class="p">()</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;token is valid --&gt; </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">valid</span><span class="p">)</span>
<span class="k">return</span> <span class="n">valid</span>
<div class="viewcode-block" id="get_token">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.link_token.get_token">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_token</span><span class="p">()</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns current token. If there is no currently active token a new token</span>
<span class="sd"> is generated randomly and stored in the Valkey DB. Without without a</span>
<span class="sd"> database connection, string &quot;12345678&quot; is returned.</span>
<span class="sd"> - :py:obj:`TOKEN_LIVE_TIME`</span>
<span class="sd"> - :py:obj:`TOKEN_KEY`</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">valkey_client</span> <span class="o">=</span> <span class="n">valkeydb</span><span class="o">.</span><span class="n">get_valkey_client</span><span class="p">()</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="c1"># This function is also called when limiter is inactive / no valkey DB</span>
<span class="c1"># (see render function in webapp.py)</span>
<span class="k">return</span> <span class="s1">&#39;12345678&#39;</span>
<span class="n">token</span> <span class="o">=</span> <span class="n">valkey_client</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">TOKEN_KEY</span><span class="p">)</span>
<span class="k">if</span> <span class="n">token</span><span class="p">:</span>
<span class="n">token</span> <span class="o">=</span> <span class="n">token</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">&#39;UTF-8&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">token</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">string</span><span class="o">.</span><span class="n">ascii_lowercase</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">digits</span><span class="p">)</span> <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">16</span><span class="p">))</span>
<span class="n">valkey_client</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">TOKEN_KEY</span><span class="p">,</span> <span class="n">token</span><span class="p">,</span> <span class="n">ex</span><span class="o">=</span><span class="n">TOKEN_LIVE_TIME</span><span class="p">)</span>
<span class="k">return</span> <span class="n">token</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,283 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.botdetection.trusted_proxies &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.botdetection.trusted_proxies</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.botdetection.trusted_proxies</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Implementation of a middleware to determine the real IP of an HTTP request</span>
<span class="sd">(:py:obj:`flask.request.remote_addr`) behind a proxy chain.&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-many-branches</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">abc</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="n">IPv4Address</span><span class="p">,</span> <span class="n">IPv6Address</span><span class="p">,</span> <span class="n">ip_address</span><span class="p">,</span> <span class="n">ip_network</span><span class="p">,</span> <span class="n">IPv4Network</span><span class="p">,</span> <span class="n">IPv6Network</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">werkzeug.http</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse_list_header</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">config</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">._helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">log_error_only_once</span><span class="p">,</span> <span class="n">logger</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">_typeshed.wsgi</span><span class="w"> </span><span class="kn">import</span> <span class="n">StartResponse</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">_typeshed.wsgi</span><span class="w"> </span><span class="kn">import</span> <span class="n">WSGIApplication</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">_typeshed.wsgi</span><span class="w"> </span><span class="kn">import</span> <span class="n">WSGIEnvironment</span>
<div class="viewcode-block" id="ProxyFix">
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.ProxyFix">[docs]</a>
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">ProxyFix</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A middleware like the ProxyFix_ class, where the ``x_for`` argument is</span>
<span class="sd"> replaced by a method that determines the number of trusted proxies via the</span>
<span class="sd"> ``botdetection.trusted_proxies`` setting.</span>
<span class="sd"> .. sidebar:: :py:obj:`flask.Request.remote_addr`</span>
<span class="sd"> SearXNG uses Werkzeug&#39;s ProxyFix_ (with it default ``x_for=1``).</span>
<span class="sd"> The remote IP (:py:obj:`flask.Request.remote_addr`) of the request is taken</span>
<span class="sd"> from (first match):</span>
<span class="sd"> - X-Forwarded-For_: If the header is set, the first untrusted IP that comes</span>
<span class="sd"> before the IPs that are still part of the ``botdetection.trusted_proxies``</span>
<span class="sd"> is used.</span>
<span class="sd"> - `X-Real-IP &lt;https://github.com/searxng/searxng/issues/1237#issuecomment-1147564516&gt;`__:</span>
<span class="sd"> If X-Forwarded-For_ is not set, `X-Real-IP` is used</span>
<span class="sd"> (``botdetection.trusted_proxies`` is ignored).</span>
<span class="sd"> If none of the header is set, the REMOTE_ADDR_ from the WSGI layer is used.</span>
<span class="sd"> If (for whatever reasons) none IP can be determined, an error message is</span>
<span class="sd"> displayed and ``100::`` is used instead (:rfc:`6666`).</span>
<span class="sd"> .. _ProxyFix:</span>
<span class="sd"> https://werkzeug.palletsprojects.com/middleware/proxy_fix/</span>
<span class="sd"> .. _X-Forwarded-For:</span>
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For</span>
<span class="sd"> .. _REMOTE_ADDR:</span>
<span class="sd"> https://wsgi.readthedocs.io/en/latest/proposals-2.0.html#making-some-keys-required</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">wsgi_app</span><span class="p">:</span> <span class="s2">&quot;WSGIApplication&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">wsgi_app</span> <span class="o">=</span> <span class="n">wsgi_app</span>
<span class="k">def</span><span class="w"> </span><span class="nf">trusted_proxies</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="n">IPv4Network</span> <span class="o">|</span> <span class="n">IPv6Network</span><span class="p">]:</span>
<span class="n">cfg</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">get_global_cfg</span><span class="p">()</span>
<span class="n">proxy_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">cfg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;botdetection.trusted_proxies&quot;</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="p">[])</span>
<span class="k">return</span> <span class="p">[</span><span class="n">ip_network</span><span class="p">(</span><span class="n">net</span><span class="p">,</span> <span class="n">strict</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> <span class="k">for</span> <span class="n">net</span> <span class="ow">in</span> <span class="n">proxy_list</span><span class="p">]</span>
<span class="k">def</span><span class="w"> </span><span class="nf">trusted_remote_addr</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">x_forwarded_for</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">IPv4Address</span> <span class="o">|</span> <span class="n">IPv6Address</span><span class="p">],</span>
<span class="n">trusted_proxies</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">IPv4Network</span> <span class="o">|</span> <span class="n">IPv6Network</span><span class="p">],</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="c1"># always rtl</span>
<span class="k">for</span> <span class="n">addr</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="n">x_forwarded_for</span><span class="p">):</span>
<span class="n">trust</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">for</span> <span class="n">net</span> <span class="ow">in</span> <span class="n">trusted_proxies</span><span class="p">:</span>
<span class="k">if</span> <span class="n">addr</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="n">net</span><span class="o">.</span><span class="n">version</span> <span class="ow">and</span> <span class="n">addr</span> <span class="ow">in</span> <span class="n">net</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;trust proxy </span><span class="si">%s</span><span class="s2"> (member of </span><span class="si">%s</span><span class="s2">)&quot;</span><span class="p">,</span> <span class="n">addr</span><span class="p">,</span> <span class="n">net</span><span class="p">)</span>
<span class="n">trust</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">break</span>
<span class="c1"># client address</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">trust</span><span class="p">:</span>
<span class="k">return</span> <span class="n">addr</span><span class="o">.</span><span class="n">compressed</span>
<span class="c1"># fallback to first address</span>
<span class="k">return</span> <span class="n">x_forwarded_for</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">compressed</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">environ</span><span class="p">:</span> <span class="s2">&quot;WSGIEnvironment&quot;</span><span class="p">,</span> <span class="n">start_response</span><span class="p">:</span> <span class="s2">&quot;StartResponse&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">abc</span><span class="o">.</span><span class="n">Iterable</span><span class="p">[</span><span class="nb">bytes</span><span class="p">]:</span>
<span class="c1"># pylint: disable=too-many-statements</span>
<span class="n">trusted_proxies</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">trusted_proxies</span><span class="p">()</span>
<span class="c1"># We do not rely on the REMOTE_ADDR from the WSGI environment / the</span>
<span class="c1"># variable is first removed from the WSGI environment and explicitly set</span>
<span class="c1"># in this function!</span>
<span class="n">orig_remote_addr</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">environ</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">&quot;REMOTE_ADDR&quot;</span><span class="p">)</span>
<span class="c1"># Validate the IPs involved in this game and delete all invalid ones</span>
<span class="c1"># from the WSGI environment.</span>
<span class="k">if</span> <span class="n">orig_remote_addr</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">addr</span> <span class="o">=</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">orig_remote_addr</span><span class="p">)</span>
<span class="k">if</span> <span class="n">addr</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="mi">6</span> <span class="ow">and</span> <span class="n">addr</span><span class="o">.</span><span class="n">ipv4_mapped</span><span class="p">:</span>
<span class="n">addr</span> <span class="o">=</span> <span class="n">addr</span><span class="o">.</span><span class="n">ipv4_mapped</span>
<span class="n">orig_remote_addr</span> <span class="o">=</span> <span class="n">addr</span><span class="o">.</span><span class="n">compressed</span>
<span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;REMOTE_ADDR: </span><span class="si">%s</span><span class="s2"> / discard REMOTE_ADDR from WSGI environment&quot;</span><span class="p">,</span> <span class="n">exc</span><span class="p">)</span>
<span class="n">orig_remote_addr</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">x_real_ip</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;HTTP_X_REAL_IP&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">x_real_ip</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">addr</span> <span class="o">=</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">x_real_ip</span><span class="p">)</span>
<span class="k">if</span> <span class="n">addr</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="mi">6</span> <span class="ow">and</span> <span class="n">addr</span><span class="o">.</span><span class="n">ipv4_mapped</span><span class="p">:</span>
<span class="n">addr</span> <span class="o">=</span> <span class="n">addr</span><span class="o">.</span><span class="n">ipv4_mapped</span>
<span class="n">x_real_ip</span> <span class="o">=</span> <span class="n">addr</span><span class="o">.</span><span class="n">compressed</span>
<span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;X-Real-IP: </span><span class="si">%s</span><span class="s2"> / discard HTTP_X_REAL_IP from WSGI environment&quot;</span><span class="p">,</span> <span class="n">exc</span><span class="p">)</span>
<span class="n">environ</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">&quot;HTTP_X_REAL_IP&quot;</span><span class="p">)</span>
<span class="n">x_real_ip</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">x_forwarded_for</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">IPv4Address</span> <span class="o">|</span> <span class="n">IPv6Address</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;HTTP_X_FORWARDED_FOR&quot;</span><span class="p">):</span>
<span class="k">for</span> <span class="n">x_for_ip</span> <span class="ow">in</span> <span class="n">parse_list_header</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;HTTP_X_FORWARDED_FOR&quot;</span><span class="p">))):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">addr</span> <span class="o">=</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">x_for_ip</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;X-Forwarded-For: </span><span class="si">%s</span><span class="s2"> / discard HTTP_X_FORWARDED_FOR from WSGI environment&quot;</span><span class="p">,</span> <span class="n">exc</span><span class="p">)</span>
<span class="n">environ</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">&quot;HTTP_X_FORWARDED_FOR&quot;</span><span class="p">)</span>
<span class="n">x_forwarded_for</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">break</span>
<span class="k">if</span> <span class="n">addr</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="mi">6</span> <span class="ow">and</span> <span class="n">addr</span><span class="o">.</span><span class="n">ipv4_mapped</span><span class="p">:</span>
<span class="n">addr</span> <span class="o">=</span> <span class="n">addr</span><span class="o">.</span><span class="n">ipv4_mapped</span>
<span class="n">x_forwarded_for</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">addr</span><span class="p">)</span>
<span class="c1"># log questionable WSGI environments</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">x_forwarded_for</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">x_real_ip</span><span class="p">:</span>
<span class="n">log_error_only_once</span><span class="p">(</span><span class="s2">&quot;X-Forwarded-For nor X-Real-IP header is set!&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">x_forwarded_for</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">trusted_proxies</span><span class="p">:</span>
<span class="n">log_error_only_once</span><span class="p">(</span><span class="s2">&quot;missing botdetection.trusted_proxies config&quot;</span><span class="p">)</span>
<span class="c1"># without trusted_proxies, this variable is useless for determining</span>
<span class="c1"># the real IP</span>
<span class="n">x_forwarded_for</span> <span class="o">=</span> <span class="p">[]</span>
<span class="c1"># securing the WSGI environment variables that are adjusted</span>
<span class="n">environ</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s2">&quot;botdetection.trusted_proxies.orig&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;REMOTE_ADDR&quot;</span><span class="p">:</span> <span class="n">orig_remote_addr</span><span class="p">}})</span>
<span class="c1"># determine *the real IP*</span>
<span class="k">if</span> <span class="n">x_forwarded_for</span><span class="p">:</span>
<span class="n">environ</span><span class="p">[</span><span class="s2">&quot;REMOTE_ADDR&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">trusted_remote_addr</span><span class="p">(</span><span class="n">x_forwarded_for</span><span class="p">,</span> <span class="n">trusted_proxies</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">x_real_ip</span><span class="p">:</span>
<span class="n">environ</span><span class="p">[</span><span class="s2">&quot;REMOTE_ADDR&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">x_real_ip</span>
<span class="k">elif</span> <span class="n">orig_remote_addr</span><span class="p">:</span>
<span class="n">environ</span><span class="p">[</span><span class="s2">&quot;REMOTE_ADDR&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">orig_remote_addr</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;No remote IP could be determined, use black-hole address: 100::&quot;</span><span class="p">)</span>
<span class="n">environ</span><span class="p">[</span><span class="s2">&quot;REMOTE_ADDR&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;100::&quot;</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">_</span> <span class="o">=</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">environ</span><span class="p">[</span><span class="s2">&quot;REMOTE_ADDR&quot;</span><span class="p">])</span>
<span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;REMOTE_ADDR: </span><span class="si">%s</span><span class="s2">, use black-hole address: 100::&quot;</span><span class="p">,</span> <span class="n">exc</span><span class="p">)</span>
<span class="n">environ</span><span class="p">[</span><span class="s2">&quot;REMOTE_ADDR&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;100::&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;final REMOTE_ADDR is: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">environ</span><span class="p">[</span><span class="s2">&quot;REMOTE_ADDR&quot;</span><span class="p">])</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">wsgi_app</span><span class="p">(</span><span class="n">environ</span><span class="p">,</span> <span class="n">start_response</span><span class="p">)</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

175
_modules/searx/brand.html Normal file
View File

@@ -0,0 +1,175 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.brand &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.brand</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.brand</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Implementations needed for a branding of SearXNG.&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-few-public-methods</span>
<span class="c1"># Struct fields aren&#39;t discovered in Python 3.14</span>
<span class="c1"># - https://github.com/searxng/searxng/issues/5284</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">__future__</span><span class="w"> </span><span class="kn">import</span> <span class="n">annotations</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;SettingsBrand&quot;</span><span class="p">]</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">msgspec</span>
<span class="k">class</span><span class="w"> </span><span class="nc">BrandCustom</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">forbid_unknown_fields</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Custom settings in the brand section.&quot;&quot;&quot;</span>
<span class="n">links</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Custom entries in the footer of the WEB page: ``[title]: [link]``&quot;&quot;&quot;</span>
<div class="viewcode-block" id="SettingsBrand">
<a class="viewcode-back" href="../../admin/settings/settings_brand.html#searx.brand.SettingsBrand">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SettingsBrand</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">forbid_unknown_fields</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Options for configuring brand properties.</span>
<span class="sd"> .. code:: yaml</span>
<span class="sd"> brand:</span>
<span class="sd"> issue_url: https://github.com/searxng/searxng/issues</span>
<span class="sd"> docs_url: https://docs.searxng.org</span>
<span class="sd"> public_instances: https://searx.space</span>
<span class="sd"> wiki_url: https://github.com/searxng/searxng/wiki</span>
<span class="sd"> custom:</span>
<span class="sd"> links:</span>
<span class="sd"> Uptime: https://uptime.searxng.org/history/example-org</span>
<span class="sd"> About: https://example.org/user/about.html</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">issue_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;https://github.com/searxng/searxng/issues&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;If you host your own issue tracker change this URL.&quot;&quot;&quot;</span>
<span class="n">docs_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;https://docs.searxng.org&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;If you host your own documentation change this URL.&quot;&quot;&quot;</span>
<span class="n">public_instances</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;https://searx.space&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;If you host your own https://searx.space change this URL.&quot;&quot;&quot;</span>
<span class="n">wiki_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;https://github.com/searxng/searxng/wiki&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Link to your wiki (or ``false``)&quot;&quot;&quot;</span>
<span class="n">custom</span><span class="p">:</span> <span class="n">BrandCustom</span> <span class="o">=</span> <span class="n">msgspec</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="n">BrandCustom</span><span class="p">)</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Optional customizing.</span>
<span class="sd"> .. autoclass:: searx.brand.BrandCustom</span>
<span class="sd"> :members:</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># new_issue_url is a hackish solution tailored for only one hoster (GH). As</span>
<span class="c1"># long as we don&#39;t have a more general solution, we should support it in the</span>
<span class="c1"># given function, but it should not be expanded further.</span>
<span class="n">new_issue_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;https://github.com/searxng/searxng/issues/new&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;If you host your own issue tracker not on GitHub, then unset this URL.</span>
<span class="sd"> Note: This URL will create a pre-filled GitHub bug report form for an</span>
<span class="sd"> engine. Since this feature is implemented only for GH (and limited to</span>
<span class="sd"> engines), it will probably be replaced by another solution in the near</span>
<span class="sd"> future.</span>
<span class="sd"> &quot;&quot;&quot;</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../index.html">
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../index.html">Overview</a>
<ul>
<li><a href="../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

642
_modules/searx/cache.html Normal file
View File

@@ -0,0 +1,642 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.cache &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.cache</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.cache</h1><div class="highlight"><pre>
<span></span><span class="sd">&quot;&quot;&quot;Implementation of caching solutions.</span>
<span class="sd">- :py:obj:`searx.cache.ExpireCache` and its :py:obj:`searx.cache.ExpireCacheCfg`</span>
<span class="sd">----</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;ExpireCacheCfg&quot;</span><span class="p">,</span> <span class="s2">&quot;ExpireCacheStats&quot;</span><span class="p">,</span> <span class="s2">&quot;ExpireCache&quot;</span><span class="p">,</span> <span class="s2">&quot;ExpireCacheSQLite&quot;</span><span class="p">]</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">abc</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">collections.abc</span><span class="w"> </span><span class="kn">import</span> <span class="n">Iterator</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">dataclasses</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">datetime</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">hashlib</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">hmac</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">pickle</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">sqlite3</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">string</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">tempfile</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">msgspec</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">sqlitedb</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_setting</span>
<span class="n">log</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s2">&quot;cache&quot;</span><span class="p">)</span>
<span class="n">CacheRowType</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">TypeAlias</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">,</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span><span class="p">]</span>
<div class="viewcode-block" id="ExpireCacheCfg">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheCfg">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">ExpireCacheCfg</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">):</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Configuration of a :py:obj:`ExpireCache` cache.&quot;&quot;&quot;</span>
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Name of the cache.&quot;&quot;&quot;</span>
<span class="n">db_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;URL of the SQLite DB, the path to the database file. If unset a default</span>
<span class="sd"> DB will be created in `/tmp/sxng_cache_{self.name}.db`&quot;&quot;&quot;</span>
<span class="n">MAX_VALUE_LEN</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">10</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Max length of a *serialized* value.&quot;&quot;&quot;</span>
<span class="n">MAXHOLD_TIME</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span> <span class="c1"># 7 days</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Hold time (default in sec.), after which a value is removed from the cache.&quot;&quot;&quot;</span>
<span class="n">MAINTENANCE_PERIOD</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="c1"># 2h</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Maintenance period in seconds / when :py:obj:`MAINTENANCE_MODE` is set to</span>
<span class="sd"> ``auto``.&quot;&quot;&quot;</span>
<span class="n">MAINTENANCE_MODE</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;auto&quot;</span><span class="p">,</span> <span class="s2">&quot;off&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;auto&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Type of maintenance mode</span>
<span class="sd"> ``auto``:</span>
<span class="sd"> Maintenance is carried out automatically as part of the maintenance</span>
<span class="sd"> intervals (:py:obj:`MAINTENANCE_PERIOD`); no external process is required.</span>
<span class="sd"> ``off``:</span>
<span class="sd"> Maintenance is switched off and must be carried out by an external process</span>
<span class="sd"> if required.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">password</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">=</span> <span class="n">get_setting</span><span class="p">(</span><span class="s2">&quot;server.secret_key&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">encode</span><span class="p">()</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Password used by :py:obj:`ExpireCache.secret_hash`.</span>
<span class="sd"> The default password is taken from :ref:`secret_key &lt;server.secret_key&gt;`.</span>
<span class="sd"> When the password is changed, the hashed keys in the cache can no longer be</span>
<span class="sd"> used, which is why all values in the cache are deleted when the password is</span>
<span class="sd"> changed.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">__post_init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="c1"># if db_url is unset, use a default DB in /tmp/sxng_cache_{name}.db</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">db_url</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">db_url</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">gettempdir</span><span class="p">()</span> <span class="o">+</span> <span class="n">os</span><span class="o">.</span><span class="n">sep</span> <span class="o">+</span> <span class="sa">f</span><span class="s2">&quot;sxng_cache_</span><span class="si">{</span><span class="n">ExpireCache</span><span class="o">.</span><span class="n">normalize_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span><span class="si">}</span><span class="s2">.db&quot;</span></div>
<div class="viewcode-block" id="ExpireCacheStats">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheStats">[docs]</a>
<span class="nd">@dataclasses</span><span class="o">.</span><span class="n">dataclass</span>
<span class="k">class</span><span class="w"> </span><span class="nc">ExpireCacheStats</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Dataclass which provides information on the status of the cache.&quot;&quot;&quot;</span>
<span class="n">cached_items</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">CacheRowType</span><span class="p">]]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Values in the cache mapped by context name.</span>
<span class="sd"> .. code: python</span>
<span class="sd"> {</span>
<span class="sd"> &quot;context name&quot;: [</span>
<span class="sd"> (&quot;foo key&quot;: &quot;foo value&quot;, &lt;expire&gt;),</span>
<span class="sd"> (&quot;bar key&quot;: &quot;bar value&quot;, &lt;expire&gt;),</span>
<span class="sd"> # ...</span>
<span class="sd"> ],</span>
<span class="sd"> # ...</span>
<span class="sd"> }</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">report</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">c_ctx</span> <span class="o">=</span> <span class="mi">0</span>
<span class="n">c_kv</span> <span class="o">=</span> <span class="mi">0</span>
<span class="n">lines</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">ctx_name</span><span class="p">,</span> <span class="n">kv_list</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cached_items</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">c_ctx</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">kv_list</span><span class="p">:</span>
<span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;[</span><span class="si">{</span><span class="n">ctx_name</span><span class="si">:</span><span class="s2">20s</span><span class="si">}</span><span class="s2">] empty&quot;</span><span class="p">)</span>
<span class="k">continue</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">expire</span> <span class="ow">in</span> <span class="n">kv_list</span><span class="p">:</span>
<span class="n">valid_until</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">if</span> <span class="n">expire</span><span class="p">:</span>
<span class="n">valid_until</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">expire</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S&quot;</span><span class="p">)</span>
<span class="n">c_kv</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;[</span><span class="si">{</span><span class="n">ctx_name</span><span class="si">:</span><span class="s2">20s</span><span class="si">}</span><span class="s2">] </span><span class="si">{</span><span class="n">valid_until</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">key</span><span class="si">:</span><span class="s2">12</span><span class="si">}</span><span class="s2">&quot;</span> <span class="sa">f</span><span class="s2">&quot; --&gt; (</span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">value</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">) </span><span class="si">{</span><span class="n">value</span><span class="si">}</span><span class="s2"> &quot;</span><span class="p">)</span>
<span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Number of contexts: </span><span class="si">{</span><span class="n">c_ctx</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;number of key/value pairs: </span><span class="si">{</span><span class="n">c_kv</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">lines</span><span class="p">)</span></div>
<div class="viewcode-block" id="ExpireCache">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">ExpireCache</span><span class="p">(</span><span class="n">abc</span><span class="o">.</span><span class="n">ABC</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Abstract base class for the implementation of a key/value cache</span>
<span class="sd"> with expire date.&quot;&quot;&quot;</span>
<span class="n">cfg</span><span class="p">:</span> <span class="n">ExpireCacheCfg</span>
<span class="n">hash_token</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;hash_token&quot;</span>
<div class="viewcode-block" id="ExpireCache.set">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache.set">[docs]</a>
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">,</span> <span class="n">expire</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">ctx</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Set *key* to *value*. To set a timeout on key use argument</span>
<span class="sd"> ``expire`` (in sec.). If expire is unset the default is taken from</span>
<span class="sd"> :py:obj:`ExpireCacheCfg.MAXHOLD_TIME`. After the timeout has expired,</span>
<span class="sd"> the key will automatically be deleted.</span>
<span class="sd"> The ``ctx`` argument specifies the context of the ``key``. A key is</span>
<span class="sd"> only unique in its context.</span>
<span class="sd"> The concrete implementations of this abstraction determine how the</span>
<span class="sd"> context is mapped in the connected database. In SQL databases, for</span>
<span class="sd"> example, the context is a DB table or in a Key/Value DB it could be</span>
<span class="sd"> a prefix for the key.</span>
<span class="sd"> If the context is not specified (the default is ``None``) then a</span>
<span class="sd"> default context should be used, e.g. a default table for SQL databases</span>
<span class="sd"> or a default prefix in a Key/Value DB.</span>
<span class="sd"> &quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="ExpireCache.get">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache.get">[docs]</a>
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">ctx</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Return *value* of *key*. If key is unset, ``None`` is returned.&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="ExpireCache.maintenance">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache.maintenance">[docs]</a>
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">truncate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Performs maintenance on the cache.</span>
<span class="sd"> ``force``:</span>
<span class="sd"> Maintenance should be carried out even if the maintenance interval has</span>
<span class="sd"> not yet been reached.</span>
<span class="sd"> ``truncate``:</span>
<span class="sd"> Truncate the entire cache, which is necessary, for example, if the</span>
<span class="sd"> password has changed.</span>
<span class="sd"> &quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="ExpireCache.state">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache.state">[docs]</a>
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ExpireCacheStats</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns a :py:obj:`ExpireCacheStats`, which provides information</span>
<span class="sd"> about the status of the cache.&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="ExpireCache.build_cache">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache.build_cache">[docs]</a>
<span class="nd">@staticmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">build_cache</span><span class="p">(</span><span class="n">cfg</span><span class="p">:</span> <span class="n">ExpireCacheCfg</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ExpireCacheSQLite&quot;</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Factory to build a caching instance.</span>
<span class="sd"> .. note::</span>
<span class="sd"> Currently, only the SQLite adapter is available, but other database</span>
<span class="sd"> types could be implemented in the future, e.g. a Valkey (Redis)</span>
<span class="sd"> adapter.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">ExpireCacheSQLite</span><span class="p">(</span><span class="n">cfg</span><span class="p">)</span></div>
<div class="viewcode-block" id="ExpireCache.normalize_name">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache.normalize_name">[docs]</a>
<span class="nd">@staticmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">normalize_name</span><span class="p">(</span><span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns a normalized name that can be used as a file name or as a SQL</span>
<span class="sd"> table name (is used, for example, to normalize the context name).&quot;&quot;&quot;</span>
<span class="n">_valid</span> <span class="o">=</span> <span class="s2">&quot;-_.&quot;</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">ascii_letters</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">digits</span>
<span class="k">return</span> <span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">name</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">_valid</span><span class="p">])</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bytes</span><span class="p">:</span>
<span class="n">dump</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="k">return</span> <span class="n">dump</span>
<span class="k">def</span><span class="w"> </span><span class="nf">deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="nb">bytes</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
<span class="n">obj</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="k">return</span> <span class="n">obj</span>
<div class="viewcode-block" id="ExpireCache.secret_hash">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache.secret_hash">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">secret_hash</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Creates a hash of the argument ``name``. The hash value is formed</span>
<span class="sd"> from the ``name`` combined with the :py:obj:`password</span>
<span class="sd"> &lt;ExpireCacheCfg.password&gt;`. Can be used, for example, to make the</span>
<span class="sd"> ``key`` stored in the DB unreadable for third parties.&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="nb">bytes</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
<span class="n">m</span> <span class="o">=</span> <span class="n">hmac</span><span class="o">.</span><span class="n">new</span><span class="p">(</span><span class="n">name</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> <span class="n">digestmod</span><span class="o">=</span><span class="s1">&#39;sha256&#39;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">m</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span></div>
</div>
<div class="viewcode-block" id="ExpireCacheSQLite">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">ExpireCacheSQLite</span><span class="p">(</span><span class="n">sqlitedb</span><span class="o">.</span><span class="n">SQLiteAppl</span><span class="p">,</span> <span class="n">ExpireCache</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Cache that manages key/value pairs in a SQLite DB. The DB model in the</span>
<span class="sd"> SQLite DB is implemented in abstract class :py:obj:`SQLiteAppl</span>
<span class="sd"> &lt;searx.sqlitedb.SQLiteAppl&gt;`.</span>
<span class="sd"> The following configurations are required / supported:</span>
<span class="sd"> - :py:obj:`ExpireCacheCfg.db_url`</span>
<span class="sd"> - :py:obj:`ExpireCacheCfg.MAXHOLD_TIME`</span>
<span class="sd"> - :py:obj:`ExpireCacheCfg.MAINTENANCE_PERIOD`</span>
<span class="sd"> - :py:obj:`ExpireCacheCfg.MAINTENANCE_MODE`</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">DB_SCHEMA</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span>
<span class="c1"># The key/value tables will be created on demand by self.create_table</span>
<span class="n">DDL_CREATE_TABLES</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">CACHE_TABLE_PREFIX</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;CACHE-TABLE&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">ExpireCacheCfg</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;An instance of the SQLite expire cache is build up from a</span>
<span class="sd"> :py:obj:`config &lt;ExpireCacheCfg&gt;`.&quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="p">:</span> <span class="n">ExpireCacheCfg</span> <span class="o">=</span> <span class="n">cfg</span>
<span class="k">if</span> <span class="n">cfg</span><span class="o">.</span><span class="n">db_url</span> <span class="o">==</span> <span class="s2">&quot;:memory:&quot;</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">critical</span><span class="p">(</span><span class="s2">&quot;don&#39;t use SQLite DB in :memory: in production!!&quot;</span><span class="p">)</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">cfg</span><span class="o">.</span><span class="n">db_url</span><span class="p">)</span>
<div class="viewcode-block" id="ExpireCacheSQLite.init">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.init">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn</span><span class="p">:</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">Connection</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="n">ret_val</span> <span class="o">=</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">init</span><span class="p">(</span><span class="n">conn</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">ret_val</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="n">new</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">password</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
<span class="n">old</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hash_token</span><span class="p">)</span>
<span class="k">if</span> <span class="n">old</span> <span class="o">!=</span> <span class="n">new</span><span class="p">:</span>
<span class="k">if</span> <span class="n">old</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;[</span><span class="si">%s</span><span class="s2">] hash token changed: truncate all cache tables&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">maintenance</span><span class="p">(</span><span class="n">force</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">truncate</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hash_token</span><span class="p">,</span> <span class="n">new</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span></div>
<div class="viewcode-block" id="ExpireCacheSQLite.maintenance">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.maintenance">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">truncate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">force</span> <span class="ow">and</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_maintenance_time</span><span class="p">:</span>
<span class="c1"># log.debug(&quot;no maintenance required yet, next maintenance interval is in the future&quot;)</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="c1"># Prevent parallel DB maintenance cycles from other DB connections</span>
<span class="c1"># (e.g. in multi thread or process environments).</span>
<span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s2">&quot;LAST_MAINTENANCE&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span> <span class="c1"># hint: this (also) sets the m_time of the property!</span>
<span class="k">if</span> <span class="n">truncate</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">truncate_tables</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">table_names</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="c1"># drop items by expire time stamp ..</span>
<span class="n">expire</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span>
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
<span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_names</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;DELETE FROM </span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2"> WHERE expire &lt; ?&quot;</span><span class="p">,</span> <span class="p">(</span><span class="n">expire</span><span class="p">,))</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;deleted </span><span class="si">%s</span><span class="s2"> keys from table </span><span class="si">%s</span><span class="s2"> (expire date reached)&quot;</span><span class="p">,</span> <span class="n">res</span><span class="o">.</span><span class="n">rowcount</span><span class="p">,</span> <span class="n">table</span><span class="p">)</span>
<span class="c1"># Vacuuming the WALs</span>
<span class="c1"># https://www.theunterminatedstring.com/sqlite-vacuuming/</span>
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">&quot;PRAGMA wal_checkpoint(TRUNCATE)&quot;</span><span class="p">)</span>
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">return</span> <span class="kc">True</span></div>
<div class="viewcode-block" id="ExpireCacheSQLite.create_table">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.create_table">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">create_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Create DB ``table`` if it has not yet been created, no recreates are</span>
<span class="sd"> initiated if the table already exists.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">table</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_names</span><span class="p">:</span>
<span class="c1"># log.debug(&quot;key/value table %s exists in DB (no need to recreate)&quot;, table)</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;key/value table &#39;</span><span class="si">%s</span><span class="s2">&#39; NOT exists in DB -&gt; create DB table ..&quot;</span><span class="p">,</span> <span class="n">table</span><span class="p">)</span>
<span class="n">sql_table</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
<span class="p">[</span>
<span class="sa">f</span><span class="s2">&quot;CREATE TABLE IF NOT EXISTS </span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2"> (&quot;</span><span class="p">,</span>
<span class="s2">&quot; key TEXT,&quot;</span><span class="p">,</span>
<span class="s2">&quot; value BLOB,&quot;</span><span class="p">,</span>
<span class="sa">f</span><span class="s2">&quot; expire INTEGER DEFAULT (strftime(&#39;%s&#39;, &#39;now&#39;) + </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAXHOLD_TIME</span><span class="si">}</span><span class="s2">),&quot;</span><span class="p">,</span>
<span class="s2">&quot;PRIMARY KEY (key))&quot;</span><span class="p">,</span>
<span class="p">]</span>
<span class="p">)</span>
<span class="n">sql_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;CREATE INDEX IF NOT EXISTS index_expire_</span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2"> ON </span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2">(expire);&quot;</span>
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql_table</span><span class="p">)</span>
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql_index</span><span class="p">)</span>
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">CACHE_TABLE_PREFIX</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">table</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span></div>
<span class="nd">@property</span>
<span class="k">def</span><span class="w"> </span><span class="nf">table_names</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;List of key/value tables already created in the DB.&quot;&quot;&quot;</span>
<span class="n">sql</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;SELECT value FROM properties WHERE name LIKE &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">CACHE_TABLE_PREFIX</span><span class="si">}</span><span class="s2">%%&#39;&quot;</span>
<span class="n">rows</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span><span class="o">.</span><span class="n">fetchall</span><span class="p">()</span> <span class="ow">or</span> <span class="p">[]</span>
<span class="k">return</span> <span class="p">[</span><span class="n">r</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">]</span>
<span class="k">def</span><span class="w"> </span><span class="nf">truncate_tables</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table_names</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]):</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;truncate table: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot;,&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">table_names</span><span class="p">))</span>
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
<span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="n">table_names</span><span class="p">:</span>
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;DELETE FROM </span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="nd">@property</span>
<span class="k">def</span><span class="w"> </span><span class="nf">next_maintenance_time</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns (unix epoch) time of the next maintenance.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAINTENANCE_PERIOD</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">m_time</span><span class="p">(</span><span class="s2">&quot;LAST_MAINTENANCE&quot;</span><span class="p">,</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()))</span>
<span class="c1"># implement ABC methods of ExpireCache</span>
<div class="viewcode-block" id="ExpireCacheSQLite.set">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.set">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">,</span> <span class="n">expire</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">ctx</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Set key/value in DB table given by argument ``ctx``. If expire is</span>
<span class="sd"> unset the default is taken from :py:obj:`ExpireCacheCfg.MAXHOLD_TIME`.</span>
<span class="sd"> If ``ctx`` argument is ``None`` (the default), a table name is</span>
<span class="sd"> generated from the :py:obj:`ExpireCacheCfg.name`. If DB table does not</span>
<span class="sd"> exists, it will be created (on demand) by :py:obj:`self.create_table</span>
<span class="sd"> &lt;ExpireCacheSQLite.create_table&gt;`.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">c</span><span class="p">,</span> <span class="n">err_msg_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_setmany</span><span class="p">([(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">expire</span><span class="p">)],</span> <span class="n">ctx</span><span class="o">=</span><span class="n">ctx</span><span class="p">)</span>
<span class="k">if</span> <span class="n">c</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%s</span><span class="s2"> -- </span><span class="si">%s</span><span class="s2">: key &#39;</span><span class="si">%s</span><span class="s2">&#39; updated or inserted (</span><span class="si">%s</span><span class="s2"> errors)&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">ctx</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">err_msg_list</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">for</span> <span class="n">msg</span> <span class="ow">in</span> <span class="n">err_msg_list</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%s</span><span class="s2"> -- </span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">ctx</span><span class="p">,</span> <span class="n">msg</span><span class="p">)</span>
<span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">c</span><span class="p">)</span></div>
<div class="viewcode-block" id="ExpireCacheSQLite.setmany">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.setmany">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">setmany</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">opt_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">CacheRowType</span><span class="p">],</span>
<span class="n">ctx</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Efficient bootload of the cache from a list of options. The list</span>
<span class="sd"> contains tuples with the arguments described in</span>
<span class="sd"> :py:obj:`ExpireCacheSQLite.set`.&quot;&quot;&quot;</span>
<span class="n">_start</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
<span class="n">c</span><span class="p">,</span> <span class="n">err_msg_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_setmany</span><span class="p">(</span><span class="n">opt_list</span><span class="o">=</span><span class="n">opt_list</span><span class="p">,</span> <span class="n">ctx</span><span class="o">=</span><span class="n">ctx</span><span class="p">)</span>
<span class="n">_end</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
<span class="k">for</span> <span class="n">msg</span> <span class="ow">in</span> <span class="n">err_msg_list</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%s</span><span class="s2"> -- </span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">ctx</span><span class="p">,</span> <span class="n">msg</span><span class="p">)</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
<span class="s2">&quot;</span><span class="si">%s</span><span class="s2"> -- </span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2">/</span><span class="si">%s</span><span class="s2"> key/value pairs updated or inserted in </span><span class="si">%s</span><span class="s2"> sec (</span><span class="si">%s</span><span class="s2"> errors)&quot;</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
<span class="n">ctx</span><span class="p">,</span>
<span class="n">c</span><span class="p">,</span>
<span class="nb">len</span><span class="p">(</span><span class="n">opt_list</span><span class="p">),</span>
<span class="n">_end</span> <span class="o">-</span> <span class="n">_start</span><span class="p">,</span>
<span class="nb">len</span><span class="p">(</span><span class="n">err_msg_list</span><span class="p">),</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">c</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">_setmany</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">opt_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">CacheRowType</span><span class="p">],</span>
<span class="n">ctx</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
<span class="n">table</span> <span class="o">=</span> <span class="n">ctx</span>
<span class="bp">self</span><span class="o">.</span><span class="n">maintenance</span><span class="p">()</span>
<span class="n">table_name</span> <span class="o">=</span> <span class="n">table</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">table_name</span><span class="p">:</span>
<span class="n">table_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">normalize_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">create_table</span><span class="p">(</span><span class="n">table_name</span><span class="p">)</span>
<span class="n">sql_str</span> <span class="o">=</span> <span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;INSERT INTO </span><span class="si">{</span><span class="n">table_name</span><span class="si">}</span><span class="s2"> (key, value, expire) VALUES (?, ?, ?)&quot;</span>
<span class="sa">f</span><span class="s2">&quot; ON CONFLICT DO &quot;</span>
<span class="sa">f</span><span class="s2">&quot;UPDATE SET value=?, expire=?&quot;</span>
<span class="p">)</span>
<span class="n">sql_rows</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span>
<span class="nb">tuple</span><span class="p">[</span>
<span class="nb">str</span><span class="p">,</span> <span class="c1"># key</span>
<span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">,</span> <span class="c1"># value</span>
<span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="c1"># expire</span>
<span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">,</span> <span class="c1"># value</span>
<span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="c1"># expire</span>
<span class="p">]</span>
<span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">err_msg_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">_val</span><span class="p">,</span> <span class="n">expire</span> <span class="ow">in</span> <span class="n">opt_list</span><span class="p">:</span>
<span class="n">value</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">serialize</span><span class="p">(</span><span class="n">value</span><span class="o">=</span><span class="n">_val</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">value</span><span class="p">)</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAX_VALUE_LEN</span><span class="p">:</span>
<span class="n">err_msg_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2">.key=&#39;</span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">&#39; - serialized value too big to cache (len: </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">value</span><span class="p">)</span><span class="si">}</span><span class="s2">) &quot;</span><span class="p">)</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">expire</span><span class="p">:</span>
<span class="n">expire</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAXHOLD_TIME</span>
<span class="n">expire</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span> <span class="o">+</span> <span class="n">expire</span>
<span class="c1"># positional arguments of the INSERT INTO statement</span>
<span class="n">sql_args</span> <span class="o">=</span> <span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">expire</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">expire</span><span class="p">)</span>
<span class="n">sql_rows</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">sql_args</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">sql_rows</span><span class="p">:</span>
<span class="k">return</span> <span class="mi">0</span><span class="p">,</span> <span class="n">err_msg_list</span>
<span class="k">if</span> <span class="n">table</span><span class="p">:</span>
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">executemany</span><span class="p">(</span><span class="n">sql_str</span><span class="p">,</span> <span class="n">sql_rows</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
<span class="n">conn</span><span class="o">.</span><span class="n">executemany</span><span class="p">(</span><span class="n">sql_str</span><span class="p">,</span> <span class="n">sql_rows</span><span class="p">)</span>
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">sql_rows</span><span class="p">),</span> <span class="n">err_msg_list</span>
<div class="viewcode-block" id="ExpireCacheSQLite.get">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.get">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">ctx</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get value of ``key`` from table given by argument ``ctx``. If</span>
<span class="sd"> ``ctx`` argument is ``None`` (the default), a table name is generated</span>
<span class="sd"> from the :py:obj:`ExpireCacheCfg.name`. If ``key`` not exists (in</span>
<span class="sd"> table), the ``default`` value is returned.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">table</span> <span class="o">=</span> <span class="n">ctx</span>
<span class="bp">self</span><span class="o">.</span><span class="n">maintenance</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">table</span><span class="p">:</span>
<span class="n">table</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">normalize_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="k">if</span> <span class="n">table</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_names</span><span class="p">:</span>
<span class="k">return</span> <span class="n">default</span>
<span class="n">sql</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;SELECT value FROM </span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2"> WHERE key = ?&quot;</span>
<span class="n">row</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="p">(</span><span class="n">key</span><span class="p">,))</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
<span class="k">if</span> <span class="n">row</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">default</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">deserialize</span><span class="p">(</span><span class="n">row</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span></div>
<div class="viewcode-block" id="ExpireCacheSQLite.pairs">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.pairs">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">pairs</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ctx</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Iterator</span><span class="p">[</span><span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Iterate over key/value pairs from table given by argument ``ctx``.</span>
<span class="sd"> If ``ctx`` argument is ``None`` (the default), a table name is</span>
<span class="sd"> generated from the :py:obj:`ExpireCacheCfg.name`.&quot;&quot;&quot;</span>
<span class="n">table</span> <span class="o">=</span> <span class="n">ctx</span>
<span class="bp">self</span><span class="o">.</span><span class="n">maintenance</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">table</span><span class="p">:</span>
<span class="n">table</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">normalize_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="k">if</span> <span class="n">table</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_names</span><span class="p">:</span>
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;SELECT key, value FROM </span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">):</span>
<span class="k">yield</span> <span class="n">row</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">deserialize</span><span class="p">(</span><span class="n">row</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span></div>
<div class="viewcode-block" id="ExpireCacheSQLite.state">
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.state">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ExpireCacheStats</span><span class="p">:</span>
<span class="n">cached_items</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">CacheRowType</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_names</span><span class="p">:</span>
<span class="n">cached_items</span><span class="p">[</span><span class="n">table</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;SELECT key, value, expire FROM </span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">):</span>
<span class="n">cached_items</span><span class="p">[</span><span class="n">table</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">row</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">deserialize</span><span class="p">(</span><span class="n">row</span><span class="p">[</span><span class="mi">1</span><span class="p">]),</span> <span class="n">row</span><span class="p">[</span><span class="mi">2</span><span class="p">]))</span>
<span class="k">return</span> <span class="n">ExpireCacheStats</span><span class="p">(</span><span class="n">cached_items</span><span class="o">=</span><span class="n">cached_items</span><span class="p">)</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../index.html">
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../index.html">Overview</a>
<ul>
<li><a href="../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,482 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.enginelib &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.enginelib</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.enginelib</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Implementations of the framework for the SearXNG engines.</span>
<span class="sd">- :py:obj:`searx.enginelib.EngineCache`</span>
<span class="sd">- :py:obj:`searx.enginelib.Engine`</span>
<span class="sd">- :py:obj:`searx.enginelib.traits`</span>
<span class="sd">There is a command line for developer purposes and for deeper analysis. Here is</span>
<span class="sd">an example in which the command line is called in the development environment::</span>
<span class="sd"> $ ./manage pyenv.cmd bash --norc --noprofile</span>
<span class="sd"> (py3) python -m searx.enginelib --help</span>
<span class="sd">.. hint::</span>
<span class="sd"> The long term goal is to modularize all implementations of the engine</span>
<span class="sd"> framework here in this Python package. ToDo:</span>
<span class="sd"> - move implementations of the :ref:`searx.engines loader` to a new module in</span>
<span class="sd"> the :py:obj:`searx.enginelib` namespace.</span>
<span class="sd">-----</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;EngineCache&quot;</span><span class="p">,</span> <span class="s2">&quot;Engine&quot;</span><span class="p">,</span> <span class="s2">&quot;ENGINES_CACHE&quot;</span><span class="p">]</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">abc</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">collections.abc</span><span class="w"> </span><span class="kn">import</span> <span class="n">Callable</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">string</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typer</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">..cache</span><span class="w"> </span><span class="kn">import</span> <span class="n">ExpireCacheSQLite</span><span class="p">,</span> <span class="n">ExpireCacheCfg</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib</span><span class="w"> </span><span class="kn">import</span> <span class="n">traits</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OfflineParamTypes</span><span class="p">,</span> <span class="n">OnlineParamTypes</span>
<span class="n">ENGINES_CACHE</span><span class="p">:</span> <span class="n">ExpireCacheSQLite</span> <span class="o">=</span> <span class="n">ExpireCacheSQLite</span><span class="o">.</span><span class="n">build_cache</span><span class="p">(</span>
<span class="n">ExpireCacheCfg</span><span class="p">(</span>
<span class="n">name</span><span class="o">=</span><span class="s2">&quot;ENGINES_CACHE&quot;</span><span class="p">,</span>
<span class="n">MAXHOLD_TIME</span><span class="o">=</span><span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span><span class="p">,</span> <span class="c1"># 7 days</span>
<span class="n">MAINTENANCE_PERIOD</span><span class="o">=</span><span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span><span class="p">,</span> <span class="c1"># 2h</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="sd">&quot;&quot;&quot;Global :py:obj:`searx.cache.ExpireCacheSQLite` instance where the cached</span>
<span class="sd">values from all engines are stored. The `MAXHOLD_TIME` is 7 days and the</span>
<span class="sd">`MAINTENANCE_PERIOD` is set to two hours.&quot;&quot;&quot;</span>
<span class="n">app</span> <span class="o">=</span> <span class="n">typer</span><span class="o">.</span><span class="n">Typer</span><span class="p">()</span>
<span class="nd">@app</span><span class="o">.</span><span class="n">command</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">():</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Show state for the caches of the engines.&quot;&quot;&quot;</span>
<span class="n">title</span> <span class="o">=</span> <span class="s2">&quot;cache tables and key/values&quot;</span>
<span class="nb">print</span><span class="p">(</span><span class="n">title</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">title</span><span class="p">))</span>
<span class="nb">print</span><span class="p">(</span><span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">state</span><span class="p">()</span><span class="o">.</span><span class="n">report</span><span class="p">())</span>
<span class="nb">print</span><span class="p">()</span>
<span class="n">title</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;properties of </span><span class="si">{</span><span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="nb">print</span><span class="p">(</span><span class="n">title</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">title</span><span class="p">))</span>
<span class="nb">print</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">properties</span><span class="p">))</span>
<span class="nd">@app</span><span class="o">.</span><span class="n">command</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Carry out maintenance on cache of the engines.&quot;&quot;&quot;</span>
<span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">maintenance</span><span class="p">(</span><span class="n">force</span><span class="o">=</span><span class="n">force</span><span class="p">)</span>
<div class="viewcode-block" id="EngineCache">
<a class="viewcode-back" href="../../dev/engines/enginelib.html#searx.enginelib.EngineCache">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">EngineCache</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Persistent (SQLite) key/value cache that deletes its values again after</span>
<span class="sd"> ``expire`` seconds (default/max: :py:obj:`MAXHOLD_TIME</span>
<span class="sd"> &lt;searx.cache.ExpireCacheCfg.MAXHOLD_TIME&gt;`). This class is a wrapper around</span>
<span class="sd"> :py:obj:`ENGINES_CACHE` (:py:obj:`ExpireCacheSQLite</span>
<span class="sd"> &lt;searx.cache.ExpireCacheSQLite&gt;`).</span>
<span class="sd"> In the :origin:`searx/engines/demo_offline.py` engine you can find an</span>
<span class="sd"> exemplary implementation of such a cache other examples are implemented</span>
<span class="sd"> in:</span>
<span class="sd"> - :origin:`searx/engines/radio_browser.py`</span>
<span class="sd"> - :origin:`searx/engines/soundcloud.py`</span>
<span class="sd"> - :origin:`searx/engines/startpage.py`</span>
<span class="sd"> .. code: python</span>
<span class="sd"> from searx.enginelib import EngineCache</span>
<span class="sd"> CACHE: EngineCache</span>
<span class="sd"> def init(engine_settings):</span>
<span class="sd"> global CACHE</span>
<span class="sd"> CACHE = EngineCache(engine_settings[&quot;name&quot;])</span>
<span class="sd"> def request(query, params):</span>
<span class="sd"> token = CACHE.get(key=&quot;token&quot;)</span>
<span class="sd"> if token is None:</span>
<span class="sd"> token = get_token()</span>
<span class="sd"> # cache token of this engine for 1h</span>
<span class="sd"> CACHE.set(key=&quot;token&quot;, value=token, expire=3600)</span>
<span class="sd"> ...</span>
<span class="sd"> For introspection of the DB, jump into developer environment and run command to</span>
<span class="sd"> show cache state::</span>
<span class="sd"> $ ./manage pyenv.cmd bash --norc --noprofile</span>
<span class="sd"> (py3) python -m searx.enginelib cache state</span>
<span class="sd"> cache tables and key/values</span>
<span class="sd"> ===========================</span>
<span class="sd"> [demo_offline ] 2025-04-22 11:32:50 count --&gt; (int) 4</span>
<span class="sd"> [startpage ] 2025-04-22 12:32:30 SC_CODE --&gt; (str) fSOBnhEMlDfE20</span>
<span class="sd"> [duckduckgo ] 2025-04-22 12:32:31 4dff493e.... --&gt; (str) 4-128634958369380006627592672385352473325</span>
<span class="sd"> [duckduckgo ] 2025-04-22 12:40:06 3e2583e2.... --&gt; (str) 4-263126175288871260472289814259666848451</span>
<span class="sd"> [radio_browser ] 2025-04-23 11:33:08 servers --&gt; (list) [&#39;https://de2.api.radio-browser.info&#39;, ...]</span>
<span class="sd"> [soundcloud ] 2025-04-29 11:40:06 guest_client_id --&gt; (str) EjkRJG0BLNEZquRiPZYdNtJdyGtTuHdp</span>
<span class="sd"> [wolframalpha ] 2025-04-22 12:40:06 code --&gt; (str) 5aa79f86205ad26188e0e26e28fb7ae7</span>
<span class="sd"> number of tables: 6</span>
<span class="sd"> number of key/value pairs: 7</span>
<span class="sd"> In the &quot;cache tables and key/values&quot; section, the table name (engine name) is at</span>
<span class="sd"> first position on the second there is the calculated expire date and on the</span>
<span class="sd"> third and fourth position the key/value is shown.</span>
<span class="sd"> About duckduckgo: The *vqd coode* of ddg depends on the query term and therefore</span>
<span class="sd"> the key is a hash value of the query term (to not to store the raw query term).</span>
<span class="sd"> In the &quot;properties of ENGINES_CACHE&quot; section all properties of the SQLiteAppl /</span>
<span class="sd"> ExpireCache and their last modification date are shown::</span>
<span class="sd"> properties of ENGINES_CACHE</span>
<span class="sd"> ===========================</span>
<span class="sd"> [last modified: 2025-04-22 11:32:27] DB_SCHEMA : 1</span>
<span class="sd"> [last modified: 2025-04-22 11:32:27] LAST_MAINTENANCE :</span>
<span class="sd"> [last modified: 2025-04-22 11:32:27] crypt_hash : ca612e3566fdfd7cf7efe...</span>
<span class="sd"> [last modified: 2025-04-22 11:32:30] CACHE-TABLE--demo_offline: demo_offline</span>
<span class="sd"> [last modified: 2025-04-22 11:32:30] CACHE-TABLE--startpage: startpage</span>
<span class="sd"> [last modified: 2025-04-22 11:32:31] CACHE-TABLE--duckduckgo: duckduckgo</span>
<span class="sd"> [last modified: 2025-04-22 11:33:08] CACHE-TABLE--radio_browser: radio_browser</span>
<span class="sd"> [last modified: 2025-04-22 11:40:06] CACHE-TABLE--soundcloud: soundcloud</span>
<span class="sd"> [last modified: 2025-04-22 11:40:06] CACHE-TABLE--wolframalpha: wolframalpha</span>
<span class="sd"> These properties provide information about the state of the ExpireCache and</span>
<span class="sd"> control the behavior. For example, the maintenance intervals are controlled by</span>
<span class="sd"> the last modification date of the LAST_MAINTENANCE property and the hash value</span>
<span class="sd"> of the password can be used to detect whether the password has been changed (in</span>
<span class="sd"> this case the DB entries can no longer be decrypted and the entire cache must be</span>
<span class="sd"> discarded).</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">engine_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">expire</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">expire</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">expire</span> <span class="ow">or</span> <span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAXHOLD_TIME</span>
<span class="n">_valid</span> <span class="o">=</span> <span class="s2">&quot;-_.&quot;</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">ascii_letters</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">digits</span>
<span class="bp">self</span><span class="o">.</span><span class="n">table_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">c</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">_valid</span> <span class="k">else</span> <span class="s2">&quot;_&quot;</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">engine_name</span><span class="p">])</span>
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">,</span> <span class="n">expire</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="k">return</span> <span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">set</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span>
<span class="n">value</span><span class="o">=</span><span class="n">value</span><span class="p">,</span>
<span class="n">expire</span><span class="o">=</span><span class="n">expire</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">expire</span><span class="p">,</span>
<span class="n">ctx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_name</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
<span class="k">return</span> <span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">default</span><span class="p">,</span> <span class="n">ctx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_name</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">secret_hash</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="k">return</span> <span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">secret_hash</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">)</span></div>
<div class="viewcode-block" id="Engine">
<a class="viewcode-back" href="../../dev/engines/enginelib.html#searx.enginelib.Engine">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">Engine</span><span class="p">(</span><span class="n">abc</span><span class="o">.</span><span class="n">ABC</span><span class="p">):</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Class of engine instances build from YAML settings.</span>
<span class="sd"> Further documentation see :ref:`general engine configuration`.</span>
<span class="sd"> .. hint::</span>
<span class="sd"> This class is currently never initialized and only used for type hinting.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">logger</span><span class="p">:</span> <span class="n">logging</span><span class="o">.</span><span class="n">Logger</span>
<span class="c1"># Common options in the engine module</span>
<span class="n">engine_type</span><span class="p">:</span> <span class="nb">str</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Type of the engine (:ref:`searx.search.processors`)&quot;&quot;&quot;</span>
<span class="n">paging</span><span class="p">:</span> <span class="nb">bool</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Engine supports multiple pages.&quot;&quot;&quot;</span>
<span class="n">max_page</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;If the engine supports paging, then this is the value for the last page</span>
<span class="sd"> that is still supported. ``0`` means unlimited numbers of pages.&quot;&quot;&quot;</span>
<span class="n">time_range_support</span><span class="p">:</span> <span class="nb">bool</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Engine supports search time range.&quot;&quot;&quot;</span>
<span class="n">safesearch</span><span class="p">:</span> <span class="nb">bool</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Engine supports SafeSearch&quot;&quot;&quot;</span>
<span class="n">language_support</span><span class="p">:</span> <span class="nb">bool</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Engine supports languages (locales) search.&quot;&quot;&quot;</span>
<span class="n">language</span><span class="p">:</span> <span class="nb">str</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;For an engine, when there is ``language: ...`` in the YAML settings the engine</span>
<span class="sd"> does support only this one language:</span>
<span class="sd"> .. code:: yaml</span>
<span class="sd"> - name: google french</span>
<span class="sd"> engine: google</span>
<span class="sd"> language: fr</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">region</span><span class="p">:</span> <span class="nb">str</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;For an engine, when there is ``region: ...`` in the YAML settings the engine</span>
<span class="sd"> does support only this one region::</span>
<span class="sd"> .. code:: yaml</span>
<span class="sd"> - name: google belgium</span>
<span class="sd"> engine: google</span>
<span class="sd"> region: fr-BE</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">fetch_traits</span><span class="p">:</span> <span class="s2">&quot;Callable[[EngineTraits, bool], None]&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Function to to fetch engine&#39;s traits from origin.&quot;&quot;&quot;</span>
<span class="n">traits</span><span class="p">:</span> <span class="s2">&quot;traits.EngineTraits&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Traits of the engine.&quot;&quot;&quot;</span>
<span class="c1"># settings.yml</span>
<span class="n">categories</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Specifies to which :ref:`engine categories` the engine should be added.&quot;&quot;&quot;</span>
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Name that will be used across SearXNG to define this engine. In settings, on</span>
<span class="sd"> the result page ..&quot;&quot;&quot;</span>
<span class="n">engine</span><span class="p">:</span> <span class="nb">str</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Name of the python file used to handle requests and responses to and from</span>
<span class="sd"> this search engine (file name from :origin:`searx/engines` without</span>
<span class="sd"> ``.py``).&quot;&quot;&quot;</span>
<span class="n">enable_http</span><span class="p">:</span> <span class="nb">bool</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Enable HTTP (by default only HTTPS is enabled).&quot;&quot;&quot;</span>
<span class="n">shortcut</span><span class="p">:</span> <span class="nb">str</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Code used to execute bang requests (``!foo``)&quot;&quot;&quot;</span>
<span class="n">timeout</span><span class="p">:</span> <span class="nb">float</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Specific timeout for search-engine.&quot;&quot;&quot;</span>
<span class="n">display_error_messages</span><span class="p">:</span> <span class="nb">bool</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Display error messages on the web UI.&quot;&quot;&quot;</span>
<span class="n">proxies</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Set proxies for a specific engine (YAML):</span>
<span class="sd"> .. code:: yaml</span>
<span class="sd"> proxies :</span>
<span class="sd"> http: socks5://proxy:port</span>
<span class="sd"> https: socks5://proxy:port</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">disabled</span><span class="p">:</span> <span class="nb">bool</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;To disable by default the engine, but not deleting it. It will allow the</span>
<span class="sd"> user to manually activate it in the settings.&quot;&quot;&quot;</span>
<span class="n">inactive</span><span class="p">:</span> <span class="nb">bool</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Remove the engine from the settings (*disabled &amp; removed*).&quot;&quot;&quot;</span>
<span class="n">about</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Additional fields describing the engine.</span>
<span class="sd"> .. code:: yaml</span>
<span class="sd"> about:</span>
<span class="sd"> website: https://example.com</span>
<span class="sd"> wikidata_id: Q306656</span>
<span class="sd"> official_api_documentation: https://example.com/api-doc</span>
<span class="sd"> use_official_api: true</span>
<span class="sd"> require_api_key: true</span>
<span class="sd"> results: HTML</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">using_tor_proxy</span><span class="p">:</span> <span class="nb">bool</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Using tor proxy (``true``) or not (``false``) for this engine.&quot;&quot;&quot;</span>
<span class="n">send_accept_language_header</span><span class="p">:</span> <span class="nb">bool</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;When this option is activated, the language (locale) that is selected by</span>
<span class="sd"> the user is used to build and send a ``Accept-Language`` header in the</span>
<span class="sd"> request to the origin search engine.&quot;&quot;&quot;</span>
<span class="n">tokens</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A list of secret tokens to make this engine *private*, more details see</span>
<span class="sd"> :ref:`private engines`.&quot;&quot;&quot;</span>
<span class="n">weight</span><span class="p">:</span> <span class="nb">int</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Weighting of the results of this engine (:ref:`weight &lt;settings engines&gt;`).&quot;&quot;&quot;</span>
<div class="viewcode-block" id="Engine.setup">
<a class="viewcode-back" href="../../dev/engines/enginelib.html#searx.enginelib.Engine.setup">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Dynamic setup of the engine settings.</span>
<span class="sd"> With this method, the engine&#39;s setup is carried out. For example, to</span>
<span class="sd"> check or dynamically adapt the values handed over in the parameter</span>
<span class="sd"> ``engine_settings``. The return value (True/False) indicates whether</span>
<span class="sd"> the setup was successful and the engine can be built or rejected.</span>
<span class="sd"> The method is optional and is called synchronously as part of the</span>
<span class="sd"> initialization of the service and is therefore only suitable for simple</span>
<span class="sd"> (local) exams/changes at the engine setting. The :py:obj:`Engine.init`</span>
<span class="sd"> method must be used for longer tasks in which values of a remote must be</span>
<span class="sd"> determined, for example.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="kc">True</span></div>
<div class="viewcode-block" id="Engine.init">
<a class="viewcode-back" href="../../dev/engines/enginelib.html#searx.enginelib.Engine.init">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Initialization of the engine.</span>
<span class="sd"> The method is optional and asynchronous (in a thread). It is suitable,</span>
<span class="sd"> for example, for setting up a cache (for the engine) or for querying</span>
<span class="sd"> values (required by the engine) from a remote.</span>
<span class="sd"> Whether the initialization was successful can be indicated by the return</span>
<span class="sd"> value ``True`` or even ``False``.</span>
<span class="sd"> - If no return value is given from this init method (``None``), this is</span>
<span class="sd"> equivalent to ``True``.</span>
<span class="sd"> - If an exception is thrown as part of the initialization, this is</span>
<span class="sd"> equivalent to ``False``.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="kc">True</span></div>
<div class="viewcode-block" id="Engine.search">
<a class="viewcode-back" href="../../dev/engines/enginelib.html#searx.enginelib.Engine.search">[docs]</a>
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">&quot;OfflineParamTypes&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;EngineResults&quot;</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Search method of the ``offline`` engines&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="Engine.request">
<a class="viewcode-back" href="../../dev/engines/enginelib.html#searx.enginelib.Engine.request">[docs]</a>
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">&quot;OnlineParamTypes&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Method to build the parameters for the request of an ``online``</span>
<span class="sd"> engine.&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="Engine.response">
<a class="viewcode-back" href="../../dev/engines/enginelib.html#searx.enginelib.Engine.response">[docs]</a>
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resp</span><span class="p">:</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;EngineResults&quot;</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Method to parse the response of an ``online`` engine.&quot;&quot;&quot;</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../index.html">
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../index.html">Overview</a>
<ul>
<li><a href="../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,410 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.enginelib.traits &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../enginelib.html" accesskey="U">searx.enginelib</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.enginelib.traits</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.enginelib.traits</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Engine&#39;s traits are fetched from the origin engines and stored in a JSON file</span>
<span class="sd">in the *data folder*. Most often traits are languages and region codes and</span>
<span class="sd">their mapping from SearXNG&#39;s representation to the representation in the origin</span>
<span class="sd">search engine. For new traits new properties can be added to the class</span>
<span class="sd">:py:class:`EngineTraits`.</span>
<span class="sd">To load traits from the persistence :py:obj:`EngineTraitsMap.from_data` can be</span>
<span class="sd">used.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">dataclasses</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">types</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">pathlib</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">locales</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.data</span><span class="w"> </span><span class="kn">import</span> <span class="n">data_dir</span><span class="p">,</span> <span class="n">ENGINE_TRAITS</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">Engine</span>
<div class="viewcode-block" id="EngineTraitsEncoder">
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraitsEncoder">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">EngineTraitsEncoder</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">JSONEncoder</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Encodes :class:`EngineTraits` to a serializable object, see</span>
<span class="sd"> :class:`json.JSONEncoder`.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="EngineTraitsEncoder.default">
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraitsEncoder.default">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">default</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">o</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Return dictionary of a :class:`EngineTraits` object.&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">o</span><span class="p">,</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="k">return</span> <span class="n">o</span><span class="o">.</span><span class="vm">__dict__</span>
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">default</span><span class="p">(</span><span class="n">o</span><span class="p">)</span></div>
</div>
<div class="viewcode-block" id="EngineTraits">
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraits">[docs]</a>
<span class="nd">@dataclasses</span><span class="o">.</span><span class="n">dataclass</span>
<span class="k">class</span><span class="w"> </span><span class="nc">EngineTraits</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The class is intended to be instantiated for each engine.&quot;&quot;&quot;</span>
<span class="n">regions</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">dataclasses</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">dict</span><span class="p">)</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Maps SearXNG&#39;s internal representation of a region to the one of the engine.</span>
<span class="sd"> SearXNG&#39;s internal representation can be parsed by babel and the value is</span>
<span class="sd"> send to the engine:</span>
<span class="sd"> .. code:: python</span>
<span class="sd"> regions ={</span>
<span class="sd"> &#39;fr-BE&#39; : &lt;engine&#39;s region name&gt;,</span>
<span class="sd"> }</span>
<span class="sd"> for key, egnine_region regions.items():</span>
<span class="sd"> searxng_region = babel.Locale.parse(key, sep=&#39;-&#39;)</span>
<span class="sd"> ...</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">languages</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">dataclasses</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">dict</span><span class="p">)</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Maps SearXNG&#39;s internal representation of a language to the one of the engine.</span>
<span class="sd"> SearXNG&#39;s internal representation can be parsed by babel and the value is</span>
<span class="sd"> send to the engine:</span>
<span class="sd"> .. code:: python</span>
<span class="sd"> languages = {</span>
<span class="sd"> &#39;ca&#39; : &lt;engine&#39;s language name&gt;,</span>
<span class="sd"> }</span>
<span class="sd"> for key, egnine_lang in languages.items():</span>
<span class="sd"> searxng_lang = babel.Locale.parse(key)</span>
<span class="sd"> ...</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">all_locale</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;To which locale value SearXNG&#39;s ``all`` language is mapped (shown a &quot;Default</span>
<span class="sd"> language&quot;).</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">data_type</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s1">&#39;traits_v1&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;traits_v1&#39;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Data type, default is &#39;traits_v1&#39;.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">custom</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">dataclasses</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">dict</span><span class="p">)</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A place to store engine&#39;s custom traits, not related to the SearXNG core.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<div class="viewcode-block" id="EngineTraits.get_language">
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraits.get_language">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_language</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">searxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Return engine&#39;s language string that *best fits* to SearXNG&#39;s locale.</span>
<span class="sd"> :param searxng_locale: SearXNG&#39;s internal representation of locale</span>
<span class="sd"> selected by the user.</span>
<span class="sd"> :param default: engine&#39;s default language</span>
<span class="sd"> The *best fits* rules are implemented in</span>
<span class="sd"> :py:obj:`searx.locales.get_engine_locale`. Except for the special value ``all``</span>
<span class="sd"> which is determined from :py:obj:`EngineTraits.all_locale`.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">searxng_locale</span> <span class="o">==</span> <span class="s1">&#39;all&#39;</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_locale</span>
<span class="k">return</span> <span class="n">locales</span><span class="o">.</span><span class="n">get_engine_locale</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">languages</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">default</span><span class="p">)</span></div>
<div class="viewcode-block" id="EngineTraits.get_region">
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraits.get_region">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_region</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">searxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Return engine&#39;s region string that best fits to SearXNG&#39;s locale.</span>
<span class="sd"> :param searxng_locale: SearXNG&#39;s internal representation of locale</span>
<span class="sd"> selected by the user.</span>
<span class="sd"> :param default: engine&#39;s default region</span>
<span class="sd"> The *best fits* rules are implemented in</span>
<span class="sd"> :py:obj:`searx.locales.get_engine_locale`. Except for the special value ``all``</span>
<span class="sd"> which is determined from :py:obj:`EngineTraits.all_locale`.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">searxng_locale</span> <span class="o">==</span> <span class="s1">&#39;all&#39;</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_locale</span>
<span class="k">return</span> <span class="n">locales</span><span class="o">.</span><span class="n">get_engine_locale</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">regions</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">default</span><span class="p">)</span></div>
<div class="viewcode-block" id="EngineTraits.is_locale_supported">
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraits.is_locale_supported">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">is_locale_supported</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">searxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A *locale* (SearXNG&#39;s internal representation) is considered to be</span>
<span class="sd"> supported by the engine if the *region* or the *language* is supported</span>
<span class="sd"> by the engine.</span>
<span class="sd"> For verification the functions :py:func:`EngineTraits.get_region` and</span>
<span class="sd"> :py:func:`EngineTraits.get_language` are used.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">data_type</span> <span class="o">==</span> <span class="s1">&#39;traits_v1&#39;</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">))</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;engine traits of type </span><span class="si">%s</span><span class="s1"> is unknown&#39;</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">data_type</span><span class="p">)</span></div>
<div class="viewcode-block" id="EngineTraits.copy">
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraits.copy">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">copy</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Create a copy of the dataclass object.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">EngineTraits</span><span class="p">(</span><span class="o">**</span><span class="n">dataclasses</span><span class="o">.</span><span class="n">asdict</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span> <span class="c1"># type: ignore</span></div>
<div class="viewcode-block" id="EngineTraits.fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraits.fetch_traits">[docs]</a>
<span class="nd">@classmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">engine</span><span class="p">:</span> <span class="s2">&quot;Engine | types.ModuleType&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;EngineTraits | None&quot;</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Call a function ``fetch_traits(engine_traits)`` from engines namespace to fetch</span>
<span class="sd"> and set properties from the origin engine in the object ``engine_traits``. If</span>
<span class="sd"> function does not exists, ``None`` is returned.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">fetch_traits</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">&#39;fetch_traits&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="n">engine_traits</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">fetch_traits</span><span class="p">:</span>
<span class="n">engine_traits</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">()</span>
<span class="n">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">)</span>
<span class="k">return</span> <span class="n">engine_traits</span></div>
<div class="viewcode-block" id="EngineTraits.set_traits">
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraits.set_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">set_traits</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">engine</span><span class="p">:</span> <span class="s2">&quot;Engine | types.ModuleType&quot;</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Set traits from self object in a :py:obj:`.Engine` namespace.</span>
<span class="sd"> :param engine: engine instance build by :py:func:`searx.engines.load_engine`</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">data_type</span> <span class="o">==</span> <span class="s1">&#39;traits_v1&#39;</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_set_traits_v1</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;engine traits of type </span><span class="si">%s</span><span class="s1"> is unknown&#39;</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">data_type</span><span class="p">)</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">_set_traits_v1</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">engine</span><span class="p">:</span> <span class="s2">&quot;Engine | types.ModuleType&quot;</span><span class="p">):</span>
<span class="c1"># For an engine, when there is `language: ...` in the YAML settings the engine</span>
<span class="c1"># does support only this one language (region)::</span>
<span class="c1">#</span>
<span class="c1"># - name: google italian</span>
<span class="c1"># engine: google</span>
<span class="c1"># language: it</span>
<span class="c1"># region: it-IT</span>
<span class="n">traits</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">_msg</span> <span class="o">=</span> <span class="s2">&quot;settings.yml - engine: &#39;</span><span class="si">%s</span><span class="s2">&#39; / </span><span class="si">%s</span><span class="s2">: &#39;</span><span class="si">%s</span><span class="s2">&#39; not supported&quot;</span>
<span class="n">languages</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">languages</span>
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">&#39;language&#39;</span><span class="p">):</span>
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">language</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">languages</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">_msg</span> <span class="o">%</span> <span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="s1">&#39;language&#39;</span><span class="p">,</span> <span class="n">engine</span><span class="o">.</span><span class="n">language</span><span class="p">))</span>
<span class="n">traits</span><span class="o">.</span><span class="n">languages</span> <span class="o">=</span> <span class="p">{</span><span class="n">engine</span><span class="o">.</span><span class="n">language</span><span class="p">:</span> <span class="n">languages</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">language</span><span class="p">]}</span>
<span class="n">regions</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">regions</span>
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">&#39;region&#39;</span><span class="p">):</span>
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">region</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">regions</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">_msg</span> <span class="o">%</span> <span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="s1">&#39;region&#39;</span><span class="p">,</span> <span class="n">engine</span><span class="o">.</span><span class="n">region</span><span class="p">))</span>
<span class="n">traits</span><span class="o">.</span><span class="n">regions</span> <span class="o">=</span> <span class="p">{</span><span class="n">engine</span><span class="o">.</span><span class="n">region</span><span class="p">:</span> <span class="n">regions</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">region</span><span class="p">]}</span>
<span class="n">engine</span><span class="o">.</span><span class="n">language_support</span> <span class="o">=</span> <span class="nb">bool</span><span class="p">(</span><span class="n">traits</span><span class="o">.</span><span class="n">languages</span> <span class="ow">or</span> <span class="n">traits</span><span class="o">.</span><span class="n">regions</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="c1"># set the copied &amp; modified traits in engine&#39;s namespace</span>
<span class="n">engine</span><span class="o">.</span><span class="n">traits</span> <span class="o">=</span> <span class="n">traits</span> <span class="c1"># pyright: ignore[reportAttributeAccessIssue]</span></div>
<div class="viewcode-block" id="EngineTraitsMap">
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraitsMap">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">EngineTraitsMap</span><span class="p">(</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">EngineTraits</span><span class="p">]):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A python dictionary to map :class:`EngineTraits` by engine name.&quot;&quot;&quot;</span>
<span class="n">ENGINE_TRAITS_FILE</span><span class="p">:</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span> <span class="o">=</span> <span class="p">(</span><span class="n">data_dir</span> <span class="o">/</span> <span class="s1">&#39;engine_traits.json&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">resolve</span><span class="p">()</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;File with persistence of the :py:obj:`EngineTraitsMap`.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="EngineTraitsMap.save_data">
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraitsMap.save_data">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">save_data</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Store EngineTraitsMap in in file :py:obj:`self.ENGINE_TRAITS_FILE`&quot;&quot;&quot;</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ENGINE_TRAITS_FILE</span><span class="p">,</span> <span class="s1">&#39;w&#39;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">f</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">sort_keys</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="bp">cls</span><span class="o">=</span><span class="n">EngineTraitsEncoder</span><span class="p">)</span></div>
<div class="viewcode-block" id="EngineTraitsMap.from_data">
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraitsMap.from_data">[docs]</a>
<span class="nd">@classmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">from_data</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s1">&#39;EngineTraitsMap&#39;</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Instantiate :class:`EngineTraitsMap` object from :py:obj:`ENGINE_TRAITS`&quot;&quot;&quot;</span>
<span class="n">obj</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">()</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">ENGINE_TRAITS</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">obj</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">EngineTraits</span><span class="p">(</span><span class="o">**</span><span class="n">v</span><span class="p">)</span>
<span class="k">return</span> <span class="n">obj</span></div>
<span class="nd">@classmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">log</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Callable</span><span class="p">[[</span><span class="nb">str</span><span class="p">],</span> <span class="kc">None</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="s1">&#39;EngineTraitsMap&#39;</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">engines</span> <span class="c1"># pylint: disable=cyclic-import, import-outside-toplevel</span>
<span class="n">names</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">engines</span><span class="o">.</span><span class="n">engines</span><span class="p">)</span>
<span class="n">names</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span>
<span class="n">obj</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">()</span>
<span class="k">for</span> <span class="n">engine_name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">:</span>
<span class="n">engine</span><span class="p">:</span> <span class="n">Engine</span> <span class="o">|</span> <span class="n">types</span><span class="o">.</span><span class="n">ModuleType</span> <span class="o">=</span> <span class="n">engines</span><span class="o">.</span><span class="n">engines</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]</span>
<span class="n">traits</span> <span class="o">=</span> <span class="kc">None</span>
<span class="c1"># pylint: disable=broad-exception-caught</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">traits</span> <span class="o">=</span> <span class="n">EngineTraits</span><span class="o">.</span><span class="n">fetch_traits</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
<span class="n">log</span><span class="p">(</span><span class="s2">&quot;FATAL: while fetch_traits </span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="n">exc</span><span class="p">))</span>
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;FORCE&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;on&#39;</span><span class="p">,</span> <span class="s1">&#39;true&#39;</span><span class="p">,</span> <span class="s1">&#39;1&#39;</span><span class="p">]:</span>
<span class="k">raise</span>
<span class="n">v</span> <span class="o">=</span> <span class="n">ENGINE_TRAITS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">engine_name</span><span class="p">)</span>
<span class="k">if</span> <span class="n">v</span><span class="p">:</span>
<span class="n">log</span><span class="p">(</span><span class="s2">&quot;FORCE: re-use old values from fetch_traits - ENGINE_TRAITS[</span><span class="si">%s</span><span class="s2">]&quot;</span> <span class="o">%</span> <span class="n">engine_name</span><span class="p">)</span>
<span class="n">traits</span> <span class="o">=</span> <span class="n">EngineTraits</span><span class="p">(</span><span class="o">**</span><span class="n">v</span><span class="p">)</span>
<span class="k">if</span> <span class="n">traits</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">log</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%-20s</span><span class="s2">: SearXNG languages --&gt; </span><span class="si">%s</span><span class="s2"> &quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">traits</span><span class="o">.</span><span class="n">languages</span><span class="p">)))</span>
<span class="n">log</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%-20s</span><span class="s2">: SearXNG regions --&gt; </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">traits</span><span class="o">.</span><span class="n">regions</span><span class="p">)))</span>
<span class="n">obj</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">traits</span>
<span class="k">return</span> <span class="n">obj</span>
<div class="viewcode-block" id="EngineTraitsMap.set_traits">
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraitsMap.set_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">set_traits</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">engine</span><span class="p">:</span> <span class="s2">&quot;Engine | types.ModuleType&quot;</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Set traits in a :py:obj:`Engine` namespace.</span>
<span class="sd"> :param engine: engine instance build by :py:func:`searx.engines.load_engine`</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">engine_traits</span> <span class="o">=</span> <span class="n">EngineTraits</span><span class="p">(</span><span class="n">data_type</span><span class="o">=</span><span class="s1">&#39;traits_v1&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
<span class="n">engine_traits</span> <span class="o">=</span> <span class="bp">self</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">]</span>
<span class="k">elif</span> <span class="n">engine</span><span class="o">.</span><span class="n">engine</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
<span class="c1"># The key of the dictionary traits_map is the *engine name*</span>
<span class="c1"># configured in settings.xml. When multiple engines are configured</span>
<span class="c1"># in settings.yml to use the same origin engine (python module)</span>
<span class="c1"># these additional engines can use the languages from the origin</span>
<span class="c1"># engine. For this use the configured ``engine: ...`` from</span>
<span class="c1"># settings.yml</span>
<span class="n">engine_traits</span> <span class="o">=</span> <span class="bp">self</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">engine</span><span class="p">]</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">set_traits</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../enginelib.html">searx.enginelib</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

399
_modules/searx/engines.html Normal file
View File

@@ -0,0 +1,399 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Load and initialize the ``engines``, see :py:func:`load_engines` and register</span>
<span class="sd">:py:obj:`engine_shortcuts`.</span>
<span class="sd">usage::</span>
<span class="sd"> load_engines( settings[&#39;engines&#39;] )</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">sys</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">copy</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">os.path</span><span class="w"> </span><span class="kn">import</span> <span class="n">realpath</span><span class="p">,</span> <span class="n">dirname</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">types</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">inspect</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span><span class="p">,</span> <span class="n">settings</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">load_module</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Engine</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">&#39;engines&#39;</span><span class="p">)</span>
<span class="n">ENGINE_DIR</span> <span class="o">=</span> <span class="n">dirname</span><span class="p">(</span><span class="n">realpath</span><span class="p">(</span><span class="vm">__file__</span><span class="p">))</span>
<span class="c1"># Defaults for the namespace of an engine module, see load_engine()</span>
<span class="n">ENGINE_DEFAULT_ARGS</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span> <span class="o">|</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">list</span><span class="p">[</span><span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">|</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">|</span> <span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
<span class="c1"># Common options in the engine module</span>
<span class="s2">&quot;engine_type&quot;</span><span class="p">:</span> <span class="s2">&quot;online&quot;</span><span class="p">,</span>
<span class="s2">&quot;paging&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;time_range_support&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;safesearch&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="c1"># settings.yml</span>
<span class="s2">&quot;categories&quot;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;general&quot;</span><span class="p">],</span>
<span class="s2">&quot;enable_http&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;shortcut&quot;</span><span class="p">:</span> <span class="s2">&quot;-&quot;</span><span class="p">,</span>
<span class="s2">&quot;timeout&quot;</span><span class="p">:</span> <span class="n">settings</span><span class="p">[</span><span class="s2">&quot;outgoing&quot;</span><span class="p">][</span><span class="s2">&quot;request_timeout&quot;</span><span class="p">],</span>
<span class="s2">&quot;display_error_messages&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;disabled&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;inactive&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;about&quot;</span><span class="p">:</span> <span class="p">{},</span>
<span class="s2">&quot;using_tor_proxy&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;send_accept_language_header&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;tokens&quot;</span><span class="p">:</span> <span class="p">[],</span>
<span class="s2">&quot;max_page&quot;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># set automatically when an engine does not have any tab category</span>
<span class="n">DEFAULT_CATEGORY</span> <span class="o">=</span> <span class="s1">&#39;other&#39;</span>
<span class="n">categories</span><span class="p">:</span> <span class="s2">&quot;dict[str, list[Engine|types.ModuleType]]&quot;</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;general&#39;</span><span class="p">:</span> <span class="p">[]}</span>
<span class="n">engines</span><span class="p">:</span> <span class="s2">&quot;dict[str, Engine | types.ModuleType]&quot;</span> <span class="o">=</span> <span class="p">{}</span>
<span class="sd">&quot;&quot;&quot;Global registered engine instances.&quot;&quot;&quot;</span>
<span class="n">engine_shortcuts</span> <span class="o">=</span> <span class="p">{}</span>
<span class="sd">&quot;&quot;&quot;Simple map of registered *shortcuts* to name of the engine (or ``None``).</span>
<span class="sd">::</span>
<span class="sd"> engine_shortcuts[engine.shortcut] = engine.name</span>
<span class="sd">:meta hide-value:</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">check_engine_module</span><span class="p">(</span><span class="n">module</span><span class="p">:</span> <span class="n">types</span><span class="o">.</span><span class="n">ModuleType</span><span class="p">):</span>
<span class="c1"># probe unintentional name collisions / for example name collisions caused</span>
<span class="c1"># by import statements in the engine module ..</span>
<span class="c1"># network: https://github.com/searxng/searxng/issues/762#issuecomment-1605323861</span>
<span class="n">obj</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="s1">&#39;network&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">obj</span> <span class="ow">and</span> <span class="n">inspect</span><span class="o">.</span><span class="n">ismodule</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
<span class="n">msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;type of </span><span class="si">{</span><span class="n">module</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s1">.network is a module (</span><span class="si">{</span><span class="n">obj</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s1">), expected a string&#39;</span>
<span class="c1"># logger.error(msg)</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
<div class="viewcode-block" id="load_engine">
<a class="viewcode-back" href="../../dev/engines/engines.html#searx.engines.load_engine">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">load_engine</span><span class="p">(</span><span class="n">engine_data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="s2">&quot;Engine | types.ModuleType | None&quot;</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Load engine from ``engine_data``.</span>
<span class="sd"> :param dict engine_data: Attributes from YAML ``settings:engines/&lt;engine&gt;``</span>
<span class="sd"> :return: initialized namespace of the ``&lt;engine&gt;``.</span>
<span class="sd"> 1. create a namespace and load module of the ``&lt;engine&gt;``</span>
<span class="sd"> 2. update namespace with the defaults from :py:obj:`ENGINE_DEFAULT_ARGS`</span>
<span class="sd"> 3. update namespace with values from ``engine_data``</span>
<span class="sd"> If engine *is active*, return namespace of the engine, otherwise return</span>
<span class="sd"> ``None``.</span>
<span class="sd"> This function also returns ``None`` if initialization of the namespace fails</span>
<span class="sd"> for one of the following reasons:</span>
<span class="sd"> - engine name contains underscore</span>
<span class="sd"> - engine name is not lowercase</span>
<span class="sd"> - required attribute is not set :py:func:`is_missing_required_attributes`</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-many-return-statements</span>
<span class="n">engine_name</span> <span class="o">=</span> <span class="n">engine_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;name&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">engine_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;An engine does not have a &quot;name&quot; field&#39;</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">if</span> <span class="s1">&#39;_&#39;</span> <span class="ow">in</span> <span class="n">engine_name</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;Engine name contains underscore: &quot;</span><span class="si">{}</span><span class="s1">&quot;&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine_name</span><span class="p">))</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">engine_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">!=</span> <span class="n">engine_name</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s1">&#39;Engine name is not lowercase: &quot;</span><span class="si">{}</span><span class="s1">&quot;, converting to lowercase&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine_name</span><span class="p">))</span>
<span class="n">engine_name</span> <span class="o">=</span> <span class="n">engine_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="n">engine_data</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_name</span>
<span class="c1"># load_module</span>
<span class="n">module_name</span> <span class="o">=</span> <span class="n">engine_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;engine&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">module_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;The &quot;engine&quot; field is missing for the engine named &quot;</span><span class="si">{}</span><span class="s1">&quot;&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine_name</span><span class="p">))</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">engine</span> <span class="o">=</span> <span class="n">load_module</span><span class="p">(</span><span class="n">module_name</span> <span class="o">+</span> <span class="s1">&#39;.py&#39;</span><span class="p">,</span> <span class="n">ENGINE_DIR</span><span class="p">)</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">SyntaxError</span><span class="p">,</span> <span class="ne">KeyboardInterrupt</span><span class="p">,</span> <span class="ne">SystemExit</span><span class="p">,</span> <span class="ne">SystemError</span><span class="p">,</span> <span class="ne">ImportError</span><span class="p">,</span> <span class="ne">RuntimeError</span><span class="p">):</span>
<span class="n">logger</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s1">&#39;Fatal exception in engine &quot;</span><span class="si">{}</span><span class="s1">&quot;&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">module_name</span><span class="p">))</span>
<span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">BaseException</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s1">&#39;Cannot load engine &quot;</span><span class="si">{}</span><span class="s1">&quot;&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">module_name</span><span class="p">))</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">check_engine_module</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
<span class="n">update_engine_attributes</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">engine_data</span><span class="p">)</span>
<span class="n">update_attributes_for_tor</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
<span class="c1"># avoid cyclic imports</span>
<span class="c1"># pylint: disable=import-outside-toplevel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraitsMap</span>
<span class="n">trait_map</span> <span class="o">=</span> <span class="n">EngineTraitsMap</span><span class="o">.</span><span class="n">from_data</span><span class="p">()</span>
<span class="n">trait_map</span><span class="o">.</span><span class="n">set_traits</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_engine_active</span><span class="p">(</span><span class="n">engine</span><span class="p">):</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">is_missing_required_attributes</span><span class="p">(</span><span class="n">engine</span><span class="p">):</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">set_loggers</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">engine_name</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">call_engine_setup</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">engine_data</span><span class="p">):</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">any</span><span class="p">(</span><span class="n">cat</span> <span class="ow">in</span> <span class="n">settings</span><span class="p">[</span><span class="s1">&#39;categories_as_tabs&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">cat</span> <span class="ow">in</span> <span class="n">engine</span><span class="o">.</span><span class="n">categories</span><span class="p">):</span>
<span class="n">engine</span><span class="o">.</span><span class="n">categories</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">DEFAULT_CATEGORY</span><span class="p">)</span>
<span class="k">return</span> <span class="n">engine</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">set_loggers</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">&quot;Engine|types.ModuleType&quot;</span><span class="p">,</span> <span class="n">engine_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="c1"># set the logger for engine</span>
<span class="n">engine</span><span class="o">.</span><span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="n">engine_name</span><span class="p">)</span>
<span class="c1"># the engine may have load some other engines</span>
<span class="c1"># may sure the logger is initialized</span>
<span class="c1"># use sys.modules.copy() to avoid &quot;RuntimeError: dictionary changed size during iteration&quot;</span>
<span class="c1"># see https://github.com/python/cpython/issues/89516</span>
<span class="c1"># and https://docs.python.org/3.10/library/sys.html#sys.modules</span>
<span class="n">modules</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="k">for</span> <span class="n">module_name</span><span class="p">,</span> <span class="n">module</span> <span class="ow">in</span> <span class="n">modules</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="p">(</span>
<span class="n">module_name</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;searx.engines&quot;</span><span class="p">)</span>
<span class="ow">and</span> <span class="n">module_name</span> <span class="o">!=</span> <span class="s2">&quot;searx.engines.__init__&quot;</span>
<span class="ow">and</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="s2">&quot;logger&quot;</span><span class="p">)</span>
<span class="p">):</span>
<span class="n">module_engine_name</span> <span class="o">=</span> <span class="n">module_name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
<span class="n">module</span><span class="o">.</span><span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="n">module_engine_name</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="k">def</span><span class="w"> </span><span class="nf">update_engine_attributes</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">&quot;Engine | types.ModuleType&quot;</span><span class="p">,</span> <span class="n">engine_data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]):</span>
<span class="c1"># set engine attributes from engine_data</span>
<span class="k">for</span> <span class="n">param_name</span><span class="p">,</span> <span class="n">param_value</span> <span class="ow">in</span> <span class="n">engine_data</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="n">param_name</span> <span class="o">==</span> <span class="s1">&#39;categories&#39;</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">param_value</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="n">param_value</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="nb">str</span><span class="o">.</span><span class="n">strip</span><span class="p">,</span> <span class="n">param_value</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)))</span>
<span class="n">engine</span><span class="o">.</span><span class="n">categories</span> <span class="o">=</span> <span class="n">param_value</span> <span class="c1"># type: ignore</span>
<span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">&#39;about&#39;</span><span class="p">)</span> <span class="ow">and</span> <span class="n">param_name</span> <span class="o">==</span> <span class="s1">&#39;about&#39;</span><span class="p">:</span>
<span class="n">engine</span><span class="o">.</span><span class="n">about</span> <span class="o">=</span> <span class="p">{</span><span class="o">**</span><span class="n">engine</span><span class="o">.</span><span class="n">about</span><span class="p">,</span> <span class="o">**</span><span class="n">engine_data</span><span class="p">[</span><span class="s1">&#39;about&#39;</span><span class="p">]}</span> <span class="c1"># type: ignore</span>
<span class="k">else</span><span class="p">:</span>
<span class="nb">setattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">param_name</span><span class="p">,</span> <span class="n">param_value</span><span class="p">)</span>
<span class="c1"># set default attributes</span>
<span class="k">for</span> <span class="n">arg_name</span><span class="p">,</span> <span class="n">arg_value</span> <span class="ow">in</span> <span class="n">ENGINE_DEFAULT_ARGS</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">arg_name</span><span class="p">):</span>
<span class="nb">setattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">arg_name</span><span class="p">,</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">arg_value</span><span class="p">))</span>
<span class="k">def</span><span class="w"> </span><span class="nf">update_attributes_for_tor</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">&quot;Engine | types.ModuleType&quot;</span><span class="p">):</span>
<span class="k">if</span> <span class="n">using_tor_proxy</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">&#39;onion_url&#39;</span><span class="p">):</span>
<span class="n">engine</span><span class="o">.</span><span class="n">search_url</span> <span class="o">=</span> <span class="n">engine</span><span class="o">.</span><span class="n">onion_url</span> <span class="o">+</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">&#39;search_path&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">engine</span><span class="o">.</span><span class="n">timeout</span> <span class="o">+=</span> <span class="n">settings</span><span class="p">[</span><span class="s1">&#39;outgoing&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;extra_proxy_timeout&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<div class="viewcode-block" id="is_missing_required_attributes">
<a class="viewcode-back" href="../../dev/engines/engines.html#searx.engines.is_missing_required_attributes">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">is_missing_required_attributes</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">&quot;Engine | types.ModuleType&quot;</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;An attribute is required when its name doesn&#39;t start with ``_`` (underline).</span>
<span class="sd"> Required attributes must not be ``None``.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">missing</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">for</span> <span class="n">engine_attr</span> <span class="ow">in</span> <span class="nb">dir</span><span class="p">(</span><span class="n">engine</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">engine_attr</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">engine_attr</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;Missing engine config attribute: &quot;</span><span class="si">{0}</span><span class="s1">.</span><span class="si">{1}</span><span class="s1">&quot;&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">engine_attr</span><span class="p">))</span>
<span class="n">missing</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">return</span> <span class="n">missing</span></div>
<div class="viewcode-block" id="using_tor_proxy">
<a class="viewcode-back" href="../../dev/engines/engines.html#searx.engines.using_tor_proxy">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">using_tor_proxy</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">&quot;Engine | types.ModuleType&quot;</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Return True if the engine configuration declares to use Tor.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">settings</span><span class="p">[</span><span class="s1">&#39;outgoing&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;using_tor_proxy&#39;</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">&#39;using_tor_proxy&#39;</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">is_engine_active</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">&quot;Engine | types.ModuleType&quot;</span><span class="p">):</span>
<span class="c1"># check if engine is inactive</span>
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">inactive</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="c1"># exclude onion engines if not using tor</span>
<span class="k">if</span> <span class="s1">&#39;onions&#39;</span> <span class="ow">in</span> <span class="n">engine</span><span class="o">.</span><span class="n">categories</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">using_tor_proxy</span><span class="p">(</span><span class="n">engine</span><span class="p">):</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="k">def</span><span class="w"> </span><span class="nf">call_engine_setup</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">&quot;Engine | types.ModuleType&quot;</span><span class="p">,</span> <span class="n">engine_data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="n">setup_ok</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">setup_func</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s2">&quot;setup&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">setup_func</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">setup_ok</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">elif</span> <span class="ow">not</span> <span class="nb">callable</span><span class="p">(</span><span class="n">setup_func</span><span class="p">):</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;engine&#39;s setup method isn&#39;t a callable (is of type: </span><span class="si">%s</span><span class="s2">)&quot;</span><span class="p">,</span> <span class="nb">type</span><span class="p">(</span><span class="n">setup_func</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">setup_ok</span> <span class="o">=</span> <span class="n">engine</span><span class="o">.</span><span class="n">setup</span><span class="p">(</span><span class="n">engine_data</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
<span class="n">logger</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s1">&#39;exception : </span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">setup_ok</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%s</span><span class="s2">: Engine setup was not successful, engine is set to inactive.&quot;</span><span class="p">,</span> <span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="k">return</span> <span class="n">setup_ok</span>
<span class="k">def</span><span class="w"> </span><span class="nf">register_engine</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">&quot;Engine | types.ModuleType&quot;</span><span class="p">):</span>
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">name</span> <span class="ow">in</span> <span class="n">engines</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;Engine config error: ambiguous name: </span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">))</span>
<span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
<span class="n">engines</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine</span>
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">shortcut</span> <span class="ow">in</span> <span class="n">engine_shortcuts</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;Engine config error: ambiguous shortcut: </span><span class="si">{0}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">shortcut</span><span class="p">))</span>
<span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
<span class="n">engine_shortcuts</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">shortcut</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine</span><span class="o">.</span><span class="n">name</span>
<span class="k">for</span> <span class="n">category_name</span> <span class="ow">in</span> <span class="n">engine</span><span class="o">.</span><span class="n">categories</span><span class="p">:</span>
<span class="n">categories</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="n">category_name</span><span class="p">,</span> <span class="p">[])</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
<div class="viewcode-block" id="load_engines">
<a class="viewcode-back" href="../../dev/engines/engines.html#searx.engines.load_engines">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">load_engines</span><span class="p">(</span><span class="n">engine_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;usage: ``engine_list = settings[&#39;engines&#39;]``&quot;&quot;&quot;</span>
<span class="n">engines</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span>
<span class="n">engine_shortcuts</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span>
<span class="n">categories</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span>
<span class="n">categories</span><span class="p">[</span><span class="s1">&#39;general&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">engine_data</span> <span class="ow">in</span> <span class="n">engine_list</span><span class="p">:</span>
<span class="k">if</span> <span class="n">engine_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;inactive&quot;</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">engine</span> <span class="o">=</span> <span class="n">load_engine</span><span class="p">(</span><span class="n">engine_data</span><span class="p">)</span>
<span class="k">if</span> <span class="n">engine</span><span class="p">:</span>
<span class="n">register_engine</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># if an engine can&#39;t be loaded (if for example the engine is missing</span>
<span class="c1"># tor or some other requirements) its set to inactive!</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;loading engine </span><span class="si">%s</span><span class="s2"> failed: set engine to inactive!&quot;</span><span class="p">,</span> <span class="n">engine_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;name&quot;</span><span class="p">,</span> <span class="s2">&quot;???&quot;</span><span class="p">))</span>
<span class="n">engine_data</span><span class="p">[</span><span class="s2">&quot;inactive&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">return</span> <span class="n">engines</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../index.html">
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../index.html">Overview</a>
<ul>
<li><a href="../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,326 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.annas_archive &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.annas_archive</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.annas_archive</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;`Anna&#39;s Archive`_ is a free non-profit online shadow library metasearch</span>
<span class="sd">engine providing access to a variety of book resources (also via IPFS), created</span>
<span class="sd">by a team of anonymous archivists (AnnaArchivist_).</span>
<span class="sd">.. _Anna&#39;s Archive: https://annas-archive.org/</span>
<span class="sd">.. _AnnaArchivist: https://annas-software.org/AnnaArchivist/annas-archive</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">The engine has the following additional settings:</span>
<span class="sd">- :py:obj:`aa_content`</span>
<span class="sd">- :py:obj:`aa_ext`</span>
<span class="sd">- :py:obj:`aa_sort`</span>
<span class="sd">With this options a SearXNG maintainer is able to configure **additional**</span>
<span class="sd">engines for specific searches in Anna&#39;s Archive. For example a engine to search</span>
<span class="sd">for *newest* articles and journals (PDF) / by shortcut ``!aaa &lt;search-term&gt;``.</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: annas articles</span>
<span class="sd"> engine: annas_archive</span>
<span class="sd"> categories = [&quot;general&quot;, &quot;articles&quot;]</span>
<span class="sd"> shortcut: aaa</span>
<span class="sd"> aa_content: &quot;magazine&quot;</span>
<span class="sd"> aa_ext: &quot;pdf&quot;</span>
<span class="sd"> aa_sort: &quot;newest&quot;</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml.etree</span><span class="w"> </span><span class="kn">import</span> <span class="n">ElementBase</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">eval_xpath_getindex</span><span class="p">,</span> <span class="n">eval_xpath_list</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.data</span><span class="w"> </span><span class="kn">import</span> <span class="n">ENGINE_TRAITS</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineXPathException</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
<span class="c1"># about</span>
<span class="n">about</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s2">&quot;https://annas-archive.org/&quot;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s2">&quot;Q115288326&quot;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s2">&quot;HTML&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;files&quot;</span><span class="p">,</span> <span class="s2">&quot;books&quot;</span><span class="p">]</span>
<span class="n">paging</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
<span class="c1"># search-url</span>
<span class="n">base_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;https://annas-archive.org&quot;</span>
<span class="n">aa_content</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Anan&#39;s search form field **Content** / possible values::</span>
<span class="sd"> book_fiction, book_unknown, book_nonfiction,</span>
<span class="sd"> book_comic, magazine, standards_document</span>
<span class="sd">To not filter use an empty string (default).</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">aa_sort</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Sort Anna&#39;s results, possible values::</span>
<span class="sd"> newest, oldest, largest, smallest</span>
<span class="sd">To sort by *most relevant* use an empty string (default).&quot;&quot;&quot;</span>
<span class="n">aa_ext</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Filter Anna&#39;s results by a file ending. Common filters for example are</span>
<span class="sd">``pdf`` and ``epub``.</span>
<span class="sd">.. note::</span>
<span class="sd"> Anna&#39;s Archive is a beta release: Filter results by file extension does not</span>
<span class="sd"> really work on Anna&#39;s Archive.</span>
<span class="sd">&quot;&quot;&quot;</span>
<div class="viewcode-block" id="setup">
<a class="viewcode-back" href="../../../dev/engines/online/annas_archive.html#searx.engines.annas_archive.setup">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Check of engine&#39;s settings.&quot;&quot;&quot;</span>
<span class="n">traits</span> <span class="o">=</span> <span class="n">EngineTraits</span><span class="p">(</span><span class="o">**</span><span class="n">ENGINE_TRAITS</span><span class="p">[</span><span class="s2">&quot;annas archive&quot;</span><span class="p">])</span>
<span class="k">if</span> <span class="n">aa_content</span> <span class="ow">and</span> <span class="n">aa_content</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;content&quot;</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;invalid setting content: </span><span class="si">{</span><span class="n">aa_content</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">aa_sort</span> <span class="ow">and</span> <span class="n">aa_sort</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;sort&quot;</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;invalid setting sort: </span><span class="si">{</span><span class="n">aa_sort</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">aa_ext</span> <span class="ow">and</span> <span class="n">aa_ext</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;ext&quot;</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;invalid setting ext: </span><span class="si">{</span><span class="n">aa_ext</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">&quot;OnlineParams&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">lang</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;searxng_locale&quot;</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span>
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;lang&quot;</span><span class="p">:</span> <span class="n">lang</span><span class="p">,</span>
<span class="s2">&quot;content&quot;</span><span class="p">:</span> <span class="n">aa_content</span><span class="p">,</span>
<span class="s2">&quot;ext&quot;</span><span class="p">:</span> <span class="n">aa_ext</span><span class="p">,</span>
<span class="s2">&quot;sort&quot;</span><span class="p">:</span> <span class="n">aa_sort</span><span class="p">,</span>
<span class="s2">&quot;q&quot;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s2">&quot;page&quot;</span><span class="p">:</span> <span class="n">params</span><span class="p">[</span><span class="s2">&quot;pageno&quot;</span><span class="p">],</span>
<span class="p">}</span>
<span class="c1"># filter out None and empty values</span>
<span class="n">filtered_args</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">args</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">v</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">/search?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">filtered_args</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="c1"># The rendering of the WEB page is strange; positions of Anna&#39;s result page</span>
<span class="c1"># are enclosed in SGML comments. These comments are *uncommented* by some</span>
<span class="c1"># JS code, see query of class &quot;.js-scroll-hidden&quot; in Anna&#39;s HTML template:</span>
<span class="c1"># https://annas-software.org/AnnaArchivist/annas-archive/-/blob/main/allthethings/templates/macros/md5_list.html</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//main//div[contains(@class, &#39;js-aarecord-list-outer&#39;)]/div&quot;</span><span class="p">):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">kwargs</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">_get_result</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">except</span> <span class="n">SearxEngineXPathException</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Paper</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">))</span>
<span class="k">return</span> <span class="n">res</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_get_result</span><span class="p">(</span><span class="n">item</span><span class="p">:</span> <span class="n">ElementBase</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]:</span>
<span class="k">return</span> <span class="p">{</span>
<span class="s2">&quot;url&quot;</span><span class="p">:</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">&quot;./a/@href&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span>
<span class="s2">&quot;title&quot;</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">&quot;./div//a[starts-with(@href, &#39;/md5&#39;)]&quot;</span><span class="p">)),</span>
<span class="s2">&quot;authors&quot;</span><span class="p">:</span> <span class="p">[</span><span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">&quot;.//a[starts-with(@href, &#39;/search&#39;)]&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">))],</span>
<span class="s2">&quot;publisher&quot;</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span>
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">&quot;.//a[starts-with(@href, &#39;/search&#39;)]&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span>
<span class="p">),</span>
<span class="s2">&quot;content&quot;</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">&quot;.//div[contains(@class, &#39;relative&#39;)]&quot;</span><span class="p">)),</span>
<span class="s2">&quot;thumbnail&quot;</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">&quot;.//img/@src&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
<span class="p">}</span>
<div class="viewcode-block" id="fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/online/annas_archive.html#searx.engines.annas_archive.fetch_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Fetch languages and other search arguments from Anna&#39;s search form.&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=import-outside-toplevel</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;content&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;ext&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;sort&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">base_url</span> <span class="o">+</span> <span class="s2">&quot;/search&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;Response from Anna&#39;s search page is not OK.&quot;</span><span class="p">)</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="c1"># supported language codes</span>
<span class="n">lang_map</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//form//input[@name=&#39;lang&#39;]&quot;</span><span class="p">):</span>
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">eng_lang</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="s2">&quot;_empty&quot;</span><span class="p">,</span> <span class="s2">&quot;nl-BE&quot;</span><span class="p">,</span> <span class="s2">&quot;und&quot;</span><span class="p">)</span> <span class="ow">or</span> <span class="n">eng_lang</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;anti__&quot;</span><span class="p">):</span>
<span class="k">continue</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">lang_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_lang</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">),</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot;-&quot;</span><span class="p">)</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="c1"># silently ignore unknown languages</span>
<span class="c1"># print(&quot;ERROR: %s -&gt; %s is unknown by babel&quot; % (x.get(&quot;data-name&quot;), eng_lang))</span>
<span class="k">continue</span>
<span class="n">sxng_lang</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_lang</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_lang</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_lang</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_lang</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_lang</span>
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//form//input[@name=&#39;content&#39;]&quot;</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;anti__&quot;</span><span class="p">):</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;content&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">))</span>
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//form//input[@name=&#39;ext&#39;]&quot;</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;anti__&quot;</span><span class="p">):</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;ext&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">))</span>
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//form//select[@name=&#39;sort&#39;]//option&quot;</span><span class="p">):</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;sort&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">))</span>
<span class="c1"># for better diff; sort the persistence of these traits</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;content&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;ext&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;sort&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,257 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.archlinux &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.archlinux</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.archlinux</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">Arch Linux Wiki</span>
<span class="sd">~~~~~~~~~~~~~~~</span>
<span class="sd">This implementation does not use a official API: Mediawiki provides API, but</span>
<span class="sd">Arch Wiki blocks access to it.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span><span class="p">,</span> <span class="n">urljoin</span><span class="p">,</span> <span class="n">urlparse</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">lxml</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">,</span> <span class="n">eval_xpath_getindex</span><span class="p">,</span> <span class="n">searxng_useragent</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://wiki.archlinux.org/&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q101445877&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;HTML&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;it&#39;</span><span class="p">,</span> <span class="s1">&#39;software wikis&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">main_wiki</span> <span class="o">=</span> <span class="s1">&#39;wiki.archlinux.org&#39;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="n">sxng_lang</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">netloc</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;wiki_netloc&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_lang</span><span class="p">,</span> <span class="n">main_wiki</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">title</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;title&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_lang</span><span class="p">,</span> <span class="s1">&#39;Special:Search&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;https://&#39;</span> <span class="o">+</span> <span class="n">netloc</span> <span class="o">+</span> <span class="s1">&#39;/index.php?&#39;</span>
<span class="n">offset</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">20</span>
<span class="k">if</span> <span class="n">netloc</span> <span class="o">==</span> <span class="n">main_wiki</span><span class="p">:</span>
<span class="n">eng_lang</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">sxng_lang</span><span class="p">,</span> <span class="s1">&#39;English&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">query</span> <span class="o">+=</span> <span class="s1">&#39; (&#39;</span> <span class="o">+</span> <span class="n">eng_lang</span> <span class="o">+</span> <span class="s1">&#39;)&#39;</span>
<span class="c1"># wiki.archlinux.org is protected by anubis</span>
<span class="c1"># - https://github.com/searxng/searxng/issues/4646#issuecomment-2817848019</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;User-Agent&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">searxng_useragent</span><span class="p">()</span>
<span class="k">elif</span> <span class="n">netloc</span> <span class="o">==</span> <span class="s1">&#39;wiki.archlinuxcn.org&#39;</span><span class="p">:</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;https://&#39;</span> <span class="o">+</span> <span class="n">netloc</span> <span class="o">+</span> <span class="s1">&#39;/wzh/index.php?&#39;</span>
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;search&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
<span class="s1">&#39;limit&#39;</span><span class="p">:</span> <span class="mi">20</span><span class="p">,</span>
<span class="s1">&#39;offset&#39;</span><span class="p">:</span> <span class="n">offset</span><span class="p">,</span>
<span class="s1">&#39;profile&#39;</span><span class="p">:</span> <span class="s1">&#39;default&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
<span class="k">return</span> <span class="n">params</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="c1"># get the base URL for the language in which request was made</span>
<span class="n">sxng_lang</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">netloc</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;wiki_netloc&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_lang</span><span class="p">,</span> <span class="n">main_wiki</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;https://&#39;</span> <span class="o">+</span> <span class="n">netloc</span> <span class="o">+</span> <span class="s1">&#39;/index.php?&#39;</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">&#39;//ul[@class=&quot;mw-search-results&quot;]/li&#39;</span><span class="p">):</span>
<span class="n">link</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//div[@class=&quot;mw-search-result-heading&quot;]/a&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;.//div[@class=&quot;searchresult&quot;]&#39;</span><span class="p">))</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">urljoin</span><span class="p">(</span><span class="n">base_url</span><span class="p">,</span> <span class="n">link</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;href&#39;</span><span class="p">)),</span> <span class="c1"># type: ignore</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">link</span><span class="p">),</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span>
<div class="viewcode-block" id="fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/online/archlinux.html#searx.engines.archlinux.fetch_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Fetch languages from Archlinux-Wiki. The location of the Wiki address of a</span>
<span class="sd"> language is mapped in a :py:obj:`custom field</span>
<span class="sd"> &lt;searx.enginelib.traits.EngineTraits.custom&gt;` (``wiki_netloc``). Depending</span>
<span class="sd"> on the location, the ``title`` argument in the request is translated.</span>
<span class="sd"> .. code:: python</span>
<span class="sd"> &quot;custom&quot;: {</span>
<span class="sd"> &quot;wiki_netloc&quot;: {</span>
<span class="sd"> &quot;de&quot;: &quot;wiki.archlinux.de&quot;,</span>
<span class="sd"> # ...</span>
<span class="sd"> &quot;zh&quot;: &quot;wiki.archlinuxcn.org&quot;</span>
<span class="sd"> }</span>
<span class="sd"> &quot;title&quot;: {</span>
<span class="sd"> &quot;de&quot;: &quot;Spezial:Suche&quot;,</span>
<span class="sd"> # ...</span>
<span class="sd"> &quot;zh&quot;: &quot;Special:\u641c\u7d22&quot;</span>
<span class="sd"> },</span>
<span class="sd"> },</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=import-outside-toplevel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;wiki_netloc&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;title&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">title_map</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;de&#39;</span><span class="p">:</span> <span class="s1">&#39;Spezial:Suche&#39;</span><span class="p">,</span>
<span class="s1">&#39;fa&#39;</span><span class="p">:</span> <span class="s1">&#39;ویژه:جستجو&#39;</span><span class="p">,</span>
<span class="s1">&#39;ja&#39;</span><span class="p">:</span> <span class="s1">&#39;特別:検索&#39;</span><span class="p">,</span>
<span class="s1">&#39;zh&#39;</span><span class="p">:</span> <span class="s1">&#39;Special:搜索&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">&#39;https://wiki.archlinux.org/&#39;</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: response from wiki.archlinux.org is not OK.&quot;</span><span class="p">)</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//a[@class=&#39;interlanguage-link-target&#39;]&quot;</span><span class="p">):</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">a</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;lang&#39;</span><span class="p">),</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;-&#39;</span><span class="p">))</span>
<span class="c1"># zh_Hans --&gt; zh</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">sxng_tag</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">netloc</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">a</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;href&#39;</span><span class="p">))</span><span class="o">.</span><span class="n">netloc</span>
<span class="k">if</span> <span class="n">netloc</span> <span class="o">!=</span> <span class="s1">&#39;wiki.archlinux.org&#39;</span><span class="p">:</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">title_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">title</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: title tag from </span><span class="si">%s</span><span class="s2"> (</span><span class="si">%s</span><span class="s2">) is unknown&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">netloc</span><span class="p">,</span> <span class="n">sxng_tag</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;wiki_netloc&#39;</span><span class="p">][</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">netloc</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;title&#39;</span><span class="p">][</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">title</span> <span class="c1"># type: ignore</span>
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="s2">&quot;.//span&quot;</span><span class="p">))</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span> <span class="c1"># type: ignore</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="s1">&#39;en&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;English&#39;</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,274 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.astrophysics_data_system &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.astrophysics_data_system</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.astrophysics_data_system</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;The Astrophysics Data System (ADS_) is a digital library portal for</span>
<span class="sd">researchers in astronomy and physics, operated by the Smithsonian Astrophysical</span>
<span class="sd">Observatory (SAO) under a NASA grant. The ADS_ is a solr instance, but not with</span>
<span class="sd">the standard API paths.</span>
<span class="sd">.. note::</span>
<span class="sd"> The ADS_ engine requires an :py:obj:`API key &lt;api_key&gt;`.</span>
<span class="sd">This engine uses the `search/query`_ API endpoint. Since the user&#39;s search term</span>
<span class="sd">is passed through, the `search syntax`_ of ADS can be used (at least to some</span>
<span class="sd">extent).</span>
<span class="sd">.. _ADS: https://ui.adsabs.harvard.edu</span>
<span class="sd">.. _search/query: https://ui.adsabs.harvard.edu/help/api/api-docs.html#get-/search/query</span>
<span class="sd">.. _search syntax: https://ui.adsabs.harvard.edu/help/search/search-syntax</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">The engine has the following additional settings:</span>
<span class="sd">- :py:obj:`api_key`</span>
<span class="sd">- :py:obj:`ads_sort`</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: astrophysics data system</span>
<span class="sd"> api_key: &quot;...&quot;</span>
<span class="sd"> inactive: false</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">html_to_text</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineAPIException</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s2">&quot;https://ui.adsabs.harvard.edu/&quot;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s2">&quot;Q752099&quot;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s2">&quot;https://ui.adsabs.harvard.edu/help/api/api-docs.html&quot;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s2">&quot;JSON&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;science&quot;</span><span class="p">,</span> <span class="s2">&quot;scientific publications&quot;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;https://api.adsabs.harvard.edu/v1/search/query&quot;</span>
<span class="n">api_key</span> <span class="o">=</span> <span class="s2">&quot;unset&quot;</span>
<span class="sd">&quot;&quot;&quot;Get an API token as described in https://ui.adsabs.harvard.edu/help/api&quot;&quot;&quot;</span>
<span class="n">ads_field_list</span> <span class="o">=</span> <span class="p">[</span>
<span class="s2">&quot;abstract&quot;</span><span class="p">,</span>
<span class="s2">&quot;author&quot;</span><span class="p">,</span>
<span class="s2">&quot;bibcode&quot;</span><span class="p">,</span>
<span class="s2">&quot;comment&quot;</span><span class="p">,</span>
<span class="s2">&quot;date&quot;</span><span class="p">,</span>
<span class="s2">&quot;doi&quot;</span><span class="p">,</span>
<span class="s2">&quot;isbn&quot;</span><span class="p">,</span>
<span class="s2">&quot;issn&quot;</span><span class="p">,</span>
<span class="s2">&quot;keyword&quot;</span><span class="p">,</span>
<span class="s2">&quot;page&quot;</span><span class="p">,</span>
<span class="s2">&quot;page_count&quot;</span><span class="p">,</span>
<span class="s2">&quot;page_range&quot;</span><span class="p">,</span>
<span class="s2">&quot;pub&quot;</span><span class="p">,</span>
<span class="s2">&quot;pubdate&quot;</span><span class="p">,</span>
<span class="s2">&quot;pubnote&quot;</span><span class="p">,</span>
<span class="s2">&quot;read_count&quot;</span><span class="p">,</span>
<span class="s2">&quot;title&quot;</span><span class="p">,</span>
<span class="s2">&quot;volume&quot;</span><span class="p">,</span>
<span class="s2">&quot;year&quot;</span><span class="p">,</span>
<span class="p">]</span>
<span class="sd">&quot;&quot;&quot;Set of fields to return in the response from ADS.&quot;&quot;&quot;</span>
<span class="n">ads_rows</span> <span class="o">=</span> <span class="mi">10</span>
<span class="sd">&quot;&quot;&quot;How many records to return for the ADS request.&quot;&quot;&quot;</span>
<span class="n">ads_sort</span> <span class="o">=</span> <span class="s2">&quot;read_count desc&quot;</span>
<span class="sd">&quot;&quot;&quot;The format is &#39;field&#39; + &#39;direction&#39; where direction is one of &#39;asc&#39; or &#39;desc&#39;</span>
<span class="sd">and field is any of the valid indexes.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="setup">
<a class="viewcode-back" href="../../../dev/engines/online/astrophysics_data_system.html#searx.engines.astrophysics_data_system.setup">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Initialization of the ADS_ engine, checks whether the :py:obj:`api_key`</span>
<span class="sd"> is set, otherwise the engine is inactive.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">key</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;api_key&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">key</span> <span class="ow">and</span> <span class="n">key</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">&quot;unset&quot;</span><span class="p">,</span> <span class="s2">&quot;unknown&quot;</span><span class="p">,</span> <span class="s2">&quot;...&quot;</span><span class="p">):</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Astrophysics Data System (ADS) API key is not set or invalid.&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">False</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">&quot;OnlineParams&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">args</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;q&quot;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s2">&quot;fl&quot;</span><span class="p">:</span> <span class="s2">&quot;,&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">ads_field_list</span><span class="p">),</span>
<span class="s2">&quot;rows&quot;</span><span class="p">:</span> <span class="n">ads_rows</span><span class="p">,</span>
<span class="s2">&quot;start&quot;</span><span class="p">:</span> <span class="n">ads_rows</span> <span class="o">*</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;pageno&quot;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">),</span>
<span class="p">}</span>
<span class="k">if</span> <span class="n">ads_sort</span><span class="p">:</span>
<span class="n">args</span><span class="p">[</span><span class="s2">&quot;sort&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ads_sort</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;headers&quot;</span><span class="p">][</span><span class="s2">&quot;Authorization&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;Bearer </span><span class="si">{</span><span class="n">api_key</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">json_data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">if</span> <span class="s2">&quot;error&quot;</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">SearxEngineAPIException</span><span class="p">(</span><span class="n">json_data</span><span class="p">[</span><span class="s2">&quot;error&quot;</span><span class="p">][</span><span class="s2">&quot;msg&quot;</span><span class="p">])</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_str</span><span class="p">(</span><span class="n">k</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">doc</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">))</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_list</span><span class="p">(</span><span class="n">k</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="k">return</span> <span class="n">doc</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="p">[])</span>
<span class="k">for</span> <span class="n">doc</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">&quot;response&quot;</span><span class="p">][</span><span class="s2">&quot;docs&quot;</span><span class="p">]:</span>
<span class="n">authors</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">doc</span><span class="p">[</span><span class="s2">&quot;author&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">authors</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">15</span><span class="p">:</span>
<span class="c1"># There are articles with hundreds of authors</span>
<span class="n">authors</span> <span class="o">=</span> <span class="n">authors</span><span class="p">[:</span><span class="mi">15</span><span class="p">]</span> <span class="o">+</span> <span class="p">[</span><span class="s2">&quot;et al.&quot;</span><span class="p">]</span>
<span class="n">paper</span> <span class="o">=</span> <span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Paper</span><span class="p">(</span>
<span class="n">url</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;https://ui.adsabs.harvard.edu/abs/</span><span class="si">{</span><span class="n">doc</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;bibcode&#39;</span><span class="p">)</span><span class="si">}</span><span class="s2">/&quot;</span><span class="p">,</span>
<span class="n">title</span><span class="o">=</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">_list</span><span class="p">(</span><span class="s2">&quot;title&quot;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]),</span>
<span class="n">authors</span><span class="o">=</span><span class="n">authors</span><span class="p">,</span>
<span class="n">content</span><span class="o">=</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">_str</span><span class="p">(</span><span class="s2">&quot;abstract&quot;</span><span class="p">)),</span>
<span class="n">doi</span><span class="o">=</span><span class="n">_list</span><span class="p">(</span><span class="s2">&quot;doi&quot;</span><span class="p">)[</span><span class="mi">0</span><span class="p">],</span>
<span class="n">issn</span><span class="o">=</span><span class="n">_list</span><span class="p">(</span><span class="s2">&quot;issn&quot;</span><span class="p">),</span>
<span class="n">isbn</span><span class="o">=</span><span class="n">_list</span><span class="p">(</span><span class="s2">&quot;isbn&quot;</span><span class="p">),</span>
<span class="n">tags</span><span class="o">=</span><span class="n">_list</span><span class="p">(</span><span class="s2">&quot;keyword&quot;</span><span class="p">),</span>
<span class="n">pages</span><span class="o">=</span><span class="s2">&quot;,&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_list</span><span class="p">(</span><span class="s2">&quot;page&quot;</span><span class="p">)),</span>
<span class="n">publisher</span><span class="o">=</span><span class="n">_str</span><span class="p">(</span><span class="s2">&quot;pub&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot; &quot;</span> <span class="o">+</span> <span class="n">_str</span><span class="p">(</span><span class="s2">&quot;year&quot;</span><span class="p">),</span>
<span class="n">publishedDate</span><span class="o">=</span><span class="n">datetime</span><span class="o">.</span><span class="n">fromisoformat</span><span class="p">(</span><span class="n">_str</span><span class="p">(</span><span class="s2">&quot;date&quot;</span><span class="p">)),</span>
<span class="n">volume</span><span class="o">=</span><span class="n">_str</span><span class="p">(</span><span class="s2">&quot;volume&quot;</span><span class="p">),</span>
<span class="n">views</span><span class="o">=</span><span class="n">_str</span><span class="p">(</span><span class="s2">&quot;read_count&quot;</span><span class="p">),</span>
<span class="n">comments</span><span class="o">=</span><span class="s2">&quot; / &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_list</span><span class="p">(</span><span class="s2">&quot;pubnote&quot;</span><span class="p">)),</span>
<span class="p">)</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">paper</span><span class="p">)</span>
<span class="k">return</span> <span class="n">res</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,304 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.azure &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.azure</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.azure</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Engine for Azure resources. This engine mimics the standard search bar in Azure</span>
<span class="sd">Portal (for resources and resource groups).</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">You must `register an application in Microsoft Entra ID`_ and assign it the</span>
<span class="sd">&#39;Reader&#39; role in your subscription.</span>
<span class="sd">To use this engine, add an entry similar to the following to your engine list in</span>
<span class="sd">``settings.yml``:</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: azure</span>
<span class="sd"> engine: azure</span>
<span class="sd"> ...</span>
<span class="sd"> azure_tenant_id: &quot;your_tenant_id&quot;</span>
<span class="sd"> azure_client_id: &quot;your_client_id&quot;</span>
<span class="sd"> azure_client_secret: &quot;your_client_secret&quot;</span>
<span class="sd"> azure_token_expiration_seconds: 5000</span>
<span class="sd">.. _register an application in Microsoft Entra ID:</span>
<span class="sd"> https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineCache</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">post</span> <span class="k">as</span> <span class="n">http_post</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
<span class="n">engine_type</span> <span class="o">=</span> <span class="s2">&quot;online&quot;</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;it&quot;</span><span class="p">,</span> <span class="s2">&quot;cloud&quot;</span><span class="p">]</span>
<span class="c1"># Default values, should be overridden in settings.yml</span>
<span class="n">azure_tenant_id</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="n">azure_client_id</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="n">azure_client_secret</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="n">azure_token_expiration_seconds</span> <span class="o">=</span> <span class="mi">5000</span>
<span class="sd">&quot;&quot;&quot;Time for which an auth token is valid (sec.)&quot;&quot;&quot;</span>
<span class="n">azure_batch_endpoint</span> <span class="o">=</span> <span class="s2">&quot;https://management.azure.com/batch?api-version=2020-06-01&quot;</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s2">&quot;https://www.portal.azure.com&quot;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s2">&quot;Q725967&quot;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s2">&quot;https://learn.microsoft.com/en-us/</span><span class="se">\</span>
<span class="s2"> rest/api/azure-resourcegraph/?view=rest-azureresourcegraph-resourcegraph-2024-04-01&quot;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s2">&quot;JSON&quot;</span><span class="p">,</span>
<span class="s2">&quot;language&quot;</span><span class="p">:</span> <span class="s2">&quot;en&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">CACHE</span><span class="p">:</span> <span class="n">EngineCache</span>
<span class="sd">&quot;&quot;&quot;Persistent (SQLite) key/value cache that deletes its values after ``expire``</span>
<span class="sd">seconds.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="setup">
<a class="viewcode-back" href="../../../dev/engines/online/azure.html#searx.engines.azure.setup">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Initialization of the engine.</span>
<span class="sd"> - Instantiate a cache for this engine (:py:obj:`CACHE`).</span>
<span class="sd"> - Checks whether the tenant_id, client_id and client_secret are set,</span>
<span class="sd"> otherwise the engine is inactive.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">global</span> <span class="n">CACHE</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="n">CACHE</span> <span class="o">=</span> <span class="n">EngineCache</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">[</span><span class="s2">&quot;name&quot;</span><span class="p">])</span>
<span class="n">missing_opts</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">opt</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">&quot;azure_tenant_id&quot;</span><span class="p">,</span> <span class="s2">&quot;azure_client_id&quot;</span><span class="p">,</span> <span class="s2">&quot;azure_client_secret&quot;</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">opt</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">):</span>
<span class="n">missing_opts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">opt</span><span class="p">)</span>
<span class="k">if</span> <span class="n">missing_opts</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;missing values for options: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">missing_opts</span><span class="p">))</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="k">return</span> <span class="kc">True</span></div>
<div class="viewcode-block" id="authenticate">
<a class="viewcode-back" href="../../../dev/engines/online/azure.html#searx.engines.azure.authenticate">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">authenticate</span><span class="p">(</span><span class="n">t_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">c_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">c_secret</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Authenticates to Azure using Oauth2 Client Credentials Flow and returns</span>
<span class="sd"> an access token.&quot;&quot;&quot;</span>
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;https://login.microsoftonline.com/</span><span class="si">{</span><span class="n">t_id</span><span class="si">}</span><span class="s2">/oauth2/v2.0/token&quot;</span>
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;client_id&quot;</span><span class="p">:</span> <span class="n">c_id</span><span class="p">,</span>
<span class="s2">&quot;client_secret&quot;</span><span class="p">:</span> <span class="n">c_secret</span><span class="p">,</span>
<span class="s2">&quot;grant_type&quot;</span><span class="p">:</span> <span class="s2">&quot;client_credentials&quot;</span><span class="p">,</span>
<span class="s2">&quot;scope&quot;</span><span class="p">:</span> <span class="s2">&quot;https://management.azure.com/.default&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">resp</span><span class="p">:</span> <span class="n">SXNG_Response</span> <span class="o">=</span> <span class="n">http_post</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">body</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">!=</span> <span class="mi">200</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Azure authentication failed (status </span><span class="si">{</span><span class="n">resp</span><span class="o">.</span><span class="n">status_code</span><span class="si">}</span><span class="s2">): </span><span class="si">{</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()[</span><span class="s2">&quot;access_token&quot;</span><span class="p">]</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">get_auth_token</span><span class="p">(</span><span class="n">t_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">c_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">c_secret</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="n">key</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;azure_tenant_id: </span><span class="si">{</span><span class="n">t_id</span><span class="si">:}</span><span class="s2">, azure_client_id: </span><span class="si">{</span><span class="n">c_id</span><span class="si">}</span><span class="s2">, azure_client_secret: </span><span class="si">{</span><span class="n">c_secret</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">token</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">CACHE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
<span class="k">if</span> <span class="n">token</span><span class="p">:</span>
<span class="k">return</span> <span class="n">token</span>
<span class="n">token</span> <span class="o">=</span> <span class="n">authenticate</span><span class="p">(</span><span class="n">t_id</span><span class="p">,</span> <span class="n">c_id</span><span class="p">,</span> <span class="n">c_secret</span><span class="p">)</span>
<span class="n">CACHE</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">token</span><span class="p">,</span> <span class="n">expire</span><span class="o">=</span><span class="n">azure_token_expiration_seconds</span><span class="p">)</span>
<span class="k">return</span> <span class="n">token</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">&quot;OnlineParams&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">token</span> <span class="o">=</span> <span class="n">get_auth_token</span><span class="p">(</span><span class="n">azure_tenant_id</span><span class="p">,</span> <span class="n">azure_client_id</span><span class="p">,</span> <span class="n">azure_client_secret</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">azure_batch_endpoint</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;method&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;POST&quot;</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;headers&quot;</span><span class="p">][</span><span class="s2">&quot;Authorization&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;Bearer </span><span class="si">{</span><span class="n">token</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;headers&quot;</span><span class="p">][</span><span class="s2">&quot;Content-Type&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;application/json&quot;</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;json&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;requests&quot;</span><span class="p">:</span> <span class="p">[</span>
<span class="p">{</span>
<span class="s2">&quot;url&quot;</span><span class="p">:</span> <span class="s2">&quot;/providers/Microsoft.ResourceGraph/resources?api-version=2024-04-01&quot;</span><span class="p">,</span>
<span class="s2">&quot;httpMethod&quot;</span><span class="p">:</span> <span class="s2">&quot;POST&quot;</span><span class="p">,</span>
<span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;resourceGroups&quot;</span><span class="p">,</span>
<span class="s2">&quot;requestHeaderDetails&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;commandName&quot;</span><span class="p">:</span> <span class="s2">&quot;Microsoft.ResourceGraph&quot;</span><span class="p">},</span>
<span class="s2">&quot;content&quot;</span><span class="p">:</span> <span class="p">{</span>
<span class="s2">&quot;query&quot;</span><span class="p">:</span> <span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;ResourceContainers&quot;</span>
<span class="sa">f</span><span class="s2">&quot; | where (name contains (&#39;</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s2">&#39;))&quot;</span>
<span class="sa">f</span><span class="s2">&quot; | where (type =~ (&#39;Microsoft.Resources/subscriptions/resourcegroups&#39;))&quot;</span>
<span class="sa">f</span><span class="s2">&quot; | project id,name,type,kind,subscriptionId,resourceGroup&quot;</span>
<span class="sa">f</span><span class="s2">&quot; | extend matchscore = name startswith &#39;</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s2">&#39;&quot;</span>
<span class="sa">f</span><span class="s2">&quot; | extend normalizedName = tolower(tostring(name))&quot;</span>
<span class="sa">f</span><span class="s2">&quot; | sort by matchscore desc, normalizedName asc&quot;</span>
<span class="sa">f</span><span class="s2">&quot; | take 30&quot;</span>
<span class="p">)</span>
<span class="p">},</span>
<span class="p">},</span>
<span class="p">{</span>
<span class="s2">&quot;url&quot;</span><span class="p">:</span> <span class="s2">&quot;/providers/Microsoft.ResourceGraph/resources?api-version=2024-04-01&quot;</span><span class="p">,</span>
<span class="s2">&quot;httpMethod&quot;</span><span class="p">:</span> <span class="s2">&quot;POST&quot;</span><span class="p">,</span>
<span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;resources&quot;</span><span class="p">,</span>
<span class="s2">&quot;requestHeaderDetails&quot;</span><span class="p">:</span> <span class="p">{</span>
<span class="s2">&quot;commandName&quot;</span><span class="p">:</span> <span class="s2">&quot;Microsoft.ResourceGraph&quot;</span><span class="p">,</span>
<span class="p">},</span>
<span class="s2">&quot;content&quot;</span><span class="p">:</span> <span class="p">{</span>
<span class="s2">&quot;query&quot;</span><span class="p">:</span> <span class="sa">f</span><span class="s2">&quot;Resources | where name contains &#39;</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s2">&#39; | take 30&quot;</span><span class="p">,</span>
<span class="p">},</span>
<span class="p">},</span>
<span class="p">]</span>
<span class="p">}</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">&quot;responses&quot;</span><span class="p">]:</span>
<span class="k">if</span> <span class="n">result</span><span class="p">[</span><span class="s2">&quot;name&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;resourceGroups&quot;</span><span class="p">:</span>
<span class="k">for</span> <span class="n">data</span> <span class="ow">in</span> <span class="n">result</span><span class="p">[</span><span class="s2">&quot;content&quot;</span><span class="p">][</span><span class="s2">&quot;data&quot;</span><span class="p">]:</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">MainResult</span><span class="p">(</span>
<span class="n">url</span><span class="o">=</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;https://portal.azure.com/#@/resource&quot;</span>
<span class="sa">f</span><span class="s2">&quot;/subscriptions/</span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;subscriptionId&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">/resourceGroups/</span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">/overview&quot;</span>
<span class="p">),</span>
<span class="n">title</span><span class="o">=</span><span class="n">data</span><span class="p">[</span><span class="s2">&quot;name&quot;</span><span class="p">],</span>
<span class="n">content</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;Resource Group in Subscription: </span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;subscriptionId&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">elif</span> <span class="n">result</span><span class="p">[</span><span class="s2">&quot;name&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;resources&quot;</span><span class="p">:</span>
<span class="k">for</span> <span class="n">data</span> <span class="ow">in</span> <span class="n">result</span><span class="p">[</span><span class="s2">&quot;content&quot;</span><span class="p">][</span><span class="s2">&quot;data&quot;</span><span class="p">]:</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">MainResult</span><span class="p">(</span>
<span class="n">url</span><span class="o">=</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;https://portal.azure.com/#@/resource&quot;</span>
<span class="sa">f</span><span class="s2">&quot;/subscriptions/</span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;subscriptionId&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">/resourceGroups/</span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;resourceGroup&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="sa">f</span><span class="s2">&quot;/providers/</span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">/overview&quot;</span>
<span class="p">),</span>
<span class="n">title</span><span class="o">=</span><span class="n">data</span><span class="p">[</span><span class="s2">&quot;name&quot;</span><span class="p">],</span>
<span class="n">content</span><span class="o">=</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;Resource of type </span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2"> in Subscription:&quot;</span>
<span class="sa">f</span><span class="s2">&quot; </span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;subscriptionId&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">, Resource Group: </span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;resourceGroup&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="p">),</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">res</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,393 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.bing &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.bing</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.bing</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;This is the implementation of the Bing-WEB engine. Some of this</span>
<span class="sd">implementations are shared by other engines:</span>
<span class="sd">- :ref:`bing images engine`</span>
<span class="sd">- :ref:`bing news engine`</span>
<span class="sd">- :ref:`bing videos engine`</span>
<span class="sd">On the `preference page`_ Bing offers a lot of languages an regions (see section</span>
<span class="sd">LANGUAGE and COUNTRY/REGION). The Language is the language of the UI, we need</span>
<span class="sd">in SearXNG to get the translations of data such as *&quot;published last week&quot;*.</span>
<span class="sd">There is a description of the official search-APIs_, unfortunately this is not</span>
<span class="sd">the API we can use or that bing itself would use. You can look up some things</span>
<span class="sd">in the API to get a better picture of bing, but the value specifications like</span>
<span class="sd">the market codes are usually outdated or at least no longer used by bing itself.</span>
<span class="sd">The market codes have been harmonized and are identical for web, video and</span>
<span class="sd">images. The news area has also been harmonized with the other categories. Only</span>
<span class="sd">political adjustments still seem to be made -- for example, there is no news</span>
<span class="sd">category for the Chinese market.</span>
<span class="sd">.. _preference page: https://www.bing.com/account/general</span>
<span class="sd">.. _search-APIs: https://learn.microsoft.com/en-us/bing/search-apis/</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-many-branches, invalid-name</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">base64</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse_qs</span><span class="p">,</span> <span class="n">urlencode</span><span class="p">,</span> <span class="n">urlparse</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.languages</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">,</span> <span class="n">eval_xpath_getindex</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span><span class="p">,</span> <span class="n">region_tag</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineAPIException</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.bing.com&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q182496&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.microsoft.com/en-us/bing/apis/bing-web-search-api&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;HTML&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;general&#39;</span><span class="p">,</span> <span class="s1">&#39;web&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">200</span>
<span class="sd">&quot;&quot;&quot;200 pages maximum (``&amp;first=1991``)&quot;&quot;&quot;</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
<span class="sd">&quot;&quot;&quot;Bing results are always SFW. To get NSFW links from bing some age</span>
<span class="sd">verification by a cookie is needed / thats not possible in SearXNG.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;https://www.bing.com/search&#39;</span>
<span class="sd">&quot;&quot;&quot;Bing (Web) search URL&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_page_offset</span><span class="p">(</span><span class="n">pageno</span><span class="p">):</span>
<span class="k">return</span> <span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">pageno</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span> <span class="o">+</span> <span class="mi">1</span>
<span class="k">def</span><span class="w"> </span><span class="nf">set_bing_cookies</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">engine_language</span><span class="p">,</span> <span class="n">engine_region</span><span class="p">):</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">][</span><span class="s1">&#39;_EDGE_CD&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;m=</span><span class="si">{</span><span class="n">engine_region</span><span class="si">}</span><span class="s1">&amp;u=</span><span class="si">{</span><span class="n">engine_language</span><span class="si">}</span><span class="s1">&#39;</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">][</span><span class="s1">&#39;_EDGE_S&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;mkt=</span><span class="si">{</span><span class="n">engine_region</span><span class="si">}</span><span class="s1">&amp;ui=</span><span class="si">{</span><span class="n">engine_language</span><span class="si">}</span><span class="s1">&#39;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;bing cookies: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">])</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Assemble a Bing-Web request.&quot;&quot;&quot;</span>
<span class="n">engine_region</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">engine_language</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="s1">&#39;en&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">set_bing_cookies</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">engine_language</span><span class="p">,</span> <span class="n">engine_region</span><span class="p">)</span>
<span class="n">page</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;pageno&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">query_params</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="c1"># if arg &#39;pq&#39; is missed, sometimes on page 4 we get results from page 1,</span>
<span class="c1"># don&#39;t ask why it is only sometimes / its M$ and they have never been</span>
<span class="c1"># deterministic ;)</span>
<span class="s1">&#39;pq&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># To get correct page, arg first and this arg FORM is needed, the value PERE</span>
<span class="c1"># is on page 2, on page 3 its PERE1 and on page 4 its PERE2 .. and so forth.</span>
<span class="c1"># The &#39;first&#39; arg should never send on page 1.</span>
<span class="k">if</span> <span class="n">page</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">query_params</span><span class="p">[</span><span class="s1">&#39;first&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">_page_offset</span><span class="p">(</span><span class="n">page</span><span class="p">)</span> <span class="c1"># see also arg FORM</span>
<span class="k">if</span> <span class="n">page</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
<span class="n">query_params</span><span class="p">[</span><span class="s1">&#39;FORM&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;PERE&#39;</span>
<span class="k">elif</span> <span class="n">page</span> <span class="o">&gt;</span> <span class="mi">2</span><span class="p">:</span>
<span class="n">query_params</span><span class="p">[</span><span class="s1">&#39;FORM&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;PERE</span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">page</span> <span class="o">-</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s1">?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">query_params</span><span class="p">)</span><span class="si">}</span><span class="s1">&#39;</span>
<span class="k">if</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;time_range&#39;</span><span class="p">):</span>
<span class="n">unix_day</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">/</span> <span class="mi">86400</span><span class="p">)</span>
<span class="n">time_ranges</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;day&#39;</span><span class="p">:</span> <span class="s1">&#39;1&#39;</span><span class="p">,</span> <span class="s1">&#39;week&#39;</span><span class="p">:</span> <span class="s1">&#39;2&#39;</span><span class="p">,</span> <span class="s1">&#39;month&#39;</span><span class="p">:</span> <span class="s1">&#39;3&#39;</span><span class="p">,</span> <span class="s1">&#39;year&#39;</span><span class="p">:</span> <span class="sa">f</span><span class="s1">&#39;5_</span><span class="si">{</span><span class="n">unix_day</span><span class="o">-</span><span class="mi">365</span><span class="si">}</span><span class="s1">_</span><span class="si">{</span><span class="n">unix_day</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">}</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="sa">f</span><span class="s1">&#39;&amp;filters=ex1:&quot;ez</span><span class="si">{</span><span class="n">time_ranges</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;time_range&quot;</span><span class="p">]]</span><span class="si">}</span><span class="s1">&quot;&#39;</span>
<span class="c1"># in some regions where geoblocking is employed (e.g. China),</span>
<span class="c1"># www.bing.com redirects to the regional version of Bing</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;allow_redirects&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">return</span> <span class="n">params</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="c1"># pylint: disable=too-many-locals</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">result_len</span> <span class="o">=</span> <span class="mi">0</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="c1"># parse results again if nothing is found yet</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">&#39;//ol[@id=&quot;b_results&quot;]/li[contains(@class, &quot;b_algo&quot;)]&#39;</span><span class="p">):</span>
<span class="n">link</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//h2/a&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">link</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">link</span><span class="o">.</span><span class="n">attrib</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;href&#39;</span><span class="p">)</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">link</span><span class="p">)</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//p&#39;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">content</span><span class="p">:</span>
<span class="c1"># Make sure that the element is free of:</span>
<span class="c1"># &lt;span class=&quot;algoSlug_icon&quot; # data-priority=&quot;2&quot;&gt;Web&lt;/span&gt;</span>
<span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="n">p</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;.//span[@class=&quot;algoSlug_icon&quot;]&#39;</span><span class="p">):</span>
<span class="n">e</span><span class="o">.</span><span class="n">getparent</span><span class="p">()</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
<span class="c1"># get the real URL</span>
<span class="k">if</span> <span class="n">url</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;https://www.bing.com/ck/a?&#39;</span><span class="p">):</span>
<span class="c1"># get the first value of u parameter</span>
<span class="n">url_query</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span><span class="o">.</span><span class="n">query</span>
<span class="n">parsed_url_query</span> <span class="o">=</span> <span class="n">parse_qs</span><span class="p">(</span><span class="n">url_query</span><span class="p">)</span>
<span class="n">param_u</span> <span class="o">=</span> <span class="n">parsed_url_query</span><span class="p">[</span><span class="s2">&quot;u&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
<span class="c1"># remove &quot;a1&quot; in front</span>
<span class="n">encoded_url</span> <span class="o">=</span> <span class="n">param_u</span><span class="p">[</span><span class="mi">2</span><span class="p">:]</span>
<span class="c1"># add padding</span>
<span class="n">encoded_url</span> <span class="o">=</span> <span class="n">encoded_url</span> <span class="o">+</span> <span class="s1">&#39;=&#39;</span> <span class="o">*</span> <span class="p">(</span><span class="o">-</span><span class="nb">len</span><span class="p">(</span><span class="n">encoded_url</span><span class="p">)</span> <span class="o">%</span> <span class="mi">4</span><span class="p">)</span>
<span class="c1"># decode base64 encoded URL</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">base64</span><span class="o">.</span><span class="n">urlsafe_b64decode</span><span class="p">(</span><span class="n">encoded_url</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
<span class="c1"># append result</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span> <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">})</span>
<span class="c1"># get number_of_results</span>
<span class="k">if</span> <span class="n">results</span><span class="p">:</span>
<span class="n">result_len_container</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">&#39;//span[@class=&quot;sb_count&quot;]//text()&#39;</span><span class="p">))</span>
<span class="k">if</span> <span class="s2">&quot;-&quot;</span> <span class="ow">in</span> <span class="n">result_len_container</span><span class="p">:</span>
<span class="n">start_str</span><span class="p">,</span> <span class="n">result_len_container</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;-\d+&#39;</span><span class="p">,</span> <span class="n">result_len_container</span><span class="p">)</span>
<span class="n">start</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">start_str</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">start</span> <span class="o">=</span> <span class="mi">1</span>
<span class="n">result_len_container</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s1">&#39;[^0-9]&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="n">result_len_container</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">result_len_container</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">result_len</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">result_len_container</span><span class="p">)</span>
<span class="n">expected_start</span> <span class="o">=</span> <span class="n">_page_offset</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;pageno&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
<span class="k">if</span> <span class="n">expected_start</span> <span class="o">!=</span> <span class="n">start</span><span class="p">:</span>
<span class="k">if</span> <span class="n">expected_start</span> <span class="o">&gt;</span> <span class="n">result_len</span><span class="p">:</span>
<span class="c1"># Avoid reading more results than available.</span>
<span class="c1"># For example, if there is 100 results from some search and we try to get results from 120 to 130,</span>
<span class="c1"># Bing will send back the results from 0 to 10 and no error.</span>
<span class="c1"># If we compare results count with the first parameter of the request we can avoid this &quot;invalid&quot;</span>
<span class="c1"># results.</span>
<span class="k">return</span> <span class="p">[]</span>
<span class="c1"># Sometimes Bing will send back the first result page instead of the requested page as a rate limiting</span>
<span class="c1"># measure.</span>
<span class="n">msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;Expected results to start at </span><span class="si">{</span><span class="n">expected_start</span><span class="si">}</span><span class="s2">, but got results starting at </span><span class="si">{</span><span class="n">start</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">raise</span> <span class="n">SearxEngineAPIException</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;number_of_results&#39;</span><span class="p">:</span> <span class="n">result_len</span><span class="p">})</span>
<span class="k">return</span> <span class="n">results</span>
<div class="viewcode-block" id="fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing.fetch_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Fetch languages and regions from Bing-Web.&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=import-outside-toplevel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">gen_useragent</span>
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="n">gen_useragent</span><span class="p">(),</span>
<span class="s2">&quot;Accept&quot;</span><span class="p">:</span> <span class="s2">&quot;text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8&quot;</span><span class="p">,</span>
<span class="s2">&quot;Accept-Language&quot;</span><span class="p">:</span> <span class="s2">&quot;en-US;q=0.5,en;q=0.3&quot;</span><span class="p">,</span>
<span class="s2">&quot;DNT&quot;</span><span class="p">:</span> <span class="s2">&quot;1&quot;</span><span class="p">,</span>
<span class="s2">&quot;Connection&quot;</span><span class="p">:</span> <span class="s2">&quot;keep-alive&quot;</span><span class="p">,</span>
<span class="s2">&quot;Upgrade-Insecure-Requests&quot;</span><span class="p">:</span> <span class="s2">&quot;1&quot;</span><span class="p">,</span>
<span class="s2">&quot;Sec-GPC&quot;</span><span class="p">:</span> <span class="s2">&quot;1&quot;</span><span class="p">,</span>
<span class="s2">&quot;Cache-Control&quot;</span><span class="p">:</span> <span class="s2">&quot;max-age=0&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s2">&quot;https://www.bing.com/account/general&quot;</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: response from bing is not OK.&quot;</span><span class="p">)</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="c1"># languages</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="s1">&#39;zh&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;zh-hans&#39;</span>
<span class="n">map_lang</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;prs&#39;</span><span class="p">:</span> <span class="s1">&#39;fa-AF&#39;</span><span class="p">,</span> <span class="s1">&#39;en&#39;</span><span class="p">:</span> <span class="s1">&#39;en-us&#39;</span><span class="p">}</span>
<span class="n">bing_ui_lang_map</span> <span class="o">=</span> <span class="p">{</span>
<span class="c1"># HINT: this list probably needs to be supplemented</span>
<span class="s1">&#39;en&#39;</span><span class="p">:</span> <span class="s1">&#39;us&#39;</span><span class="p">,</span> <span class="c1"># en --&gt; en-us</span>
<span class="s1">&#39;da&#39;</span><span class="p">:</span> <span class="s1">&#39;dk&#39;</span><span class="p">,</span> <span class="c1"># da --&gt; da-dk</span>
<span class="p">}</span>
<span class="k">for</span> <span class="n">href</span> <span class="ow">in</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">&#39;//div[@id=&quot;language-section-content&quot;]//div[@class=&quot;languageItem&quot;]/a/@href&#39;</span><span class="p">):</span>
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">parse_qs</span><span class="p">(</span><span class="n">urlparse</span><span class="p">(</span><span class="n">href</span><span class="p">)</span><span class="o">.</span><span class="n">query</span><span class="p">)[</span><span class="s1">&#39;setlang&#39;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
<span class="n">babel_lang</span> <span class="o">=</span> <span class="n">map_lang</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_lang</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">babel_lang</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">,</span> <span class="s1">&#39;_&#39;</span><span class="p">)))</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: language (</span><span class="si">%s</span><span class="s2">) is unknown by babel&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">babel_lang</span><span class="p">))</span>
<span class="k">continue</span>
<span class="c1"># Language (e.g. &#39;en&#39; or &#39;de&#39;) from https://www.bing.com/account/general</span>
<span class="c1"># is converted by bing to &#39;en-us&#39; or &#39;de-de&#39;. But only if there is not</span>
<span class="c1"># already a &#39;-&#39; delemitter in the language. For instance &#39;pt-PT&#39; --&gt;</span>
<span class="c1"># &#39;pt-pt&#39; and &#39;pt-br&#39; --&gt; &#39;pt-br&#39;</span>
<span class="n">bing_ui_lang</span> <span class="o">=</span> <span class="n">eng_lang</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="k">if</span> <span class="s1">&#39;-&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">bing_ui_lang</span><span class="p">:</span>
<span class="n">bing_ui_lang</span> <span class="o">=</span> <span class="n">bing_ui_lang</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="n">bing_ui_lang_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">bing_ui_lang</span><span class="p">,</span> <span class="n">bing_ui_lang</span><span class="p">)</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">bing_ui_lang</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">{</span><span class="n">sxng_tag</span><span class="si">}</span><span class="s2"> --&gt; </span><span class="si">{</span><span class="n">conflict</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="n">bing_ui_lang</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">bing_ui_lang</span>
<span class="c1"># regions (aka &quot;market codes&quot;)</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="s1">&#39;zh-CN&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;zh-cn&#39;</span>
<span class="n">map_market_codes</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;zh-hk&#39;</span><span class="p">:</span> <span class="s1">&#39;en-hk&#39;</span><span class="p">,</span> <span class="c1"># not sure why, but at M$ this is the market code for Hongkong</span>
<span class="p">}</span>
<span class="k">for</span> <span class="n">href</span> <span class="ow">in</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">&#39;//div[@id=&quot;region-section-content&quot;]//div[@class=&quot;regionItem&quot;]/a/@href&#39;</span><span class="p">):</span>
<span class="n">cc_tag</span> <span class="o">=</span> <span class="n">parse_qs</span><span class="p">(</span><span class="n">urlparse</span><span class="p">(</span><span class="n">href</span><span class="p">)</span><span class="o">.</span><span class="n">query</span><span class="p">)[</span><span class="s1">&#39;cc&#39;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
<span class="k">if</span> <span class="n">cc_tag</span> <span class="o">==</span> <span class="s1">&#39;clear&#39;</span><span class="p">:</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="n">cc_tag</span>
<span class="k">continue</span>
<span class="c1"># add market codes from official languages of the country ..</span>
<span class="k">for</span> <span class="n">lang_tag</span> <span class="ow">in</span> <span class="n">babel</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get_official_languages</span><span class="p">(</span><span class="n">cc_tag</span><span class="p">,</span> <span class="n">de_facto</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="k">if</span> <span class="n">lang_tag</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
<span class="c1"># print(&quot;ignore lang: %s &lt;-- %s&quot; % (cc_tag, lang_tag))</span>
<span class="k">continue</span>
<span class="n">lang_tag</span> <span class="o">=</span> <span class="n">lang_tag</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="c1"># zh_Hant --&gt; zh</span>
<span class="n">market_code</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">lang_tag</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">cc_tag</span><span class="si">}</span><span class="s2">&quot;</span> <span class="c1"># zh-tw</span>
<span class="n">market_code</span> <span class="o">=</span> <span class="n">map_market_codes</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">market_code</span><span class="p">,</span> <span class="n">market_code</span><span class="p">)</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">%s</span><span class="s1">_</span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">lang_tag</span><span class="p">,</span> <span class="n">cc_tag</span><span class="o">.</span><span class="n">upper</span><span class="p">())))</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">market_code</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">market_code</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">market_code</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,210 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.bing_images &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.bing_images</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.bing_images</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Bing-Images: description see :py:obj:`searx.engines.bing`.&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=invalid-name</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.bing</span><span class="w"> </span><span class="kn">import</span> <span class="n">set_bing_cookies</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.bing</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="c1"># pylint: disable=unused-import</span>
<span class="c1"># about</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.bing.com/images&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q182496&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.microsoft.com/en-us/bing/apis/bing-image-search-api&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;HTML&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;images&#39;</span><span class="p">,</span> <span class="s1">&#39;web&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;https://www.bing.com/images/async&#39;</span>
<span class="sd">&quot;&quot;&quot;Bing (Images) search URL&quot;&quot;&quot;</span>
<span class="n">time_map</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;day&#39;</span><span class="p">:</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span><span class="p">,</span>
<span class="s1">&#39;week&#39;</span><span class="p">:</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span><span class="p">,</span>
<span class="s1">&#39;month&#39;</span><span class="p">:</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">31</span><span class="p">,</span>
<span class="s1">&#39;year&#39;</span><span class="p">:</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">365</span><span class="p">,</span>
<span class="p">}</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing_images.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Assemble a Bing-Image request.&quot;&quot;&quot;</span>
<span class="n">engine_region</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">engine_language</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="s1">&#39;en&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">set_bing_cookies</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">engine_language</span><span class="p">,</span> <span class="n">engine_region</span><span class="p">)</span>
<span class="c1"># build URL query</span>
<span class="c1"># - example: https://www.bing.com/images/async?q=foo&amp;async=content&amp;first=1&amp;count=35</span>
<span class="n">query_params</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;async&#39;</span><span class="p">:</span> <span class="s1">&#39;1&#39;</span><span class="p">,</span>
<span class="c1"># to simplify the page count lets use the default of 35 images per page</span>
<span class="s1">&#39;first&#39;</span><span class="p">:</span> <span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;pageno&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">35</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
<span class="s1">&#39;count&#39;</span><span class="p">:</span> <span class="mi">35</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># time range</span>
<span class="c1"># - example: one year (525600 minutes) &#39;qft=+filterui:age-lt525600&#39;</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]:</span>
<span class="n">query_params</span><span class="p">[</span><span class="s1">&#39;qft&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;filterui:age-lt</span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">time_map</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]]</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s1">&#39;?&#39;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">query_params</span><span class="p">)</span>
<span class="k">return</span> <span class="n">params</span></div>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing_images.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get response from Bing-Images&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;//ul[contains(@class, &quot;dgControl_list&quot;)]/li&#39;</span><span class="p">):</span>
<span class="n">metadata</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;.//a[@class=&quot;iusc&quot;]/@m&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">metadata</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">metadata</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;.//a[@class=&quot;iusc&quot;]/@m&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
<span class="n">title</span> <span class="o">=</span> <span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;.//div[@class=&quot;infnmpt&quot;]//a/text()&#39;</span><span class="p">))</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="n">img_format</span> <span class="o">=</span> <span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;.//div[@class=&quot;imgpt&quot;]/div/span/text()&#39;</span><span class="p">))</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot; · &quot;</span><span class="p">)</span>
<span class="n">source</span> <span class="o">=</span> <span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;.//div[@class=&quot;imgpt&quot;]//div[@class=&quot;lnkw&quot;]//a/text()&#39;</span><span class="p">))</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;template&#39;</span><span class="p">:</span> <span class="s1">&#39;images.html&#39;</span><span class="p">,</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">metadata</span><span class="p">[</span><span class="s1">&#39;purl&#39;</span><span class="p">],</span>
<span class="s1">&#39;thumbnail_src&#39;</span><span class="p">:</span> <span class="n">metadata</span><span class="p">[</span><span class="s1">&#39;turl&#39;</span><span class="p">],</span>
<span class="s1">&#39;img_src&#39;</span><span class="p">:</span> <span class="n">metadata</span><span class="p">[</span><span class="s1">&#39;murl&#39;</span><span class="p">],</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;desc&#39;</span><span class="p">),</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
<span class="s1">&#39;source&#39;</span><span class="p">:</span> <span class="n">source</span><span class="p">,</span>
<span class="s1">&#39;resolution&#39;</span><span class="p">:</span> <span class="n">img_format</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
<span class="s1">&#39;img_format&#39;</span><span class="p">:</span> <span class="n">img_format</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">img_format</span><span class="p">)</span> <span class="o">&gt;=</span> <span class="mi">2</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,268 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.bing_news &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.bing_news</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.bing_news</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Bing-News: description see :py:obj:`searx.engines.bing`.</span>
<span class="sd">.. hint::</span>
<span class="sd"> Bing News is *different* in some ways!</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=invalid-name</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">,</span> <span class="n">eval_xpath_getindex</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.bing</span><span class="w"> </span><span class="kn">import</span> <span class="n">set_bing_cookies</span>
<span class="c1"># about</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.bing.com/news&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q2878637&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.microsoft.com/en-us/bing/apis/bing-news-search-api&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;RSS&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;news&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="sd">&quot;&quot;&quot;If go through the pages and there are actually no new results for another</span>
<span class="sd">page, then bing returns the results from the last page again.&quot;&quot;&quot;</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">time_map</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;day&#39;</span><span class="p">:</span> <span class="s1">&#39;interval=&quot;4&quot;&#39;</span><span class="p">,</span>
<span class="s1">&#39;week&#39;</span><span class="p">:</span> <span class="s1">&#39;interval=&quot;7&quot;&#39;</span><span class="p">,</span>
<span class="s1">&#39;month&#39;</span><span class="p">:</span> <span class="s1">&#39;interval=&quot;9&quot;&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="sd">&quot;&quot;&quot;A string &#39;4&#39; means *last hour*. We use *last hour* for ``day`` here since the</span>
<span class="sd">difference of *last day* and *last week* in the result list is just marginally.</span>
<span class="sd">Bing does not have news range ``year`` / we use ``month`` instead.&quot;&quot;&quot;</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;https://www.bing.com/news/infinitescrollajax&#39;</span>
<span class="sd">&quot;&quot;&quot;Bing (News) search URL&quot;&quot;&quot;</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing_news.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Assemble a Bing-News request.&quot;&quot;&quot;</span>
<span class="n">engine_region</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">engine_language</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="s1">&#39;en&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">set_bing_cookies</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">engine_language</span><span class="p">,</span> <span class="n">engine_region</span><span class="p">)</span>
<span class="c1"># build URL query</span>
<span class="c1">#</span>
<span class="c1"># example: https://www.bing.com/news/infinitescrollajax?q=london&amp;first=1</span>
<span class="n">page</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;pageno&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span> <span class="o">-</span> <span class="mi">1</span>
<span class="n">query_params</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;InfiniteScroll&#39;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
<span class="c1"># to simplify the page count lets use the default of 10 images per page</span>
<span class="s1">&#39;first&#39;</span><span class="p">:</span> <span class="n">page</span> <span class="o">*</span> <span class="mi">10</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
<span class="s1">&#39;SFX&#39;</span><span class="p">:</span> <span class="n">page</span><span class="p">,</span>
<span class="s1">&#39;form&#39;</span><span class="p">:</span> <span class="s1">&#39;PTFTNR&#39;</span><span class="p">,</span>
<span class="s1">&#39;setlang&#39;</span><span class="p">:</span> <span class="n">engine_region</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">],</span>
<span class="s1">&#39;cc&#39;</span><span class="p">:</span> <span class="n">engine_region</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span>
<span class="p">}</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]:</span>
<span class="n">query_params</span><span class="p">[</span><span class="s1">&#39;qft&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">time_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">],</span> <span class="s1">&#39;interval=&quot;9&quot;&#39;</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s1">&#39;?&#39;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">query_params</span><span class="p">)</span>
<span class="k">return</span> <span class="n">params</span></div>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing_news.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get response from Bing-Video&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">for</span> <span class="n">newsitem</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">&#39;//div[contains(@class, &quot;newsitem&quot;)]&#39;</span><span class="p">):</span>
<span class="n">link</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">newsitem</span><span class="p">,</span> <span class="s1">&#39;.//a[@class=&quot;title&quot;]&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">link</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">link</span><span class="o">.</span><span class="n">attrib</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;href&#39;</span><span class="p">)</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">link</span><span class="p">)</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">newsitem</span><span class="p">,</span> <span class="s1">&#39;.//div[@class=&quot;snippet&quot;]&#39;</span><span class="p">))</span>
<span class="n">metadata</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">source</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">newsitem</span><span class="p">,</span> <span class="s1">&#39;.//div[contains(@class, &quot;source&quot;)]&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">source</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="p">(</span>
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="s1">&#39;.//span[@aria-label]/@aria-label&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
<span class="c1"># eval_xpath_getindex(source, &#39;.//a&#39;, 0, None),</span>
<span class="c1"># eval_xpath_getindex(source, &#39;./div/span&#39;, 3, None),</span>
<span class="n">link</span><span class="o">.</span><span class="n">attrib</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;data-author&#39;</span><span class="p">),</span>
<span class="p">):</span>
<span class="k">if</span> <span class="n">item</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">t</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">if</span> <span class="n">t</span> <span class="ow">and</span> <span class="n">t</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span>
<span class="n">metadata</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">strip</span><span class="p">())</span>
<span class="n">metadata</span> <span class="o">=</span> <span class="s1">&#39; | &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">metadata</span><span class="p">)</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">imagelink</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">newsitem</span><span class="p">,</span> <span class="s1">&#39;.//a[@class=&quot;imagelink&quot;]//img&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">imagelink</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">imagelink</span><span class="o">.</span><span class="n">attrib</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;src&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">thumbnail</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;https://www.bing.com&quot;</span><span class="p">):</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="s1">&#39;https://www.bing.com/&#39;</span> <span class="o">+</span> <span class="n">thumbnail</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
<span class="s1">&#39;thumbnail&#39;</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
<span class="s1">&#39;metadata&#39;</span><span class="p">:</span> <span class="n">metadata</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span></div>
<div class="viewcode-block" id="fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing_news.fetch_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Fetch languages and regions from Bing-News.&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=import-outside-toplevel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.bing</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="k">as</span> <span class="n">_f</span>
<span class="n">_f</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">)</span>
<span class="c1"># fix market codes not known by bing news:</span>
<span class="c1"># In bing the market code &#39;zh-cn&#39; exists, but there is no &#39;news&#39; category in</span>
<span class="c1"># bing for this market. Alternatively we use the the market code from Honk</span>
<span class="c1"># Kong. Even if this is not correct, it is better than having no hits at</span>
<span class="c1"># all, or sending false queries to bing that could raise the suspicion of a</span>
<span class="c1"># bot.</span>
<span class="c1"># HINT: &#39;en-hk&#39; is the region code it does not indicate the language en!!</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="s1">&#39;zh-CN&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;en-hk&#39;</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,202 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.bing_videos &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.bing_videos</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.bing_videos</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="c1"># pylint: disable=invalid-name</span>
<span class="sd">&quot;&quot;&quot;Bing-Videos: description see :py:obj:`searx.engines.bing`.&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.bing</span><span class="w"> </span><span class="kn">import</span> <span class="n">set_bing_cookies</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.bing</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="c1"># pylint: disable=unused-import</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.bing_images</span><span class="w"> </span><span class="kn">import</span> <span class="n">time_map</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.bing.com/videos&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q4914152&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.microsoft.com/en-us/bing/apis/bing-video-search-api&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;HTML&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;videos&#39;</span><span class="p">,</span> <span class="s1">&#39;web&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;https://www.bing.com/videos/asyncv2&#39;</span>
<span class="sd">&quot;&quot;&quot;Bing (Videos) async search URL.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing_videos.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Assemble a Bing-Video request.&quot;&quot;&quot;</span>
<span class="n">engine_region</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">engine_language</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="s1">&#39;en&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">set_bing_cookies</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">engine_language</span><span class="p">,</span> <span class="n">engine_region</span><span class="p">)</span>
<span class="c1"># build URL query</span>
<span class="c1">#</span>
<span class="c1"># example: https://www.bing.com/videos/asyncv2?q=foo&amp;async=content&amp;first=1&amp;count=35</span>
<span class="n">query_params</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;async&#39;</span><span class="p">:</span> <span class="s1">&#39;content&#39;</span><span class="p">,</span>
<span class="c1"># to simplify the page count lets use the default of 35 images per page</span>
<span class="s1">&#39;first&#39;</span><span class="p">:</span> <span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;pageno&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">35</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
<span class="s1">&#39;count&#39;</span><span class="p">:</span> <span class="mi">35</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># time range</span>
<span class="c1">#</span>
<span class="c1"># example: one week (10080 minutes) &#39;&amp;qft= filterui:videoage-lt10080&#39; &#39;&amp;form=VRFLTR&#39;</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]:</span>
<span class="n">query_params</span><span class="p">[</span><span class="s1">&#39;form&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;VRFLTR&#39;</span>
<span class="n">query_params</span><span class="p">[</span><span class="s1">&#39;qft&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39; filterui:videoage-lt</span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">time_map</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]]</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s1">&#39;?&#39;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">query_params</span><span class="p">)</span>
<span class="k">return</span> <span class="n">params</span></div>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing_videos.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get response from Bing-Video&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;//div[@class=&quot;dg_u&quot;]//div[contains(@id, &quot;mc_vtvc_video&quot;)]&#39;</span><span class="p">):</span>
<span class="n">metadata</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;.//div[@class=&quot;vrhdata&quot;]/@vrhm&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
<span class="n">info</span> <span class="o">=</span> <span class="s1">&#39; - &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;.//div[@class=&quot;mc_vtvc_meta_block&quot;]//span/text()&#39;</span><span class="p">))</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="n">content</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="si">{0}</span><span class="s1"> - </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s1">&#39;du&#39;</span><span class="p">],</span> <span class="n">info</span><span class="p">)</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;.//div[contains(@class, &quot;mc_vtvc_th&quot;)]//img/@src&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">metadata</span><span class="p">[</span><span class="s1">&#39;murl&#39;</span><span class="p">],</span>
<span class="s1">&#39;thumbnail&#39;</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;vt&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">),</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
<span class="s1">&#39;template&#39;</span><span class="p">:</span> <span class="s1">&#39;videos.html&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,596 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.brave &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.brave</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.brave</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Brave supports the categories listed in :py:obj:`brave_category` (General,</span>
<span class="sd">news, videos, images). The support of :py:obj:`paging` and :py:obj:`time range</span>
<span class="sd">&lt;time_range_support&gt;` is limited (see remarks).</span>
<span class="sd">Configured ``brave`` engines:</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: brave</span>
<span class="sd"> engine: brave</span>
<span class="sd"> ...</span>
<span class="sd"> brave_category: search</span>
<span class="sd"> time_range_support: true</span>
<span class="sd"> paging: true</span>
<span class="sd"> - name: brave.images</span>
<span class="sd"> engine: brave</span>
<span class="sd"> ...</span>
<span class="sd"> brave_category: images</span>
<span class="sd"> - name: brave.videos</span>
<span class="sd"> engine: brave</span>
<span class="sd"> ...</span>
<span class="sd"> brave_category: videos</span>
<span class="sd"> - name: brave.news</span>
<span class="sd"> engine: brave</span>
<span class="sd"> ...</span>
<span class="sd"> brave_category: news</span>
<span class="sd"> - name: brave.goggles</span>
<span class="sd"> time_range_support: true</span>
<span class="sd"> paging: true</span>
<span class="sd"> ...</span>
<span class="sd"> brave_category: goggles</span>
<span class="sd">.. _brave regions:</span>
<span class="sd">Brave regions</span>
<span class="sd">=============</span>
<span class="sd">Brave uses two-digit tags for the regions like ``ca`` while SearXNG deals with</span>
<span class="sd">locales. To get a mapping, all *officiat de-facto* languages of the Brave</span>
<span class="sd">region are mapped to regions in SearXNG (see :py:obj:`babel</span>
<span class="sd">&lt;babel.languages.get_official_languages&gt;`):</span>
<span class="sd">.. code:: python</span>
<span class="sd"> &quot;regions&quot;: {</span>
<span class="sd"> ..</span>
<span class="sd"> &quot;en-CA&quot;: &quot;ca&quot;,</span>
<span class="sd"> &quot;fr-CA&quot;: &quot;ca&quot;,</span>
<span class="sd"> ..</span>
<span class="sd"> }</span>
<span class="sd">.. note::</span>
<span class="sd"> The language (aka region) support of Brave&#39;s index is limited to very basic</span>
<span class="sd"> languages. The search results for languages like Chinese or Arabic are of</span>
<span class="sd"> low quality.</span>
<span class="sd">.. _brave googles:</span>
<span class="sd">Brave Goggles</span>
<span class="sd">=============</span>
<span class="sd">.. _list of Goggles: https://search.brave.com/goggles/discover</span>
<span class="sd">.. _Goggles Whitepaper: https://brave.com/static-assets/files/goggles.pdf</span>
<span class="sd">.. _Goggles Quickstart: https://github.com/brave/goggles-quickstart</span>
<span class="sd">Goggles allow you to choose, alter, or extend the ranking of Brave Search</span>
<span class="sd">results (`Goggles Whitepaper`_). Goggles are openly developed by the community</span>
<span class="sd">of Brave Search users.</span>
<span class="sd">Select from the `list of Goggles`_ people have published, or create your own</span>
<span class="sd">(`Goggles Quickstart`_).</span>
<span class="sd">.. _brave languages:</span>
<span class="sd">Brave languages</span>
<span class="sd">===============</span>
<span class="sd">Brave&#39;s language support is limited to the UI (menus, area local notations,</span>
<span class="sd">etc). Brave&#39;s index only seems to support a locale, but it does not seem to</span>
<span class="sd">support any languages in its index. The choice of available languages is very</span>
<span class="sd">small (and its not clear to me where the difference in UI is when switching</span>
<span class="sd">from en-us to en-ca or en-gb).</span>
<span class="sd">In the :py:obj:`EngineTraits object &lt;searx.enginelib.traits.EngineTraits&gt;` the</span>
<span class="sd">UI languages are stored in a custom field named ``ui_lang``:</span>
<span class="sd">.. code:: python</span>
<span class="sd"> &quot;custom&quot;: {</span>
<span class="sd"> &quot;ui_lang&quot;: {</span>
<span class="sd"> &quot;ca&quot;: &quot;ca&quot;,</span>
<span class="sd"> &quot;de-DE&quot;: &quot;de-de&quot;,</span>
<span class="sd"> &quot;en-CA&quot;: &quot;en-ca&quot;,</span>
<span class="sd"> &quot;en-GB&quot;: &quot;en-gb&quot;,</span>
<span class="sd"> &quot;en-US&quot;: &quot;en-us&quot;,</span>
<span class="sd"> &quot;es&quot;: &quot;es&quot;,</span>
<span class="sd"> &quot;fr-CA&quot;: &quot;fr-ca&quot;,</span>
<span class="sd"> &quot;fr-FR&quot;: &quot;fr-fr&quot;,</span>
<span class="sd"> &quot;ja-JP&quot;: &quot;ja-jp&quot;,</span>
<span class="sd"> &quot;pt-BR&quot;: &quot;pt-br&quot;,</span>
<span class="sd"> &quot;sq-AL&quot;: &quot;sq-al&quot;</span>
<span class="sd"> }</span>
<span class="sd"> },</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">urlencode</span><span class="p">,</span>
<span class="n">urlparse</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil</span><span class="w"> </span><span class="kn">import</span> <span class="n">parser</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">locales</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">extract_text</span><span class="p">,</span>
<span class="n">eval_xpath_list</span><span class="p">,</span>
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
<span class="n">js_obj_str_to_python</span><span class="p">,</span>
<span class="n">js_obj_str_to_json_str</span><span class="p">,</span>
<span class="n">get_embeded_stream_url</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s2">&quot;https://search.brave.com/&quot;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s2">&quot;Q22906900&quot;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s2">&quot;HTML&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;https://search.brave.com/&quot;</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">brave_category</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;search&quot;</span><span class="p">,</span> <span class="s2">&quot;videos&quot;</span><span class="p">,</span> <span class="s2">&quot;images&quot;</span><span class="p">,</span> <span class="s2">&quot;news&quot;</span><span class="p">,</span> <span class="s2">&quot;goggles&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;search&quot;</span>
<span class="sd">&quot;&quot;&quot;Brave supports common web-search, videos, images, news, and goggles search.</span>
<span class="sd">- ``search``: Common WEB search</span>
<span class="sd">- ``videos``: search for videos</span>
<span class="sd">- ``images``: search for images</span>
<span class="sd">- ``news``: search for news</span>
<span class="sd">- ``goggles``: Common WEB search with custom rules, requires a :py:obj:`Goggles` URL.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">Goggles</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;This should be a URL ending in ``.goggle``&quot;&quot;&quot;</span>
<span class="n">brave_spellcheck</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&quot;&quot;&quot;Brave supports some kind of spell checking. When activated, Brave tries to</span>
<span class="sd">fix typos, e.g. it searches for ``food`` when the user queries for ``fooh``. In</span>
<span class="sd">the UI of Brave the user gets warned about this, since we can not warn the user</span>
<span class="sd">in SearXNG, the spellchecking is disabled by default.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&quot;&quot;&quot;Brave only supports paging in :py:obj:`brave_category` ``search`` (UI</span>
<span class="sd">category All) and in the goggles category.&quot;&quot;&quot;</span>
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">10</span>
<span class="sd">&quot;&quot;&quot;Tested 9 pages maximum (``&amp;offset=8``), to be save max is set to 10. Trying</span>
<span class="sd">to do more won&#39;t return any result and you will most likely be flagged as a bot.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">safesearch_map</span> <span class="o">=</span> <span class="p">{</span><span class="mi">2</span><span class="p">:</span> <span class="s2">&quot;strict&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s2">&quot;moderate&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">:</span> <span class="s2">&quot;off&quot;</span><span class="p">}</span> <span class="c1"># cookie: safesearch=off</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&quot;&quot;&quot;Brave only supports time-range in :py:obj:`brave_category` ``search`` (UI</span>
<span class="sd">category All) and in the goggles category.&quot;&quot;&quot;</span>
<span class="n">time_range_map</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;day&quot;</span><span class="p">:</span> <span class="s2">&quot;pd&quot;</span><span class="p">,</span>
<span class="s2">&quot;week&quot;</span><span class="p">:</span> <span class="s2">&quot;pw&quot;</span><span class="p">,</span>
<span class="s2">&quot;month&quot;</span><span class="p">:</span> <span class="s2">&quot;pm&quot;</span><span class="p">,</span>
<span class="s2">&quot;year&quot;</span><span class="p">:</span> <span class="s2">&quot;py&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">args</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;q&quot;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s2">&quot;source&quot;</span><span class="p">:</span> <span class="s2">&quot;web&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="k">if</span> <span class="n">brave_spellcheck</span><span class="p">:</span>
<span class="n">args</span><span class="p">[</span><span class="s2">&quot;spellcheck&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;1&quot;</span>
<span class="k">if</span> <span class="n">brave_category</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">&quot;search&quot;</span><span class="p">,</span> <span class="s2">&quot;goggles&quot;</span><span class="p">):</span>
<span class="k">if</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;pageno&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">args</span><span class="p">[</span><span class="s2">&quot;offset&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;pageno&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span>
<span class="k">if</span> <span class="n">time_range_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;time_range&quot;</span><span class="p">]):</span>
<span class="n">args</span><span class="p">[</span><span class="s2">&quot;tf&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">time_range_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;time_range&quot;</span><span class="p">])</span>
<span class="k">if</span> <span class="n">brave_category</span> <span class="o">==</span> <span class="s2">&quot;goggles&quot;</span><span class="p">:</span>
<span class="n">args</span><span class="p">[</span><span class="s2">&quot;goggles_id&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">Goggles</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;headers&quot;</span><span class="p">][</span><span class="s2">&quot;Accept-Encoding&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;gzip, deflate&quot;</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">base_url</span><span class="si">}{</span><span class="n">brave_category</span><span class="si">}</span><span class="s2">?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;url </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">])</span>
<span class="c1"># set properties in the cookies</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;cookies&quot;</span><span class="p">][</span><span class="s2">&quot;safesearch&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">safesearch_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;safesearch&quot;</span><span class="p">],</span> <span class="s2">&quot;off&quot;</span><span class="p">)</span>
<span class="c1"># the useLocation is IP based, we use cookie &quot;country&quot; for the region</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;cookies&quot;</span><span class="p">][</span><span class="s2">&quot;useLocation&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;0&quot;</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;cookies&quot;</span><span class="p">][</span><span class="s2">&quot;summarizer&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;0&quot;</span>
<span class="n">engine_region</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;searxng_locale&quot;</span><span class="p">],</span> <span class="s2">&quot;all&quot;</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;cookies&quot;</span><span class="p">][</span><span class="s2">&quot;country&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_region</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;-&quot;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="c1"># type: ignore</span>
<span class="n">ui_lang</span> <span class="o">=</span> <span class="n">locales</span><span class="o">.</span><span class="n">get_engine_locale</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;searxng_locale&quot;</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;ui_lang&quot;</span><span class="p">],</span> <span class="s2">&quot;en-us&quot;</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;cookies&quot;</span><span class="p">][</span><span class="s2">&quot;ui_lang&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ui_lang</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;cookies </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s2">&quot;cookies&quot;</span><span class="p">])</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_extract_published_date</span><span class="p">(</span><span class="n">published_date_raw</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">):</span>
<span class="k">if</span> <span class="n">published_date_raw</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">return</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">published_date_raw</span><span class="p">)</span>
<span class="k">except</span> <span class="n">parser</span><span class="o">.</span><span class="n">ParserError</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">def</span><span class="w"> </span><span class="nf">extract_json_data</span><span class="p">(</span><span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]:</span>
<span class="c1"># Example script source containing the data:</span>
<span class="c1">#</span>
<span class="c1"># kit.start(app, element, {</span>
<span class="c1"># node_ids: [0, 19],</span>
<span class="c1"># data: [{type:&quot;data&quot;,data: .... [&quot;q&quot;,&quot;goggles_id&quot;],route:1,url:1}}]</span>
<span class="c1"># ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^</span>
<span class="n">text</span> <span class="o">=</span> <span class="n">text</span><span class="p">[</span><span class="n">text</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s2">&quot;&lt;script&quot;</span><span class="p">)</span> <span class="p">:</span> <span class="n">text</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s2">&quot;&lt;/script&quot;</span><span class="p">)]</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">text</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;can&#39;t find JS/JSON data in the given text&quot;</span><span class="p">)</span>
<span class="n">start</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s2">&quot;data: [{&quot;</span><span class="p">)</span>
<span class="n">end</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">rindex</span><span class="p">(</span><span class="s2">&quot;}}]&quot;</span><span class="p">)</span>
<span class="n">js_obj_str</span> <span class="o">=</span> <span class="n">text</span><span class="p">[</span><span class="n">start</span><span class="p">:</span><span class="n">end</span><span class="p">]</span>
<span class="n">js_obj_str</span> <span class="o">=</span> <span class="s2">&quot;{&quot;</span> <span class="o">+</span> <span class="n">js_obj_str</span> <span class="o">+</span> <span class="s2">&quot;}}]}&quot;</span>
<span class="c1"># js_obj_str = js_obj_str.replace(&quot;\xa0&quot;, &quot;&quot;) # remove ASCII for &amp;nbsp;</span>
<span class="c1"># js_obj_str = js_obj_str.replace(r&quot;\u003C&quot;, &quot;&lt;&quot;).replace(r&quot;\u003c&quot;, &quot;&lt;&quot;) # fix broken HTML tags in strings</span>
<span class="n">json_str</span> <span class="o">=</span> <span class="n">js_obj_str_to_json_str</span><span class="p">(</span><span class="n">js_obj_str</span><span class="p">)</span>
<span class="n">data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">json_str</span><span class="p">)</span>
<span class="k">return</span> <span class="n">data</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="n">SXNG_Response</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="k">if</span> <span class="n">brave_category</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">&#39;search&#39;</span><span class="p">,</span> <span class="s1">&#39;goggles&#39;</span><span class="p">):</span>
<span class="k">return</span> <span class="n">_parse_search</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<span class="k">if</span> <span class="n">brave_category</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">&#39;news&#39;</span><span class="p">):</span>
<span class="k">return</span> <span class="n">_parse_news</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<span class="c1"># Example script source containing the data:</span>
<span class="c1">#</span>
<span class="c1"># kit.start(app, element, {</span>
<span class="c1"># node_ids: [0, 19],</span>
<span class="c1"># data: [{type:&quot;data&quot;,data: .... [&quot;q&quot;,&quot;goggles_id&quot;],route:1,url:1}}]</span>
<span class="c1"># ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^</span>
<span class="n">json_data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">extract_json_data</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="n">json_resp</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">json_data</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="mi">1</span><span class="p">][</span><span class="s2">&quot;data&quot;</span><span class="p">][</span><span class="s1">&#39;body&#39;</span><span class="p">][</span><span class="s1">&#39;response&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">brave_category</span> <span class="o">==</span> <span class="s1">&#39;images&#39;</span><span class="p">:</span>
<span class="k">return</span> <span class="n">_parse_images</span><span class="p">(</span><span class="n">json_resp</span><span class="p">)</span>
<span class="k">if</span> <span class="n">brave_category</span> <span class="o">==</span> <span class="s1">&#39;videos&#39;</span><span class="p">:</span>
<span class="k">return</span> <span class="n">_parse_videos</span><span class="p">(</span><span class="n">json_resp</span><span class="p">)</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Unsupported brave category: </span><span class="si">{</span><span class="n">brave_category</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_search</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="n">SXNG_Response</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//div[contains(@class, &#39;snippet &#39;)]&quot;</span><span class="p">):</span>
<span class="n">url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//a/@href&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
<span class="n">title_tag</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//div[contains(@class, &#39;title&#39;)]&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">url</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">title_tag</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span><span class="o">.</span><span class="n">netloc</span><span class="p">:</span> <span class="c1"># partial url likely means it&#39;s an ad</span>
<span class="k">continue</span>
<span class="n">content</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="n">pub_date</span> <span class="o">=</span> <span class="kc">None</span>
<span class="c1"># there are other classes like &#39;site-name-content&#39; we don&#39;t want to match,</span>
<span class="c1"># however only using contains(@class, &#39;content&#39;) would e.g. also match `site-name-content`</span>
<span class="c1"># thus, we explicitly also require the spaces as class separator</span>
<span class="n">_content</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//div[contains(concat(&#39; &#39;, @class, &#39; &#39;), &#39; content &#39;)]&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">_content</span><span class="p">):</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">_content</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">_pub_date</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span>
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">_content</span><span class="p">,</span> <span class="s2">&quot;.//span[contains(@class, &#39;t-secondary&#39;)]&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">_pub_date</span><span class="p">:</span>
<span class="n">pub_date</span> <span class="o">=</span> <span class="n">_extract_published_date</span><span class="p">(</span><span class="n">_pub_date</span><span class="p">)</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">content</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="n">_pub_date</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">&quot;- </span><span class="se">\n\t</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">thumbnail</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//a[contains(@class, &#39;thumbnail&#39;)]//img/@src&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="n">item</span> <span class="o">=</span> <span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span>
<span class="n">template</span><span class="o">=</span><span class="s2">&quot;default.html&quot;</span><span class="p">,</span>
<span class="n">url</span><span class="o">=</span><span class="n">url</span><span class="p">,</span>
<span class="n">title</span><span class="o">=</span><span class="n">extract_text</span><span class="p">(</span><span class="n">title_tag</span><span class="p">),</span>
<span class="n">content</span><span class="o">=</span><span class="n">content</span><span class="p">,</span>
<span class="n">publishedDate</span><span class="o">=</span><span class="n">pub_date</span><span class="p">,</span>
<span class="n">thumbnail</span><span class="o">=</span><span class="n">thumbnail</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="n">video_tag</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span>
<span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//div[contains(@class, &#39;video-snippet&#39;) and @data-macro=&#39;video&#39;]&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="p">[]</span>
<span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">video_tag</span><span class="p">):</span>
<span class="c1"># In my tests a video tag in the WEB search was most often not a</span>
<span class="c1"># video, except the ones from youtube ..</span>
<span class="n">iframe_src</span> <span class="o">=</span> <span class="n">get_embeded_stream_url</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
<span class="k">if</span> <span class="n">iframe_src</span><span class="p">:</span>
<span class="n">item</span><span class="p">[</span><span class="s2">&quot;iframe_src&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">iframe_src</span>
<span class="n">item</span><span class="p">[</span><span class="s2">&quot;template&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;videos.html&quot;</span>
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//a[contains(@class, &#39;related-query&#39;)]&quot;</span><span class="p">):</span>
<span class="n">res</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">({</span><span class="s1">&#39;suggestion&#39;</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)}))</span>
<span class="k">return</span> <span class="n">res</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_news</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="n">SXNG_Response</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//div[contains(@class, &#39;results&#39;)]//div[@data-type=&#39;news&#39;]&quot;</span><span class="p">):</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//a[contains(@class, &#39;result-header&#39;)]/@href&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">url</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//span[contains(@class, &#39;snippet-title&#39;)]&quot;</span><span class="p">)</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//p[contains(@class, &#39;desc&#39;)]&quot;</span><span class="p">)</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//div[contains(@class, &#39;image-wrapper&#39;)]//img/@src&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="n">item</span> <span class="o">=</span> <span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span>
<span class="n">template</span><span class="o">=</span><span class="s2">&quot;default.html&quot;</span><span class="p">,</span>
<span class="n">url</span><span class="o">=</span><span class="n">url</span><span class="p">,</span>
<span class="n">title</span><span class="o">=</span><span class="n">extract_text</span><span class="p">(</span><span class="n">title</span><span class="p">),</span>
<span class="n">thumbnail</span><span class="o">=</span><span class="n">thumbnail</span><span class="p">,</span>
<span class="n">content</span><span class="o">=</span><span class="n">extract_text</span><span class="p">(</span><span class="n">content</span><span class="p">),</span>
<span class="p">)</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">return</span> <span class="n">res</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_images</span><span class="p">(</span><span class="n">json_resp</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_resp</span><span class="p">[</span><span class="s2">&quot;results&quot;</span><span class="p">]:</span>
<span class="n">item</span> <span class="o">=</span> <span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span>
<span class="n">template</span><span class="o">=</span><span class="s2">&quot;images.html&quot;</span><span class="p">,</span>
<span class="n">url</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">],</span>
<span class="n">title</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;title&quot;</span><span class="p">],</span>
<span class="n">source</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">],</span>
<span class="n">img_src</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;properties&quot;</span><span class="p">][</span><span class="s2">&quot;url&quot;</span><span class="p">],</span>
<span class="n">thumbnail_src</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;thumbnail&quot;</span><span class="p">][</span><span class="s2">&quot;src&quot;</span><span class="p">],</span>
<span class="p">)</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">return</span> <span class="n">res</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_videos</span><span class="p">(</span><span class="n">json_resp</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_resp</span><span class="p">[</span><span class="s2">&quot;results&quot;</span><span class="p">]:</span>
<span class="n">item</span> <span class="o">=</span> <span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span>
<span class="n">template</span><span class="o">=</span><span class="s2">&quot;videos.html&quot;</span><span class="p">,</span>
<span class="n">url</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">],</span>
<span class="n">title</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;title&quot;</span><span class="p">],</span>
<span class="n">content</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;description&quot;</span><span class="p">],</span>
<span class="n">length</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;video&quot;</span><span class="p">][</span><span class="s2">&quot;duration&quot;</span><span class="p">],</span>
<span class="n">duration</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;video&quot;</span><span class="p">][</span><span class="s2">&quot;duration&quot;</span><span class="p">],</span>
<span class="n">publishedDate</span><span class="o">=</span><span class="n">_extract_published_date</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;age&quot;</span><span class="p">]),</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">result</span><span class="p">[</span><span class="s2">&quot;thumbnail&quot;</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">item</span><span class="p">[</span><span class="s2">&quot;thumbnail&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">&quot;thumbnail&quot;</span><span class="p">][</span><span class="s2">&quot;src&quot;</span><span class="p">]</span>
<span class="n">iframe_src</span> <span class="o">=</span> <span class="n">get_embeded_stream_url</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">])</span>
<span class="k">if</span> <span class="n">iframe_src</span><span class="p">:</span>
<span class="n">item</span><span class="p">[</span><span class="s2">&quot;iframe_src&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">iframe_src</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">return</span> <span class="n">res</span>
<div class="viewcode-block" id="fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/online/brave.html#searx.engines.brave.fetch_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Fetch :ref:`languages &lt;brave languages&gt;` and :ref:`regions &lt;brave</span>
<span class="sd"> regions&gt;` from Brave.&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=import-outside-toplevel, too-many-branches</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.languages</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">region_tag</span><span class="p">,</span> <span class="n">language_tag</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;ui_lang&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">lang_map</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;no&#39;</span><span class="p">:</span> <span class="s1">&#39;nb&#39;</span><span class="p">}</span> <span class="c1"># norway</span>
<span class="c1"># languages (UI)</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">&#39;https://search.brave.com/settings&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: response from Brave is not OK.&quot;</span><span class="p">)</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">for</span> <span class="n">option</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s2">&quot;//section//option[@value=&#39;en-us&#39;]/../option&quot;</span><span class="p">):</span>
<span class="n">ui_lang</span> <span class="o">=</span> <span class="n">option</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">l</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">ui_lang</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot;-&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">l</span><span class="o">.</span><span class="n">territory</span><span class="p">:</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">ui_lang</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot;-&quot;</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">ui_lang</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot;-&quot;</span><span class="p">))</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: can&#39;t determine babel locale of Brave&#39;s (UI) language </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">ui_lang</span><span class="p">)</span>
<span class="k">continue</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;ui_lang&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">ui_lang</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">ui_lang</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;ui_lang&quot;</span><span class="p">][</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">ui_lang</span>
<span class="c1"># search regions of brave</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s2">&quot;https://cdn.search.brave.com/serp/v2/_app/immutable/chunks/parameters.734c106a.js&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: response from Brave is not OK.&quot;</span><span class="p">)</span>
<span class="n">country_js</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">[</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s2">&quot;options:{all&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="s2">&quot;options:&quot;</span><span class="p">)</span> <span class="p">:]</span>
<span class="n">country_js</span> <span class="o">=</span> <span class="n">country_js</span><span class="p">[:</span> <span class="n">country_js</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s2">&quot;},k={default&quot;</span><span class="p">)]</span>
<span class="n">country_tags</span> <span class="o">=</span> <span class="n">js_obj_str_to_python</span><span class="p">(</span><span class="n">country_js</span><span class="p">)</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">country_tags</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="n">k</span> <span class="o">==</span> <span class="s2">&quot;all&quot;</span><span class="p">:</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="s2">&quot;all&quot;</span>
<span class="k">continue</span>
<span class="n">country_tag</span> <span class="o">=</span> <span class="n">v</span><span class="p">[</span><span class="s2">&quot;value&quot;</span><span class="p">]</span>
<span class="c1"># add official languages of the country ..</span>
<span class="k">for</span> <span class="n">lang_tag</span> <span class="ow">in</span> <span class="n">babel</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get_official_languages</span><span class="p">(</span><span class="n">country_tag</span><span class="p">,</span> <span class="n">de_facto</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="n">lang_tag</span> <span class="o">=</span> <span class="n">lang_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">lang_tag</span><span class="p">,</span> <span class="n">lang_tag</span><span class="p">)</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%s</span><span class="s2">_</span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">lang_tag</span><span class="p">,</span> <span class="n">country_tag</span><span class="o">.</span><span class="n">upper</span><span class="p">())))</span>
<span class="c1"># print(&quot;%-20s: %s &lt;-- %s&quot; % (v[&quot;label&quot;], country_tag, sxng_tag))</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">country_tag</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">country_tag</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">country_tag</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,354 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.command &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.command</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.command</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;With *command engines* administrators can run engines to integrate arbitrary</span>
<span class="sd">shell commands.</span>
<span class="sd">.. attention::</span>
<span class="sd"> When creating and enabling a ``command`` engine on a public instance, you</span>
<span class="sd"> must be careful to avoid leaking private data.</span>
<span class="sd">The easiest solution is to limit the access by setting ``tokens`` as described</span>
<span class="sd">in section :ref:`private engines`. The engine base is flexible. Only your</span>
<span class="sd">imagination can limit the power of this engine (and maybe security concerns).</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">The following options are available:</span>
<span class="sd">``command``:</span>
<span class="sd"> A comma separated list of the elements of the command. A special token</span>
<span class="sd"> ``{{QUERY}}`` tells where to put the search terms of the user. Example:</span>
<span class="sd"> .. code:: yaml</span>
<span class="sd"> [&#39;ls&#39;, &#39;-l&#39;, &#39;-h&#39;, &#39;{{QUERY}}&#39;]</span>
<span class="sd">``delimiter``:</span>
<span class="sd"> A mapping containing a delimiter ``char`` and the *titles* of each element in</span>
<span class="sd"> ``keys``.</span>
<span class="sd">``parse_regex``:</span>
<span class="sd"> A dict containing the regular expressions for each result key.</span>
<span class="sd">``query_type``:</span>
<span class="sd"> The expected type of user search terms. Possible values: ``path`` and</span>
<span class="sd"> ``enum``.</span>
<span class="sd"> ``path``:</span>
<span class="sd"> Checks if the user provided path is inside the working directory. If not,</span>
<span class="sd"> the query is not executed.</span>
<span class="sd"> ``enum``:</span>
<span class="sd"> Is a list of allowed search terms. If the user submits something which is</span>
<span class="sd"> not included in the list, the query returns an error.</span>
<span class="sd">``query_enum``:</span>
<span class="sd"> A list containing allowed search terms if ``query_type`` is set to ``enum``.</span>
<span class="sd">``working_dir``:</span>
<span class="sd"> The directory where the command has to be executed. Default: ``./``.</span>
<span class="sd">``result_separator``:</span>
<span class="sd"> The character that separates results. Default: ``\\n``.</span>
<span class="sd">Example</span>
<span class="sd">=======</span>
<span class="sd">The example engine below can be used to find files with a specific name in the</span>
<span class="sd">configured working directory:</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: find</span>
<span class="sd"> engine: command</span>
<span class="sd"> command: [&#39;find&#39;, &#39;.&#39;, &#39;-name&#39;, &#39;{{QUERY}}&#39;]</span>
<span class="sd"> query_type: path</span>
<span class="sd"> shortcut: fnd</span>
<span class="sd"> delimiter:</span>
<span class="sd"> chars: &#39; &#39;</span>
<span class="sd"> keys: [&#39;line&#39;]</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">os.path</span><span class="w"> </span><span class="kn">import</span> <span class="n">expanduser</span><span class="p">,</span> <span class="n">isabs</span><span class="p">,</span> <span class="n">realpath</span><span class="p">,</span> <span class="n">commonprefix</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">shlex</span><span class="w"> </span><span class="kn">import</span> <span class="n">split</span> <span class="k">as</span> <span class="n">shlex_split</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">subprocess</span><span class="w"> </span><span class="kn">import</span> <span class="n">Popen</span><span class="p">,</span> <span class="n">PIPE</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">threading</span><span class="w"> </span><span class="kn">import</span> <span class="n">Thread</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="n">engine_type</span> <span class="o">=</span> <span class="s1">&#39;offline&#39;</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">command</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">delimiter</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">parse_regex</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">query_type</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="n">query_enum</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">environment_variables</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">working_dir</span> <span class="o">=</span> <span class="n">realpath</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
<span class="n">result_separator</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span>
<span class="n">timeout</span> <span class="o">=</span> <span class="mf">4.0</span>
<span class="n">_command_logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">&#39;command&#39;</span><span class="p">)</span>
<span class="n">_compiled_parse_regex</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">):</span>
<span class="n">check_parsing_options</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">)</span>
<span class="k">if</span> <span class="s1">&#39;command&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;engine command : missing configuration key: command&#39;</span><span class="p">)</span>
<span class="k">global</span> <span class="n">command</span><span class="p">,</span> <span class="n">working_dir</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">,</span> <span class="n">parse_regex</span><span class="p">,</span> <span class="n">environment_variables</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="n">command</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">&#39;command&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="s1">&#39;working_dir&#39;</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
<span class="n">working_dir</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">&#39;working_dir&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">isabs</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">[</span><span class="s1">&#39;working_dir&#39;</span><span class="p">]):</span>
<span class="n">working_dir</span> <span class="o">=</span> <span class="n">realpath</span><span class="p">(</span><span class="n">working_dir</span><span class="p">)</span>
<span class="k">if</span> <span class="s1">&#39;parse_regex&#39;</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
<span class="n">parse_regex</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">&#39;parse_regex&#39;</span><span class="p">]</span>
<span class="k">for</span> <span class="n">result_key</span><span class="p">,</span> <span class="n">regex</span> <span class="ow">in</span> <span class="n">parse_regex</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">_compiled_parse_regex</span><span class="p">[</span><span class="n">result_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">regex</span><span class="p">,</span> <span class="n">flags</span><span class="o">=</span><span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="k">if</span> <span class="s1">&#39;delimiter&#39;</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
<span class="n">delimiter</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">&#39;delimiter&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="s1">&#39;environment_variables&#39;</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
<span class="n">environment_variables</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">&#39;environment_variables&#39;</span><span class="p">]</span>
<span class="k">def</span><span class="w"> </span><span class="nf">search</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">cmd</span> <span class="o">=</span> <span class="n">_get_command_to_run</span><span class="p">(</span><span class="n">query</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">cmd</span><span class="p">:</span>
<span class="k">return</span> <span class="n">res</span>
<span class="n">reader_thread</span> <span class="o">=</span> <span class="n">Thread</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="n">_get_results_from_process</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="p">(</span><span class="n">res</span><span class="p">,</span> <span class="n">cmd</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]))</span>
<span class="n">reader_thread</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
<span class="n">reader_thread</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
<span class="k">return</span> <span class="n">res</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_get_command_to_run</span><span class="p">(</span><span class="n">query</span><span class="p">):</span>
<span class="n">params</span> <span class="o">=</span> <span class="n">shlex_split</span><span class="p">(</span><span class="n">query</span><span class="p">)</span>
<span class="n">__check_query_params</span><span class="p">(</span><span class="n">params</span><span class="p">)</span>
<span class="n">cmd</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">command</span><span class="p">:</span>
<span class="k">if</span> <span class="n">c</span> <span class="o">==</span> <span class="s1">&#39;{{QUERY}}&#39;</span><span class="p">:</span>
<span class="n">cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">params</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">cmd</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">c</span><span class="p">)</span>
<span class="k">return</span> <span class="n">cmd</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_get_results_from_process</span><span class="p">(</span><span class="n">res</span><span class="p">:</span> <span class="n">EngineResults</span><span class="p">,</span> <span class="n">cmd</span><span class="p">,</span> <span class="n">pageno</span><span class="p">):</span>
<span class="n">leftover</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="n">count</span> <span class="o">=</span> <span class="mi">0</span>
<span class="n">start</span><span class="p">,</span> <span class="n">end</span> <span class="o">=</span> <span class="n">__get_results_limits</span><span class="p">(</span><span class="n">pageno</span><span class="p">)</span>
<span class="k">with</span> <span class="n">Popen</span><span class="p">(</span><span class="n">cmd</span><span class="p">,</span> <span class="n">stdout</span><span class="o">=</span><span class="n">PIPE</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">PIPE</span><span class="p">,</span> <span class="n">env</span><span class="o">=</span><span class="n">environment_variables</span><span class="p">)</span> <span class="k">as</span> <span class="n">process</span><span class="p">:</span>
<span class="n">line</span> <span class="o">=</span> <span class="n">process</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
<span class="k">while</span> <span class="n">line</span><span class="p">:</span>
<span class="n">buf</span> <span class="o">=</span> <span class="n">leftover</span> <span class="o">+</span> <span class="n">line</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
<span class="n">raw_results</span> <span class="o">=</span> <span class="n">buf</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">result_separator</span><span class="p">)</span>
<span class="k">if</span> <span class="n">raw_results</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]:</span>
<span class="n">leftover</span> <span class="o">=</span> <span class="n">raw_results</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
<span class="n">raw_results</span> <span class="o">=</span> <span class="n">raw_results</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
<span class="k">for</span> <span class="n">raw_result</span> <span class="ow">in</span> <span class="n">raw_results</span><span class="p">:</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">__parse_single_result</span><span class="p">(</span><span class="n">raw_result</span><span class="p">)</span>
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">_command_logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;skipped result:&#39;</span><span class="p">,</span> <span class="n">raw_result</span><span class="p">)</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="n">start</span> <span class="o">&lt;=</span> <span class="n">count</span> <span class="ow">and</span> <span class="n">count</span> <span class="o">&lt;=</span> <span class="n">end</span><span class="p">:</span> <span class="c1"># pylint: disable=chained-comparison</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">KeyValue</span><span class="p">(</span><span class="n">kvmap</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
<span class="n">count</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="k">if</span> <span class="n">end</span> <span class="o">&lt;</span> <span class="n">count</span><span class="p">:</span>
<span class="k">return</span> <span class="n">res</span>
<span class="n">line</span> <span class="o">=</span> <span class="n">process</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
<span class="n">return_code</span> <span class="o">=</span> <span class="n">process</span><span class="o">.</span><span class="n">wait</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
<span class="k">if</span> <span class="n">return_code</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s1">&#39;non-zero return code when running command&#39;</span><span class="p">,</span> <span class="n">cmd</span><span class="p">,</span> <span class="n">return_code</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">def</span><span class="w"> </span><span class="nf">__get_results_limits</span><span class="p">(</span><span class="n">pageno</span><span class="p">):</span>
<span class="n">start</span> <span class="o">=</span> <span class="p">(</span><span class="n">pageno</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span>
<span class="n">end</span> <span class="o">=</span> <span class="n">start</span> <span class="o">+</span> <span class="mi">9</span>
<span class="k">return</span> <span class="n">start</span><span class="p">,</span> <span class="n">end</span>
<span class="k">def</span><span class="w"> </span><span class="nf">__check_query_params</span><span class="p">(</span><span class="n">params</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">query_type</span><span class="p">:</span>
<span class="k">return</span>
<span class="k">if</span> <span class="n">query_type</span> <span class="o">==</span> <span class="s1">&#39;path&#39;</span><span class="p">:</span>
<span class="n">query_path</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
<span class="n">query_path</span> <span class="o">=</span> <span class="n">expanduser</span><span class="p">(</span><span class="n">query_path</span><span class="p">)</span>
<span class="k">if</span> <span class="n">commonprefix</span><span class="p">([</span><span class="n">realpath</span><span class="p">(</span><span class="n">query_path</span><span class="p">),</span> <span class="n">working_dir</span><span class="p">])</span> <span class="o">!=</span> <span class="n">working_dir</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;requested path is outside of configured working directory&#39;</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">query_type</span> <span class="o">==</span> <span class="s1">&#39;enum&#39;</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">query_enum</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">for</span> <span class="n">param</span> <span class="ow">in</span> <span class="n">params</span><span class="p">:</span>
<span class="k">if</span> <span class="n">param</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">query_enum</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;submitted query params is not allowed&#39;</span><span class="p">,</span> <span class="n">param</span><span class="p">,</span> <span class="s1">&#39;allowed params:&#39;</span><span class="p">,</span> <span class="n">query_enum</span><span class="p">)</span>
<div class="viewcode-block" id="check_parsing_options">
<a class="viewcode-back" href="../../../dev/engines/offline/command-line-engines.html#searx.engines.command.check_parsing_options">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">check_parsing_options</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Checks if delimiter based parsing or regex parsing is configured correctly&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="s1">&#39;delimiter&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine_settings</span> <span class="ow">and</span> <span class="s1">&#39;parse_regex&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;failed to init settings for parsing lines: missing delimiter or parse_regex&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="s1">&#39;delimiter&#39;</span> <span class="ow">in</span> <span class="n">engine_settings</span> <span class="ow">and</span> <span class="s1">&#39;parse_regex&#39;</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;failed to init settings for parsing lines: too many settings&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="s1">&#39;delimiter&#39;</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
<span class="k">if</span> <span class="s1">&#39;chars&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">&#39;delimiter&#39;</span><span class="p">]</span> <span class="ow">or</span> <span class="s1">&#39;keys&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">&#39;delimiter&#39;</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">ValueError</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">__parse_single_result</span><span class="p">(</span><span class="n">raw_result</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Parses command line output based on configuration&quot;&quot;&quot;</span>
<span class="n">result</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">if</span> <span class="n">delimiter</span><span class="p">:</span>
<span class="n">elements</span> <span class="o">=</span> <span class="n">raw_result</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">delimiter</span><span class="p">[</span><span class="s1">&#39;chars&#39;</span><span class="p">],</span> <span class="n">maxsplit</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="n">delimiter</span><span class="p">[</span><span class="s1">&#39;keys&#39;</span><span class="p">])</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">elements</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">delimiter</span><span class="p">[</span><span class="s1">&#39;keys&#39;</span><span class="p">]):</span>
<span class="k">return</span> <span class="p">{}</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">elements</span><span class="p">)):</span> <span class="c1"># pylint: disable=consider-using-enumerate</span>
<span class="n">result</span><span class="p">[</span><span class="n">delimiter</span><span class="p">[</span><span class="s1">&#39;keys&#39;</span><span class="p">][</span><span class="n">i</span><span class="p">]]</span> <span class="o">=</span> <span class="n">elements</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
<span class="k">if</span> <span class="n">parse_regex</span><span class="p">:</span>
<span class="k">for</span> <span class="n">result_key</span><span class="p">,</span> <span class="n">regex</span> <span class="ow">in</span> <span class="n">_compiled_parse_regex</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">found</span> <span class="o">=</span> <span class="n">regex</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">raw_result</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">found</span><span class="p">:</span>
<span class="k">return</span> <span class="p">{}</span>
<span class="n">result</span><span class="p">[</span><span class="n">result_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">raw_result</span><span class="p">[</span><span class="n">found</span><span class="o">.</span><span class="n">start</span><span class="p">()</span> <span class="p">:</span> <span class="n">found</span><span class="o">.</span><span class="n">end</span><span class="p">()]</span>
<span class="k">return</span> <span class="n">result</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,272 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.core &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.core</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.core</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;CORE_ (COnnecting REpositories) provides a comprehensive bibliographic</span>
<span class="sd">database of the worlds scholarly literature, collecting and indexing</span>
<span class="sd">research from repositories and journals.</span>
<span class="sd">.. _CORE: https://core.ac.uk/about</span>
<span class="sd">.. note::</span>
<span class="sd"> The CORE engine requires an :py:obj:`API key &lt;api_key&gt;`.</span>
<span class="sd">.. _core engine config:</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">The engine has the following additional settings:</span>
<span class="sd">- :py:obj:`api_key`</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: core.ac.uk</span>
<span class="sd"> api_key: &quot;...&quot;</span>
<span class="sd"> inactive: false</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s2">&quot;https://core.ac.uk&quot;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s2">&quot;Q22661180&quot;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s2">&quot;https://api.core.ac.uk/docs/v3&quot;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s2">&quot;JSON&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">api_key</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;For an API key register at https://core.ac.uk/services/api and insert</span>
<span class="sd">the API key in the engine :ref:`core engine config`.&quot;&quot;&quot;</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;science&quot;</span><span class="p">,</span> <span class="s2">&quot;scientific publications&quot;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">nb_per_page</span> <span class="o">=</span> <span class="mi">10</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;https://api.core.ac.uk/v3/search/works/&quot;</span>
<div class="viewcode-block" id="setup">
<a class="viewcode-back" href="../../../dev/engines/online/core.html#searx.engines.core.setup">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Initialization of the CORE_ engine, checks whether the :py:obj:`api_key`</span>
<span class="sd"> is set, otherwise the engine is inactive.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">key</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;api_key&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">key</span> <span class="ow">and</span> <span class="n">key</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">&quot;unset&quot;</span><span class="p">,</span> <span class="s2">&quot;unknown&quot;</span><span class="p">,</span> <span class="s2">&quot;...&quot;</span><span class="p">):</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;CORE&#39;s API key is not set or invalid.&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">False</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">&quot;OnlineParams&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="c1"># API v3 uses different parameters</span>
<span class="n">search_params</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;q&quot;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s2">&quot;offset&quot;</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;pageno&quot;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">nb_per_page</span><span class="p">,</span>
<span class="s2">&quot;limit&quot;</span><span class="p">:</span> <span class="n">nb_per_page</span><span class="p">,</span>
<span class="s2">&quot;sort&quot;</span><span class="p">:</span> <span class="s2">&quot;relevance&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s2">&quot;?&quot;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">search_params</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;headers&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;Authorization&quot;</span><span class="p">:</span> <span class="sa">f</span><span class="s2">&quot;Bearer </span><span class="si">{</span><span class="n">api_key</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">}</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="c1"># pylint: disable=too-many-branches</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;results&quot;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="c1"># Get title</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;title&quot;</span><span class="p">):</span>
<span class="k">continue</span>
<span class="c1"># Get URL - try different options</span>
<span class="n">url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="c1"># Try DOI first</span>
<span class="n">doi</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;doi&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">doi</span><span class="p">:</span>
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;https://doi.org/</span><span class="si">{</span><span class="n">doi</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">if</span> <span class="n">url</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;doi&quot;</span><span class="p">):</span>
<span class="c1"># use the DOI reference</span>
<span class="n">url</span> <span class="o">=</span> <span class="s2">&quot;https://doi.org/&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;doi&quot;</span><span class="p">])</span>
<span class="k">elif</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;id&quot;</span><span class="p">):</span>
<span class="n">url</span> <span class="o">=</span> <span class="s2">&quot;https://core.ac.uk/works/&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;id&quot;</span><span class="p">])</span>
<span class="k">elif</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;downloadUrl&quot;</span><span class="p">):</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">&quot;downloadUrl&quot;</span><span class="p">]</span>
<span class="k">elif</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;sourceFulltextUrls&quot;</span><span class="p">):</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">&quot;sourceFulltextUrls&quot;</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">continue</span>
<span class="c1"># Published date</span>
<span class="n">published_date</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">raw_date</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;publishedDate&quot;</span><span class="p">)</span> <span class="ow">or</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;depositedDate&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">raw_date</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">published_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromisoformat</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;publishedDate&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;Z&quot;</span><span class="p">,</span> <span class="s2">&quot;+00:00&quot;</span><span class="p">))</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">AttributeError</span><span class="p">):</span>
<span class="k">pass</span>
<span class="c1"># Handle journals</span>
<span class="n">journals</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;journals&quot;</span><span class="p">):</span>
<span class="n">journals</span> <span class="o">=</span> <span class="p">[</span><span class="n">j</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;title&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="n">result</span><span class="p">[</span><span class="s2">&quot;journals&quot;</span><span class="p">]</span> <span class="k">if</span> <span class="n">j</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;title&quot;</span><span class="p">)]</span>
<span class="c1"># Handle publisher</span>
<span class="n">publisher</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;publisher&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">&quot;&#39;&quot;</span><span class="p">)</span>
<span class="c1"># Handle authors</span>
<span class="n">authors</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;authors&quot;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">i</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;name&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">name</span><span class="p">:</span>
<span class="n">authors</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Paper</span><span class="p">(</span>
<span class="n">title</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;title&quot;</span><span class="p">),</span>
<span class="n">url</span><span class="o">=</span><span class="n">url</span><span class="p">,</span>
<span class="n">content</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;fullText&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">&quot;&quot;</span><span class="p">,</span>
<span class="n">tags</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;fieldOfStudy&quot;</span><span class="p">,</span> <span class="p">[]),</span>
<span class="n">publishedDate</span><span class="o">=</span><span class="n">published_date</span><span class="p">,</span>
<span class="nb">type</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;documentType&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">&quot;&quot;</span><span class="p">,</span>
<span class="n">authors</span><span class="o">=</span><span class="n">authors</span><span class="p">,</span>
<span class="n">editor</span><span class="o">=</span><span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;contributors&quot;</span><span class="p">,</span> <span class="p">[])),</span>
<span class="n">publisher</span><span class="o">=</span><span class="n">publisher</span><span class="p">,</span>
<span class="n">journal</span><span class="o">=</span><span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">journals</span><span class="p">),</span>
<span class="n">doi</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;doi&quot;</span><span class="p">),</span>
<span class="n">pdf_url</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;downloadUrl&quot;</span><span class="p">,</span> <span class="p">{})</span> <span class="ow">or</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;sourceFulltextUrls&quot;</span><span class="p">,</span> <span class="p">{}),</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">res</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,353 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.dailymotion &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.dailymotion</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.dailymotion</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">Dailymotion (Videos)</span>
<span class="sd">~~~~~~~~~~~~~~~~~~~~</span>
<span class="sd">.. _REST GET: https://developers.dailymotion.com/tools/</span>
<span class="sd">.. _Global API Parameters: https://developers.dailymotion.com/api/#global-parameters</span>
<span class="sd">.. _Video filters API: https://developers.dailymotion.com/api/#video-filters</span>
<span class="sd">.. _Fields selection: https://developers.dailymotion.com/api/#fields-selection</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span><span class="p">,</span> <span class="n">raise_for_httperror</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">html_to_text</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineAPIException</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">region_tag</span><span class="p">,</span> <span class="n">language_tag</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="c1"># about</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.dailymotion.com&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q769222&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.dailymotion.com/developer&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;JSON&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;videos&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">number_of_results</span> <span class="o">=</span> <span class="mi">10</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">time_delta_dict</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;day&quot;</span><span class="p">:</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">),</span>
<span class="s2">&quot;week&quot;</span><span class="p">:</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">7</span><span class="p">),</span>
<span class="s2">&quot;month&quot;</span><span class="p">:</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">31</span><span class="p">),</span>
<span class="s2">&quot;year&quot;</span><span class="p">:</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">365</span><span class="p">),</span>
<span class="p">}</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">safesearch_params</span> <span class="o">=</span> <span class="p">{</span>
<span class="mi">2</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;is_created_for_kids&#39;</span><span class="p">:</span> <span class="s1">&#39;true&#39;</span><span class="p">},</span>
<span class="mi">1</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;is_created_for_kids&#39;</span><span class="p">:</span> <span class="s1">&#39;true&#39;</span><span class="p">},</span>
<span class="mi">0</span><span class="p">:</span> <span class="p">{},</span>
<span class="p">}</span>
<span class="sd">&quot;&quot;&quot;True if this video is &quot;Created for Kids&quot; / intends to target an audience</span>
<span class="sd">under the age of 16 (``is_created_for_kids`` in `Video filters API`_ )</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">family_filter_map</span> <span class="o">=</span> <span class="p">{</span>
<span class="mi">2</span><span class="p">:</span> <span class="s1">&#39;true&#39;</span><span class="p">,</span>
<span class="mi">1</span><span class="p">:</span> <span class="s1">&#39;true&#39;</span><span class="p">,</span>
<span class="mi">0</span><span class="p">:</span> <span class="s1">&#39;false&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="sd">&quot;&quot;&quot;By default, the family filter is turned on. Setting this parameter to</span>
<span class="sd">``false`` will stop filtering-out explicit content from searches and global</span>
<span class="sd">contexts (``family_filter`` in `Global API Parameters`_ ).</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">result_fields</span> <span class="o">=</span> <span class="p">[</span>
<span class="s1">&#39;allow_embed&#39;</span><span class="p">,</span>
<span class="s1">&#39;description&#39;</span><span class="p">,</span>
<span class="s1">&#39;title&#39;</span><span class="p">,</span>
<span class="s1">&#39;created_time&#39;</span><span class="p">,</span>
<span class="s1">&#39;duration&#39;</span><span class="p">,</span>
<span class="s1">&#39;url&#39;</span><span class="p">,</span>
<span class="s1">&#39;thumbnail_360_url&#39;</span><span class="p">,</span>
<span class="s1">&#39;id&#39;</span><span class="p">,</span>
<span class="p">]</span>
<span class="sd">&quot;&quot;&quot;`Fields selection`_, by default, a few fields are returned. To request more</span>
<span class="sd">specific fields, the ``fields`` parameter is used with the list of fields</span>
<span class="sd">SearXNG needs in the response to build a video result list.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">search_url</span> <span class="o">=</span> <span class="s1">&#39;https://api.dailymotion.com/videos?&#39;</span>
<span class="sd">&quot;&quot;&quot;URL to retrieve a list of videos.</span>
<span class="sd">- `REST GET`_</span>
<span class="sd">- `Global API Parameters`_</span>
<span class="sd">- `Video filters API`_</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">iframe_src</span> <span class="o">=</span> <span class="s2">&quot;https://www.dailymotion.com/embed/video/</span><span class="si">{video_id}</span><span class="s2">&quot;</span>
<span class="sd">&quot;&quot;&quot;URL template to embed video in SearXNG&#39;s result list.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">query</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="n">eng_region</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="s1">&#39;en_US&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="s1">&#39;en&#39;</span><span class="p">)</span>
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;search&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;family_filter&#39;</span><span class="p">:</span> <span class="n">family_filter_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">],</span> <span class="s1">&#39;false&#39;</span><span class="p">),</span>
<span class="s1">&#39;thumbnail_ratio&#39;</span><span class="p">:</span> <span class="s1">&#39;original&#39;</span><span class="p">,</span> <span class="c1"># original|widescreen|square</span>
<span class="c1"># https://developers.dailymotion.com/api/#video-filters</span>
<span class="s1">&#39;languages&#39;</span><span class="p">:</span> <span class="n">eng_lang</span><span class="p">,</span>
<span class="s1">&#39;page&#39;</span><span class="p">:</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">],</span>
<span class="s1">&#39;password_protected&#39;</span><span class="p">:</span> <span class="s1">&#39;false&#39;</span><span class="p">,</span>
<span class="s1">&#39;private&#39;</span><span class="p">:</span> <span class="s1">&#39;false&#39;</span><span class="p">,</span>
<span class="s1">&#39;sort&#39;</span><span class="p">:</span> <span class="s1">&#39;relevance&#39;</span><span class="p">,</span>
<span class="s1">&#39;limit&#39;</span><span class="p">:</span> <span class="n">number_of_results</span><span class="p">,</span>
<span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="s1">&#39;,&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result_fields</span><span class="p">),</span>
<span class="p">}</span>
<span class="n">args</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">safesearch_params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">],</span> <span class="p">{}))</span>
<span class="c1"># Don&#39;t add localization and country arguments if the user does select a</span>
<span class="c1"># language (:de, :en, ..)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">))</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="c1"># https://developers.dailymotion.com/api/#global-parameters</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;localization&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_region</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;country&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_region</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>
<span class="c1"># Insufficient rights for the `ams_country&#39; parameter of route `GET /videos&#39;</span>
<span class="c1"># &#39;ams_country&#39;: eng_region.split(&#39;_&#39;)[1],</span>
<span class="n">time_delta</span> <span class="o">=</span> <span class="n">time_delta_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;time_range&quot;</span><span class="p">])</span>
<span class="k">if</span> <span class="n">time_delta</span><span class="p">:</span>
<span class="n">created_after</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">time_delta</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;created_after&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="n">created_after</span><span class="p">)</span>
<span class="n">query_str</span> <span class="o">=</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span> <span class="o">+</span> <span class="n">query_str</span>
<span class="k">return</span> <span class="n">params</span>
<span class="c1"># get response from search-request</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">search_res</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="c1"># check for an API error</span>
<span class="k">if</span> <span class="s1">&#39;error&#39;</span> <span class="ow">in</span> <span class="n">search_res</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">SearxEngineAPIException</span><span class="p">(</span><span class="n">search_res</span><span class="p">[</span><span class="s1">&#39;error&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;message&#39;</span><span class="p">))</span>
<span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<span class="c1"># parse results</span>
<span class="k">for</span> <span class="n">res</span> <span class="ow">in</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;list&#39;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">res</span><span class="p">[</span><span class="s1">&#39;title&#39;</span><span class="p">]</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">res</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">html_to_text</span><span class="p">(</span><span class="n">res</span><span class="p">[</span><span class="s1">&#39;description&#39;</span><span class="p">])</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">300</span><span class="p">:</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">content</span><span class="p">[:</span><span class="mi">300</span><span class="p">]</span> <span class="o">+</span> <span class="s1">&#39;...&#39;</span>
<span class="n">publishedDate</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">res</span><span class="p">[</span><span class="s1">&#39;created_time&#39;</span><span class="p">],</span> <span class="kc">None</span><span class="p">)</span>
<span class="n">length</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">gmtime</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;duration&#39;</span><span class="p">))</span>
<span class="k">if</span> <span class="n">length</span><span class="o">.</span><span class="n">tm_hour</span><span class="p">:</span>
<span class="n">length</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%H:%M:%S&quot;</span><span class="p">,</span> <span class="n">length</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">length</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%M:%S&quot;</span><span class="p">,</span> <span class="n">length</span><span class="p">)</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">res</span><span class="p">[</span><span class="s1">&#39;thumbnail_360_url&#39;</span><span class="p">]</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">thumbnail</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;http://&quot;</span><span class="p">,</span> <span class="s2">&quot;https://&quot;</span><span class="p">)</span>
<span class="n">item</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;template&#39;</span><span class="p">:</span> <span class="s1">&#39;videos.html&#39;</span><span class="p">,</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
<span class="s1">&#39;publishedDate&#39;</span><span class="p">:</span> <span class="n">publishedDate</span><span class="p">,</span>
<span class="s1">&#39;length&#39;</span><span class="p">:</span> <span class="n">length</span><span class="p">,</span>
<span class="s1">&#39;thumbnail&#39;</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># HINT: no mater what the value is, without API token videos can&#39;t shown</span>
<span class="c1"># embedded</span>
<span class="k">if</span> <span class="n">res</span><span class="p">[</span><span class="s1">&#39;allow_embed&#39;</span><span class="p">]:</span>
<span class="n">item</span><span class="p">[</span><span class="s1">&#39;iframe_src&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">iframe_src</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">video_id</span><span class="o">=</span><span class="n">res</span><span class="p">[</span><span class="s1">&#39;id&#39;</span><span class="p">])</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="c1"># return results</span>
<span class="k">return</span> <span class="n">results</span>
<div class="viewcode-block" id="fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/online/dailymotion.html#searx.engines.dailymotion.fetch_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Fetch locales &amp; languages from dailymotion.</span>
<span class="sd"> Locales fetched from `api/locales &lt;https://api.dailymotion.com/locales&gt;`_.</span>
<span class="sd"> There are duplications in the locale codes returned from Dailymotion which</span>
<span class="sd"> can be ignored::</span>
<span class="sd"> en_EN --&gt; en_GB, en_US</span>
<span class="sd"> ar_AA --&gt; ar_EG, ar_AE, ar_SA</span>
<span class="sd"> The language list `api/languages &lt;https://api.dailymotion.com/languages&gt;`_</span>
<span class="sd"> contains over 7000 *languages* codes (see PR1071_). We use only those</span>
<span class="sd"> language codes that are used in the locales.</span>
<span class="sd"> .. _PR1071: https://github.com/searxng/searxng/pull/1071</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">&#39;https://api.dailymotion.com/locales&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: response from dailymotion/locales is not OK.&quot;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()[</span><span class="s1">&#39;list&#39;</span><span class="p">]:</span> <span class="c1"># type: ignore</span>
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">&#39;locale&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">eng_tag</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">&#39;en_EN&#39;</span><span class="p">,</span> <span class="s1">&#39;ar_AA&#39;</span><span class="p">):</span>
<span class="k">continue</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">))</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: item unknown --&gt; </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">item</span><span class="p">)</span>
<span class="k">continue</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
<span class="n">locale_lang_list</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="o">.</span><span class="n">values</span><span class="p">()]</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">&#39;https://api.dailymotion.com/languages&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: response from dailymotion/languages is not OK.&quot;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()[</span><span class="s1">&#39;list&#39;</span><span class="p">]:</span> <span class="c1"># type: ignore</span>
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">&#39;code&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">eng_tag</span> <span class="ow">in</span> <span class="n">locale_lang_list</span><span class="p">:</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">))</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,233 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.demo_offline &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.demo_offline</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.demo_offline</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Within this module we implement a *demo offline engine*. Do not look to</span>
<span class="sd">close to the implementation, its just a simple example.</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">To get in use of this *demo* engine add the following entry to your engines list</span>
<span class="sd">in ``settings.yml``:</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: my offline engine</span>
<span class="sd"> engine: demo_offline</span>
<span class="sd"> shortcut: demo</span>
<span class="sd"> disabled: false</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineCache</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">RequestParams</span>
<span class="n">engine_type</span> <span class="o">=</span> <span class="s2">&quot;offline&quot;</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;general&quot;</span><span class="p">]</span>
<span class="n">disabled</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">timeout</span> <span class="o">=</span> <span class="mf">2.0</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s2">&quot;JSON&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># if there is a need for globals, use a leading underline</span>
<span class="n">_my_offline_engine</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="n">CACHE</span><span class="p">:</span> <span class="n">EngineCache</span>
<span class="sd">&quot;&quot;&quot;Persistent (SQLite) key/value cache that deletes its values after ``expire``</span>
<span class="sd">seconds.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="setup">
<a class="viewcode-back" href="../../../dev/engines/demo/demo_offline.html#searx.engines.demo_offline.setup">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Dynamic setup of the engine settings.</span>
<span class="sd"> The origin of this demo engine is a simple json string which is loaded in</span>
<span class="sd"> this example while the engine is initialized.</span>
<span class="sd"> For more details see :py:obj:`searx.enginelib.Engine.setup`.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">global</span> <span class="n">_my_offline_engine</span><span class="p">,</span> <span class="n">CACHE</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="n">CACHE</span> <span class="o">=</span> <span class="n">EngineCache</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">[</span><span class="s2">&quot;name&quot;</span><span class="p">])</span>
<span class="n">_my_offline_engine</span> <span class="o">=</span> <span class="p">(</span>
<span class="s1">&#39;[ {&quot;value&quot;: &quot;</span><span class="si">%s</span><span class="s1">&quot;}&#39;</span>
<span class="s1">&#39;, {&quot;value&quot;:&quot;first item&quot;}&#39;</span>
<span class="s1">&#39;, {&quot;value&quot;:&quot;second item&quot;}&#39;</span>
<span class="s1">&#39;, {&quot;value&quot;:&quot;third item&quot;}&#39;</span>
<span class="s1">&#39;]&#39;</span> <span class="o">%</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;name&#39;</span><span class="p">)</span>
<span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span></div>
<div class="viewcode-block" id="init">
<a class="viewcode-back" href="../../../dev/engines/demo/demo_offline.html#searx.engines.demo_offline.init">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Initialization of the engine.</span>
<span class="sd"> For more details see :py:obj:`searx.enginelib.Engine.init`.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="kc">True</span></div>
<div class="viewcode-block" id="search">
<a class="viewcode-back" href="../../../dev/engines/demo/demo_offline.html#searx.engines.demo_offline.search">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">search</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">&quot;RequestParams&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Query (offline) engine and return results. Assemble the list of results</span>
<span class="sd"> from your local engine.</span>
<span class="sd"> In this demo engine we ignore the &#39;query&#39; term, usual you would pass the</span>
<span class="sd"> &#39;query&#39; term to your local engine to filter out the results.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">count</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">CACHE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;count&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
<span class="n">data_rows</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">_my_offline_engine</span><span class="p">)</span>
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">data_rows</span><span class="p">:</span>
<span class="n">count</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="n">kvmap</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;language&#39;</span><span class="p">:</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span>
<span class="s1">&#39;value&#39;</span><span class="p">:</span> <span class="n">row</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">),</span>
<span class="p">}</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">KeyValue</span><span class="p">(</span>
<span class="n">caption</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;Demo Offline Engine Result #</span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
<span class="n">key_title</span><span class="o">=</span><span class="s2">&quot;Name&quot;</span><span class="p">,</span>
<span class="n">value_title</span><span class="o">=</span><span class="s2">&quot;Value&quot;</span><span class="p">,</span>
<span class="n">kvmap</span><span class="o">=</span><span class="n">kvmap</span><span class="p">,</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span><span class="n">number_of_results</span><span class="o">=</span><span class="n">count</span><span class="p">))</span>
<span class="c1"># cache counter value for 20sec</span>
<span class="n">CACHE</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s2">&quot;count&quot;</span><span class="p">,</span> <span class="n">count</span><span class="p">,</span> <span class="n">expire</span><span class="o">=</span><span class="mi">20</span><span class="p">)</span>
<span class="k">return</span> <span class="n">res</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,242 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.demo_online &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.demo_online</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.demo_online</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Within this module we implement a *demo online engine*. Do not look to</span>
<span class="sd">close to the implementation, its just a simple example which queries `The Art</span>
<span class="sd">Institute of Chicago &lt;https://www.artic.edu&gt;`_</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">To get in use of this *demo* engine add the following entry to your engines</span>
<span class="sd">list in ``settings.yml``:</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: my online engine</span>
<span class="sd"> engine: demo_online</span>
<span class="sd"> shortcut: demo</span>
<span class="sd"> disabled: false</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
<span class="n">engine_type</span> <span class="o">=</span> <span class="s2">&quot;online&quot;</span>
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;general&quot;</span><span class="p">]</span>
<span class="n">disabled</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">timeout</span> <span class="o">=</span> <span class="mf">2.0</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;images&quot;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">page_size</span> <span class="o">=</span> <span class="mi">20</span>
<span class="n">search_api</span> <span class="o">=</span> <span class="s2">&quot;https://api.artic.edu/api/v1/artworks/search&quot;</span>
<span class="n">image_api</span> <span class="o">=</span> <span class="s2">&quot;https://www.artic.edu/iiif/2/&quot;</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s2">&quot;https://www.artic.edu&quot;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s2">&quot;Q239303&quot;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s2">&quot;http://api.artic.edu/docs/&quot;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s2">&quot;JSON&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># if there is a need for globals, use a leading underline</span>
<span class="n">_my_online_engine</span> <span class="o">=</span> <span class="kc">None</span>
<div class="viewcode-block" id="setup">
<a class="viewcode-back" href="../../../dev/engines/demo/demo_online.html#searx.engines.demo_online.setup">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="s2">&quot;OnlineParams&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Dynamic setup of the engine settings.</span>
<span class="sd"> For more details see :py:obj:`searx.enginelib.Engine.setup`.&quot;&quot;&quot;</span>
<span class="k">global</span> <span class="n">_my_online_engine</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="n">_my_online_engine</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;name&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span></div>
<div class="viewcode-block" id="init">
<a class="viewcode-back" href="../../../dev/engines/demo/demo_online.html#searx.engines.demo_online.init">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Initialization of the engine.</span>
<span class="sd"> For more details see :py:obj:`searx.enginelib.Engine.init`.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="kc">True</span></div>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/demo/demo_online.html#searx.engines.demo_online.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">&quot;OnlineParams&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Build up the ``params`` for the online request. In this example we build a</span>
<span class="sd"> URL to fetch images from `artic.edu &lt;https://artic.edu&gt;`__.&quot;&quot;&quot;</span>
<span class="n">args</span> <span class="o">=</span> <span class="n">urlencode</span><span class="p">(</span>
<span class="p">{</span>
<span class="s2">&quot;q&quot;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s2">&quot;page&quot;</span><span class="p">:</span> <span class="n">params</span><span class="p">[</span><span class="s2">&quot;pageno&quot;</span><span class="p">],</span>
<span class="s2">&quot;fields&quot;</span><span class="p">:</span> <span class="s2">&quot;id,title,artist_display,medium_display,image_id,date_display,dimensions,artist_titles&quot;</span><span class="p">,</span>
<span class="s2">&quot;limit&quot;</span><span class="p">:</span> <span class="n">page_size</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">search_api</span><span class="si">}</span><span class="s2">?</span><span class="si">{</span><span class="n">args</span><span class="si">}</span><span class="s2">&quot;</span></div>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/demo/demo_online.html#searx.engines.demo_online.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Parse out the result items from the response. In this example we parse the</span>
<span class="sd"> response from `api.artic.edu &lt;https://artic.edu&gt;`__ and filter out all</span>
<span class="sd"> images.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span>
<span class="n">answer</span><span class="o">=</span><span class="s2">&quot;this is a dummy answer ..&quot;</span><span class="p">,</span>
<span class="n">url</span><span class="o">=</span><span class="s2">&quot;https://example.org&quot;</span><span class="p">,</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">&quot;data&quot;</span><span class="p">]:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">result</span><span class="p">[</span><span class="s2">&quot;image_id&quot;</span><span class="p">]:</span>
<span class="k">continue</span>
<span class="n">kwargs</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;url&quot;</span><span class="p">:</span> <span class="s2">&quot;https://artic.edu/artworks/</span><span class="si">%(id)s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">result</span><span class="p">,</span>
<span class="s2">&quot;title&quot;</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s2">&quot;title&quot;</span><span class="p">]</span> <span class="o">+</span> <span class="s2">&quot; (</span><span class="si">%(date_display)s</span><span class="s2">) // </span><span class="si">%(artist_display)s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">result</span><span class="p">,</span>
<span class="s2">&quot;content&quot;</span><span class="p">:</span> <span class="s2">&quot;</span><span class="si">%(medium_display)s</span><span class="s2"> // </span><span class="si">%(dimensions)s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">result</span><span class="p">,</span>
<span class="s2">&quot;author&quot;</span><span class="p">:</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;artist_titles&quot;</span><span class="p">]),</span>
<span class="s2">&quot;img_src&quot;</span><span class="p">:</span> <span class="n">image_api</span> <span class="o">+</span> <span class="s2">&quot;/</span><span class="si">%(image_id)s</span><span class="s2">/full/843,/0/default.jpg&quot;</span> <span class="o">%</span> <span class="n">result</span><span class="p">,</span>
<span class="s2">&quot;template&quot;</span><span class="p">:</span> <span class="s2">&quot;images.html&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">))</span>
<span class="k">return</span> <span class="n">res</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,606 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.duckduckgo &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.duckduckgo</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.duckduckgo</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">DuckDuckGo WEB</span>
<span class="sd">~~~~~~~~~~~~~~</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">quote_plus</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">lxml.html</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">locales</span><span class="p">,</span>
<span class="n">external_bang</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">eval_xpath</span><span class="p">,</span>
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
<span class="n">extr</span><span class="p">,</span>
<span class="n">extract_text</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineCache</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineCaptchaException</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://lite.duckduckgo.com/lite/&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q12805&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;HTML&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
<span class="sd">&quot;&quot;&quot;DuckDuckGo-Lite tries to guess user&#39;s preferred language from the HTTP</span>
<span class="sd">``Accept-Language``. Optional the user can select a region filter (but not a</span>
<span class="sd">language).</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;general&#39;</span><span class="p">,</span> <span class="s1">&#39;web&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span> <span class="c1"># user can&#39;t select but the results are filtered</span>
<span class="n">url</span> <span class="o">=</span> <span class="s2">&quot;https://html.duckduckgo.com/html/&quot;</span>
<span class="n">time_range_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;day&#39;</span><span class="p">:</span> <span class="s1">&#39;d&#39;</span><span class="p">,</span> <span class="s1">&#39;week&#39;</span><span class="p">:</span> <span class="s1">&#39;w&#39;</span><span class="p">,</span> <span class="s1">&#39;month&#39;</span><span class="p">:</span> <span class="s1">&#39;m&#39;</span><span class="p">,</span> <span class="s1">&#39;year&#39;</span><span class="p">:</span> <span class="s1">&#39;y&#39;</span><span class="p">}</span>
<span class="n">form_data</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;v&#39;</span><span class="p">:</span> <span class="s1">&#39;l&#39;</span><span class="p">,</span> <span class="s1">&#39;api&#39;</span><span class="p">:</span> <span class="s1">&#39;d.js&#39;</span><span class="p">,</span> <span class="s1">&#39;o&#39;</span><span class="p">:</span> <span class="s1">&#39;json&#39;</span><span class="p">}</span>
<span class="n">_CACHE</span><span class="p">:</span> <span class="n">EngineCache</span> <span class="o">=</span> <span class="kc">None</span> <span class="c1"># type: ignore</span>
<span class="sd">&quot;&quot;&quot;Persistent (SQLite) key/value cache that deletes its values after ``expire``</span>
<span class="sd">seconds.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_cache</span><span class="p">():</span>
<span class="k">global</span> <span class="n">_CACHE</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="k">if</span> <span class="n">_CACHE</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">_CACHE</span> <span class="o">=</span> <span class="n">EngineCache</span><span class="p">(</span><span class="s2">&quot;duckduckgo&quot;</span><span class="p">)</span> <span class="c1"># type:ignore</span>
<span class="k">return</span> <span class="n">_CACHE</span>
<div class="viewcode-block" id="get_vqd">
<a class="viewcode-back" href="../../../dev/engines/online/duckduckgo.html#searx.engines.duckduckgo.get_vqd">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_vqd</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">region</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">force_request</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns the ``vqd`` that fits to the *query*.</span>
<span class="sd"> :param query: The query term</span>
<span class="sd"> :param region: DDG&#39;s region code</span>
<span class="sd"> :param force_request: force a request to get a vqd value from DDG</span>
<span class="sd"> TL;DR; the ``vqd`` value is needed to pass DDG&#39;s bot protection and is used</span>
<span class="sd"> by all request to DDG:</span>
<span class="sd"> - DuckDuckGo Lite: ``https://lite.duckduckgo.com/lite`` (POST form data)</span>
<span class="sd"> - DuckDuckGo Web: ``https://links.duckduckgo.com/d.js?q=...&amp;vqd=...``</span>
<span class="sd"> - DuckDuckGo Images: ``https://duckduckgo.com/i.js??q=...&amp;vqd=...``</span>
<span class="sd"> - DuckDuckGo Videos: ``https://duckduckgo.com/v.js??q=...&amp;vqd=...``</span>
<span class="sd"> - DuckDuckGo News: ``https://duckduckgo.com/news.js??q=...&amp;vqd=...``</span>
<span class="sd"> DDG&#39;s bot detection is sensitive to the ``vqd`` value. For some search terms</span>
<span class="sd"> (such as extremely long search terms that are often sent by bots), no ``vqd``</span>
<span class="sd"> value can be determined.</span>
<span class="sd"> If SearXNG cannot determine a ``vqd`` value, then no request should go out</span>
<span class="sd"> to DDG.</span>
<span class="sd"> .. attention::</span>
<span class="sd"> A request with a wrong ``vqd`` value leads to DDG temporarily putting</span>
<span class="sd"> SearXNG&#39;s IP on a block list.</span>
<span class="sd"> Requests from IPs in this block list run into timeouts. Not sure, but it</span>
<span class="sd"> seems the block list is a sliding window: to get my IP rid from the bot list</span>
<span class="sd"> I had to cool down my IP for 1h (send no requests from that IP to DDG).</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">cache</span> <span class="o">=</span> <span class="n">get_cache</span><span class="p">()</span>
<span class="n">key</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">secret_hash</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s2">//</span><span class="si">{</span><span class="n">region</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">)</span>
<span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">force_request</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;vqd: re-use cached value: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
<span class="k">return</span> <span class="n">value</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;vqd: request value from from duckduckgo.com&quot;</span><span class="p">)</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;https://duckduckgo.com/?q=</span><span class="si">{</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">query</span><span class="p">)</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span> <span class="c1"># type: ignore</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">extr</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="s1">&#39;vqd=&quot;&#39;</span><span class="p">,</span> <span class="s1">&#39;&quot;&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="k">if</span> <span class="n">value</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;vqd value from duckduckgo.com request: &#39;</span><span class="si">%s</span><span class="s2">&#39;&quot;</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;vqd: can&#39;t parse value from ddg response (return empty string)&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="s2">&quot;&quot;</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;vqd: got HTTP </span><span class="si">%s</span><span class="s2"> from duckduckgo.com&quot;</span><span class="p">,</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span><span class="p">)</span>
<span class="k">if</span> <span class="n">value</span><span class="p">:</span>
<span class="n">cache</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">value</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;none vqd value from duckduckgo.com: HTTP </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span><span class="p">)</span>
<span class="k">return</span> <span class="n">value</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">set_vqd</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">region</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="n">cache</span> <span class="o">=</span> <span class="n">get_cache</span><span class="p">()</span>
<span class="n">key</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">secret_hash</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s2">//</span><span class="si">{</span><span class="n">region</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">cache</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">value</span><span class="p">,</span> <span class="n">expire</span><span class="o">=</span><span class="mi">3600</span><span class="p">)</span>
<div class="viewcode-block" id="get_ddg_lang">
<a class="viewcode-back" href="../../../dev/engines/online/duckduckgo.html#searx.engines.duckduckgo.get_ddg_lang">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_ddg_lang</span><span class="p">(</span><span class="n">eng_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s1">&#39;en_US&#39;</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get DuckDuckGo&#39;s language identifier from SearXNG&#39;s locale.</span>
<span class="sd"> DuckDuckGo defines its languages by region codes (see</span>
<span class="sd"> :py:obj:`fetch_traits`).</span>
<span class="sd"> To get region and language of a DDG service use:</span>
<span class="sd"> .. code: python</span>
<span class="sd"> eng_region = traits.get_region(params[&#39;searxng_locale&#39;], traits.all_locale)</span>
<span class="sd"> eng_lang = get_ddg_lang(traits, params[&#39;searxng_locale&#39;])</span>
<span class="sd"> It might confuse, but the ``l`` value of the cookie is what SearXNG calls</span>
<span class="sd"> the *region*:</span>
<span class="sd"> .. code:: python</span>
<span class="sd"> # !ddi paris :es-AR --&gt; {&#39;ad&#39;: &#39;es_AR&#39;, &#39;ah&#39;: &#39;ar-es&#39;, &#39;l&#39;: &#39;ar-es&#39;}</span>
<span class="sd"> params[&#39;cookies&#39;][&#39;ad&#39;] = eng_lang</span>
<span class="sd"> params[&#39;cookies&#39;][&#39;ah&#39;] = eng_region</span>
<span class="sd"> params[&#39;cookies&#39;][&#39;l&#39;] = eng_region</span>
<span class="sd"> .. hint::</span>
<span class="sd"> `DDG-lite &lt;https://lite.duckduckgo.com/lite&gt;`__ and the *no Javascript*</span>
<span class="sd"> page https://html.duckduckgo.com/html do not offer a language selection</span>
<span class="sd"> to the user, only a region can be selected by the user (``eng_region``</span>
<span class="sd"> from the example above). DDG-lite and *no Javascript* store the selected</span>
<span class="sd"> region in a cookie::</span>
<span class="sd"> params[&#39;cookies&#39;][&#39;kl&#39;] = eng_region # &#39;ar-es&#39;</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;lang_region&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span> <span class="c1"># type: ignore</span>
<span class="n">sxng_locale</span><span class="p">,</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">default</span><span class="p">)</span>
<span class="p">)</span></div>
<span class="n">ddg_reg_map</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;tw-tzh&#39;</span><span class="p">:</span> <span class="s1">&#39;zh_TW&#39;</span><span class="p">,</span>
<span class="s1">&#39;hk-tzh&#39;</span><span class="p">:</span> <span class="s1">&#39;zh_HK&#39;</span><span class="p">,</span>
<span class="s1">&#39;ct-ca&#39;</span><span class="p">:</span> <span class="s1">&#39;skip&#39;</span><span class="p">,</span> <span class="c1"># ct-ca and es-ca both map to ca_ES</span>
<span class="s1">&#39;es-ca&#39;</span><span class="p">:</span> <span class="s1">&#39;ca_ES&#39;</span><span class="p">,</span>
<span class="s1">&#39;id-en&#39;</span><span class="p">:</span> <span class="s1">&#39;id_ID&#39;</span><span class="p">,</span>
<span class="s1">&#39;no-no&#39;</span><span class="p">:</span> <span class="s1">&#39;nb_NO&#39;</span><span class="p">,</span>
<span class="s1">&#39;jp-jp&#39;</span><span class="p">:</span> <span class="s1">&#39;ja_JP&#39;</span><span class="p">,</span>
<span class="s1">&#39;kr-kr&#39;</span><span class="p">:</span> <span class="s1">&#39;ko_KR&#39;</span><span class="p">,</span>
<span class="s1">&#39;xa-ar&#39;</span><span class="p">:</span> <span class="s1">&#39;ar_SA&#39;</span><span class="p">,</span>
<span class="s1">&#39;sl-sl&#39;</span><span class="p">:</span> <span class="s1">&#39;sl_SI&#39;</span><span class="p">,</span>
<span class="s1">&#39;th-en&#39;</span><span class="p">:</span> <span class="s1">&#39;th_TH&#39;</span><span class="p">,</span>
<span class="s1">&#39;vn-en&#39;</span><span class="p">:</span> <span class="s1">&#39;vi_VN&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">ddg_lang_map</span> <span class="o">=</span> <span class="p">{</span>
<span class="c1"># use ar --&gt; ar_EG (Egypt&#39;s arabic)</span>
<span class="s2">&quot;ar_DZ&quot;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="s2">&quot;ar_JO&quot;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="s2">&quot;ar_SA&quot;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="c1"># use bn --&gt; bn_BD</span>
<span class="s1">&#39;bn_IN&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="c1"># use de --&gt; de_DE</span>
<span class="s1">&#39;de_CH&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="c1"># use en --&gt; en_US,</span>
<span class="s1">&#39;en_AU&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="s1">&#39;en_CA&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="s1">&#39;en_GB&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="c1"># Esperanto</span>
<span class="s1">&#39;eo_XX&#39;</span><span class="p">:</span> <span class="s1">&#39;eo&#39;</span><span class="p">,</span>
<span class="c1"># use es --&gt; es_ES,</span>
<span class="s1">&#39;es_AR&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="s1">&#39;es_CL&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="s1">&#39;es_CO&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="s1">&#39;es_CR&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="s1">&#39;es_EC&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="s1">&#39;es_MX&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="s1">&#39;es_PE&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="s1">&#39;es_UY&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="s1">&#39;es_VE&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="c1"># use fr --&gt; rf_FR</span>
<span class="s1">&#39;fr_CA&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="s1">&#39;fr_CH&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="s1">&#39;fr_BE&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="c1"># use nl --&gt; nl_NL</span>
<span class="s1">&#39;nl_BE&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="c1"># use pt --&gt; pt_PT</span>
<span class="s1">&#39;pt_BR&#39;</span><span class="p">:</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">,</span>
<span class="c1"># skip these languages</span>
<span class="s1">&#39;od_IN&#39;</span><span class="p">:</span> <span class="s1">&#39;skip&#39;</span><span class="p">,</span>
<span class="s1">&#39;io_XX&#39;</span><span class="p">:</span> <span class="s1">&#39;skip&#39;</span><span class="p">,</span>
<span class="s1">&#39;tokipona_XX&#39;</span><span class="p">:</span> <span class="s1">&#39;skip&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="k">def</span><span class="w"> </span><span class="nf">quote_ddg_bangs</span><span class="p">(</span><span class="n">query</span><span class="p">):</span>
<span class="c1"># quote ddg bangs</span>
<span class="n">query_parts</span> <span class="o">=</span> <span class="p">[]</span>
<span class="c1"># for val in re.split(r&#39;(\s+)&#39;, query):</span>
<span class="k">for</span> <span class="n">val</span> <span class="ow">in</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;(\s+)&#39;</span><span class="p">,</span> <span class="n">query</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">val</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="n">val</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;!&#39;</span><span class="p">)</span> <span class="ow">and</span> <span class="n">external_bang</span><span class="o">.</span><span class="n">get_node</span><span class="p">(</span><span class="n">external_bang</span><span class="o">.</span><span class="n">EXTERNAL_BANGS</span><span class="p">,</span> <span class="n">val</span><span class="p">[</span><span class="mi">1</span><span class="p">:]):</span>
<span class="n">val</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;&#39;</span><span class="si">{</span><span class="n">val</span><span class="si">}</span><span class="s2">&#39;&quot;</span>
<span class="n">query_parts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">val</span><span class="p">)</span>
<span class="k">return</span> <span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">query_parts</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">quote_ddg_bangs</span><span class="p">(</span><span class="n">query</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">query</span><span class="p">)</span> <span class="o">&gt;=</span> <span class="mi">500</span><span class="p">:</span>
<span class="c1"># DDG does not accept queries with more than 499 chars</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">return</span>
<span class="n">eng_region</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="c1"># Note: The API is reverse-engineered from DuckDuckGo&#39;s HTML webpage</span>
<span class="c1"># (https://html.duckduckgo.com/html/) and may be subject to additional bot detection mechanisms</span>
<span class="c1"># and breaking changes in the future.</span>
<span class="c1">#</span>
<span class="c1"># The params[&#39;data&#39;] dictionary can have the following key parameters, in this order:</span>
<span class="c1"># - q (str): Search query string</span>
<span class="c1"># - b (str): Beginning parameter - empty string for first page requests</span>
<span class="c1"># - s (int): Search offset for pagination</span>
<span class="c1"># - nextParams (str): Continuation parameters from previous page response, typically empty</span>
<span class="c1"># - v (str): Typically &#39;l&#39; for subsequent pages</span>
<span class="c1"># - o (str): Output format, typically &#39;json&#39;</span>
<span class="c1"># - dc (int): Display count - value equal to offset (s) + 1</span>
<span class="c1"># - api (str): API endpoint identifier, typically &#39;d.js&#39;</span>
<span class="c1"># - vqd (str): Validation query digest</span>
<span class="c1"># - kl (str): Keyboard language/region code (e.g., &#39;en-us&#39;)</span>
<span class="c1"># - df (str): Time filter, maps to values like &#39;d&#39; (day), &#39;w&#39; (week), &#39;m&#39; (month), &#39;y&#39; (year)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;q&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">query</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;b&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">elif</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">&gt;=</span> <span class="mi">2</span><span class="p">:</span>
<span class="n">offset</span> <span class="o">=</span> <span class="mi">10</span> <span class="o">+</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">2</span><span class="p">)</span> <span class="o">*</span> <span class="mi">15</span> <span class="c1"># Page 2 = 10, Page 3+ = 10 + n*15</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;s&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">offset</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;nextParams&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">form_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;nextParams&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;v&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">form_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;v&#39;</span><span class="p">,</span> <span class="s1">&#39;l&#39;</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;o&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">form_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;o&#39;</span><span class="p">,</span> <span class="s1">&#39;json&#39;</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;dc&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">offset</span> <span class="o">+</span> <span class="mi">1</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;api&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">form_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;api&#39;</span><span class="p">,</span> <span class="s1">&#39;d.js&#39;</span><span class="p">)</span>
<span class="c1"># vqd is required to request other pages after the first one</span>
<span class="n">vqd</span> <span class="o">=</span> <span class="n">get_vqd</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">eng_region</span><span class="p">,</span> <span class="n">force_request</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="k">if</span> <span class="n">vqd</span><span class="p">:</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;vqd&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">vqd</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># Don&#39;t try to call follow up pages without a vqd value.</span>
<span class="c1"># DDG recognizes this as a request from a bot. This lowers the</span>
<span class="c1"># reputation of the SearXNG IP and DDG starts to activate CAPTCHAs.</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">return</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;zh&quot;</span><span class="p">):</span>
<span class="c1"># Some locales (at least China) do not have a &quot;next page&quot; button and DDG</span>
<span class="c1"># will return a HTTP/2 403 Forbidden for a request of such a page.</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">return</span>
<span class="c1"># Put empty kl in form data if language/region set to all</span>
<span class="k">if</span> <span class="n">eng_region</span> <span class="o">==</span> <span class="s2">&quot;wt-wt&quot;</span><span class="p">:</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;kl&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;kl&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_region</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;df&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;df&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]]</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">][</span><span class="s1">&#39;df&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]]</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">][</span><span class="s1">&#39;kl&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_region</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">url</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;method&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;POST&#39;</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Content-Type&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;application/x-www-form-urlencoded&#39;</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Referer&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">url</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Sec-Fetch-Dest&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;document&quot;</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Sec-Fetch-Mode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;navigate&quot;</span> <span class="c1"># at least this one is used by ddg&#39;s bot detection</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Sec-Fetch-Site&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;same-origin&quot;</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Sec-Fetch-User&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;?1&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;param headers: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">])</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;param data: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">])</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;param cookies: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">])</span>
<div class="viewcode-block" id="is_ddg_captcha">
<a class="viewcode-back" href="../../../dev/engines/online/duckduckgo.html#searx.engines.duckduckgo.is_ddg_captcha">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">is_ddg_captcha</span><span class="p">(</span><span class="n">dom</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;In case of CAPTCHA ddg response its own *not a Robot* dialog and is not</span>
<span class="sd"> redirected to a CAPTCHA page.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//form[@id=&#39;challenge-form&#39;]&quot;</span><span class="p">))</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">303</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">doc</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">if</span> <span class="n">is_ddg_captcha</span><span class="p">(</span><span class="n">doc</span><span class="p">):</span>
<span class="c1"># set suspend time to zero is OK --&gt; ddg does not block the IP</span>
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">(</span><span class="n">suspended_time</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">message</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;CAPTCHA (</span><span class="si">{</span><span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;kl&#39;</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span><span class="p">)</span>
<span class="n">form</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">doc</span><span class="p">,</span> <span class="s1">&#39;//input[@name=&quot;vqd&quot;]/..&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">form</span><span class="p">):</span>
<span class="c1"># some locales (at least China) does not have a &quot;next page&quot; button</span>
<span class="n">form</span> <span class="o">=</span> <span class="n">form</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">form_vqd</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">form</span><span class="p">,</span> <span class="s1">&#39;//input[@name=&quot;vqd&quot;]/@value&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">set_vqd</span><span class="p">(</span>
<span class="n">query</span><span class="o">=</span><span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;q&#39;</span><span class="p">],</span>
<span class="n">region</span><span class="o">=</span><span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">][</span><span class="s1">&#39;kl&#39;</span><span class="p">],</span>
<span class="n">value</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">form_vqd</span><span class="p">),</span>
<span class="p">)</span>
<span class="c1"># just select &quot;web-result&quot; and ignore results of class &quot;result--ad result--ad--small&quot;</span>
<span class="k">for</span> <span class="n">div_result</span> <span class="ow">in</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">doc</span><span class="p">,</span> <span class="s1">&#39;//div[@id=&quot;links&quot;]/div[contains(@class, &quot;web-result&quot;)]&#39;</span><span class="p">):</span>
<span class="n">item</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">div_result</span><span class="p">,</span> <span class="s1">&#39;.//h2/a&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">title</span><span class="p">:</span>
<span class="c1"># this is the &quot;No results.&quot; item in the result list</span>
<span class="k">continue</span>
<span class="n">item</span><span class="p">[</span><span class="s2">&quot;title&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">title</span><span class="p">)</span>
<span class="n">item</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">div_result</span><span class="p">,</span> <span class="s1">&#39;.//h2/a/@href&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">item</span><span class="p">[</span><span class="s2">&quot;content&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span>
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">div_result</span><span class="p">,</span> <span class="s1">&#39;.//a[contains(@class, &quot;result__snippet&quot;)]&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">[])</span>
<span class="p">)</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="n">zero_click_info_xpath</span> <span class="o">=</span> <span class="s1">&#39;//div[@id=&quot;zero_click_abstract&quot;]&#39;</span>
<span class="n">zero_click</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">doc</span><span class="p">,</span> <span class="n">zero_click_info_xpath</span><span class="p">))</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="c1"># type: ignore</span>
<span class="k">if</span> <span class="n">zero_click</span> <span class="ow">and</span> <span class="p">(</span>
<span class="s2">&quot;Your IP address is&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">zero_click</span>
<span class="ow">and</span> <span class="s2">&quot;Your user agent:&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">zero_click</span>
<span class="ow">and</span> <span class="s2">&quot;URL Decoded:&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">zero_click</span>
<span class="p">):</span>
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
<span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span>
<span class="n">answer</span><span class="o">=</span><span class="n">zero_click</span><span class="p">,</span>
<span class="n">url</span><span class="o">=</span><span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">doc</span><span class="p">,</span> <span class="s1">&#39;//div[@id=&quot;zero_click_abstract&quot;]/a/@href&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> <span class="c1"># type: ignore</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span>
<div class="viewcode-block" id="fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/online/duckduckgo.html#searx.engines.duckduckgo.fetch_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Fetch languages &amp; regions from DuckDuckGo.</span>
<span class="sd"> SearXNG&#39;s ``all`` locale maps DuckDuckGo&#39;s &quot;Alle regions&quot; (``wt-wt``).</span>
<span class="sd"> DuckDuckGo&#39;s language &quot;Browsers preferred language&quot; (``wt_WT``) makes no</span>
<span class="sd"> sense in a SearXNG request since SearXNG&#39;s ``all`` will not add a</span>
<span class="sd"> ``Accept-Language`` HTTP header. The value in ``engine_traits.all_locale``</span>
<span class="sd"> is ``wt-wt`` (the region).</span>
<span class="sd"> Beside regions DuckDuckGo also defines its languages by region codes. By</span>
<span class="sd"> example these are the english languages in DuckDuckGo:</span>
<span class="sd"> - en_US</span>
<span class="sd"> - en_AU</span>
<span class="sd"> - en_CA</span>
<span class="sd"> - en_GB</span>
<span class="sd"> The function :py:obj:`get_ddg_lang` evaluates DuckDuckGo&#39;s language from</span>
<span class="sd"> SearXNG&#39;s locale.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-many-branches, too-many-statements, disable=import-outside-toplevel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">js_obj_str_to_python</span>
<span class="c1"># fetch regions</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="s1">&#39;wt-wt&#39;</span>
<span class="c1"># updated from u661.js to u.7669f071a13a7daa57cb / should be updated automatically?</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">&#39;https://duckduckgo.com/dist/util/u.7669f071a13a7daa57cb.js&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: response from DuckDuckGo is not OK.&quot;</span><span class="p">)</span>
<span class="n">js_code</span> <span class="o">=</span> <span class="n">extr</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="s1">&#39;regions:&#39;</span><span class="p">,</span> <span class="s1">&#39;,snippetLengths&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">regions</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">js_code</span><span class="p">)</span>
<span class="k">for</span> <span class="n">eng_tag</span><span class="p">,</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">regions</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="n">eng_tag</span> <span class="o">==</span> <span class="s1">&#39;wt-wt&#39;</span><span class="p">:</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="s1">&#39;wt-wt&#39;</span>
<span class="k">continue</span>
<span class="n">region</span> <span class="o">=</span> <span class="n">ddg_reg_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">region</span> <span class="o">==</span> <span class="s1">&#39;skip&#39;</span><span class="p">:</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">region</span><span class="p">:</span>
<span class="n">eng_territory</span><span class="p">,</span> <span class="n">eng_lang</span> <span class="o">=</span> <span class="n">eng_tag</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)</span>
<span class="n">region</span> <span class="o">=</span> <span class="n">eng_lang</span> <span class="o">+</span> <span class="s1">&#39;_&#39;</span> <span class="o">+</span> <span class="n">eng_territory</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">locales</span><span class="o">.</span><span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">region</span><span class="p">))</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: </span><span class="si">%s</span><span class="s2"> (</span><span class="si">%s</span><span class="s2">) -&gt; </span><span class="si">%s</span><span class="s2"> is unknown by babel&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">,</span> <span class="n">region</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
<span class="c1"># fetch languages</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;lang_region&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">js_code</span> <span class="o">=</span> <span class="n">extr</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="s1">&#39;languages:&#39;</span><span class="p">,</span> <span class="s1">&#39;,regions&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">languages</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">js_obj_str_to_python</span><span class="p">(</span><span class="n">js_code</span><span class="p">)</span>
<span class="k">for</span> <span class="n">eng_lang</span><span class="p">,</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">languages</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="n">eng_lang</span> <span class="o">==</span> <span class="s1">&#39;wt_WT&#39;</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">babel_tag</span> <span class="o">=</span> <span class="n">ddg_lang_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_lang</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">)</span>
<span class="k">if</span> <span class="n">babel_tag</span> <span class="o">==</span> <span class="s1">&#39;skip&#39;</span><span class="p">:</span>
<span class="k">continue</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">if</span> <span class="n">babel_tag</span> <span class="o">==</span> <span class="s1">&#39;lang_region&#39;</span><span class="p">:</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">locales</span><span class="o">.</span><span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">eng_lang</span><span class="p">))</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;lang_region&#39;</span><span class="p">][</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_lang</span>
<span class="k">continue</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">locales</span><span class="o">.</span><span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">babel_tag</span><span class="p">))</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: language </span><span class="si">%s</span><span class="s2"> (</span><span class="si">%s</span><span class="s2">) is unknown by babel&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_lang</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_lang</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,371 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.duckduckgo_definitions &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.duckduckgo_definitions</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.duckduckgo_definitions</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">DuckDuckGo Instant Answer API</span>
<span class="sd">~~~~~~~~~~~~~~~~~~~~~~~~~~~~~</span>
<span class="sd">The `DDG-API &lt;https://duckduckgo.com/api&gt;`__ is no longer documented but from</span>
<span class="sd">reverse engineering we can see that some services (e.g. instant answers) still</span>
<span class="sd">in use from the DDG search engine.</span>
<span class="sd">As far we can say the *instant answers* API does not support languages, or at</span>
<span class="sd">least we could not find out how language support should work. It seems that</span>
<span class="sd">most of the features are based on English terms.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span><span class="p">,</span> <span class="n">urlparse</span><span class="p">,</span> <span class="n">urljoin</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.data</span><span class="w"> </span><span class="kn">import</span> <span class="n">WIKIDATA_UNITS</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">html_to_text</span><span class="p">,</span> <span class="n">get_string_replaces_function</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.external_urls</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_external_url</span><span class="p">,</span> <span class="n">get_earth_coordinates_url</span><span class="p">,</span> <span class="n">area_to_osm_zoom</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="c1"># about</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://duckduckgo.com/&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q12805&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://duckduckgo.com/api&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;JSON&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">URL</span> <span class="o">=</span> <span class="s1">&#39;https://api.duckduckgo.com/&#39;</span> <span class="o">+</span> <span class="s1">&#39;?</span><span class="si">{query}</span><span class="s1">&amp;format=json&amp;pretty=0&amp;no_redirect=1&amp;d=1&#39;</span>
<span class="n">WIKIDATA_PREFIX</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;http://www.wikidata.org/entity/&#39;</span><span class="p">,</span> <span class="s1">&#39;https://www.wikidata.org/entity/&#39;</span><span class="p">]</span>
<span class="n">replace_http_by_https</span> <span class="o">=</span> <span class="n">get_string_replaces_function</span><span class="p">({</span><span class="s1">&#39;http:&#39;</span><span class="p">:</span> <span class="s1">&#39;https:&#39;</span><span class="p">})</span>
<div class="viewcode-block" id="is_broken_text">
<a class="viewcode-back" href="../../../dev/engines/online/duckduckgo.html#searx.engines.duckduckgo_definitions.is_broken_text">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">is_broken_text</span><span class="p">(</span><span class="n">text</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;duckduckgo may return something like ``&lt;a href=&quot;xxxx&quot;&gt;http://somewhere Related website&lt;a/&gt;``</span>
<span class="sd"> The href URL is broken, the &quot;Related website&quot; may contains some HTML.</span>
<span class="sd"> The best solution seems to ignore these results.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">text</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;http&#39;</span><span class="p">)</span> <span class="ow">and</span> <span class="s1">&#39; &#39;</span> <span class="ow">in</span> <span class="n">text</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">result_to_text</span><span class="p">(</span><span class="n">text</span><span class="p">,</span> <span class="n">htmlResult</span><span class="p">):</span>
<span class="c1"># TODO : remove result ending with &quot;Meaning&quot; or &quot;Category&quot; # pylint: disable=fixme</span>
<span class="n">result</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">htmlResult</span><span class="p">)</span>
<span class="n">a</span> <span class="o">=</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;//a&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">a</span><span class="p">)</span> <span class="o">&gt;=</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">a</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">text</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_broken_text</span><span class="p">(</span><span class="n">result</span><span class="p">):</span>
<span class="k">return</span> <span class="n">result</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">URL</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">}))</span>
<span class="k">return</span> <span class="n">params</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="c1"># pylint: disable=too-many-locals, too-many-branches, too-many-statements</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">search_res</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="c1"># search_res.get(&#39;Entity&#39;) possible values (not exhaustive) :</span>
<span class="c1"># * continent / country / department / location / waterfall</span>
<span class="c1"># * actor / musician / artist</span>
<span class="c1"># * book / performing art / film / television / media franchise / concert tour / playwright</span>
<span class="c1"># * prepared food</span>
<span class="c1"># * website / software / os / programming language / file format / software engineer</span>
<span class="c1"># * company</span>
<span class="n">content</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="n">heading</span> <span class="o">=</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Heading&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">attributes</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">urls</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">infobox_id</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">relatedTopics</span> <span class="o">=</span> <span class="p">[]</span>
<span class="c1"># add answer if there is one</span>
<span class="n">answer</span> <span class="o">=</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Answer&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">answer</span><span class="p">:</span>
<span class="n">answer_type</span> <span class="o">=</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AnswerType&#39;</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;AnswerType=&quot;</span><span class="si">%s</span><span class="s1">&quot; Answer=&quot;</span><span class="si">%s</span><span class="s1">&quot;&#39;</span><span class="p">,</span> <span class="n">answer_type</span><span class="p">,</span> <span class="n">answer</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">answer</span><span class="p">,</span> <span class="nb">str</span><span class="p">)</span> <span class="ow">and</span> <span class="n">answer_type</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;calc&#39;</span><span class="p">,</span> <span class="s1">&#39;ip&#39;</span><span class="p">]:</span>
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
<span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span>
<span class="n">answer</span><span class="o">=</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">answer</span><span class="p">),</span>
<span class="n">url</span><span class="o">=</span><span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AbstractURL&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">),</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="c1"># add infobox</span>
<span class="k">if</span> <span class="s1">&#39;Definition&#39;</span> <span class="ow">in</span> <span class="n">search_res</span><span class="p">:</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">content</span> <span class="o">+</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Definition&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="s1">&#39;Abstract&#39;</span> <span class="ow">in</span> <span class="n">search_res</span><span class="p">:</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">content</span> <span class="o">+</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Abstract&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="c1"># image</span>
<span class="n">image</span> <span class="o">=</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Image&#39;</span><span class="p">)</span>
<span class="n">image</span> <span class="o">=</span> <span class="kc">None</span> <span class="k">if</span> <span class="n">image</span> <span class="o">==</span> <span class="s1">&#39;&#39;</span> <span class="k">else</span> <span class="n">image</span>
<span class="k">if</span> <span class="n">image</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">image</span><span class="p">)</span><span class="o">.</span><span class="n">netloc</span> <span class="o">==</span> <span class="s1">&#39;&#39;</span><span class="p">:</span>
<span class="n">image</span> <span class="o">=</span> <span class="n">urljoin</span><span class="p">(</span><span class="s1">&#39;https://duckduckgo.com&#39;</span><span class="p">,</span> <span class="n">image</span><span class="p">)</span>
<span class="c1"># urls</span>
<span class="c1"># Official website, Wikipedia page</span>
<span class="k">for</span> <span class="n">ddg_result</span> <span class="ow">in</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Results&#39;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="n">firstURL</span> <span class="o">=</span> <span class="n">ddg_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;FirstURL&#39;</span><span class="p">)</span>
<span class="n">text</span> <span class="o">=</span> <span class="n">ddg_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Text&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">firstURL</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">text</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">text</span><span class="p">,</span> <span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">firstURL</span><span class="p">})</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">heading</span><span class="p">,</span> <span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">firstURL</span><span class="p">})</span>
<span class="c1"># related topics</span>
<span class="k">for</span> <span class="n">ddg_result</span> <span class="ow">in</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;RelatedTopics&#39;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="k">if</span> <span class="s1">&#39;FirstURL&#39;</span> <span class="ow">in</span> <span class="n">ddg_result</span><span class="p">:</span>
<span class="n">firstURL</span> <span class="o">=</span> <span class="n">ddg_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;FirstURL&#39;</span><span class="p">)</span>
<span class="n">text</span> <span class="o">=</span> <span class="n">ddg_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Text&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_broken_text</span><span class="p">(</span><span class="n">text</span><span class="p">):</span>
<span class="n">suggestion</span> <span class="o">=</span> <span class="n">result_to_text</span><span class="p">(</span><span class="n">text</span><span class="p">,</span> <span class="n">ddg_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Result&#39;</span><span class="p">))</span>
<span class="k">if</span> <span class="n">suggestion</span> <span class="o">!=</span> <span class="n">heading</span> <span class="ow">and</span> <span class="n">suggestion</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;suggestion&#39;</span><span class="p">:</span> <span class="n">suggestion</span><span class="p">})</span>
<span class="k">elif</span> <span class="s1">&#39;Topics&#39;</span> <span class="ow">in</span> <span class="n">ddg_result</span><span class="p">:</span>
<span class="n">suggestions</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">relatedTopics</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;name&#39;</span><span class="p">:</span> <span class="n">ddg_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Name&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">),</span> <span class="s1">&#39;suggestions&#39;</span><span class="p">:</span> <span class="n">suggestions</span><span class="p">})</span>
<span class="k">for</span> <span class="n">topic_result</span> <span class="ow">in</span> <span class="n">ddg_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Topics&#39;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="n">suggestion</span> <span class="o">=</span> <span class="n">result_to_text</span><span class="p">(</span><span class="n">topic_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Text&#39;</span><span class="p">),</span> <span class="n">topic_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Result&#39;</span><span class="p">))</span>
<span class="k">if</span> <span class="n">suggestion</span> <span class="o">!=</span> <span class="n">heading</span> <span class="ow">and</span> <span class="n">suggestion</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">suggestions</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)</span>
<span class="c1"># abstract</span>
<span class="n">abstractURL</span> <span class="o">=</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AbstractURL&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">abstractURL</span> <span class="o">!=</span> <span class="s1">&#39;&#39;</span><span class="p">:</span>
<span class="c1"># add as result ? problem always in english</span>
<span class="n">infobox_id</span> <span class="o">=</span> <span class="n">abstractURL</span>
<span class="n">urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;AbstractSource&#39;</span><span class="p">),</span> <span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">abstractURL</span><span class="p">,</span> <span class="s1">&#39;official&#39;</span><span class="p">:</span> <span class="kc">True</span><span class="p">})</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">abstractURL</span><span class="p">,</span> <span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">heading</span><span class="p">})</span>
<span class="c1"># definition</span>
<span class="n">definitionURL</span> <span class="o">=</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;DefinitionURL&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">definitionURL</span> <span class="o">!=</span> <span class="s1">&#39;&#39;</span><span class="p">:</span>
<span class="c1"># add as result ? as answer ? problem always in english</span>
<span class="n">infobox_id</span> <span class="o">=</span> <span class="n">definitionURL</span>
<span class="n">urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;DefinitionSource&#39;</span><span class="p">),</span> <span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">definitionURL</span><span class="p">})</span>
<span class="c1"># to merge with wikidata&#39;s infobox</span>
<span class="k">if</span> <span class="n">infobox_id</span><span class="p">:</span>
<span class="n">infobox_id</span> <span class="o">=</span> <span class="n">replace_http_by_https</span><span class="p">(</span><span class="n">infobox_id</span><span class="p">)</span>
<span class="c1"># attributes</span>
<span class="c1"># some will be converted to urls</span>
<span class="k">if</span> <span class="s1">&#39;Infobox&#39;</span> <span class="ow">in</span> <span class="n">search_res</span><span class="p">:</span>
<span class="n">infobox</span> <span class="o">=</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Infobox&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="s1">&#39;content&#39;</span> <span class="ow">in</span> <span class="n">infobox</span><span class="p">:</span>
<span class="n">osm_zoom</span> <span class="o">=</span> <span class="mi">17</span>
<span class="n">coordinates</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">for</span> <span class="n">info</span> <span class="ow">in</span> <span class="n">infobox</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;content&#39;</span><span class="p">):</span>
<span class="n">data_type</span> <span class="o">=</span> <span class="n">info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;data_type&#39;</span><span class="p">)</span>
<span class="n">data_label</span> <span class="o">=</span> <span class="n">info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;label&#39;</span><span class="p">)</span>
<span class="n">data_value</span> <span class="o">=</span> <span class="n">info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;value&#39;</span><span class="p">)</span>
<span class="c1"># Workaround: ddg may return a double quote</span>
<span class="k">if</span> <span class="n">data_value</span> <span class="o">==</span> <span class="s1">&#39;&quot;&quot;&#39;</span><span class="p">:</span>
<span class="k">continue</span>
<span class="c1"># Is it an external URL ?</span>
<span class="c1"># * imdb_id / facebook_profile / youtube_channel / youtube_video / twitter_profile</span>
<span class="c1"># * instagram_profile / rotten_tomatoes / spotify_artist_id / itunes_artist_id / soundcloud_id</span>
<span class="c1"># * netflix_id</span>
<span class="n">external_url</span> <span class="o">=</span> <span class="n">get_external_url</span><span class="p">(</span><span class="n">data_type</span><span class="p">,</span> <span class="n">data_value</span><span class="p">)</span>
<span class="k">if</span> <span class="n">external_url</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">data_label</span><span class="p">,</span> <span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">external_url</span><span class="p">})</span>
<span class="k">elif</span> <span class="n">data_type</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;instance&#39;</span><span class="p">,</span> <span class="s1">&#39;wiki_maps_trigger&#39;</span><span class="p">,</span> <span class="s1">&#39;google_play_artist_id&#39;</span><span class="p">]:</span>
<span class="c1"># ignore instance: Wikidata value from &quot;Instance Of&quot; (Qxxxx)</span>
<span class="c1"># ignore wiki_maps_trigger: reference to a javascript</span>
<span class="c1"># ignore google_play_artist_id: service shutdown</span>
<span class="k">pass</span>
<span class="k">elif</span> <span class="n">data_type</span> <span class="o">==</span> <span class="s1">&#39;string&#39;</span> <span class="ow">and</span> <span class="n">data_label</span> <span class="o">==</span> <span class="s1">&#39;Website&#39;</span><span class="p">:</span>
<span class="c1"># There is already an URL for the website</span>
<span class="k">pass</span>
<span class="k">elif</span> <span class="n">data_type</span> <span class="o">==</span> <span class="s1">&#39;area&#39;</span><span class="p">:</span>
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;label&#39;</span><span class="p">:</span> <span class="n">data_label</span><span class="p">,</span> <span class="s1">&#39;value&#39;</span><span class="p">:</span> <span class="n">area_to_str</span><span class="p">(</span><span class="n">data_value</span><span class="p">),</span> <span class="s1">&#39;entity&#39;</span><span class="p">:</span> <span class="s1">&#39;P2046&#39;</span><span class="p">})</span>
<span class="n">osm_zoom</span> <span class="o">=</span> <span class="n">area_to_osm_zoom</span><span class="p">(</span><span class="n">data_value</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;amount&#39;</span><span class="p">))</span>
<span class="k">elif</span> <span class="n">data_type</span> <span class="o">==</span> <span class="s1">&#39;coordinates&#39;</span><span class="p">:</span>
<span class="k">if</span> <span class="n">data_value</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;globe&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s1">&#39;http://www.wikidata.org/entity/Q2&#39;</span><span class="p">:</span>
<span class="c1"># coordinate on Earth</span>
<span class="c1"># get the zoom information from the area</span>
<span class="n">coordinates</span> <span class="o">=</span> <span class="n">info</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># coordinate NOT on Earth</span>
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;label&#39;</span><span class="p">:</span> <span class="n">data_label</span><span class="p">,</span> <span class="s1">&#39;value&#39;</span><span class="p">:</span> <span class="n">data_value</span><span class="p">,</span> <span class="s1">&#39;entity&#39;</span><span class="p">:</span> <span class="s1">&#39;P625&#39;</span><span class="p">})</span>
<span class="k">elif</span> <span class="n">data_type</span> <span class="o">==</span> <span class="s1">&#39;string&#39;</span><span class="p">:</span>
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;label&#39;</span><span class="p">:</span> <span class="n">data_label</span><span class="p">,</span> <span class="s1">&#39;value&#39;</span><span class="p">:</span> <span class="n">data_value</span><span class="p">})</span>
<span class="k">if</span> <span class="n">coordinates</span><span class="p">:</span>
<span class="n">data_label</span> <span class="o">=</span> <span class="n">coordinates</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;label&#39;</span><span class="p">)</span>
<span class="n">data_value</span> <span class="o">=</span> <span class="n">coordinates</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;value&#39;</span><span class="p">)</span>
<span class="n">latitude</span> <span class="o">=</span> <span class="n">data_value</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;latitude&#39;</span><span class="p">)</span>
<span class="n">longitude</span> <span class="o">=</span> <span class="n">data_value</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;longitude&#39;</span><span class="p">)</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">get_earth_coordinates_url</span><span class="p">(</span><span class="n">latitude</span><span class="p">,</span> <span class="n">longitude</span><span class="p">,</span> <span class="n">osm_zoom</span><span class="p">)</span>
<span class="n">urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="s1">&#39;OpenStreetMap&#39;</span><span class="p">,</span> <span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">&#39;entity&#39;</span><span class="p">:</span> <span class="s1">&#39;P625&#39;</span><span class="p">})</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">heading</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="c1"># TODO get infobox.meta.value where .label=&#39;article_title&#39; # pylint: disable=fixme</span>
<span class="k">if</span> <span class="n">image</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">attributes</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">urls</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">relatedTopics</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">urls</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;url&#39;</span><span class="p">],</span> <span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">heading</span><span class="p">,</span> <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">})</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;infobox&#39;</span><span class="p">:</span> <span class="n">heading</span><span class="p">,</span>
<span class="s1">&#39;id&#39;</span><span class="p">:</span> <span class="n">infobox_id</span><span class="p">,</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
<span class="s1">&#39;img_src&#39;</span><span class="p">:</span> <span class="n">image</span><span class="p">,</span>
<span class="s1">&#39;attributes&#39;</span><span class="p">:</span> <span class="n">attributes</span><span class="p">,</span>
<span class="s1">&#39;urls&#39;</span><span class="p">:</span> <span class="n">urls</span><span class="p">,</span>
<span class="s1">&#39;relatedTopics&#39;</span><span class="p">:</span> <span class="n">relatedTopics</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">unit_to_str</span><span class="p">(</span><span class="n">unit</span><span class="p">):</span>
<span class="k">for</span> <span class="n">prefix</span> <span class="ow">in</span> <span class="n">WIKIDATA_PREFIX</span><span class="p">:</span>
<span class="k">if</span> <span class="n">unit</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">prefix</span><span class="p">):</span>
<span class="n">wikidata_entity</span> <span class="o">=</span> <span class="n">unit</span><span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="n">prefix</span><span class="p">)</span> <span class="p">:]</span>
<span class="n">real_unit</span> <span class="o">=</span> <span class="n">WIKIDATA_UNITS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">wikidata_entity</span><span class="p">)</span>
<span class="k">if</span> <span class="n">real_unit</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">unit</span>
<span class="k">return</span> <span class="n">real_unit</span><span class="p">[</span><span class="s1">&#39;symbol&#39;</span><span class="p">]</span>
<span class="k">return</span> <span class="n">unit</span>
<div class="viewcode-block" id="area_to_str">
<a class="viewcode-back" href="../../../dev/engines/online/duckduckgo.html#searx.engines.duckduckgo_definitions.area_to_str">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">area_to_str</span><span class="p">(</span><span class="n">area</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;parse ``{&#39;unit&#39;: &#39;https://www.wikidata.org/entity/Q712226&#39;, &#39;amount&#39;: &#39;+20.99&#39;}``&quot;&quot;&quot;</span>
<span class="n">unit</span> <span class="o">=</span> <span class="n">unit_to_str</span><span class="p">(</span><span class="n">area</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;unit&#39;</span><span class="p">))</span>
<span class="k">if</span> <span class="n">unit</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">amount</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">area</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;amount&#39;</span><span class="p">))</span>
<span class="k">return</span> <span class="s1">&#39;</span><span class="si">{}</span><span class="s1"> </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">amount</span><span class="p">,</span> <span class="n">unit</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="k">pass</span>
<span class="k">return</span> <span class="s1">&#39;</span><span class="si">{}</span><span class="s1"> </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">area</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;amount&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">),</span> <span class="n">area</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;unit&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">))</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,360 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.github_code &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.github_code</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.github_code</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-lat_er</span>
<span class="sd">&quot;&quot;&quot;GitHub code search with `search syntax`_ as described in `Constructing a</span>
<span class="sd">search query`_ in the documentation of GitHub&#39;s REST API.</span>
<span class="sd">.. _search syntax:</span>
<span class="sd"> https://docs.github.com/en/search-github/getting-started-with-searching-on-github/understanding-the-search-syntax</span>
<span class="sd">.. _Constructing a search query:</span>
<span class="sd"> https://docs.github.com/en/rest/search/search?apiVersion=2022-11-28#constructing-a-search-query</span>
<span class="sd">.. _Github REST API for code search:</span>
<span class="sd"> https://docs.github.com/en/rest/search/search?apiVersion=2022-11-28#search-code</span>
<span class="sd">.. _Github REST API auth for code search:</span>
<span class="sd"> https://docs.github.com/en/rest/search/search?apiVersion=2022-11-28#search-code--fine-grained-access-tokens</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">The engine has the following mandatory setting:</span>
<span class="sd">- :py:obj:`ghc_auth`</span>
<span class="sd"> Change the authentication method used when using the API, defaults to none.</span>
<span class="sd">Optional settings are:</span>
<span class="sd">- :py:obj:`ghc_highlight_matching_lines`</span>
<span class="sd"> Control the highlighting of the matched text (turns off/on).</span>
<span class="sd">- :py:obj:`ghc_strip_new_lines`</span>
<span class="sd"> Strip new lines at the start or end of each code fragment.</span>
<span class="sd">- :py:obj:`ghc_strip_whitespace`</span>
<span class="sd"> Strip any whitespace at the start or end of each code fragment.</span>
<span class="sd">- :py:obj:`ghc_insert_block_separator`</span>
<span class="sd"> Add a `...` between each code fragment before merging them.</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: github code</span>
<span class="sd"> engine: github_code</span>
<span class="sd"> shortcut: ghc</span>
<span class="sd"> ghc_auth:</span>
<span class="sd"> type: &quot;none&quot;</span>
<span class="sd"> - name: github code</span>
<span class="sd"> engine: github_code</span>
<span class="sd"> shortcut: ghc</span>
<span class="sd"> ghc_auth:</span>
<span class="sd"> type: &quot;personal_access_token&quot;</span>
<span class="sd"> token: &quot;&lt;token&gt;&quot;</span>
<span class="sd"> ghc_highlight_matching_lines: true</span>
<span class="sd"> ghc_strip_whitespace: true</span>
<span class="sd"> ghc_strip_new_lines: true</span>
<span class="sd"> - name: github code</span>
<span class="sd"> engine: github_code</span>
<span class="sd"> shortcut: ghc</span>
<span class="sd"> ghc_auth:</span>
<span class="sd"> type: &quot;bearer&quot;</span>
<span class="sd"> token: &quot;&lt;token&gt;&quot;</span>
<span class="sd">Implementation</span>
<span class="sd">===============</span>
<span class="sd">GitHub does not return the code line indices alongside the code fragment in the</span>
<span class="sd">search API. Since these are not super important for the user experience all the</span>
<span class="sd">code lines are just relabeled (starting from 1) and appended (a disjoint set of</span>
<span class="sd">code blocks in a single file might be returned from the API).</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">raise_for_httperror</span>
<span class="c1"># about</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://github.com/&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q364&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://docs.github.com/en/rest/search/search?apiVersion=2022-11-28#search-code&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;JSON&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;code&#39;</span><span class="p">]</span>
<span class="n">search_url</span> <span class="o">=</span> <span class="s1">&#39;https://api.github.com/search/code?sort=indexed&amp;</span><span class="si">{query}</span><span class="s1">&amp;</span><span class="si">{page}</span><span class="s1">&#39;</span>
<span class="c1"># https://docs.github.com/en/rest/search/search?apiVersion=2022-11-28#text-match-metadata</span>
<span class="n">accept_header</span> <span class="o">=</span> <span class="s1">&#39;application/vnd.github.text-match+json&#39;</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">ghc_auth</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;none&quot;</span><span class="p">,</span>
<span class="s2">&quot;token&quot;</span><span class="p">:</span> <span class="s2">&quot;&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="sd">&quot;&quot;&quot;Change the method of authenticating to the github API.</span>
<span class="sd">``type`` needs to be one of ``none``, ``personal_access_token``, or ``bearer``.</span>
<span class="sd">When type is not `none` a token is expected to be passed as well in</span>
<span class="sd">``auth.token``.</span>
<span class="sd">If there is any privacy concerns about generating a token, one can use the API</span>
<span class="sd">without authentication. The calls will be heavily rate limited, this is what the</span>
<span class="sd">API returns on such calls::</span>
<span class="sd"> API rate limit exceeded for &lt;redacted ip&gt;.</span>
<span class="sd"> (But here&#39;s the good news: Authenticated requests get a higher rate limit)</span>
<span class="sd">The personal access token or a bearer for an org or a group can be generated [in</span>
<span class="sd">the `GitHub settings`_.</span>
<span class="sd">.. _GitHub settings:</span>
<span class="sd"> https://docs.github.com/en/rest/search/search?apiVersion=2022-11-28#search-code--fine-grained-access-tokens</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">ghc_highlight_matching_lines</span> <span class="o">=</span> <span class="kc">True</span>
<span class="sd">&quot;&quot;&quot;Highlight the matching code lines.&quot;&quot;&quot;</span>
<span class="n">ghc_strip_new_lines</span> <span class="o">=</span> <span class="kc">True</span>
<span class="sd">&quot;&quot;&quot;Strip leading and trailing newlines for each returned fragment.</span>
<span class="sd">Single file might return multiple code fragments.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">ghc_strip_whitespace</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&quot;&quot;&quot;Strip all leading and trailing whitespace for each returned fragment.</span>
<span class="sd">Single file might return multiple code fragments. Enabling this might break</span>
<span class="sd">code indentation.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">ghc_api_version</span> <span class="o">=</span> <span class="s2">&quot;2022-11-28&quot;</span>
<span class="sd">&quot;&quot;&quot;The version of the GitHub REST API.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">ghc_insert_block_separator</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&quot;&quot;&quot;Each file possibly consists of more than one code block that matches the</span>
<span class="sd">search, if this is set to true, the blocks will be separated with ``...`` line.</span>
<span class="sd">This might break the lexer and thus result in the lack of code highlighting.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">}),</span> <span class="n">page</span><span class="o">=</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;page&#39;</span><span class="p">:</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]}))</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Accept&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">accept_header</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;X-GitHub-Api-Version&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ghc_api_version</span>
<span class="k">if</span> <span class="n">ghc_auth</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;none&quot;</span><span class="p">:</span>
<span class="c1"># Without the auth header the query fails, so add a dummy instead.</span>
<span class="c1"># Queries without auth are heavily rate limited.</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Authorization&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;placeholder&quot;</span>
<span class="k">if</span> <span class="n">ghc_auth</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;personal_access_token&quot;</span><span class="p">:</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Authorization&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;token </span><span class="si">{</span><span class="n">ghc_auth</span><span class="p">[</span><span class="s1">&#39;token&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">if</span> <span class="n">ghc_auth</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;bearer&quot;</span><span class="p">:</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Authorization&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;Bearer </span><span class="si">{</span><span class="n">ghc_auth</span><span class="p">[</span><span class="s1">&#39;token&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;raise_for_httperror&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
<div class="viewcode-block" id="extract_code">
<a class="viewcode-back" href="../../../dev/engines/online/github_code.html#searx.engines.github_code.extract_code">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">extract_code</span><span class="p">(</span><span class="n">code_matches</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]])</span> <span class="o">-&gt;</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">set</span><span class="p">[</span><span class="nb">int</span><span class="p">]]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Iterate over multiple possible matches, for each extract a code fragment.</span>
<span class="sd"> Github additionally sends context for _word_ highlights; pygments supports</span>
<span class="sd"> highlighting lines, as such we calculate which lines to highlight while</span>
<span class="sd"> traversing the text.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">lines</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">highlighted_lines_index</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">match</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">code_matches</span><span class="p">):</span>
<span class="k">if</span> <span class="n">i</span> <span class="o">&gt;</span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">ghc_insert_block_separator</span><span class="p">:</span>
<span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;...&quot;</span><span class="p">)</span>
<span class="n">buffer</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">highlight_groups</span> <span class="o">=</span> <span class="p">[</span><span class="n">highlight_group</span><span class="p">[</span><span class="s1">&#39;indices&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">highlight_group</span> <span class="ow">in</span> <span class="n">match</span><span class="p">[</span><span class="s1">&#39;matches&#39;</span><span class="p">]]</span>
<span class="n">code</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">match</span><span class="p">[</span><span class="s1">&#39;fragment&#39;</span><span class="p">]</span>
<span class="n">original_code_lenght</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">code</span><span class="p">)</span>
<span class="k">if</span> <span class="n">ghc_strip_whitespace</span><span class="p">:</span>
<span class="n">code</span> <span class="o">=</span> <span class="n">code</span><span class="o">.</span><span class="n">lstrip</span><span class="p">()</span>
<span class="k">if</span> <span class="n">ghc_strip_new_lines</span><span class="p">:</span>
<span class="n">code</span> <span class="o">=</span> <span class="n">code</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">offset</span> <span class="o">=</span> <span class="n">original_code_lenght</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">code</span><span class="p">)</span>
<span class="k">if</span> <span class="n">ghc_strip_whitespace</span><span class="p">:</span>
<span class="n">code</span> <span class="o">=</span> <span class="n">code</span><span class="o">.</span><span class="n">rstrip</span><span class="p">()</span>
<span class="k">if</span> <span class="n">ghc_strip_new_lines</span><span class="p">:</span>
<span class="n">code</span> <span class="o">=</span> <span class="n">code</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">letter</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">code</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">highlight_groups</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="c1"># the API ensures these are sorted already, and we have a</span>
<span class="c1"># guaranteed match in the code (all indices are in the range 0</span>
<span class="c1"># and len(fragment)), so only check the first highlight group</span>
<span class="p">[</span><span class="n">after</span><span class="p">,</span> <span class="n">before</span><span class="p">]</span> <span class="o">=</span> <span class="n">highlight_groups</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">if</span> <span class="n">after</span> <span class="o">&lt;=</span> <span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="n">offset</span><span class="p">)</span> <span class="o">&lt;</span> <span class="n">before</span><span class="p">:</span>
<span class="c1"># pygments enumerates lines from 1, highlight the next line</span>
<span class="n">highlighted_lines_index</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">lines</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">highlight_groups</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
<span class="k">if</span> <span class="n">letter</span> <span class="o">==</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">:</span>
<span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">buffer</span><span class="p">))</span>
<span class="n">buffer</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">continue</span>
<span class="n">buffer</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">letter</span><span class="p">)</span>
<span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">buffer</span><span class="p">))</span>
<span class="k">return</span> <span class="n">lines</span><span class="p">,</span> <span class="n">highlighted_lines_index</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="n">SXNG_Response</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">422</span><span class="p">:</span>
<span class="c1"># on a invalid search term the status code 422 &quot;Unprocessable Content&quot;</span>
<span class="c1"># is returned / e.g. search term is &quot;user: foo&quot; instead &quot;user:foo&quot;</span>
<span class="k">return</span> <span class="n">res</span>
<span class="c1"># raise for other errors</span>
<span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;items&#39;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="n">repo</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">&#39;repository&#39;</span><span class="p">]</span> <span class="c1"># pyright: ignore[reportAny]</span>
<span class="n">text_matches</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">&#39;text_matches&#39;</span><span class="p">]</span> <span class="c1"># pyright: ignore[reportAny]</span>
<span class="c1"># ensure picking only the code contents in the blob</span>
<span class="n">code_matches</span> <span class="o">=</span> <span class="p">[</span>
<span class="n">match</span> <span class="k">for</span> <span class="n">match</span> <span class="ow">in</span> <span class="n">text_matches</span> <span class="k">if</span> <span class="n">match</span><span class="p">[</span><span class="s2">&quot;object_type&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;FileContent&quot;</span> <span class="ow">and</span> <span class="n">match</span><span class="p">[</span><span class="s2">&quot;property&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;content&quot;</span>
<span class="p">]</span>
<span class="n">lines</span><span class="p">,</span> <span class="n">highlighted_lines_index</span> <span class="o">=</span> <span class="n">extract_code</span><span class="p">(</span><span class="n">code_matches</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">ghc_highlight_matching_lines</span><span class="p">:</span>
<span class="n">highlighted_lines_index</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Code</span><span class="p">(</span>
<span class="n">url</span><span class="o">=</span><span class="n">item</span><span class="p">[</span><span class="s2">&quot;html_url&quot;</span><span class="p">],</span> <span class="c1"># pyright: ignore[reportAny]</span>
<span class="n">title</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">repo</span><span class="p">[</span><span class="s1">&#39;full_name&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2"> · </span><span class="si">{</span><span class="n">item</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
<span class="n">filename</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">item</span><span class="p">[</span><span class="s1">&#39;path&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
<span class="n">content</span><span class="o">=</span><span class="n">repo</span><span class="p">[</span><span class="s1">&#39;description&#39;</span><span class="p">],</span>
<span class="n">repository</span><span class="o">=</span><span class="n">repo</span><span class="p">[</span><span class="s1">&#39;html_url&#39;</span><span class="p">],</span>
<span class="n">codelines</span><span class="o">=</span><span class="p">[(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span> <span class="k">for</span> <span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">lines</span><span class="p">)],</span>
<span class="n">hl_lines</span><span class="o">=</span><span class="n">highlighted_lines_index</span><span class="p">,</span>
<span class="n">strip_whitespace</span><span class="o">=</span><span class="n">ghc_strip_whitespace</span><span class="p">,</span>
<span class="n">strip_new_lines</span><span class="o">=</span><span class="n">ghc_strip_new_lines</span><span class="p">,</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">res</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,640 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.google &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.google</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.google</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;This is the implementation of the Google WEB engine. Some of this</span>
<span class="sd">implementations (manly the :py:obj:`get_google_info`) are shared by other</span>
<span class="sd">engines:</span>
<span class="sd">- :ref:`google images engine`</span>
<span class="sd">- :ref:`google news engine`</span>
<span class="sd">- :ref:`google videos engine`</span>
<span class="sd">- :ref:`google scholar engine`</span>
<span class="sd">- :ref:`google autocomplete`</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">random</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">string</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.core</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.languages</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">,</span> <span class="n">eval_xpath_getindex</span><span class="p">,</span> <span class="n">gen_gsa_useragent</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span><span class="p">,</span> <span class="n">region_tag</span><span class="p">,</span> <span class="n">get_official_locales</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineCaptchaException</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.google.com&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q9366&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://developers.google.com/custom-search/&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;HTML&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;general&#39;</span><span class="p">,</span> <span class="s1">&#39;web&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">50</span>
<span class="sd">&quot;&quot;&quot;`Google max 50 pages`_</span>
<span class="sd">.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">time_range_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;day&#39;</span><span class="p">:</span> <span class="s1">&#39;d&#39;</span><span class="p">,</span> <span class="s1">&#39;week&#39;</span><span class="p">:</span> <span class="s1">&#39;w&#39;</span><span class="p">,</span> <span class="s1">&#39;month&#39;</span><span class="p">:</span> <span class="s1">&#39;m&#39;</span><span class="p">,</span> <span class="s1">&#39;year&#39;</span><span class="p">:</span> <span class="s1">&#39;y&#39;</span><span class="p">}</span>
<span class="c1"># Filter results. 0: None, 1: Moderate, 2: Strict</span>
<span class="n">filter_mapping</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">&#39;off&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">&#39;medium&#39;</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">&#39;high&#39;</span><span class="p">}</span>
<span class="c1"># specific xpath variables</span>
<span class="c1"># ------------------------</span>
<span class="c1"># Suggestions are links placed in a *card-section*, we extract only the text</span>
<span class="c1"># from the links not the links itself.</span>
<span class="n">suggestion_xpath</span> <span class="o">=</span> <span class="s1">&#39;//div[contains(@class, &quot;ouy7Mc&quot;)]//a&#39;</span>
<span class="n">_arcid_range</span> <span class="o">=</span> <span class="n">string</span><span class="o">.</span><span class="n">ascii_letters</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">digits</span> <span class="o">+</span> <span class="s2">&quot;_-&quot;</span>
<span class="n">_arcid_random</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<div class="viewcode-block" id="ui_async">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google.ui_async">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">ui_async</span><span class="p">(</span><span class="n">start</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Format of the response from UI&#39;s async request.</span>
<span class="sd"> - ``arc_id:&lt;...&gt;,use_ac:true,_fmt:prog``</span>
<span class="sd"> The arc_id is random generated every hour.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">global</span> <span class="n">_arcid_random</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="n">use_ac</span> <span class="o">=</span> <span class="s2">&quot;use_ac:true&quot;</span>
<span class="c1"># _fmt:html returns a HTTP 500 when user search for celebrities like</span>
<span class="c1"># &#39;!google natasha allegri&#39; or &#39;!google chris evans&#39;</span>
<span class="n">_fmt</span> <span class="o">=</span> <span class="s2">&quot;_fmt:prog&quot;</span>
<span class="c1"># create a new random arc_id every hour</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">_arcid_random</span> <span class="ow">or</span> <span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span> <span class="o">-</span> <span class="n">_arcid_random</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="o">&gt;</span> <span class="mi">3600</span><span class="p">:</span>
<span class="n">_arcid_random</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">choices</span><span class="p">(</span><span class="n">_arcid_range</span><span class="p">,</span> <span class="n">k</span><span class="o">=</span><span class="mi">23</span><span class="p">)),</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()))</span>
<span class="n">arc_id</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;arc_id:srp_</span><span class="si">{</span><span class="n">_arcid_random</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="si">}</span><span class="s2">_1</span><span class="si">{</span><span class="n">start</span><span class="si">:</span><span class="s2">02</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">return</span> <span class="s2">&quot;,&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">arc_id</span><span class="p">,</span> <span class="n">use_ac</span><span class="p">,</span> <span class="n">_fmt</span><span class="p">])</span></div>
<div class="viewcode-block" id="get_google_info">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google.get_google_info">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_google_info</span><span class="p">(</span><span class="n">params</span><span class="p">:</span> <span class="s2">&quot;OnlineParams&quot;</span><span class="p">,</span> <span class="n">eng_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Composing various (language) properties for the google engines (:ref:`google</span>
<span class="sd"> API`).</span>
<span class="sd"> This function is called by the various google engines (:ref:`google web</span>
<span class="sd"> engine`, :ref:`google images engine`, :ref:`google news engine` and</span>
<span class="sd"> :ref:`google videos engine`).</span>
<span class="sd"> :param dict param: Request parameters of the engine. At least</span>
<span class="sd"> a ``searxng_locale`` key should be in the dictionary.</span>
<span class="sd"> :param eng_traits: Engine&#39;s traits fetched from google preferences</span>
<span class="sd"> (:py:obj:`searx.enginelib.traits.EngineTraits`)</span>
<span class="sd"> :rtype: dict</span>
<span class="sd"> :returns:</span>
<span class="sd"> Py-Dictionary with the key/value pairs:</span>
<span class="sd"> language:</span>
<span class="sd"> The language code that is used by google (e.g. ``lang_en`` or</span>
<span class="sd"> ``lang_zh-TW``)</span>
<span class="sd"> country:</span>
<span class="sd"> The country code that is used by google (e.g. ``US`` or ``TW``)</span>
<span class="sd"> locale:</span>
<span class="sd"> A instance of :py:obj:`babel.core.Locale` build from the</span>
<span class="sd"> ``searxng_locale`` value.</span>
<span class="sd"> subdomain:</span>
<span class="sd"> Google subdomain :py:obj:`google_domains` that fits to the country</span>
<span class="sd"> code.</span>
<span class="sd"> params:</span>
<span class="sd"> Py-Dictionary with additional request arguments (can be passed to</span>
<span class="sd"> :py:func:`urllib.parse.urlencode`).</span>
<span class="sd"> - ``hl`` parameter: specifies the interface language of user interface.</span>
<span class="sd"> - ``lr`` parameter: restricts search results to documents written in</span>
<span class="sd"> a particular language.</span>
<span class="sd"> - ``cr`` parameter: restricts search results to documents</span>
<span class="sd"> originating in a particular country.</span>
<span class="sd"> - ``ie`` parameter: sets the character encoding scheme that should</span>
<span class="sd"> be used to interpret the query string (&#39;utf8&#39;).</span>
<span class="sd"> - ``oe`` parameter: sets the character encoding scheme that should</span>
<span class="sd"> be used to decode the XML result (&#39;utf8&#39;).</span>
<span class="sd"> headers:</span>
<span class="sd"> Py-Dictionary with additional HTTP headers (can be passed to</span>
<span class="sd"> request&#39;s headers)</span>
<span class="sd"> - ``Accept: &#39;*/*``</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">ret_val</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;language&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s1">&#39;country&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s1">&#39;subdomain&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s1">&#39;params&#39;</span><span class="p">:</span> <span class="p">{},</span>
<span class="s1">&#39;headers&#39;</span><span class="p">:</span> <span class="p">{},</span>
<span class="s1">&#39;cookies&#39;</span><span class="p">:</span> <span class="p">{},</span>
<span class="s1">&#39;locale&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">sxng_locale</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">,</span> <span class="s1">&#39;all&#39;</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;-&#39;</span><span class="p">)</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="n">locale</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="s1">&#39;lang_en&#39;</span><span class="p">)</span>
<span class="n">lang_code</span> <span class="o">=</span> <span class="n">eng_lang</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="c1"># lang_zh-TW --&gt; zh-TW / lang_en --&gt; en</span>
<span class="n">country</span> <span class="o">=</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span>
<span class="c1"># Test zh_hans &amp; zh_hant --&gt; in the topmost links in the result list of list</span>
<span class="c1"># TW and HK you should a find wiktionary.org zh_hant link. In the result</span>
<span class="c1"># list of zh-CN should not be no hant link instead you should find</span>
<span class="c1"># zh.m.wikipedia.org/zh somewhere in the top.</span>
<span class="c1"># &#39;!go 日 :zh-TW&#39; --&gt; https://zh.m.wiktionary.org/zh-hant/%E6%97%A5</span>
<span class="c1"># &#39;!go 日 :zh-CN&#39; --&gt; https://zh.m.wikipedia.org/zh/%E6%97%A5</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s1">&#39;language&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_lang</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s1">&#39;country&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">country</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s1">&#39;locale&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">locale</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s1">&#39;subdomain&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;supported_domains&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">country</span><span class="o">.</span><span class="n">upper</span><span class="p">(),</span> <span class="s1">&#39;www.google.com&#39;</span><span class="p">)</span>
<span class="c1"># hl parameter:</span>
<span class="c1"># The hl parameter specifies the interface language (host language) of</span>
<span class="c1"># your user interface. To improve the performance and the quality of your</span>
<span class="c1"># search results, you are strongly encouraged to set this parameter</span>
<span class="c1"># explicitly.</span>
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#hlsp</span>
<span class="c1"># The Interface Language:</span>
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results_appendices#interfaceLanguages</span>
<span class="c1"># https://github.com/searxng/searxng/issues/2515#issuecomment-1607150817</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;hl&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">lang_code</span><span class="si">}</span><span class="s1">-</span><span class="si">{</span><span class="n">country</span><span class="si">}</span><span class="s1">&#39;</span>
<span class="c1"># lr parameter:</span>
<span class="c1"># The lr (language restrict) parameter restricts search results to</span>
<span class="c1"># documents written in a particular language.</span>
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#lrsp</span>
<span class="c1"># Language Collection Values:</span>
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results_appendices#languageCollections</span>
<span class="c1">#</span>
<span class="c1"># To select &#39;all&#39; languages an empty &#39;lr&#39; value is used.</span>
<span class="c1">#</span>
<span class="c1"># Different to other google services, Google Scholar supports to select more</span>
<span class="c1"># than one language. The languages are separated by a pipe &#39;|&#39; (logical OR).</span>
<span class="c1"># By example: &amp;lr=lang_zh-TW%7Clang_de selects articles written in</span>
<span class="c1"># traditional chinese OR german language.</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_lang</span>
<span class="k">if</span> <span class="n">sxng_locale</span> <span class="o">==</span> <span class="s1">&#39;all&#39;</span><span class="p">:</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="c1"># cr parameter:</span>
<span class="c1"># The cr parameter restricts search results to documents originating in a</span>
<span class="c1"># particular country.</span>
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#crsp</span>
<span class="c1"># specify a region (country) only if a region is given in the selected</span>
<span class="c1"># locale --&gt; https://github.com/searxng/searxng/issues/2672</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;cr&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">sxng_locale</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">))</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;cr&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;country&#39;</span> <span class="o">+</span> <span class="n">country</span>
<span class="c1"># gl parameter: (mandatory by Google News)</span>
<span class="c1"># The gl parameter value is a two-letter country code. For WebSearch</span>
<span class="c1"># results, the gl parameter boosts search results whose country of origin</span>
<span class="c1"># matches the parameter value. See the Country Codes section for a list of</span>
<span class="c1"># valid values.</span>
<span class="c1"># Specifying a gl parameter value in WebSearch requests should improve the</span>
<span class="c1"># relevance of results. This is particularly true for international</span>
<span class="c1"># customers and, even more specifically, for customers in English-speaking</span>
<span class="c1"># countries other than the United States.</span>
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#glsp</span>
<span class="c1"># https://github.com/searxng/searxng/issues/2515#issuecomment-1606294635</span>
<span class="c1"># ret_val[&#39;params&#39;][&#39;gl&#39;] = country</span>
<span class="c1"># ie parameter:</span>
<span class="c1"># The ie parameter sets the character encoding scheme that should be used</span>
<span class="c1"># to interpret the query string. The default ie value is latin1.</span>
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#iesp</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;ie&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;utf8&#39;</span>
<span class="c1"># oe parameter:</span>
<span class="c1"># The oe parameter sets the character encoding scheme that should be used</span>
<span class="c1"># to decode the XML result. The default oe value is latin1.</span>
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#oesp</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;oe&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;utf8&#39;</span>
<span class="c1"># num parameter:</span>
<span class="c1"># The num parameter identifies the number of search results to return.</span>
<span class="c1"># The default num value is 10, and the maximum value is 20. If you request</span>
<span class="c1"># more than 20 results, only 20 results will be returned.</span>
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#numsp</span>
<span class="c1"># HINT: seems to have no effect (tested in google WEB &amp; Images)</span>
<span class="c1"># ret_val[&#39;params&#39;][&#39;num&#39;] = 20</span>
<span class="c1"># HTTP headers</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Accept&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;*/*&#39;</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;User-Agent&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">gen_gsa_useragent</span><span class="p">()</span>
<span class="c1"># Cookies</span>
<span class="c1"># - https://github.com/searxng/searxng/pull/1679#issuecomment-1235432746</span>
<span class="c1"># - https://github.com/searxng/searxng/issues/1555</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">][</span><span class="s1">&#39;CONSENT&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;YES+&quot;</span>
<span class="k">return</span> <span class="n">ret_val</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">url</span><span class="o">.</span><span class="n">host</span> <span class="o">==</span> <span class="s1">&#39;sorry.google.com&#39;</span> <span class="ow">or</span> <span class="n">resp</span><span class="o">.</span><span class="n">url</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;/sorry&#39;</span><span class="p">):</span>
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">()</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">&quot;OnlineParams&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Google search request&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=line-too-long</span>
<span class="n">start</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span>
<span class="n">str_async</span> <span class="o">=</span> <span class="n">ui_async</span><span class="p">(</span><span class="n">start</span><span class="p">)</span>
<span class="n">google_info</span> <span class="o">=</span> <span class="n">get_google_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">traits</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;ARC_ID: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">str_async</span><span class="p">)</span>
<span class="c1"># https://www.google.de/search?q=corona&amp;hl=de&amp;lr=lang_de&amp;start=0&amp;tbs=qdr%3Ad&amp;safe=medium</span>
<span class="n">query_url</span> <span class="o">=</span> <span class="p">(</span>
<span class="s1">&#39;https://&#39;</span>
<span class="o">+</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;subdomain&#39;</span><span class="p">]</span>
<span class="o">+</span> <span class="s1">&#39;/search&#39;</span>
<span class="o">+</span> <span class="s2">&quot;?&quot;</span>
<span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="o">**</span><span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">],</span>
<span class="s1">&#39;filter&#39;</span><span class="p">:</span> <span class="s1">&#39;0&#39;</span><span class="p">,</span>
<span class="s1">&#39;start&#39;</span><span class="p">:</span> <span class="n">start</span><span class="p">,</span>
<span class="c1"># &#39;vet&#39;: &#39;12ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0QxK8CegQIARAC..i&#39;,</span>
<span class="c1"># &#39;ved&#39;: &#39;2ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0Q_skCegQIARAG&#39;,</span>
<span class="c1"># &#39;cs&#39; : 1,</span>
<span class="c1"># &#39;sa&#39;: &#39;N&#39;,</span>
<span class="c1"># &#39;yv&#39;: 3,</span>
<span class="c1"># &#39;prmd&#39;: &#39;vin&#39;,</span>
<span class="c1"># &#39;ei&#39;: &#39;GASaY6TxOcy_xc8PtYeY6AE&#39;,</span>
<span class="c1"># &#39;sa&#39;: &#39;N&#39;,</span>
<span class="c1"># &#39;sstk&#39;: &#39;AcOHfVkD7sWCSAheZi-0tx_09XDO55gTWY0JNq3_V26cNN-c8lfD45aZYPI8s_Bqp8s57AHz5pxchDtAGCA_cikAWSjy9kw3kgg&#39;</span>
<span class="c1"># formally known as use_mobile_ui</span>
<span class="s1">&#39;asearch&#39;</span><span class="p">:</span> <span class="s1">&#39;arc&#39;</span><span class="p">,</span>
<span class="s1">&#39;async&#39;</span><span class="p">:</span> <span class="n">str_async</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">&#39;&amp;&#39;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;tbs&#39;</span><span class="p">:</span> <span class="s1">&#39;qdr:&#39;</span> <span class="o">+</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]]})</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]:</span>
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">&#39;&amp;&#39;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;safe&#39;</span><span class="p">:</span> <span class="n">filter_mapping</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]]})</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_url</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">]</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">])</span></div>
<span class="c1"># =26;[3,&quot;dimg_ZNMiZPCqE4apxc8P3a2tuAQ_137&quot;]a87;data:image/jpeg;base64,/9j/4AAQSkZJRgABA</span>
<span class="c1"># ...6T+9Nl4cnD+gr9OK8I56/tX3l86nWYw//2Q==26;</span>
<span class="n">RE_DATA_IMAGE</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;&quot;(dimg_[^&quot;]*)&quot;[^;]*;(data:image[^;]*;[^;]*);&#39;</span><span class="p">)</span>
<span class="n">RE_DATA_IMAGE_end</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;&quot;(dimg_[^&quot;]*)&quot;[^;]*;(data:image[^;]*;[^;]*)$&#39;</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">parse_data_images</span><span class="p">(</span><span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="n">data_image_map</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">for</span> <span class="n">img_id</span><span class="p">,</span> <span class="n">data_image</span> <span class="ow">in</span> <span class="n">RE_DATA_IMAGE</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">text</span><span class="p">):</span>
<span class="n">end_pos</span> <span class="o">=</span> <span class="n">data_image</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s1">&#39;=&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">end_pos</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">data_image</span> <span class="o">=</span> <span class="n">data_image</span><span class="p">[:</span> <span class="n">end_pos</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span>
<span class="n">data_image_map</span><span class="p">[</span><span class="n">img_id</span><span class="p">]</span> <span class="o">=</span> <span class="n">data_image</span>
<span class="n">last</span> <span class="o">=</span> <span class="n">RE_DATA_IMAGE_end</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
<span class="k">if</span> <span class="n">last</span><span class="p">:</span>
<span class="n">data_image_map</span><span class="p">[</span><span class="n">last</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)]</span> <span class="o">=</span> <span class="n">last</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;data:image objects --&gt; </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="nb">list</span><span class="p">(</span><span class="n">data_image_map</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span>
<span class="k">return</span> <span class="n">data_image_map</span>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get response from google&#39;s search request&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-many-branches, too-many-statements</span>
<span class="n">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<span class="n">data_image_map</span> <span class="o">=</span> <span class="n">parse_data_images</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="c1"># convert the text to dom</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="c1"># parse results</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">&#39;.//div[contains(@class, &quot;MjjYud&quot;)]&#39;</span><span class="p">):</span>
<span class="c1"># pylint: disable=too-many-nested-blocks</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">title_tag</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//div[contains(@role, &quot;link&quot;)]&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">title_tag</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="c1"># this not one of the common google results *section*</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;ignoring item from the result_xpath list: missing title&#39;</span><span class="p">)</span>
<span class="k">continue</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">title_tag</span><span class="p">)</span>
<span class="n">raw_url</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//a/@href&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">raw_url</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;ignoring item from the result_xpath list: missing url of title &quot;</span><span class="si">%s</span><span class="s1">&quot;&#39;</span><span class="p">,</span> <span class="n">title</span><span class="p">)</span>
<span class="k">continue</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">raw_url</span><span class="p">[</span><span class="mi">7</span><span class="p">:]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;&amp;sa=U&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="c1"># remove the google redirector</span>
<span class="n">content_nodes</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//div[contains(@data-sncf, &quot;1&quot;)]&#39;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">content_nodes</span><span class="p">:</span>
<span class="k">for</span> <span class="n">script</span> <span class="ow">in</span> <span class="n">item</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s2">&quot;.//script&quot;</span><span class="p">):</span>
<span class="n">script</span><span class="o">.</span><span class="n">getparent</span><span class="p">()</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">script</span><span class="p">)</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">content_nodes</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">content</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;ignoring item from the result_xpath list: missing content of title &quot;</span><span class="si">%s</span><span class="s1">&quot;&#39;</span><span class="p">,</span> <span class="n">title</span><span class="p">)</span>
<span class="k">continue</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">content_nodes</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;.//img/@src&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">thumbnail</span><span class="p">:</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">thumbnail</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">if</span> <span class="n">thumbnail</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;data:image&#39;</span><span class="p">):</span>
<span class="n">img_id</span> <span class="o">=</span> <span class="n">content_nodes</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;.//img/@id&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">img_id</span><span class="p">:</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">data_image_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">img_id</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span> <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span> <span class="s1">&#39;thumbnail&#39;</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">})</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">e</span><span class="p">,</span> <span class="n">exc_info</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">continue</span>
<span class="c1"># parse suggestion</span>
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">suggestion_xpath</span><span class="p">):</span>
<span class="c1"># append suggestion</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;suggestion&#39;</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)})</span>
<span class="c1"># return results</span>
<span class="k">return</span> <span class="n">results</span></div>
<span class="c1"># get supported languages from their site</span>
<span class="n">skip_countries</span> <span class="o">=</span> <span class="p">[</span>
<span class="c1"># official language of google-country not in google-languages</span>
<span class="s1">&#39;AL&#39;</span><span class="p">,</span> <span class="c1"># Albanien (sq)</span>
<span class="s1">&#39;AZ&#39;</span><span class="p">,</span> <span class="c1"># Aserbaidschan (az)</span>
<span class="s1">&#39;BD&#39;</span><span class="p">,</span> <span class="c1"># Bangladesch (bn)</span>
<span class="s1">&#39;BN&#39;</span><span class="p">,</span> <span class="c1"># Brunei Darussalam (ms)</span>
<span class="s1">&#39;BT&#39;</span><span class="p">,</span> <span class="c1"># Bhutan (dz)</span>
<span class="s1">&#39;ET&#39;</span><span class="p">,</span> <span class="c1"># Äthiopien (am)</span>
<span class="s1">&#39;GE&#39;</span><span class="p">,</span> <span class="c1"># Georgien (ka, os)</span>
<span class="s1">&#39;GL&#39;</span><span class="p">,</span> <span class="c1"># Grönland (kl)</span>
<span class="s1">&#39;KH&#39;</span><span class="p">,</span> <span class="c1"># Kambodscha (km)</span>
<span class="s1">&#39;LA&#39;</span><span class="p">,</span> <span class="c1"># Laos (lo)</span>
<span class="s1">&#39;LK&#39;</span><span class="p">,</span> <span class="c1"># Sri Lanka (si, ta)</span>
<span class="s1">&#39;ME&#39;</span><span class="p">,</span> <span class="c1"># Montenegro (sr)</span>
<span class="s1">&#39;MK&#39;</span><span class="p">,</span> <span class="c1"># Nordmazedonien (mk, sq)</span>
<span class="s1">&#39;MM&#39;</span><span class="p">,</span> <span class="c1"># Myanmar (my)</span>
<span class="s1">&#39;MN&#39;</span><span class="p">,</span> <span class="c1"># Mongolei (mn)</span>
<span class="s1">&#39;MV&#39;</span><span class="p">,</span> <span class="c1"># Malediven (dv) // dv_MV is unknown by babel</span>
<span class="s1">&#39;MY&#39;</span><span class="p">,</span> <span class="c1"># Malaysia (ms)</span>
<span class="s1">&#39;NP&#39;</span><span class="p">,</span> <span class="c1"># Nepal (ne)</span>
<span class="s1">&#39;TJ&#39;</span><span class="p">,</span> <span class="c1"># Tadschikistan (tg)</span>
<span class="s1">&#39;TM&#39;</span><span class="p">,</span> <span class="c1"># Turkmenistan (tk)</span>
<span class="s1">&#39;UZ&#39;</span><span class="p">,</span> <span class="c1"># Usbekistan (uz)</span>
<span class="p">]</span>
<div class="viewcode-block" id="fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google.fetch_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">,</span> <span class="n">add_domains</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Fetch languages from Google.&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=import-outside-toplevel, too-many-branches</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;supported_domains&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">&#39;https://www.google.com/preferences&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;Response from Google&#39;s preferences is not OK.&quot;</span><span class="p">)</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">))</span>
<span class="c1"># supported language codes</span>
<span class="n">lang_map</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;no&#39;</span><span class="p">:</span> <span class="s1">&#39;nb&#39;</span><span class="p">}</span>
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//select[@name=&#39;hl&#39;]/option&quot;</span><span class="p">):</span>
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">lang_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_lang</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">),</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;-&#39;</span><span class="p">)</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;INFO: google UI language </span><span class="si">%s</span><span class="s2"> (</span><span class="si">%s</span><span class="s2">) is unknown by babel&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">eng_lang</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;(&quot;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()))</span>
<span class="k">continue</span>
<span class="n">sxng_lang</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_lang</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_lang</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_lang</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_lang</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;lang_&#39;</span> <span class="o">+</span> <span class="n">eng_lang</span>
<span class="c1"># alias languages</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="s1">&#39;zh&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;lang_zh-CN&#39;</span>
<span class="c1"># supported region codes</span>
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//select[@name=&#39;gl&#39;]/option&quot;</span><span class="p">):</span>
<span class="n">eng_country</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">eng_country</span> <span class="ow">in</span> <span class="n">skip_countries</span><span class="p">:</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="n">eng_country</span> <span class="o">==</span> <span class="s1">&#39;ZZ&#39;</span><span class="p">:</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="s1">&#39;ZZ&#39;</span>
<span class="k">continue</span>
<span class="n">sxng_locales</span> <span class="o">=</span> <span class="n">get_official_locales</span><span class="p">(</span><span class="n">eng_country</span><span class="p">,</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">keys</span><span class="p">(),</span> <span class="n">regional</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">sxng_locales</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: can&#39;t map from google country </span><span class="si">%s</span><span class="s2"> (</span><span class="si">%s</span><span class="s2">) to a babel region.&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;data-name&#39;</span><span class="p">),</span> <span class="n">eng_country</span><span class="p">))</span>
<span class="k">continue</span>
<span class="k">for</span> <span class="n">sxng_locale</span> <span class="ow">in</span> <span class="n">sxng_locales</span><span class="p">:</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">region_tag</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">)]</span> <span class="o">=</span> <span class="n">eng_country</span>
<span class="c1"># alias regions</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="s1">&#39;zh-CN&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;HK&#39;</span>
<span class="c1"># supported domains</span>
<span class="k">if</span> <span class="n">add_domains</span><span class="p">:</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">&#39;https://www.google.com/supported_domains&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;Response from https://www.google.com/supported_domains is not OK.&quot;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">domain</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">split</span><span class="p">():</span> <span class="c1"># type: ignore</span>
<span class="n">domain</span> <span class="o">=</span> <span class="n">domain</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">domain</span> <span class="ow">or</span> <span class="n">domain</span> <span class="ow">in</span> <span class="p">[</span>
<span class="s1">&#39;.google.com&#39;</span><span class="p">,</span>
<span class="p">]:</span>
<span class="k">continue</span>
<span class="n">region</span> <span class="o">=</span> <span class="n">domain</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;supported_domains&#39;</span><span class="p">][</span><span class="n">region</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;www&#39;</span> <span class="o">+</span> <span class="n">domain</span> <span class="c1"># type: ignore</span>
<span class="k">if</span> <span class="n">region</span> <span class="o">==</span> <span class="s1">&#39;HK&#39;</span><span class="p">:</span>
<span class="c1"># There is no google.cn, we use .com.hk for zh-CN</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;supported_domains&#39;</span><span class="p">][</span><span class="s1">&#39;CN&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;www&#39;</span> <span class="o">+</span> <span class="n">domain</span> <span class="c1"># type: ignore</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,236 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.google_images &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.google_images</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.google_images</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;This is the implementation of the Google Images engine using the internal</span>
<span class="sd">Google API used by the Google Go Android app.</span>
<span class="sd">This internal API offer results in</span>
<span class="sd">- JSON (``_fmt:json``)</span>
<span class="sd">- Protobuf_ (``_fmt:pb``)</span>
<span class="sd">- Protobuf_ compressed? (``_fmt:pc``)</span>
<span class="sd">- HTML (``_fmt:html``)</span>
<span class="sd">- Protobuf_ encoded in JSON (``_fmt:jspb``).</span>
<span class="sd">.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">json</span><span class="w"> </span><span class="kn">import</span> <span class="n">loads</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="c1"># pylint: disable=unused-import</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">get_google_info</span><span class="p">,</span>
<span class="n">time_range_dict</span><span class="p">,</span>
<span class="n">detect_google_sorry</span><span class="p">,</span>
<span class="p">)</span>
<span class="c1"># about</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://images.google.com&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q521550&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://developers.google.com/custom-search&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;JSON&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;images&#39;</span><span class="p">,</span> <span class="s1">&#39;web&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">50</span>
<span class="sd">&quot;&quot;&quot;`Google max 50 pages`_</span>
<span class="sd">.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">filter_mapping</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">&#39;images&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">&#39;active&#39;</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">&#39;active&#39;</span><span class="p">}</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_images.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Google-Image search request&quot;&quot;&quot;</span>
<span class="n">google_info</span> <span class="o">=</span> <span class="n">get_google_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">traits</span><span class="p">)</span>
<span class="n">query_url</span> <span class="o">=</span> <span class="p">(</span>
<span class="s1">&#39;https://&#39;</span>
<span class="o">+</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;subdomain&#39;</span><span class="p">]</span>
<span class="o">+</span> <span class="s1">&#39;/search&#39;</span>
<span class="o">+</span> <span class="s1">&#39;?&#39;</span>
<span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span> <span class="s1">&#39;tbm&#39;</span><span class="p">:</span> <span class="s2">&quot;isch&quot;</span><span class="p">,</span> <span class="o">**</span><span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">],</span> <span class="s1">&#39;asearch&#39;</span><span class="p">:</span> <span class="s1">&#39;isch&#39;</span><span class="p">})</span>
<span class="c1"># don&#39;t urlencode this because wildly different AND bad results</span>
<span class="c1"># pagination uses Zero-based numbering</span>
<span class="o">+</span> <span class="sa">f</span><span class="s1">&#39;&amp;async=_fmt:json,p:1,ijn:</span><span class="si">{</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;pageno&quot;</span><span class="p">]</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s1">&#39;</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">&#39;&amp;&#39;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;tbs&#39;</span><span class="p">:</span> <span class="s1">&#39;qdr:&#39;</span> <span class="o">+</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]]})</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]:</span>
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">&#39;&amp;&#39;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;safe&#39;</span><span class="p">:</span> <span class="n">filter_mapping</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]]})</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_url</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">]</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">])</span>
<span class="c1"># this ua will allow getting ~50 results instead of 10. #1641</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;User-Agent&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
<span class="s1">&#39;NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12;&#39;</span> <span class="sa">f</span><span class="s1">&#39; </span><span class="si">{</span><span class="n">google_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;country&quot;</span><span class="p">,</span><span class="w"> </span><span class="s2">&quot;US&quot;</span><span class="p">)</span><span class="si">}</span><span class="s1">) gzip&#39;</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">params</span></div>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_images.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get response from google&#39;s search request&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<span class="n">json_start</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;{&quot;ischj&quot;:&#39;</span><span class="p">)</span>
<span class="n">json_data</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">[</span><span class="n">json_start</span><span class="p">:])</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">&quot;ischj&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;metadata&quot;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="n">result_item</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;result&quot;</span><span class="p">][</span><span class="s2">&quot;referrer_url&quot;</span><span class="p">],</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;result&quot;</span><span class="p">][</span><span class="s2">&quot;page_title&quot;</span><span class="p">],</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;text_in_grid&quot;</span><span class="p">][</span><span class="s2">&quot;snippet&quot;</span><span class="p">],</span>
<span class="s1">&#39;source&#39;</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;result&quot;</span><span class="p">][</span><span class="s2">&quot;site_title&quot;</span><span class="p">],</span>
<span class="s1">&#39;resolution&#39;</span><span class="p">:</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">item</span><span class="p">[</span><span class="s2">&quot;original_image&quot;</span><span class="p">][</span><span class="s2">&quot;width&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1"> x </span><span class="si">{</span><span class="n">item</span><span class="p">[</span><span class="s2">&quot;original_image&quot;</span><span class="p">][</span><span class="s2">&quot;height&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="s1">&#39;img_src&#39;</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;original_image&quot;</span><span class="p">][</span><span class="s2">&quot;url&quot;</span><span class="p">],</span>
<span class="s1">&#39;thumbnail_src&#39;</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;thumbnail&quot;</span><span class="p">][</span><span class="s2">&quot;url&quot;</span><span class="p">],</span>
<span class="s1">&#39;template&#39;</span><span class="p">:</span> <span class="s1">&#39;images.html&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">author</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;result&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;iptc&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;creator&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">author</span><span class="p">:</span>
<span class="n">result_item</span><span class="p">[</span><span class="s1">&#39;author&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;, &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">author</span><span class="p">)</span>
<span class="n">copyright_notice</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;result&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;iptc&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;copyright_notice&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">copyright_notice</span><span class="p">:</span>
<span class="n">result_item</span><span class="p">[</span><span class="s1">&#39;source&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">&#39; | &#39;</span> <span class="o">+</span> <span class="n">copyright_notice</span>
<span class="n">freshness_date</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;result&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;freshness_date&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">freshness_date</span><span class="p">:</span>
<span class="n">result_item</span><span class="p">[</span><span class="s1">&#39;source&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">&#39; | &#39;</span> <span class="o">+</span> <span class="n">freshness_date</span>
<span class="n">file_size</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;gsa&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;file_size&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">file_size</span><span class="p">:</span>
<span class="n">result_item</span><span class="p">[</span><span class="s1">&#39;source&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">&#39; (</span><span class="si">%s</span><span class="s1">)&#39;</span> <span class="o">%</span> <span class="n">file_size</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">result_item</span><span class="p">)</span>
<span class="k">return</span> <span class="n">results</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,409 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.google_news &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.google_news</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.google_news</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;This is the implementation of the Google News engine.</span>
<span class="sd">Google News has a different region handling compared to Google WEB.</span>
<span class="sd">- the ``ceid`` argument has to be set (:py:obj:`ceid_list`)</span>
<span class="sd">- the hl_ argument has to be set correctly (and different to Google WEB)</span>
<span class="sd">- the gl_ argument is mandatory</span>
<span class="sd">If one of this argument is not set correctly, the request is redirected to</span>
<span class="sd">CONSENT dialog::</span>
<span class="sd"> https://consent.google.com/m?continue=</span>
<span class="sd">The google news API ignores some parameters from the common :ref:`google API`:</span>
<span class="sd">- num_ : the number of search results is ignored / there is no paging all</span>
<span class="sd"> results for a query term are in the first response.</span>
<span class="sd">- save_ : is ignored / Google-News results are always *SafeSearch*</span>
<span class="sd">.. _hl: https://developers.google.com/custom-search/docs/xml_results#hlsp</span>
<span class="sd">.. _gl: https://developers.google.com/custom-search/docs/xml_results#glsp</span>
<span class="sd">.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp</span>
<span class="sd">.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">base64</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">locales</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">eval_xpath</span><span class="p">,</span>
<span class="n">eval_xpath_list</span><span class="p">,</span>
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
<span class="n">extract_text</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="k">as</span> <span class="n">_fetch_traits</span> <span class="c1"># pylint: disable=unused-import</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">get_google_info</span><span class="p">,</span>
<span class="n">detect_google_sorry</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="c1"># about</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://news.google.com&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q12020&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://developers.google.com/custom-search&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;HTML&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;news&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">False</span>
<span class="c1"># Google-News results are always *SafeSearch*. Option &#39;safesearch&#39; is set to</span>
<span class="c1"># False here, otherwise checker will report safesearch-errors::</span>
<span class="c1">#</span>
<span class="c1"># safesearch : results are identical for safesearch=0 and safesearch=2</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
<span class="c1"># send_accept_language_header = True</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_news.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Google-News search request&quot;&quot;&quot;</span>
<span class="n">sxng_locale</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">,</span> <span class="s1">&#39;en-US&#39;</span><span class="p">)</span>
<span class="n">ceid</span> <span class="o">=</span> <span class="n">locales</span><span class="o">.</span><span class="n">get_engine_locale</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;ceid&#39;</span><span class="p">],</span> <span class="n">default</span><span class="o">=</span><span class="s1">&#39;US:en&#39;</span><span class="p">)</span>
<span class="n">google_info</span> <span class="o">=</span> <span class="n">get_google_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">traits</span><span class="p">)</span>
<span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;subdomain&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;news.google.com&#39;</span> <span class="c1"># google news has only one domain</span>
<span class="n">ceid_region</span><span class="p">,</span> <span class="n">ceid_lang</span> <span class="o">=</span> <span class="n">ceid</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;:&#39;</span><span class="p">)</span>
<span class="n">ceid_lang</span><span class="p">,</span> <span class="n">ceid_suffix</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">ceid_lang</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)</span>
<span class="o">+</span> <span class="p">[</span>
<span class="kc">None</span><span class="p">,</span>
<span class="p">]</span>
<span class="p">)[:</span><span class="mi">2</span><span class="p">]</span>
<span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;hl&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ceid_lang</span>
<span class="k">if</span> <span class="n">ceid_suffix</span> <span class="ow">and</span> <span class="n">ceid_suffix</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;Hans&#39;</span><span class="p">,</span> <span class="s1">&#39;Hant&#39;</span><span class="p">]:</span>
<span class="k">if</span> <span class="n">ceid_region</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="n">ceid_lang</span><span class="p">:</span>
<span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;hl&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ceid_lang</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="n">ceid_region</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;hl&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ceid_lang</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="n">ceid_suffix</span>
<span class="k">elif</span> <span class="n">ceid_region</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">!=</span> <span class="n">ceid_lang</span><span class="p">:</span>
<span class="k">if</span> <span class="n">ceid_region</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;AT&#39;</span><span class="p">,</span> <span class="s1">&#39;BE&#39;</span><span class="p">,</span> <span class="s1">&#39;CH&#39;</span><span class="p">,</span> <span class="s1">&#39;IL&#39;</span><span class="p">,</span> <span class="s1">&#39;SA&#39;</span><span class="p">,</span> <span class="s1">&#39;IN&#39;</span><span class="p">,</span> <span class="s1">&#39;BD&#39;</span><span class="p">,</span> <span class="s1">&#39;PT&#39;</span><span class="p">]:</span>
<span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;hl&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ceid_lang</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;hl&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ceid_lang</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="n">ceid_region</span>
<span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;lang_&#39;</span> <span class="o">+</span> <span class="n">ceid_lang</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">][</span><span class="s1">&#39;gl&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ceid_region</span>
<span class="n">query_url</span> <span class="o">=</span> <span class="p">(</span>
<span class="s1">&#39;https://&#39;</span>
<span class="o">+</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;subdomain&#39;</span><span class="p">]</span>
<span class="o">+</span> <span class="s2">&quot;/search?&quot;</span>
<span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="o">**</span><span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">],</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="c1"># ceid includes a &#39;:&#39; character which must not be urlencoded</span>
<span class="o">+</span> <span class="p">(</span><span class="s1">&#39;&amp;ceid=</span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">ceid</span><span class="p">)</span>
<span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_url</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">]</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">])</span>
<span class="k">return</span> <span class="n">params</span></div>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_news.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get response from google&#39;s search request&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<span class="c1"># convert the text to dom</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">&#39;//div[@class=&quot;xrnccd&quot;]&#39;</span><span class="p">):</span>
<span class="c1"># The first &lt;a&gt; tag in the &lt;article&gt; contains the link to the article</span>
<span class="c1"># The href attribute of the &lt;a&gt; tag is a google internal link, we have</span>
<span class="c1"># to decode</span>
<span class="n">href</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;./article/a/@href&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
<span class="n">href</span> <span class="o">=</span> <span class="n">href</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;?&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">href</span> <span class="o">=</span> <span class="n">href</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
<span class="n">href</span> <span class="o">=</span> <span class="n">base64</span><span class="o">.</span><span class="n">urlsafe_b64decode</span><span class="p">(</span><span class="n">href</span> <span class="o">+</span> <span class="s1">&#39;====&#39;</span><span class="p">)</span>
<span class="n">href</span> <span class="o">=</span> <span class="n">href</span><span class="p">[</span><span class="n">href</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="sa">b</span><span class="s1">&#39;http&#39;</span><span class="p">)</span> <span class="p">:]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">b</span><span class="s1">&#39;</span><span class="se">\xd2</span><span class="s1">&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">href</span> <span class="o">=</span> <span class="n">href</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;./article/h3[1]&#39;</span><span class="p">))</span>
<span class="c1"># The pub_date is mostly a string like &#39;yesterday&#39;, not a real</span>
<span class="c1"># timezone date or time. Therefore we can&#39;t use publishedDate.</span>
<span class="n">pub_date</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;./article//time&#39;</span><span class="p">))</span>
<span class="n">pub_origin</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;./article//a[@data-n-tid]&#39;</span><span class="p">))</span>
<span class="n">content</span> <span class="o">=</span> <span class="s1">&#39; / &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span><span class="n">pub_origin</span><span class="p">,</span> <span class="n">pub_date</span><span class="p">]</span> <span class="k">if</span> <span class="n">x</span><span class="p">])</span>
<span class="c1"># The image URL is located in a preceding sibling &lt;img&gt; tag, e.g.:</span>
<span class="c1"># &quot;https://lh3.googleusercontent.com/DjhQh7DMszk.....z=-p-h100-w100&quot;</span>
<span class="c1"># These URL are long but not personalized (double checked via tor).</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;preceding-sibling::a/figure/img/@src&#39;</span><span class="p">))</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">href</span><span class="p">,</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
<span class="s1">&#39;thumbnail&#39;</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="c1"># return results</span>
<span class="k">return</span> <span class="n">results</span></div>
<span class="n">ceid_list</span> <span class="o">=</span> <span class="p">[</span>
<span class="s1">&#39;AE:ar&#39;</span><span class="p">,</span>
<span class="s1">&#39;AR:es-419&#39;</span><span class="p">,</span>
<span class="s1">&#39;AT:de&#39;</span><span class="p">,</span>
<span class="s1">&#39;AU:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;BD:bn&#39;</span><span class="p">,</span>
<span class="s1">&#39;BE:fr&#39;</span><span class="p">,</span>
<span class="s1">&#39;BE:nl&#39;</span><span class="p">,</span>
<span class="s1">&#39;BG:bg&#39;</span><span class="p">,</span>
<span class="s1">&#39;BR:pt-419&#39;</span><span class="p">,</span>
<span class="s1">&#39;BW:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;CA:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;CA:fr&#39;</span><span class="p">,</span>
<span class="s1">&#39;CH:de&#39;</span><span class="p">,</span>
<span class="s1">&#39;CH:fr&#39;</span><span class="p">,</span>
<span class="s1">&#39;CL:es-419&#39;</span><span class="p">,</span>
<span class="s1">&#39;CN:zh-Hans&#39;</span><span class="p">,</span>
<span class="s1">&#39;CO:es-419&#39;</span><span class="p">,</span>
<span class="s1">&#39;CU:es-419&#39;</span><span class="p">,</span>
<span class="s1">&#39;CZ:cs&#39;</span><span class="p">,</span>
<span class="s1">&#39;DE:de&#39;</span><span class="p">,</span>
<span class="s1">&#39;EG:ar&#39;</span><span class="p">,</span>
<span class="s1">&#39;ES:es&#39;</span><span class="p">,</span>
<span class="s1">&#39;ET:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;FR:fr&#39;</span><span class="p">,</span>
<span class="s1">&#39;GB:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;GH:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;GR:el&#39;</span><span class="p">,</span>
<span class="s1">&#39;HK:zh-Hant&#39;</span><span class="p">,</span>
<span class="s1">&#39;HU:hu&#39;</span><span class="p">,</span>
<span class="s1">&#39;ID:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;ID:id&#39;</span><span class="p">,</span>
<span class="s1">&#39;IE:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;IL:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;IL:he&#39;</span><span class="p">,</span>
<span class="s1">&#39;IN:bn&#39;</span><span class="p">,</span>
<span class="s1">&#39;IN:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;IN:hi&#39;</span><span class="p">,</span>
<span class="s1">&#39;IN:ml&#39;</span><span class="p">,</span>
<span class="s1">&#39;IN:mr&#39;</span><span class="p">,</span>
<span class="s1">&#39;IN:ta&#39;</span><span class="p">,</span>
<span class="s1">&#39;IN:te&#39;</span><span class="p">,</span>
<span class="s1">&#39;IT:it&#39;</span><span class="p">,</span>
<span class="s1">&#39;JP:ja&#39;</span><span class="p">,</span>
<span class="s1">&#39;KE:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;KR:ko&#39;</span><span class="p">,</span>
<span class="s1">&#39;LB:ar&#39;</span><span class="p">,</span>
<span class="s1">&#39;LT:lt&#39;</span><span class="p">,</span>
<span class="s1">&#39;LV:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;LV:lv&#39;</span><span class="p">,</span>
<span class="s1">&#39;MA:fr&#39;</span><span class="p">,</span>
<span class="s1">&#39;MX:es-419&#39;</span><span class="p">,</span>
<span class="s1">&#39;MY:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;NA:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;NG:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;NL:nl&#39;</span><span class="p">,</span>
<span class="s1">&#39;NO:no&#39;</span><span class="p">,</span>
<span class="s1">&#39;NZ:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;PE:es-419&#39;</span><span class="p">,</span>
<span class="s1">&#39;PH:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;PK:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;PL:pl&#39;</span><span class="p">,</span>
<span class="s1">&#39;PT:pt-150&#39;</span><span class="p">,</span>
<span class="s1">&#39;RO:ro&#39;</span><span class="p">,</span>
<span class="s1">&#39;RS:sr&#39;</span><span class="p">,</span>
<span class="s1">&#39;RU:ru&#39;</span><span class="p">,</span>
<span class="s1">&#39;SA:ar&#39;</span><span class="p">,</span>
<span class="s1">&#39;SE:sv&#39;</span><span class="p">,</span>
<span class="s1">&#39;SG:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;SI:sl&#39;</span><span class="p">,</span>
<span class="s1">&#39;SK:sk&#39;</span><span class="p">,</span>
<span class="s1">&#39;SN:fr&#39;</span><span class="p">,</span>
<span class="s1">&#39;TH:th&#39;</span><span class="p">,</span>
<span class="s1">&#39;TR:tr&#39;</span><span class="p">,</span>
<span class="s1">&#39;TW:zh-Hant&#39;</span><span class="p">,</span>
<span class="s1">&#39;TZ:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;UA:ru&#39;</span><span class="p">,</span>
<span class="s1">&#39;UA:uk&#39;</span><span class="p">,</span>
<span class="s1">&#39;UG:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;US:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;US:es-419&#39;</span><span class="p">,</span>
<span class="s1">&#39;VE:es-419&#39;</span><span class="p">,</span>
<span class="s1">&#39;VN:vi&#39;</span><span class="p">,</span>
<span class="s1">&#39;ZA:en&#39;</span><span class="p">,</span>
<span class="s1">&#39;ZW:en&#39;</span><span class="p">,</span>
<span class="p">]</span>
<span class="sd">&quot;&quot;&quot;List of region/language combinations supported by Google News. Values of the</span>
<span class="sd">``ceid`` argument of the Google News REST API.&quot;&quot;&quot;</span>
<span class="n">_skip_values</span> <span class="o">=</span> <span class="p">[</span>
<span class="s1">&#39;ET:en&#39;</span><span class="p">,</span> <span class="c1"># english (ethiopia)</span>
<span class="s1">&#39;ID:en&#39;</span><span class="p">,</span> <span class="c1"># english (indonesia)</span>
<span class="s1">&#39;LV:en&#39;</span><span class="p">,</span> <span class="c1"># english (latvia)</span>
<span class="p">]</span>
<span class="n">_ceid_locale_map</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;NO:no&#39;</span><span class="p">:</span> <span class="s1">&#39;nb-NO&#39;</span><span class="p">}</span>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="n">_fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">,</span> <span class="n">add_domains</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;ceid&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">for</span> <span class="n">ceid</span> <span class="ow">in</span> <span class="n">ceid_list</span><span class="p">:</span>
<span class="k">if</span> <span class="n">ceid</span> <span class="ow">in</span> <span class="n">_skip_values</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">region</span><span class="p">,</span> <span class="n">lang</span> <span class="o">=</span> <span class="n">ceid</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;:&#39;</span><span class="p">)</span>
<span class="n">x</span> <span class="o">=</span> <span class="n">lang</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;Hant&#39;</span><span class="p">,</span> <span class="s1">&#39;Hans&#39;</span><span class="p">]:</span>
<span class="n">lang</span> <span class="o">=</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">sxng_locale</span> <span class="o">=</span> <span class="n">_ceid_locale_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">ceid</span><span class="p">,</span> <span class="n">lang</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="n">region</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;-&#39;</span><span class="p">)</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: </span><span class="si">%s</span><span class="s2"> -&gt; </span><span class="si">%s</span><span class="s2"> is unknown by babel&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">ceid</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;ceid&#39;</span><span class="p">][</span><span class="n">locales</span><span class="o">.</span><span class="n">region_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)]</span> <span class="o">=</span> <span class="n">ceid</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,364 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.google_scholar &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.google_scholar</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.google_scholar</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Google Scholar is a freely accessible web search engine that indexes the full</span>
<span class="sd">text or metadata of scholarly literature across an array of publishing formats</span>
<span class="sd">and disciplines.</span>
<span class="sd">Compared to other Google services the Scholar engine has a simple GET REST-API</span>
<span class="sd">and there does not exists ``async`` API. Even though the API slightly vintage</span>
<span class="sd">we can make use of the :ref:`google API` to assemble the arguments of the GET</span>
<span class="sd">request.</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: google scholar</span>
<span class="sd"> engine: google_scholar</span>
<span class="sd"> shortcut: gos</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">httpx</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">eval_xpath</span><span class="p">,</span>
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
<span class="n">eval_xpath_list</span><span class="p">,</span>
<span class="n">extract_text</span><span class="p">,</span>
<span class="n">ElementType</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineCaptchaException</span><span class="p">,</span> <span class="n">SearxEngineAccessDeniedException</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="c1"># pylint: disable=unused-import</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">get_google_info</span><span class="p">,</span>
<span class="n">time_range_dict</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s2">&quot;https://scholar.google.com&quot;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s2">&quot;Q494817&quot;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s2">&quot;https://developers.google.com/custom-search&quot;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s2">&quot;HTML&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;science&quot;</span><span class="p">,</span> <span class="s2">&quot;scientific publications&quot;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">50</span>
<span class="sd">&quot;&quot;&quot;`Google max 50 pages`_</span>
<span class="sd">.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">language_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_scholar.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">&quot;OnlineParams&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Google-Scholar search request&quot;&quot;&quot;</span>
<span class="n">google_info</span> <span class="o">=</span> <span class="n">get_google_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">traits</span><span class="p">)</span>
<span class="c1"># subdomain is: scholar.google.xy</span>
<span class="n">google_info</span><span class="p">[</span><span class="s2">&quot;subdomain&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">google_info</span><span class="p">[</span><span class="s2">&quot;subdomain&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;www.&quot;</span><span class="p">,</span> <span class="s2">&quot;scholar.&quot;</span><span class="p">)</span>
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;q&quot;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="o">**</span><span class="n">google_info</span><span class="p">[</span><span class="s2">&quot;params&quot;</span><span class="p">],</span>
<span class="s2">&quot;start&quot;</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;pageno&quot;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span><span class="p">,</span>
<span class="s2">&quot;as_sdt&quot;</span><span class="p">:</span> <span class="s2">&quot;2007&quot;</span><span class="p">,</span> <span class="c1"># include patents / to disable set &quot;0,5&quot;</span>
<span class="s2">&quot;as_vis&quot;</span><span class="p">:</span> <span class="s2">&quot;0&quot;</span><span class="p">,</span> <span class="c1"># include citations / to disable set &quot;1&quot;</span>
<span class="p">}</span>
<span class="n">args</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">time_range_args</span><span class="p">(</span><span class="n">params</span><span class="p">))</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;https://&quot;</span> <span class="o">+</span> <span class="n">google_info</span><span class="p">[</span><span class="s2">&quot;subdomain&quot;</span><span class="p">]</span> <span class="o">+</span> <span class="s2">&quot;/scholar?&quot;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;cookies&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">google_info</span><span class="p">[</span><span class="s2">&quot;cookies&quot;</span><span class="p">]</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;headers&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">google_info</span><span class="p">[</span><span class="s2">&quot;headers&quot;</span><span class="p">])</span></div>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_scholar.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span> <span class="c1"># pylint: disable=too-many-locals</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Parse response from Google Scholar&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="ow">in</span> <span class="p">(</span><span class="mi">301</span><span class="p">,</span> <span class="mi">302</span><span class="p">,</span> <span class="mi">303</span><span class="p">,</span> <span class="mi">307</span><span class="p">,</span> <span class="mi">308</span><span class="p">)</span> <span class="ow">and</span> <span class="s2">&quot;Location&quot;</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">headers</span><span class="p">:</span>
<span class="k">if</span> <span class="s2">&quot;/sorry/index?continue&quot;</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s2">&quot;Location&quot;</span><span class="p">]:</span>
<span class="c1"># Our systems have detected unusual traffic from your computer</span>
<span class="c1"># network. Please try again later.</span>
<span class="k">raise</span> <span class="n">SearxEngineAccessDeniedException</span><span class="p">(</span>
<span class="n">message</span><span class="o">=</span><span class="s2">&quot;google_scholar: unusual traffic detected&quot;</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">raise</span> <span class="n">httpx</span><span class="o">.</span><span class="n">TooManyRedirects</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;location </span><span class="si">{</span><span class="n">resp</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">&#39;Location&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;?&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="n">detect_google_captcha</span><span class="p">(</span><span class="n">dom</span><span class="p">)</span>
<span class="c1"># parse results</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//div[@data-rp]&quot;</span><span class="p">):</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//h3[1]//a&quot;</span><span class="p">))</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">title</span><span class="p">:</span>
<span class="c1"># this is a [ZITATION] block</span>
<span class="k">continue</span>
<span class="n">pub_type</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//span[@class=&#39;gs_ctg2&#39;]&quot;</span><span class="p">))</span> <span class="ow">or</span> <span class="s2">&quot;&quot;</span>
<span class="k">if</span> <span class="n">pub_type</span><span class="p">:</span>
<span class="n">pub_type</span> <span class="o">=</span> <span class="n">pub_type</span><span class="p">[</span><span class="mi">1</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="n">url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//h3[1]//a/@href&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
<span class="n">content</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//div[@class=&#39;gs_rs&#39;]&quot;</span><span class="p">))</span> <span class="ow">or</span> <span class="s2">&quot;&quot;</span>
<span class="n">authors</span><span class="p">,</span> <span class="n">journal</span><span class="p">,</span> <span class="n">publisher</span><span class="p">,</span> <span class="n">publishedDate</span> <span class="o">=</span> <span class="n">parse_gs_a</span><span class="p">(</span>
<span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//div[@class=&#39;gs_a&#39;]&quot;</span><span class="p">))</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">publisher</span> <span class="ow">in</span> <span class="n">url</span><span class="p">:</span>
<span class="n">publisher</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="c1"># cited by</span>
<span class="n">comments</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//div[@class=&#39;gs_fl&#39;]/a[starts-with(@href,&#39;/scholar?cites=&#39;)]&quot;</span><span class="p">))</span> <span class="ow">or</span> <span class="s2">&quot;&quot;</span>
<span class="p">)</span>
<span class="c1"># link to the html or pdf document</span>
<span class="n">html_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="n">pdf_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="n">doc_url</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//div[@class=&#39;gs_or_ggsm&#39;]/a/@href&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
<span class="n">doc_type</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;.//span[@class=&#39;gs_ctg2&#39;]&quot;</span><span class="p">))</span>
<span class="k">if</span> <span class="n">doc_type</span> <span class="o">==</span> <span class="s2">&quot;[PDF]&quot;</span><span class="p">:</span>
<span class="n">pdf_url</span> <span class="o">=</span> <span class="n">doc_url</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">html_url</span> <span class="o">=</span> <span class="n">doc_url</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Paper</span><span class="p">(</span>
<span class="nb">type</span><span class="o">=</span><span class="n">pub_type</span><span class="p">,</span>
<span class="n">url</span><span class="o">=</span><span class="n">url</span><span class="p">,</span>
<span class="n">title</span><span class="o">=</span><span class="n">title</span><span class="p">,</span>
<span class="n">authors</span><span class="o">=</span><span class="n">authors</span><span class="p">,</span>
<span class="n">publisher</span><span class="o">=</span><span class="n">publisher</span><span class="p">,</span>
<span class="n">journal</span><span class="o">=</span><span class="n">journal</span><span class="p">,</span>
<span class="n">publishedDate</span><span class="o">=</span><span class="n">publishedDate</span><span class="p">,</span>
<span class="n">content</span><span class="o">=</span><span class="n">content</span><span class="p">,</span>
<span class="n">comments</span><span class="o">=</span><span class="n">comments</span><span class="p">,</span>
<span class="n">html_url</span><span class="o">=</span><span class="n">html_url</span><span class="p">,</span>
<span class="n">pdf_url</span><span class="o">=</span><span class="n">pdf_url</span><span class="p">,</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="c1"># parse suggestion</span>
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//div[contains(@class, &#39;gs_qsuggest_wrap&#39;)]//li//a&quot;</span><span class="p">):</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span><span class="n">suggestion</span><span class="o">=</span><span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)))</span>
<span class="k">for</span> <span class="n">correction</span> <span class="ow">in</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//div[@class=&#39;gs_r gs_pda&#39;]/a&quot;</span><span class="p">):</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span><span class="n">correction</span><span class="o">=</span><span class="n">extract_text</span><span class="p">(</span><span class="n">correction</span><span class="p">)))</span>
<span class="k">return</span> <span class="n">res</span></div>
<div class="viewcode-block" id="time_range_args">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_scholar.time_range_args">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">time_range_args</span><span class="p">(</span><span class="n">params</span><span class="p">:</span> <span class="s2">&quot;OnlineParams&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns a dictionary with a time range arguments based on</span>
<span class="sd"> ``params[&quot;time_range&quot;]``.</span>
<span class="sd"> Google Scholar supports a detailed search by year. Searching by *last</span>
<span class="sd"> month* or *last week* (as offered by SearXNG) is uncommon for scientific</span>
<span class="sd"> publications and is not supported by Google Scholar.</span>
<span class="sd"> To limit the result list when the users selects a range, all the SearXNG</span>
<span class="sd"> ranges (*day*, *week*, *month*, *year*) are mapped to *year*. If no range</span>
<span class="sd"> is set an empty dictionary of arguments is returned.</span>
<span class="sd"> Example; when user selects a time range and we find ourselves in the year</span>
<span class="sd"> 2025 (current year minus one):</span>
<span class="sd"> .. code:: python</span>
<span class="sd"> { &quot;as_ylo&quot; : 2024 }</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">ret_val</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s2">&quot;time_range&quot;</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
<span class="n">ret_val</span><span class="p">[</span><span class="s2">&quot;as_ylo&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">year</span> <span class="o">-</span> <span class="mi">1</span>
<span class="k">return</span> <span class="n">ret_val</span></div>
<div class="viewcode-block" id="detect_google_captcha">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_scholar.detect_google_captcha">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">detect_google_captcha</span><span class="p">(</span><span class="n">dom</span><span class="p">:</span> <span class="n">ElementType</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;In case of CAPTCHA Google Scholar open its own *not a Robot* dialog and is</span>
<span class="sd"> not redirected to ``sorry.google.com``.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//form[@id=&#39;gs_captcha_f&#39;]&quot;</span><span class="p">):</span>
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">(</span><span class="n">message</span><span class="o">=</span><span class="s2">&quot;CAPTCHA (gs_captcha_f)&quot;</span><span class="p">)</span></div>
<div class="viewcode-block" id="parse_gs_a">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_scholar.parse_gs_a">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">parse_gs_a</span><span class="p">(</span><span class="n">text</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="n">datetime</span> <span class="o">|</span> <span class="kc">None</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Parse the text written in green.</span>
<span class="sd"> Possible formats:</span>
<span class="sd"> * &quot;{authors} - {journal}, {year} - {publisher}&quot;</span>
<span class="sd"> * &quot;{authors} - {year} - {publisher}&quot;</span>
<span class="sd"> * &quot;{authors} - {publisher}&quot;</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">text</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">text</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
<span class="k">return</span> <span class="p">[],</span> <span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="kc">None</span>
<span class="n">s_text</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot; - &quot;</span><span class="p">)</span>
<span class="n">authors</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">s_text</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;, &quot;</span><span class="p">)</span>
<span class="n">publisher</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">s_text</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">s_text</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">3</span><span class="p">:</span>
<span class="k">return</span> <span class="n">authors</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="n">publisher</span><span class="p">,</span> <span class="kc">None</span>
<span class="c1"># the format is &quot;{authors} - {journal}, {year} - {publisher}&quot; or &quot;{authors} - {year} - {publisher}&quot;</span>
<span class="c1"># get journal and year</span>
<span class="n">journal_year</span> <span class="o">=</span> <span class="n">s_text</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;, &quot;</span><span class="p">)</span>
<span class="c1"># journal is optional and may contains some coma</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">journal_year</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">journal</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">journal_year</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
<span class="k">if</span> <span class="n">journal</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
<span class="n">journal</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">journal</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="c1"># year</span>
<span class="n">year</span> <span class="o">=</span> <span class="n">journal_year</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">publishedDate</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">year</span><span class="o">.</span><span class="n">strip</span><span class="p">(),</span> <span class="s2">&quot;%Y&quot;</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="n">publishedDate</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">return</span> <span class="n">authors</span><span class="p">,</span> <span class="n">journal</span><span class="p">,</span> <span class="n">publisher</span><span class="p">,</span> <span class="n">publishedDate</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,274 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.google_videos &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.google_videos</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.google_videos</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;This is the implementation of the Google Videos engine.</span>
<span class="sd">.. admonition:: Content-Security-Policy (CSP)</span>
<span class="sd"> This engine needs to allow images from the `data URLs`_ (prefixed with the</span>
<span class="sd"> ``data:`` scheme)::</span>
<span class="sd"> Header set Content-Security-Policy &quot;img-src &#39;self&#39; data: ;&quot;</span>
<span class="sd">.. _data URLs:</span>
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span><span class="p">,</span> <span class="n">urlparse</span><span class="p">,</span> <span class="n">parse_qs</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">eval_xpath_list</span><span class="p">,</span>
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
<span class="n">extract_text</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="c1"># pylint: disable=unused-import</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">get_google_info</span><span class="p">,</span>
<span class="n">time_range_dict</span><span class="p">,</span>
<span class="n">filter_mapping</span><span class="p">,</span>
<span class="n">suggestion_xpath</span><span class="p">,</span>
<span class="n">detect_google_sorry</span><span class="p">,</span>
<span class="n">ui_async</span><span class="p">,</span>
<span class="n">parse_data_images</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_embeded_stream_url</span>
<span class="c1"># about</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.google.com&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q219885&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://developers.google.com/custom-search&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;HTML&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;videos&#39;</span><span class="p">,</span> <span class="s1">&#39;web&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">50</span>
<span class="n">language_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_videos.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Google-Video search request&quot;&quot;&quot;</span>
<span class="n">google_info</span> <span class="o">=</span> <span class="n">get_google_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">traits</span><span class="p">)</span>
<span class="n">start</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span>
<span class="n">query_url</span> <span class="o">=</span> <span class="p">(</span>
<span class="s1">&#39;https://&#39;</span>
<span class="o">+</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;subdomain&#39;</span><span class="p">]</span>
<span class="o">+</span> <span class="s1">&#39;/search&#39;</span>
<span class="o">+</span> <span class="s2">&quot;?&quot;</span>
<span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;tbm&#39;</span><span class="p">:</span> <span class="s2">&quot;vid&quot;</span><span class="p">,</span>
<span class="s1">&#39;start&#39;</span><span class="p">:</span> <span class="n">start</span><span class="p">,</span>
<span class="o">**</span><span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">],</span>
<span class="s1">&#39;asearch&#39;</span><span class="p">:</span> <span class="s1">&#39;arc&#39;</span><span class="p">,</span>
<span class="s1">&#39;async&#39;</span><span class="p">:</span> <span class="n">ui_async</span><span class="p">(</span><span class="n">start</span><span class="p">),</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">&#39;&amp;&#39;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;tbs&#39;</span><span class="p">:</span> <span class="s1">&#39;qdr:&#39;</span> <span class="o">+</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]]})</span>
<span class="k">if</span> <span class="s1">&#39;safesearch&#39;</span> <span class="ow">in</span> <span class="n">params</span><span class="p">:</span>
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">&#39;&amp;&#39;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;safe&#39;</span><span class="p">:</span> <span class="n">filter_mapping</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]]})</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_url</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">]</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">google_info</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">])</span>
<span class="k">return</span> <span class="n">params</span></div>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_videos.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get response from google&#39;s search request&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<span class="n">data_image_map</span> <span class="o">=</span> <span class="n">parse_data_images</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="c1"># convert the text to dom</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="n">result_divs</span> <span class="o">=</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">&#39;//div[contains(@class, &quot;MjjYud&quot;)]&#39;</span><span class="p">)</span>
<span class="c1"># parse results</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">result_divs</span><span class="p">:</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span>
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//h3[contains(@class, &quot;LC20lb&quot;)]&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span>
<span class="p">)</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//a[@jsname=&quot;UWckNb&quot;]/@href&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span>
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//div[contains(@class, &quot;ITZIwc&quot;)]&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span>
<span class="p">)</span>
<span class="n">pub_info</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span>
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//div[contains(@class, &quot;gqF9jc&quot;)]&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span>
<span class="p">)</span>
<span class="c1"># Broader XPath to find any &lt;img&gt; element</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//img/@src&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
<span class="n">duration</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span>
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//span[contains(@class, &quot;k1U36b&quot;)]&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span>
<span class="p">)</span>
<span class="n">video_id</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//div[@jscontroller=&quot;rTuANe&quot;]/@data-vid&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
<span class="c1"># Fallback for video_id from URL if not found via XPath</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">video_id</span> <span class="ow">and</span> <span class="n">url</span> <span class="ow">and</span> <span class="s1">&#39;youtube.com&#39;</span> <span class="ow">in</span> <span class="n">url</span><span class="p">:</span>
<span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
<span class="n">video_id</span> <span class="o">=</span> <span class="n">parse_qs</span><span class="p">(</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">query</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;v&#39;</span><span class="p">,</span> <span class="p">[</span><span class="kc">None</span><span class="p">])[</span><span class="mi">0</span><span class="p">]</span>
<span class="c1"># Handle thumbnail</span>
<span class="k">if</span> <span class="n">thumbnail</span> <span class="ow">and</span> <span class="n">thumbnail</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;data:image&#39;</span><span class="p">):</span>
<span class="n">img_id</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//img/@id&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">img_id</span> <span class="ow">and</span> <span class="n">img_id</span> <span class="ow">in</span> <span class="n">data_image_map</span><span class="p">:</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">data_image_map</span><span class="p">[</span><span class="n">img_id</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">thumbnail</span> <span class="ow">and</span> <span class="n">video_id</span><span class="p">:</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;https://img.youtube.com/vi/</span><span class="si">{</span><span class="n">video_id</span><span class="si">}</span><span class="s2">/hqdefault.jpg&quot;</span>
<span class="c1"># Handle video embed URL</span>
<span class="n">embed_url</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">video_id</span><span class="p">:</span>
<span class="n">embed_url</span> <span class="o">=</span> <span class="n">get_embeded_stream_url</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;https://www.youtube.com/watch?v=</span><span class="si">{</span><span class="n">video_id</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">url</span><span class="p">:</span>
<span class="n">embed_url</span> <span class="o">=</span> <span class="n">get_embeded_stream_url</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
<span class="c1"># Only append results with valid title and url</span>
<span class="k">if</span> <span class="n">title</span> <span class="ow">and</span> <span class="n">url</span><span class="p">:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span> <span class="ow">or</span> <span class="s1">&#39;&#39;</span><span class="p">,</span>
<span class="s1">&#39;author&#39;</span><span class="p">:</span> <span class="n">pub_info</span><span class="p">,</span>
<span class="s1">&#39;thumbnail&#39;</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
<span class="s1">&#39;length&#39;</span><span class="p">:</span> <span class="n">duration</span><span class="p">,</span>
<span class="s1">&#39;iframe_src&#39;</span><span class="p">:</span> <span class="n">embed_url</span><span class="p">,</span>
<span class="s1">&#39;template&#39;</span><span class="p">:</span> <span class="s1">&#39;videos.html&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="c1"># parse suggestion</span>
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">suggestion_xpath</span><span class="p">):</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;suggestion&#39;</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)})</span>
<span class="k">return</span> <span class="n">results</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,537 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.json_engine &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.json_engine</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.json_engine</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;The JSON engine is a *generic* engine with which it is possible to configure</span>
<span class="sd">engines in the settings.</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">Request:</span>
<span class="sd">- :py:obj:`search_url`</span>
<span class="sd">- :py:obj:`lang_all`</span>
<span class="sd">- :py:obj:`soft_max_redirects`</span>
<span class="sd">- :py:obj:`method`</span>
<span class="sd">- :py:obj:`request_body`</span>
<span class="sd">- :py:obj:`cookies`</span>
<span class="sd">- :py:obj:`headers`</span>
<span class="sd">Paging:</span>
<span class="sd">- :py:obj:`paging`</span>
<span class="sd">- :py:obj:`page_size`</span>
<span class="sd">- :py:obj:`first_page_num`</span>
<span class="sd">Time Range:</span>
<span class="sd">- :py:obj:`time_range_support`</span>
<span class="sd">- :py:obj:`time_range_url`</span>
<span class="sd">- :py:obj:`time_range_map`</span>
<span class="sd">Safe-Search:</span>
<span class="sd">- :py:obj:`safe_search_support`</span>
<span class="sd">- :py:obj:`safe_search_map`</span>
<span class="sd">Response:</span>
<span class="sd">- :py:obj:`title_html_to_text`</span>
<span class="sd">- :py:obj:`content_html_to_text`</span>
<span class="sd">- :py:obj:`no_result_for_http_status`</span>
<span class="sd">JSON query:</span>
<span class="sd">- :py:obj:`results_query`</span>
<span class="sd">- :py:obj:`url_query`</span>
<span class="sd">- :py:obj:`url_prefix`</span>
<span class="sd">- :py:obj:`title_query`</span>
<span class="sd">- :py:obj:`content_query`</span>
<span class="sd">- :py:obj:`thumbnail_query`</span>
<span class="sd">- :py:obj:`thumbnail_prefix`</span>
<span class="sd">- :py:obj:`suggestion_query`</span>
<span class="sd">Example</span>
<span class="sd">=======</span>
<span class="sd">Here is a simple example of a JSON engine configure in the :ref:`settings</span>
<span class="sd">engines` section, further read :ref:`engines-dev`.</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name : mdn</span>
<span class="sd"> engine : json_engine</span>
<span class="sd"> paging : True</span>
<span class="sd"> search_url : https://developer.mozilla.org/api/v1/search?q={query}&amp;page={pageno}</span>
<span class="sd"> results_query : documents</span>
<span class="sd"> url_query : mdn_url</span>
<span class="sd"> url_prefix : https://developer.mozilla.org</span>
<span class="sd"> title_query : title</span>
<span class="sd"> content_query : summary</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">collections.abc</span><span class="w"> </span><span class="kn">import</span> <span class="n">Iterable</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">json</span><span class="w"> </span><span class="kn">import</span> <span class="n">loads</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">to_string</span><span class="p">,</span> <span class="n">html_to_text</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">raise_for_httperror</span>
<span class="n">search_url</span> <span class="o">=</span> <span class="kc">None</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">Search URL of the engine. Example::</span>
<span class="sd"> https://example.org/?search={query}&amp;page={pageno}{time_range}{safe_search}</span>
<span class="sd">Replacements are:</span>
<span class="sd">``{query}``:</span>
<span class="sd"> Search terms from user.</span>
<span class="sd">``{pageno}``:</span>
<span class="sd"> Page number if engine supports paging :py:obj:`paging`</span>
<span class="sd">``{lang}``:</span>
<span class="sd"> ISO 639-1 language code (en, de, fr ..)</span>
<span class="sd">``{time_range}``:</span>
<span class="sd"> :py:obj:`URL parameter &lt;time_range_url&gt;` if engine :py:obj:`supports time</span>
<span class="sd"> range &lt;time_range_support&gt;`. The value for the parameter is taken from</span>
<span class="sd"> :py:obj:`time_range_map`.</span>
<span class="sd">``{safe_search}``:</span>
<span class="sd"> Safe-search :py:obj:`URL parameter &lt;safe_search_map&gt;` if engine</span>
<span class="sd"> :py:obj:`supports safe-search &lt;safe_search_support&gt;`. The ``{safe_search}``</span>
<span class="sd"> replacement is taken from the :py:obj:`safes_search_map`. Filter results::</span>
<span class="sd"> 0: none, 1: moderate, 2:strict</span>
<span class="sd"> If not supported, the URL parameter is an empty string.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">lang_all</span> <span class="o">=</span> <span class="s1">&#39;en&#39;</span>
<span class="sd">&#39;&#39;&#39;Replacement ``{lang}`` in :py:obj:`search_url` if language ``all`` is</span>
<span class="sd">selected.</span>
<span class="sd">&#39;&#39;&#39;</span>
<span class="n">no_result_for_http_status</span> <span class="o">=</span> <span class="p">[]</span>
<span class="sd">&#39;&#39;&#39;Return empty result for these HTTP status codes instead of throwing an error.</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> no_result_for_http_status: []</span>
<span class="sd">&#39;&#39;&#39;</span>
<span class="n">soft_max_redirects</span> <span class="o">=</span> <span class="mi">0</span>
<span class="sd">&#39;&#39;&#39;Maximum redirects, soft limit. Record an error but don&#39;t stop the engine&#39;&#39;&#39;</span>
<span class="n">method</span> <span class="o">=</span> <span class="s1">&#39;GET&#39;</span>
<span class="sd">&#39;&#39;&#39;Some engines might require to do POST requests for search.&#39;&#39;&#39;</span>
<span class="n">request_body</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="sd">&#39;&#39;&#39;The body of the request. This can only be used if different :py:obj:`method`</span>
<span class="sd">is set, e.g. ``POST``. For formatting see the documentation of :py:obj:`search_url`.</span>
<span class="sd">Note: Curly brackets which aren&#39;t encapsulating a replacement placeholder</span>
<span class="sd">must be escaped by doubling each ``{`` and ``}``.</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> request_body: &gt;-</span>
<span class="sd"> {{</span>
<span class="sd"> &quot;search&quot;: &quot;{query}&quot;,</span>
<span class="sd"> &quot;page&quot;: {pageno},</span>
<span class="sd"> &quot;extra&quot;: {{</span>
<span class="sd"> &quot;time_range&quot;: {time_range},</span>
<span class="sd"> &quot;rating&quot;: &quot;{safe_search}&quot;</span>
<span class="sd"> }}</span>
<span class="sd"> }}</span>
<span class="sd">&#39;&#39;&#39;</span>
<span class="n">cookies</span> <span class="o">=</span> <span class="p">{}</span>
<span class="sd">&#39;&#39;&#39;Some engines might offer different result based on cookies.</span>
<span class="sd">Possible use-case: To set safesearch cookie.&#39;&#39;&#39;</span>
<span class="n">headers</span> <span class="o">=</span> <span class="p">{}</span>
<span class="sd">&#39;&#39;&#39;Some engines might offer different result based on cookies or headers.</span>
<span class="sd">Possible use-case: To set safesearch cookie or header to moderate.&#39;&#39;&#39;</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&#39;&#39;&#39;Engine supports paging [True or False].&#39;&#39;&#39;</span>
<span class="n">page_size</span> <span class="o">=</span> <span class="mi">1</span>
<span class="sd">&#39;&#39;&#39;Number of results on each page. Only needed if the site requires not a page</span>
<span class="sd">number, but an offset.&#39;&#39;&#39;</span>
<span class="n">first_page_num</span> <span class="o">=</span> <span class="mi">1</span>
<span class="sd">&#39;&#39;&#39;Number of the first page (usually 0 or 1).&#39;&#39;&#39;</span>
<span class="n">results_query</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="sd">&#39;&#39;&#39;JSON query for the list of result items.</span>
<span class="sd">The query string is a slash `/` separated path of JSON key names.</span>
<span class="sd">Array entries can be specified using the index or can be omitted entirely,</span>
<span class="sd">in which case each entry is considered -</span>
<span class="sd">most implementations will default to the first entry in this case.</span>
<span class="sd">&#39;&#39;&#39;</span>
<span class="n">url_query</span> <span class="o">=</span> <span class="kc">None</span>
<span class="sd">&#39;&#39;&#39;JSON query of result&#39;s ``url``. For the query string documentation see :py:obj:`results_query`&#39;&#39;&#39;</span>
<span class="n">url_prefix</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&#39;&#39;&#39;String to prepend to the result&#39;s ``url``.&#39;&#39;&#39;</span>
<span class="n">title_query</span> <span class="o">=</span> <span class="kc">None</span>
<span class="sd">&#39;&#39;&#39;JSON query of result&#39;s ``title``. For the query string documentation see :py:obj:`results_query`&#39;&#39;&#39;</span>
<span class="n">content_query</span> <span class="o">=</span> <span class="kc">None</span>
<span class="sd">&#39;&#39;&#39;JSON query of result&#39;s ``content``. For the query string documentation see :py:obj:`results_query`&#39;&#39;&#39;</span>
<span class="n">thumbnail_query</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&#39;&#39;&#39;JSON query of result&#39;s ``thumbnail``. For the query string documentation see :py:obj:`results_query`&#39;&#39;&#39;</span>
<span class="n">thumbnail_prefix</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="sd">&#39;&#39;&#39;String to prepend to the result&#39;s ``thumbnail``.&#39;&#39;&#39;</span>
<span class="n">suggestion_query</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="sd">&#39;&#39;&#39;JSON query of result&#39;s ``suggestion``. For the query string documentation see :py:obj:`results_query`&#39;&#39;&#39;</span>
<span class="n">title_html_to_text</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&#39;&#39;&#39;Extract text from a HTML title string&#39;&#39;&#39;</span>
<span class="n">content_html_to_text</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&#39;&#39;&#39;Extract text from a HTML content string&#39;&#39;&#39;</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&#39;&#39;&#39;Engine supports search time range.&#39;&#39;&#39;</span>
<span class="n">time_range_url</span> <span class="o">=</span> <span class="s1">&#39;&amp;hours=</span><span class="si">{time_range_val}</span><span class="s1">&#39;</span>
<span class="sd">&#39;&#39;&#39;Time range URL parameter in the in :py:obj:`search_url`. If no time range is</span>
<span class="sd">requested by the user, the URL parameter is an empty string. The</span>
<span class="sd">``{time_range_val}`` replacement is taken from the :py:obj:`time_range_map`.</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> time_range_url : &#39;&amp;days={time_range_val}&#39;</span>
<span class="sd">&#39;&#39;&#39;</span>
<span class="n">time_range_map</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;day&#39;</span><span class="p">:</span> <span class="mi">24</span><span class="p">,</span>
<span class="s1">&#39;week&#39;</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span><span class="p">,</span>
<span class="s1">&#39;month&#39;</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">30</span><span class="p">,</span>
<span class="s1">&#39;year&#39;</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">365</span><span class="p">,</span>
<span class="p">}</span>
<span class="sd">&#39;&#39;&#39;Maps time range value from user to ``{time_range_val}`` in</span>
<span class="sd">:py:obj:`time_range_url`.</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> time_range_map:</span>
<span class="sd"> day: 1</span>
<span class="sd"> week: 7</span>
<span class="sd"> month: 30</span>
<span class="sd"> year: 365</span>
<span class="sd">&#39;&#39;&#39;</span>
<span class="n">safe_search_support</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&#39;&#39;&#39;Engine supports safe-search.&#39;&#39;&#39;</span>
<span class="n">safe_search_map</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">&#39;&amp;filter=none&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">&#39;&amp;filter=moderate&#39;</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">&#39;&amp;filter=strict&#39;</span><span class="p">}</span>
<span class="sd">&#39;&#39;&#39;Maps safe-search value to ``{safe_search}`` in :py:obj:`search_url`.</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> safesearch: true</span>
<span class="sd"> safes_search_map:</span>
<span class="sd"> 0: &#39;&amp;filter=none&#39;</span>
<span class="sd"> 1: &#39;&amp;filter=moderate&#39;</span>
<span class="sd"> 2: &#39;&amp;filter=strict&#39;</span>
<span class="sd">&#39;&#39;&#39;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">iterate</span><span class="p">(</span><span class="n">iterable</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">iterable</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="n">items</span> <span class="o">=</span> <span class="n">iterable</span><span class="o">.</span><span class="n">items</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">items</span> <span class="o">=</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">iterable</span><span class="p">)</span>
<span class="k">for</span> <span class="n">index</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">items</span><span class="p">:</span>
<span class="k">yield</span> <span class="nb">str</span><span class="p">(</span><span class="n">index</span><span class="p">),</span> <span class="n">value</span>
<span class="k">def</span><span class="w"> </span><span class="nf">is_iterable</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">Iterable</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">parse</span><span class="p">(</span><span class="n">query</span><span class="p">):</span> <span class="c1"># pylint: disable=redefined-outer-name</span>
<span class="n">q</span> <span class="o">=</span> <span class="p">[]</span> <span class="c1"># pylint: disable=invalid-name</span>
<span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">):</span>
<span class="k">if</span> <span class="n">part</span> <span class="o">==</span> <span class="s1">&#39;&#39;</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">q</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">part</span><span class="p">)</span>
<span class="k">return</span> <span class="n">q</span>
<span class="k">def</span><span class="w"> </span><span class="nf">do_query</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">q</span><span class="p">):</span> <span class="c1"># pylint: disable=invalid-name</span>
<span class="n">ret</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">q</span><span class="p">:</span>
<span class="k">return</span> <span class="n">ret</span>
<span class="n">qkey</span> <span class="o">=</span> <span class="n">q</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">iterate</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">q</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">if</span> <span class="n">key</span> <span class="o">==</span> <span class="n">qkey</span><span class="p">:</span>
<span class="n">ret</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">is_iterable</span><span class="p">(</span><span class="n">value</span><span class="p">):</span>
<span class="n">ret</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">do_query</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">q</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_iterable</span><span class="p">(</span><span class="n">value</span><span class="p">):</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="n">key</span> <span class="o">==</span> <span class="n">qkey</span><span class="p">:</span>
<span class="n">ret</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">do_query</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">q</span><span class="p">[</span><span class="mi">1</span><span class="p">:]))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">ret</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">do_query</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">q</span><span class="p">))</span>
<span class="k">return</span> <span class="n">ret</span>
<span class="k">def</span><span class="w"> </span><span class="nf">query</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">query_string</span><span class="p">):</span>
<span class="n">q</span> <span class="o">=</span> <span class="n">parse</span><span class="p">(</span><span class="n">query_string</span><span class="p">)</span>
<span class="k">return</span> <span class="n">do_query</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">q</span><span class="p">)</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/json_engine.html#searx.engines.json_engine.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span> <span class="c1"># pylint: disable=redefined-outer-name</span>
<span class="w"> </span><span class="sd">&#39;&#39;&#39;Build request parameters (see :ref:`engine request`).&#39;&#39;&#39;</span>
<span class="n">lang</span> <span class="o">=</span> <span class="n">lang_all</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;language&#39;</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">&#39;all&#39;</span><span class="p">:</span>
<span class="n">lang</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;language&#39;</span><span class="p">][:</span><span class="mi">2</span><span class="p">]</span>
<span class="n">time_range</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="k">if</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;time_range&#39;</span><span class="p">):</span>
<span class="n">time_range_val</span> <span class="o">=</span> <span class="n">time_range_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;time_range&#39;</span><span class="p">))</span>
<span class="n">time_range</span> <span class="o">=</span> <span class="n">time_range_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">time_range_val</span><span class="o">=</span><span class="n">time_range_val</span><span class="p">)</span>
<span class="n">safe_search</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]:</span>
<span class="n">safe_search</span> <span class="o">=</span> <span class="n">safe_search_map</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]]</span>
<span class="n">fp</span> <span class="o">=</span> <span class="p">{</span> <span class="c1"># pylint: disable=invalid-name</span>
<span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">})[</span><span class="mi">2</span><span class="p">:],</span>
<span class="s1">&#39;lang&#39;</span><span class="p">:</span> <span class="n">lang</span><span class="p">,</span>
<span class="s1">&#39;pageno&#39;</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">page_size</span> <span class="o">+</span> <span class="n">first_page_num</span><span class="p">,</span>
<span class="s1">&#39;time_range&#39;</span><span class="p">:</span> <span class="n">time_range</span><span class="p">,</span>
<span class="s1">&#39;safe_search&#39;</span><span class="p">:</span> <span class="n">safe_search</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">cookies</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">headers</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">fp</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;method&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">method</span>
<span class="k">if</span> <span class="n">request_body</span><span class="p">:</span>
<span class="c1"># don&#39;t url-encode the query if it&#39;s in the request body</span>
<span class="n">fp</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">query</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">request_body</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">fp</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;soft_max_redirects&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">soft_max_redirects</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;raise_for_httperror&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">return</span> <span class="n">params</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">identity</span><span class="p">(</span><span class="n">arg</span><span class="p">):</span>
<span class="k">return</span> <span class="n">arg</span>
<span class="k">def</span><span class="w"> </span><span class="nf">extract_response_info</span><span class="p">(</span><span class="n">result</span><span class="p">):</span>
<span class="n">title_filter</span> <span class="o">=</span> <span class="n">html_to_text</span> <span class="k">if</span> <span class="n">title_html_to_text</span> <span class="k">else</span> <span class="n">identity</span>
<span class="n">content_filter</span> <span class="o">=</span> <span class="n">html_to_text</span> <span class="k">if</span> <span class="n">content_html_to_text</span> <span class="k">else</span> <span class="n">identity</span>
<span class="n">tmp_result</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">query</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">url_query</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">url_prefix</span> <span class="o">+</span> <span class="n">to_string</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">query</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">title_query</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">&#39;title&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">title_filter</span><span class="p">(</span><span class="n">to_string</span><span class="p">(</span><span class="n">title</span><span class="p">))</span>
<span class="k">except</span><span class="p">:</span> <span class="c1"># pylint: disable=bare-except</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">query</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">content_query</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">&#39;content&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">content_filter</span><span class="p">(</span><span class="n">to_string</span><span class="p">(</span><span class="n">content</span><span class="p">))</span>
<span class="k">except</span><span class="p">:</span> <span class="c1"># pylint: disable=bare-except</span>
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">&#39;content&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">if</span> <span class="n">thumbnail_query</span><span class="p">:</span>
<span class="n">thumbnail_query_result</span> <span class="o">=</span> <span class="n">query</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">thumbnail_query</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">&#39;thumbnail&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">thumbnail_prefix</span> <span class="o">+</span> <span class="n">to_string</span><span class="p">(</span><span class="n">thumbnail_query_result</span><span class="p">)</span>
<span class="k">except</span><span class="p">:</span> <span class="c1"># pylint: disable=bare-except</span>
<span class="k">pass</span>
<span class="k">return</span> <span class="n">tmp_result</span>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/json_engine.html#searx.engines.json_engine.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&#39;&#39;&#39;Scrap *results* from the response (see :ref:`result types`).&#39;&#39;&#39;</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">no_result_for_http_status</span> <span class="ow">and</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="ow">in</span> <span class="n">no_result_for_http_status</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">json</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="n">is_onion</span> <span class="o">=</span> <span class="s1">&#39;onions&#39;</span> <span class="ow">in</span> <span class="n">categories</span>
<span class="k">if</span> <span class="n">results_query</span><span class="p">:</span>
<span class="n">rs</span> <span class="o">=</span> <span class="n">query</span><span class="p">(</span><span class="n">json</span><span class="p">,</span> <span class="n">results_query</span><span class="p">)</span> <span class="c1"># pylint: disable=invalid-name</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">rs</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">rs</span> <span class="o">=</span> <span class="n">rs</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="c1"># pylint: disable=invalid-name</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">rs</span> <span class="o">=</span> <span class="n">json</span> <span class="c1"># pylint: disable=invalid-name</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">rs</span><span class="p">:</span>
<span class="n">tmp_result</span> <span class="o">=</span> <span class="n">extract_response_info</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">tmp_result</span><span class="p">:</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="n">is_onion</span><span class="p">:</span>
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">&#39;is_onion&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tmp_result</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">suggestion_query</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">query</span><span class="p">(</span><span class="n">json</span><span class="p">,</span> <span class="n">suggestion_query</span><span class="p">):</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;suggestion&#39;</span><span class="p">:</span> <span class="n">suggestion</span><span class="p">})</span>
<span class="k">return</span> <span class="n">results</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,236 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.marginalia &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.marginalia</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.marginalia</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;`Marginalia Search`_ is an independent open source Internet search engine</span>
<span class="sd">operating out of Sweden. It is principally developed and operated by Viktor</span>
<span class="sd">Lofgren .</span>
<span class="sd">.. _Marginalia Search:</span>
<span class="sd"> https://about.marginalia-search.com/</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">The engine has the following required settings:</span>
<span class="sd">- :py:obj:`api_key`</span>
<span class="sd">You can configure a Marginalia engine by:</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: marginalia</span>
<span class="sd"> engine: marginalia</span>
<span class="sd"> shortcut: mar</span>
<span class="sd"> api_key: ...</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">searxng_useragent</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s2">&quot;https://marginalia.nu&quot;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s2">&quot;https://about.marginalia-search.com/article/api/&quot;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s2">&quot;JSON&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;https://api2.marginalia-search.com&quot;</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;general&quot;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">results_per_page</span> <span class="o">=</span> <span class="mi">20</span>
<span class="n">api_key</span> <span class="o">=</span> <span class="kc">None</span>
<span class="sd">&quot;&quot;&quot;To get an API key, please follow the instructions from `Key and license`_</span>
<span class="sd">.. _Key and license:</span>
<span class="sd"> https://about.marginalia-search.com/article/api/</span>
<span class="sd">&quot;&quot;&quot;</span>
<div class="viewcode-block" id="ApiSearchResult">
<a class="viewcode-back" href="../../../dev/engines/online/marginalia.html#searx.engines.marginalia.ApiSearchResult">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">ApiSearchResult</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">TypedDict</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Marginalia&#39;s ApiSearchResult_ class definition.</span>
<span class="sd"> .. _ApiSearchResult:</span>
<span class="sd"> https://github.com/MarginaliaSearch/MarginaliaSearch/blob/master/code/services-application/api-service/java/nu/marginalia/api/model/ApiSearchResult.java</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">url</span><span class="p">:</span> <span class="nb">str</span>
<span class="n">title</span><span class="p">:</span> <span class="nb">str</span>
<span class="n">description</span><span class="p">:</span> <span class="nb">str</span>
<span class="n">quality</span><span class="p">:</span> <span class="nb">float</span>
<span class="nb">format</span><span class="p">:</span> <span class="nb">str</span>
<span class="n">details</span><span class="p">:</span> <span class="nb">str</span></div>
<div class="viewcode-block" id="ApiSearchResults">
<a class="viewcode-back" href="../../../dev/engines/online/marginalia.html#searx.engines.marginalia.ApiSearchResults">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">ApiSearchResults</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">TypedDict</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Marginalia&#39;s ApiSearchResults_ class definition.</span>
<span class="sd"> .. _ApiSearchResults:</span>
<span class="sd"> https://github.com/MarginaliaSearch/MarginaliaSearch/blob/master/code/services-application/api-service/java/nu/marginalia/api/model/ApiSearchResults.java</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">license</span><span class="p">:</span> <span class="nb">str</span>
<span class="n">query</span><span class="p">:</span> <span class="nb">str</span>
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">ApiSearchResult</span><span class="p">]</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]):</span>
<span class="n">query_params</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;count&quot;</span><span class="p">:</span> <span class="n">results_per_page</span><span class="p">,</span> <span class="s2">&quot;nsfw&quot;</span><span class="p">:</span> <span class="nb">min</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;safesearch&quot;</span><span class="p">],</span> <span class="mi">1</span><span class="p">),</span> <span class="s2">&quot;query&quot;</span><span class="p">:</span> <span class="n">query</span><span class="p">}</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">/search?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">query_params</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;headers&quot;</span><span class="p">][</span><span class="s2">&quot;User-Agent&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">searxng_useragent</span><span class="p">()</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;headers&quot;</span><span class="p">][</span><span class="s2">&quot;API-Key&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">api_key</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="n">SXNG_Response</span><span class="p">):</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">resp_json</span><span class="p">:</span> <span class="n">ApiSearchResults</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span> <span class="c1"># type: ignore</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">resp_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;results&quot;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">MainResult</span><span class="p">(</span>
<span class="n">title</span><span class="o">=</span><span class="n">item</span><span class="p">[</span><span class="s2">&quot;title&quot;</span><span class="p">],</span>
<span class="n">url</span><span class="o">=</span><span class="n">item</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">],</span>
<span class="n">content</span><span class="o">=</span><span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;description&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">res</span>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]):</span>
<span class="n">_api_key</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;api_key&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">_api_key</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;missing api_key: see https://about.marginalia-search.com/article/api&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="k">if</span> <span class="n">_api_key</span> <span class="o">==</span> <span class="s2">&quot;public&quot;</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;invalid api_key (</span><span class="si">%s</span><span class="s2">): see https://about.marginalia-search.com/article/api&quot;</span><span class="p">,</span> <span class="n">api_key</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,181 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.mrs &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.mrs</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.mrs</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Matrix Rooms Search - a fully-featured, standalone, matrix rooms search service.</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">The engine has the following mandatory settings:</span>
<span class="sd">- :py:obj:`base_url`</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: MRS</span>
<span class="sd"> engine: mrs</span>
<span class="sd"> base_url: https://mrs-host</span>
<span class="sd"> ...</span>
<span class="sd">Implementation</span>
<span class="sd">==============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">quote_plus</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://matrixrooms.info&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://gitlab.com/etke.cc/mrs/api/-/blob/main/openapi.yml?ref_type=heads&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;JSON&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;social media&#39;</span><span class="p">]</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="n">matrix_url</span> <span class="o">=</span> <span class="s2">&quot;https://matrix.to&quot;</span>
<span class="n">page_size</span> <span class="o">=</span> <span class="mi">20</span>
<div class="viewcode-block" id="init">
<a class="viewcode-back" href="../../../dev/engines/online/mrs.html#searx.engines.mrs.init">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">):</span> <span class="c1"># pylint: disable=unused-argument</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The ``base_url`` must be set in the configuration, if ``base_url`` is not</span>
<span class="sd"> set, a :py:obj:`ValueError` is raised during initialization.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">base_url</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;engine MRS, base_url is unset&#39;</span><span class="p">)</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">/search/</span><span class="si">{</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">query</span><span class="p">)</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="n">page_size</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">*</span><span class="n">page_size</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">return</span> <span class="n">params</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">():</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">matrix_url</span> <span class="o">+</span> <span class="s1">&#39;/#/&#39;</span> <span class="o">+</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;alias&#39;</span><span class="p">],</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">],</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;topic&#39;</span><span class="p">]</span>
<span class="o">+</span> <span class="sa">f</span><span class="s2">&quot; // </span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;members&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2"> members&quot;</span>
<span class="o">+</span> <span class="sa">f</span><span class="s2">&quot; // </span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;alias&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="o">+</span> <span class="sa">f</span><span class="s2">&quot; // </span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;server&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
<span class="s1">&#39;thumbnail&#39;</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;avatar_url&#39;</span><span class="p">],</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,253 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.odysee &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.odysee</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.odysee</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Odysee_ is a decentralized video hosting platform.</span>
<span class="sd">.. _Odysee: https://github.com/OdyseeTeam/odysee-frontend</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="c1"># Engine metadata</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s2">&quot;https://odysee.com/&quot;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s2">&quot;Q102046570&quot;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s2">&quot;JSON&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># Engine configuration</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">results_per_page</span> <span class="o">=</span> <span class="mi">20</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;videos&#39;</span><span class="p">]</span>
<span class="c1"># Search URL (Note: lighthouse.lbry.com/search works too, and may be faster at times)</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;https://lighthouse.odysee.tv/search&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="n">time_range_dict</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;day&quot;</span><span class="p">:</span> <span class="s2">&quot;today&quot;</span><span class="p">,</span>
<span class="s2">&quot;week&quot;</span><span class="p">:</span> <span class="s2">&quot;thisweek&quot;</span><span class="p">,</span>
<span class="s2">&quot;month&quot;</span><span class="p">:</span> <span class="s2">&quot;thismonth&quot;</span><span class="p">,</span>
<span class="s2">&quot;year&quot;</span><span class="p">:</span> <span class="s2">&quot;thisyear&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">start_index</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;pageno&quot;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">results_per_page</span>
<span class="n">query_params</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;s&quot;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s2">&quot;size&quot;</span><span class="p">:</span> <span class="n">results_per_page</span><span class="p">,</span>
<span class="s2">&quot;from&quot;</span><span class="p">:</span> <span class="n">start_index</span><span class="p">,</span>
<span class="s2">&quot;include&quot;</span><span class="p">:</span> <span class="s2">&quot;channel,thumbnail_url,title,description,duration,release_time&quot;</span><span class="p">,</span>
<span class="s2">&quot;mediaType&quot;</span><span class="p">:</span> <span class="s2">&quot;video&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">lang</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">lang</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">query_params</span><span class="p">[</span><span class="s1">&#39;language&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">lang</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
<span class="n">query_params</span><span class="p">[</span><span class="s1">&#39;time_filter&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]]</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">query_params</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">return</span> <span class="n">params</span>
<span class="c1"># Format the video duration</span>
<span class="k">def</span><span class="w"> </span><span class="nf">format_duration</span><span class="p">(</span><span class="n">duration</span><span class="p">):</span>
<span class="n">seconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">duration</span><span class="p">)</span>
<span class="n">length</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">gmtime</span><span class="p">(</span><span class="n">seconds</span><span class="p">)</span>
<span class="k">if</span> <span class="n">length</span><span class="o">.</span><span class="n">tm_hour</span><span class="p">:</span>
<span class="k">return</span> <span class="n">time</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%H:%M:%S&quot;</span><span class="p">,</span> <span class="n">length</span><span class="p">)</span>
<span class="k">return</span> <span class="n">time</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%M:%S&quot;</span><span class="p">,</span> <span class="n">length</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span>
<span class="n">name</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;name&quot;</span><span class="p">]</span>
<span class="n">claim_id</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;claimId&quot;</span><span class="p">]</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;title&quot;</span><span class="p">]</span>
<span class="n">thumbnail_url</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;thumbnail_url&quot;</span><span class="p">]</span>
<span class="n">description</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;description&quot;</span><span class="p">]</span> <span class="ow">or</span> <span class="s2">&quot;&quot;</span>
<span class="n">channel</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;channel&quot;</span><span class="p">]</span>
<span class="n">release_time</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;release_time&quot;</span><span class="p">]</span>
<span class="n">duration</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;duration&quot;</span><span class="p">]</span>
<span class="n">release_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">release_time</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;T&quot;</span><span class="p">)[</span><span class="mi">0</span><span class="p">],</span> <span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">formatted_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">release_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">())</span>
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;https://odysee.com/</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">claim_id</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">iframe_url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;https://odysee.com/$/embed/</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">claim_id</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">odysee_thumbnail</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;https://thumbnails.odycdn.com/optimize/s:390:0/quality:85/plain/</span><span class="si">{</span><span class="n">thumbnail_url</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">formatted_duration</span> <span class="o">=</span> <span class="n">format_duration</span><span class="p">(</span><span class="n">duration</span><span class="p">)</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s2">&quot;title&quot;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
<span class="s2">&quot;url&quot;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
<span class="s2">&quot;content&quot;</span><span class="p">:</span> <span class="n">description</span><span class="p">,</span>
<span class="s2">&quot;author&quot;</span><span class="p">:</span> <span class="n">channel</span><span class="p">,</span>
<span class="s2">&quot;publishedDate&quot;</span><span class="p">:</span> <span class="n">formatted_date</span><span class="p">,</span>
<span class="s2">&quot;length&quot;</span><span class="p">:</span> <span class="n">formatted_duration</span><span class="p">,</span>
<span class="s2">&quot;thumbnail&quot;</span><span class="p">:</span> <span class="n">odysee_thumbnail</span><span class="p">,</span>
<span class="s2">&quot;iframe_src&quot;</span><span class="p">:</span> <span class="n">iframe_url</span><span class="p">,</span>
<span class="s2">&quot;template&quot;</span><span class="p">:</span> <span class="s2">&quot;videos.html&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span>
<div class="viewcode-block" id="fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/online/odysee.html#searx.engines.odysee.fetch_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Fetch languages from Odysee&#39;s source code.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span>
<span class="s1">&#39;https://raw.githubusercontent.com/OdyseeTeam/odysee-frontend/master/ui/constants/supported_browser_languages.js&#39;</span><span class="p">,</span> <span class="c1"># pylint: disable=line-too-long</span>
<span class="n">timeout</span><span class="o">=</span><span class="mi">60</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: can&#39;t determine languages from Odysee&quot;</span><span class="p">)</span>
<span class="k">return</span>
<span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)[</span><span class="mi">1</span><span class="p">:</span><span class="o">-</span><span class="mi">4</span><span class="p">]:</span>
<span class="n">lang_tag</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;: &quot;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;&#39;&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">lang_tag</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot;-&quot;</span><span class="p">))</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: </span><span class="si">%s</span><span class="s2"> is unknown by babel&quot;</span> <span class="o">%</span> <span class="n">lang_tag</span><span class="p">)</span>
<span class="k">continue</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">lang_tag</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">lang_tag</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">lang_tag</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,299 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.peertube &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.peertube</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.peertube</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Peertube and :py:obj:`SepiaSearch &lt;searx.engines.sepiasearch&gt;` do share</span>
<span class="sd">(more or less) the same REST API and the schema of the JSON result is identical.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil.parser</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil.relativedelta</span><span class="w"> </span><span class="kn">import</span> <span class="n">relativedelta</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">html_to_text</span><span class="p">,</span> <span class="n">humanize_number</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="c1"># pylint: disable=line-too-long</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://joinpeertube.org&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q50938515&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://docs.joinpeertube.org/api-rest-reference.html#tag/Search/operation/searchVideos&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;JSON&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;videos&quot;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;https://peer.tube&quot;</span>
<span class="sd">&quot;&quot;&quot;Base URL of the Peertube instance. A list of instances is available at:</span>
<span class="sd">- https://instances.joinpeertube.org/instances</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">time_range_table</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;day&#39;</span><span class="p">:</span> <span class="n">relativedelta</span><span class="p">(),</span>
<span class="s1">&#39;week&#39;</span><span class="p">:</span> <span class="n">relativedelta</span><span class="p">(</span><span class="n">weeks</span><span class="o">=-</span><span class="mi">1</span><span class="p">),</span>
<span class="s1">&#39;month&#39;</span><span class="p">:</span> <span class="n">relativedelta</span><span class="p">(</span><span class="n">months</span><span class="o">=-</span><span class="mi">1</span><span class="p">),</span>
<span class="s1">&#39;year&#39;</span><span class="p">:</span> <span class="n">relativedelta</span><span class="p">(</span><span class="n">years</span><span class="o">=-</span><span class="mi">1</span><span class="p">),</span>
<span class="p">}</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">safesearch_table</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">&#39;both&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">&#39;false&#39;</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">&#39;false&#39;</span><span class="p">}</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/peertube.html#searx.engines.peertube.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Assemble request for the Peertube API&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">query</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="c1"># eng_region = traits.get_region(params[&#39;searxng_locale&#39;], &#39;en_US&#39;)</span>
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="kc">None</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">base_url</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">)</span>
<span class="o">+</span> <span class="s2">&quot;/api/v1/search/videos?&quot;</span>
<span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;search&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;searchTarget&#39;</span><span class="p">:</span> <span class="s1">&#39;search-index&#39;</span><span class="p">,</span> <span class="c1"># Vidiversum</span>
<span class="s1">&#39;resultType&#39;</span><span class="p">:</span> <span class="s1">&#39;videos&#39;</span><span class="p">,</span>
<span class="s1">&#39;start&#39;</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span><span class="p">,</span>
<span class="s1">&#39;count&#39;</span><span class="p">:</span> <span class="mi">10</span><span class="p">,</span>
<span class="c1"># -createdAt: sort by date ascending / createdAt: date descending</span>
<span class="s1">&#39;sort&#39;</span><span class="p">:</span> <span class="s1">&#39;-match&#39;</span><span class="p">,</span> <span class="c1"># sort by *match descending*</span>
<span class="s1">&#39;nsfw&#39;</span><span class="p">:</span> <span class="n">safesearch_table</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]],</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">eng_lang</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">&#39;&amp;languageOneOf[]=&#39;</span> <span class="o">+</span> <span class="n">eng_lang</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">&#39;&amp;boostLanguages[]=&#39;</span> <span class="o">+</span> <span class="n">eng_lang</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_table</span><span class="p">:</span>
<span class="n">time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">date</span><span class="p">()</span> <span class="o">+</span> <span class="n">time_range_table</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]]</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">&#39;&amp;startDate=&#39;</span> <span class="o">+</span> <span class="n">time</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
<span class="k">return</span> <span class="n">params</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="k">return</span> <span class="n">video_response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<div class="viewcode-block" id="video_response">
<a class="viewcode-back" href="../../../dev/engines/online/peertube.html#searx.engines.peertube.video_response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">video_response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Parse video response from SepiaSearch and Peertube instances.&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">if</span> <span class="s1">&#39;data&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">:</span>
<span class="k">return</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">]:</span>
<span class="n">metadata</span> <span class="o">=</span> <span class="p">[</span>
<span class="n">x</span>
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span>
<span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;channel&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;displayName&#39;</span><span class="p">),</span>
<span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;channel&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;name&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="s1">&#39;@&#39;</span> <span class="o">+</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;channel&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;host&#39;</span><span class="p">),</span>
<span class="s1">&#39;, &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;tags&#39;</span><span class="p">,</span> <span class="p">[])),</span>
<span class="p">]</span>
<span class="k">if</span> <span class="n">x</span>
<span class="p">]</span>
<span class="n">duration</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;duration&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">duration</span><span class="p">:</span>
<span class="n">duration</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">seconds</span><span class="o">=</span><span class="n">duration</span><span class="p">)</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">],</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">],</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">html_to_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;description&#39;</span><span class="p">)</span> <span class="ow">or</span> <span class="s1">&#39;&#39;</span><span class="p">),</span>
<span class="s1">&#39;author&#39;</span><span class="p">:</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;account&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;displayName&#39;</span><span class="p">),</span>
<span class="s1">&#39;length&#39;</span><span class="p">:</span> <span class="n">duration</span><span class="p">,</span>
<span class="s1">&#39;views&#39;</span><span class="p">:</span> <span class="n">humanize_number</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;views&#39;</span><span class="p">]),</span>
<span class="s1">&#39;template&#39;</span><span class="p">:</span> <span class="s1">&#39;videos.html&#39;</span><span class="p">,</span>
<span class="s1">&#39;publishedDate&#39;</span><span class="p">:</span> <span class="n">parse</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;publishedAt&#39;</span><span class="p">]),</span>
<span class="s1">&#39;iframe_src&#39;</span><span class="p">:</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;embedUrl&#39;</span><span class="p">),</span>
<span class="s1">&#39;thumbnail&#39;</span><span class="p">:</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;thumbnailUrl&#39;</span><span class="p">)</span> <span class="ow">or</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;previewUrl&#39;</span><span class="p">),</span>
<span class="s1">&#39;metadata&#39;</span><span class="p">:</span> <span class="s1">&#39; | &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">metadata</span><span class="p">),</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span></div>
<div class="viewcode-block" id="fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/online/peertube.html#searx.engines.peertube.fetch_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Fetch languages from peertube&#39;s search-index source code.</span>
<span class="sd"> See videoLanguages_ in commit `8ed5c729 - Refactor and redesign client`_</span>
<span class="sd"> .. _8ed5c729 - Refactor and redesign client:</span>
<span class="sd"> https://framagit.org/framasoft/peertube/search-index/-/commit/8ed5c729</span>
<span class="sd"> .. _videoLanguages:</span>
<span class="sd"> https://framagit.org/framasoft/peertube/search-index/-/commit/8ed5c729#3d8747f9a60695c367c70bb64efba8f403721fad_0_291</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span>
<span class="s1">&#39;https://framagit.org/framasoft/peertube/search-index/-/raw/master/client/src/components/Filters.vue&#39;</span><span class="p">,</span>
<span class="c1"># the response from search-index repository is very slow</span>
<span class="n">timeout</span><span class="o">=</span><span class="mi">60</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: response from peertube is not OK.&quot;</span><span class="p">)</span>
<span class="k">return</span>
<span class="n">js_lang</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;videoLanguages \(\)[^\n]+(.*?)\]&quot;</span><span class="p">,</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">DOTALL</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">js_lang</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: can&#39;t determine languages from peertube&quot;</span><span class="p">)</span>
<span class="k">return</span>
<span class="k">for</span> <span class="n">lang</span> <span class="ow">in</span> <span class="n">re</span><span class="o">.</span><span class="n">finditer</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;\{ id: &#39;([a-z]+)&#39;, label:&quot;</span><span class="p">,</span> <span class="n">js_lang</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)):</span>
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">lang</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
<span class="k">if</span> <span class="n">eng_tag</span> <span class="o">==</span> <span class="s1">&#39;oc&#39;</span><span class="p">:</span>
<span class="c1"># Occitanis not known by babel, its closest relative is Catalan</span>
<span class="c1"># but &#39;ca&#39; is already in the list of engine_traits.languages --&gt;</span>
<span class="c1"># &#39;oc&#39; will be ignored.</span>
<span class="k">continue</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">))</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: </span><span class="si">%s</span><span class="s2"> is unknown by babel&quot;</span> <span class="o">%</span> <span class="n">eng_tag</span><span class="p">)</span>
<span class="k">continue</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="s1">&#39;zh_Hans&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;zh&#39;</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="s1">&#39;zh_Hant&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;zh&#39;</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,482 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.qwant &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.qwant</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.qwant</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;This engine uses the Qwant API (https://api.qwant.com/v3) to implement Qwant</span>
<span class="sd">-Web, -News, -Images and -Videos. The API is undocumented but can be reverse</span>
<span class="sd">engineered by reading the network log of https://www.qwant.com/ queries.</span>
<span class="sd">For Qwant&#39;s *web-search* two alternatives are implemented:</span>
<span class="sd">- ``web``: uses the :py:obj:`api_url` which returns a JSON structure</span>
<span class="sd">- ``web-lite``: uses the :py:obj:`web_lite_url` which returns a HTML page</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">The engine has the following additional settings:</span>
<span class="sd">- :py:obj:`qwant_categ`</span>
<span class="sd">This implementation is used by different qwant engines in the :ref:`settings.yml</span>
<span class="sd">&lt;settings engines&gt;`:</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: qwant</span>
<span class="sd"> qwant_categ: web-lite # alternatively use &#39;web&#39;</span>
<span class="sd"> ...</span>
<span class="sd"> - name: qwant news</span>
<span class="sd"> qwant_categ: news</span>
<span class="sd"> ...</span>
<span class="sd"> - name: qwant images</span>
<span class="sd"> qwant_categ: images</span>
<span class="sd"> ...</span>
<span class="sd"> - name: qwant videos</span>
<span class="sd"> qwant_categ: videos</span>
<span class="sd"> ...</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">datetime</span><span class="p">,</span>
<span class="n">timedelta</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">json</span><span class="w"> </span><span class="kn">import</span> <span class="n">loads</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">lxml</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">SearxEngineAPIException</span><span class="p">,</span>
<span class="n">SearxEngineTooManyRequestsException</span><span class="p">,</span>
<span class="n">SearxEngineCaptchaException</span><span class="p">,</span>
<span class="n">SearxEngineAccessDeniedException</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">raise_for_httperror</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">eval_xpath</span><span class="p">,</span>
<span class="n">eval_xpath_list</span><span class="p">,</span>
<span class="n">extract_text</span><span class="p">,</span>
<span class="n">get_embeded_stream_url</span><span class="p">,</span>
<span class="p">)</span>
<span class="c1"># about</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.qwant.com/&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q14657870&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;JSON&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">5</span>
<span class="sd">&quot;&quot;&quot;5 pages maximum (``&amp;p=5``): Trying to do more just results in an improper</span>
<span class="sd">redirect&quot;&quot;&quot;</span>
<span class="c1"># Otherwise Qwant will return 403 if not set</span>
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">qwant_categ</span> <span class="o">=</span> <span class="kc">None</span>
<span class="sd">&quot;&quot;&quot;One of ``web-lite`` (or ``web``), ``news``, ``images`` or ``videos``&quot;&quot;&quot;</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
<span class="c1"># safe_search_map = {0: &#39;&amp;safesearch=0&#39;, 1: &#39;&amp;safesearch=1&#39;, 2: &#39;&amp;safesearch=2&#39;}</span>
<span class="c1"># fmt: off</span>
<span class="n">qwant_news_locales</span> <span class="o">=</span> <span class="p">[</span>
<span class="s1">&#39;ca_ad&#39;</span><span class="p">,</span> <span class="s1">&#39;ca_es&#39;</span><span class="p">,</span> <span class="s1">&#39;ca_fr&#39;</span><span class="p">,</span> <span class="s1">&#39;co_fr&#39;</span><span class="p">,</span> <span class="s1">&#39;de_at&#39;</span><span class="p">,</span> <span class="s1">&#39;de_ch&#39;</span><span class="p">,</span> <span class="s1">&#39;de_de&#39;</span><span class="p">,</span> <span class="s1">&#39;en_au&#39;</span><span class="p">,</span>
<span class="s1">&#39;en_ca&#39;</span><span class="p">,</span> <span class="s1">&#39;en_gb&#39;</span><span class="p">,</span> <span class="s1">&#39;en_ie&#39;</span><span class="p">,</span> <span class="s1">&#39;en_my&#39;</span><span class="p">,</span> <span class="s1">&#39;en_nz&#39;</span><span class="p">,</span> <span class="s1">&#39;en_us&#39;</span><span class="p">,</span> <span class="s1">&#39;es_ad&#39;</span><span class="p">,</span> <span class="s1">&#39;es_ar&#39;</span><span class="p">,</span>
<span class="s1">&#39;es_cl&#39;</span><span class="p">,</span> <span class="s1">&#39;es_co&#39;</span><span class="p">,</span> <span class="s1">&#39;es_es&#39;</span><span class="p">,</span> <span class="s1">&#39;es_mx&#39;</span><span class="p">,</span> <span class="s1">&#39;es_pe&#39;</span><span class="p">,</span> <span class="s1">&#39;eu_es&#39;</span><span class="p">,</span> <span class="s1">&#39;eu_fr&#39;</span><span class="p">,</span> <span class="s1">&#39;fc_ca&#39;</span><span class="p">,</span>
<span class="s1">&#39;fr_ad&#39;</span><span class="p">,</span> <span class="s1">&#39;fr_be&#39;</span><span class="p">,</span> <span class="s1">&#39;fr_ca&#39;</span><span class="p">,</span> <span class="s1">&#39;fr_ch&#39;</span><span class="p">,</span> <span class="s1">&#39;fr_fr&#39;</span><span class="p">,</span> <span class="s1">&#39;it_ch&#39;</span><span class="p">,</span> <span class="s1">&#39;it_it&#39;</span><span class="p">,</span> <span class="s1">&#39;nl_be&#39;</span><span class="p">,</span>
<span class="s1">&#39;nl_nl&#39;</span><span class="p">,</span> <span class="s1">&#39;pt_ad&#39;</span><span class="p">,</span> <span class="s1">&#39;pt_pt&#39;</span><span class="p">,</span>
<span class="p">]</span>
<span class="c1"># fmt: on</span>
<span class="c1"># search-url</span>
<span class="n">api_url</span> <span class="o">=</span> <span class="s1">&#39;https://api.qwant.com/v3/search/&#39;</span>
<span class="sd">&quot;&quot;&quot;URL of Qwant&#39;s API (JSON)&quot;&quot;&quot;</span>
<span class="n">web_lite_url</span> <span class="o">=</span> <span class="s1">&#39;https://lite.qwant.com/&#39;</span>
<span class="sd">&quot;&quot;&quot;URL of Qwant-Lite (HTML)&quot;&quot;&quot;</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/qwant.html#searx.engines.qwant.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Qwant search request&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">query</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">q_locale</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;searxng_locale&quot;</span><span class="p">],</span> <span class="n">default</span><span class="o">=</span><span class="s1">&#39;en_US&#39;</span><span class="p">)</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">api_url</span> <span class="o">+</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">qwant_categ</span><span class="si">}</span><span class="s1">?&#39;</span>
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">}</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;raise_for_httperror&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">if</span> <span class="n">qwant_categ</span> <span class="o">==</span> <span class="s1">&#39;web-lite&#39;</span><span class="p">:</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">web_lite_url</span> <span class="o">+</span> <span class="s1">&#39;?&#39;</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;locale&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">q_locale</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;l&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">q_locale</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;s&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;p&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;raise_for_httperror&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">elif</span> <span class="n">qwant_categ</span> <span class="o">==</span> <span class="s1">&#39;images&#39;</span><span class="p">:</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;count&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">50</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;locale&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">q_locale</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;tgp&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">3</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;offset&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">args</span><span class="p">[</span><span class="s1">&#39;count&#39;</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span> <span class="c1"># web, news, videos</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;count&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">10</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;locale&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">q_locale</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;llm&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;false&#39;</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;tgp&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">3</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;offset&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">args</span><span class="p">[</span><span class="s1">&#39;count&#39;</span><span class="p">]</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">url</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
<span class="k">return</span> <span class="n">params</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="k">if</span> <span class="n">qwant_categ</span> <span class="o">==</span> <span class="s1">&#39;web-lite&#39;</span><span class="p">:</span>
<span class="k">return</span> <span class="n">parse_web_lite</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<span class="k">return</span> <span class="n">parse_web_api</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<div class="viewcode-block" id="parse_web_lite">
<a class="viewcode-back" href="../../../dev/engines/online/qwant.html#searx.engines.qwant.parse_web_lite">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">parse_web_lite</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Parse results from Qwant-Lite&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">&#39;//section/article&#39;</span><span class="p">):</span>
<span class="k">if</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">&quot;./span[contains(@class, &#39;tooltip&#39;)]&quot;</span><span class="p">):</span>
<span class="c1"># ignore randomly interspersed advertising adds</span>
<span class="k">continue</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">&quot;./span[contains(@class, &#39;url partner&#39;)]&quot;</span><span class="p">)),</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;./h2/a&#39;</span><span class="p">)),</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;./p&#39;</span><span class="p">)),</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span></div>
<div class="viewcode-block" id="parse_web_api">
<a class="viewcode-back" href="../../../dev/engines/online/qwant.html#searx.engines.qwant.parse_web_api">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">parse_web_api</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Parse results from Qwant&#39;s API&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-many-locals, too-many-branches, too-many-statements</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="c1"># Try to load JSON result</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">search_results</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="n">search_results</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">search_results</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;data&#39;</span><span class="p">,</span> <span class="p">{})</span>
<span class="c1"># check for an API error</span>
<span class="k">if</span> <span class="n">search_results</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;status&#39;</span><span class="p">)</span> <span class="o">!=</span> <span class="s1">&#39;success&#39;</span><span class="p">:</span>
<span class="n">error_code</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;error_code&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">error_code</span> <span class="o">==</span> <span class="mi">24</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">SearxEngineTooManyRequestsException</span><span class="p">()</span>
<span class="k">if</span> <span class="n">search_results</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;data&quot;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;error_data&quot;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;captchaUrl&quot;</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">()</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">403</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">SearxEngineAccessDeniedException</span><span class="p">()</span>
<span class="n">msg</span> <span class="o">=</span> <span class="s2">&quot;,&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;message&#39;</span><span class="p">,</span> <span class="p">[</span><span class="s1">&#39;unknown&#39;</span><span class="p">]))</span>
<span class="k">raise</span> <span class="n">SearxEngineAPIException</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">msg</span><span class="si">}</span><span class="s2"> (</span><span class="si">{</span><span class="n">error_code</span><span class="si">}</span><span class="s2">)&quot;</span><span class="p">)</span>
<span class="c1"># raise for other errors</span>
<span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<span class="k">if</span> <span class="n">qwant_categ</span> <span class="o">==</span> <span class="s1">&#39;web&#39;</span><span class="p">:</span>
<span class="c1"># The WEB query contains a list named &#39;mainline&#39;. This list can contain</span>
<span class="c1"># different result types (e.g. mainline[0][&#39;type&#39;] returns type of the</span>
<span class="c1"># result items in mainline[0][&#39;items&#39;]</span>
<span class="n">mainline</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;result&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;items&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mainline&#39;</span><span class="p">,</span> <span class="p">{})</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># Queries on News, Images and Videos do not have a list named &#39;mainline&#39;</span>
<span class="c1"># in the response. The result items are directly in the list</span>
<span class="c1"># result[&#39;items&#39;].</span>
<span class="n">mainline</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;result&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;items&#39;</span><span class="p">,</span> <span class="p">[])</span>
<span class="n">mainline</span> <span class="o">=</span> <span class="p">[</span>
<span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="n">qwant_categ</span><span class="p">,</span> <span class="s1">&#39;items&#39;</span><span class="p">:</span> <span class="n">mainline</span><span class="p">},</span>
<span class="p">]</span>
<span class="c1"># return empty array if there are no results</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">mainline</span><span class="p">:</span>
<span class="k">return</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">mainline</span><span class="p">:</span>
<span class="n">mainline_type</span> <span class="o">=</span> <span class="n">row</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;type&#39;</span><span class="p">,</span> <span class="s1">&#39;web&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">mainline_type</span> <span class="o">!=</span> <span class="n">qwant_categ</span><span class="p">:</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="n">mainline_type</span> <span class="o">==</span> <span class="s1">&#39;ads&#39;</span><span class="p">:</span>
<span class="c1"># ignore adds</span>
<span class="k">continue</span>
<span class="n">mainline_items</span> <span class="o">=</span> <span class="n">row</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;items&#39;</span><span class="p">,</span> <span class="p">[])</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">mainline_items</span><span class="p">:</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;title&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="n">res_url</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;url&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">mainline_type</span> <span class="o">==</span> <span class="s1">&#39;web&#39;</span><span class="p">:</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">&#39;desc&#39;</span><span class="p">]</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">res_url</span><span class="p">,</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">elif</span> <span class="n">mainline_type</span> <span class="o">==</span> <span class="s1">&#39;news&#39;</span><span class="p">:</span>
<span class="n">pub_date</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">&#39;date&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">pub_date</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">pub_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">pub_date</span><span class="p">)</span>
<span class="n">news_media</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;media&#39;</span><span class="p">,</span> <span class="p">[])</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">news_media</span><span class="p">:</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">news_media</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;pict&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;url&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">res_url</span><span class="p">,</span>
<span class="s1">&#39;publishedDate&#39;</span><span class="p">:</span> <span class="n">pub_date</span><span class="p">,</span>
<span class="s1">&#39;thumbnail&#39;</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">elif</span> <span class="n">mainline_type</span> <span class="o">==</span> <span class="s1">&#39;images&#39;</span><span class="p">:</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">&#39;thumbnail&#39;</span><span class="p">]</span>
<span class="n">img_src</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">&#39;media&#39;</span><span class="p">]</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">res_url</span><span class="p">,</span>
<span class="s1">&#39;template&#39;</span><span class="p">:</span> <span class="s1">&#39;images.html&#39;</span><span class="p">,</span>
<span class="s1">&#39;thumbnail_src&#39;</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
<span class="s1">&#39;img_src&#39;</span><span class="p">:</span> <span class="n">img_src</span><span class="p">,</span>
<span class="s1">&#39;resolution&#39;</span><span class="p">:</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">item</span><span class="p">[</span><span class="s1">&#39;width&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2"> x </span><span class="si">{</span><span class="n">item</span><span class="p">[</span><span class="s1">&#39;height&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
<span class="s1">&#39;img_format&#39;</span><span class="p">:</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;thumb_type&#39;</span><span class="p">),</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">elif</span> <span class="n">mainline_type</span> <span class="o">==</span> <span class="s1">&#39;videos&#39;</span><span class="p">:</span>
<span class="c1"># some videos do not have a description: while qwant-video</span>
<span class="c1"># returns an empty string, such video from a qwant-web query</span>
<span class="c1"># miss the &#39;desc&#39; key.</span>
<span class="n">d</span><span class="p">,</span> <span class="n">s</span><span class="p">,</span> <span class="n">c</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;desc&#39;</span><span class="p">),</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;source&#39;</span><span class="p">),</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;channel&#39;</span><span class="p">)</span>
<span class="n">content_parts</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">d</span><span class="p">:</span>
<span class="n">content_parts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">d</span><span class="p">)</span>
<span class="k">if</span> <span class="n">s</span><span class="p">:</span>
<span class="n">content_parts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2"> &quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Source&quot;</span><span class="p">),</span> <span class="n">s</span><span class="p">))</span>
<span class="k">if</span> <span class="n">c</span><span class="p">:</span>
<span class="n">content_parts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2"> &quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Channel&quot;</span><span class="p">),</span> <span class="n">c</span><span class="p">))</span>
<span class="n">content</span> <span class="o">=</span> <span class="s1">&#39; // &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">content_parts</span><span class="p">)</span>
<span class="n">length</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">&#39;duration&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">length</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">length</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">milliseconds</span><span class="o">=</span><span class="n">length</span><span class="p">)</span>
<span class="n">pub_date</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">&#39;date&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">pub_date</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">pub_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">pub_date</span><span class="p">)</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">&#39;thumbnail&#39;</span><span class="p">]</span>
<span class="c1"># from some locations (DE and others?) the s2 link do</span>
<span class="c1"># response a &#39;Please wait ..&#39; but does not deliver the thumbnail</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">thumbnail</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;https://s2.qwant.com&#39;</span><span class="p">,</span> <span class="s1">&#39;https://s1.qwant.com&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">res_url</span><span class="p">,</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
<span class="s1">&#39;iframe_src&#39;</span><span class="p">:</span> <span class="n">get_embeded_stream_url</span><span class="p">(</span><span class="n">res_url</span><span class="p">),</span>
<span class="s1">&#39;publishedDate&#39;</span><span class="p">:</span> <span class="n">pub_date</span><span class="p">,</span>
<span class="s1">&#39;thumbnail&#39;</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
<span class="s1">&#39;template&#39;</span><span class="p">:</span> <span class="s1">&#39;videos.html&#39;</span><span class="p">,</span>
<span class="s1">&#39;length&#39;</span><span class="p">:</span> <span class="n">length</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="c1"># pylint: disable=import-outside-toplevel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">network</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">region_tag</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extr</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">about</span><span class="p">[</span><span class="s1">&#39;website&#39;</span><span class="p">])</span>
<span class="n">json_string</span> <span class="o">=</span> <span class="n">extr</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="s1">&#39;INITIAL_PROPS = &#39;</span><span class="p">,</span> <span class="s1">&#39;&lt;/script&gt;&#39;</span><span class="p">)</span>
<span class="n">q_initial_props</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">json_string</span><span class="p">)</span>
<span class="n">q_locales</span> <span class="o">=</span> <span class="n">q_initial_props</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;locales&#39;</span><span class="p">)</span>
<span class="n">eng_tag_list</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="k">for</span> <span class="n">country</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">q_locales</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">for</span> <span class="n">lang</span> <span class="ow">in</span> <span class="n">v</span><span class="p">[</span><span class="s1">&#39;langs&#39;</span><span class="p">]:</span>
<span class="n">_locale</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{lang}</span><span class="s2">_</span><span class="si">{country}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">lang</span><span class="o">=</span><span class="n">lang</span><span class="p">,</span> <span class="n">country</span><span class="o">=</span><span class="n">country</span><span class="p">)</span>
<span class="k">if</span> <span class="n">qwant_categ</span> <span class="o">==</span> <span class="s1">&#39;news&#39;</span> <span class="ow">and</span> <span class="n">_locale</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">qwant_news_locales</span><span class="p">:</span>
<span class="c1"># qwant-news does not support all locales from qwant-web:</span>
<span class="k">continue</span>
<span class="n">eng_tag_list</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">_locale</span><span class="p">)</span>
<span class="k">for</span> <span class="n">eng_tag</span> <span class="ow">in</span> <span class="n">eng_tag_list</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;_&#39;</span><span class="p">))</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: can&#39;t determine babel locale of quant&#39;s locale </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">eng_tag</span><span class="p">)</span>
<span class="k">continue</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,329 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.radio_browser &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.radio_browser</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.radio_browser</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Search radio stations from RadioBrowser by `Advanced station search API`_.</span>
<span class="sd">.. _Advanced station search API:</span>
<span class="sd"> https://de1.api.radio-browser.info/#Advanced_station_search</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">random</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">socket</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineCache</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.radio-browser.info/&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q111664849&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://de1.api.radio-browser.info/&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;JSON&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;music&#39;</span><span class="p">,</span> <span class="s1">&#39;radio&#39;</span><span class="p">]</span>
<span class="n">number_of_results</span> <span class="o">=</span> <span class="mi">10</span>
<span class="n">station_filters</span> <span class="o">=</span> <span class="p">[]</span> <span class="c1"># [&#39;countrycode&#39;, &#39;language&#39;]</span>
<span class="sd">&quot;&quot;&quot;A list of filters to be applied to the search of radio stations. By default</span>
<span class="sd">none filters are applied. Valid filters are:</span>
<span class="sd">``language``</span>
<span class="sd"> Filter stations by selected language. For instance the ``de`` from ``:de-AU``</span>
<span class="sd"> will be translated to `german` and used in the argument ``language=``.</span>
<span class="sd">``countrycode``</span>
<span class="sd"> Filter stations by selected country. The 2-digit countrycode of the station</span>
<span class="sd"> comes from the region the user selected. For instance ``:de-AU`` will filter</span>
<span class="sd"> out all stations not in ``AU``.</span>
<span class="sd">.. note::</span>
<span class="sd"> RadioBrowser has registered a lot of languages and countrycodes unknown to</span>
<span class="sd"> :py:obj:`babel` and note that when searching for radio stations, users are</span>
<span class="sd"> more likely to search by name than by region or language.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">CACHE</span><span class="p">:</span> <span class="n">EngineCache</span>
<span class="sd">&quot;&quot;&quot;Persistent (SQLite) key/value cache that deletes its values after ``expire``</span>
<span class="sd">seconds.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">_</span><span class="p">):</span>
<span class="k">global</span> <span class="n">CACHE</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="n">CACHE</span> <span class="o">=</span> <span class="n">EngineCache</span><span class="p">(</span><span class="s2">&quot;radio_browser&quot;</span><span class="p">)</span>
<span class="n">server_list</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">server_list</span><span class="p">()</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="n">servers</span> <span class="o">=</span> <span class="n">CACHE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;servers&quot;</span><span class="p">,</span> <span class="p">[])</span>
<span class="k">if</span> <span class="n">servers</span><span class="p">:</span>
<span class="k">return</span> <span class="n">servers</span>
<span class="c1"># hint: can take up to 40sec!</span>
<span class="n">ips</span> <span class="o">=</span> <span class="n">socket</span><span class="o">.</span><span class="n">getaddrinfo</span><span class="p">(</span><span class="s2">&quot;all.api.radio-browser.info&quot;</span><span class="p">,</span> <span class="mi">80</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">socket</span><span class="o">.</span><span class="n">IPPROTO_TCP</span><span class="p">)</span>
<span class="k">for</span> <span class="n">ip_tuple</span> <span class="ow">in</span> <span class="n">ips</span><span class="p">:</span>
<span class="n">_ip</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">ip_tuple</span><span class="p">[</span><span class="mi">4</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="c1"># type: ignore</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">socket</span><span class="o">.</span><span class="n">gethostbyaddr</span><span class="p">(</span><span class="n">_ip</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">srv</span> <span class="o">=</span> <span class="s2">&quot;https://&quot;</span> <span class="o">+</span> <span class="n">url</span>
<span class="k">if</span> <span class="n">srv</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">servers</span><span class="p">:</span>
<span class="n">servers</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">srv</span><span class="p">)</span>
<span class="c1"># update server list once in 24h</span>
<span class="n">CACHE</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="s2">&quot;servers&quot;</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">servers</span><span class="p">,</span> <span class="n">expire</span><span class="o">=</span><span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span><span class="p">)</span>
<span class="k">return</span> <span class="n">servers</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="n">servers</span> <span class="o">=</span> <span class="n">server_list</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">servers</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Fetched server list is empty!&quot;</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">return</span>
<span class="n">server</span> <span class="o">=</span> <span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">servers</span><span class="p">)</span>
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;name&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;order&#39;</span><span class="p">:</span> <span class="s1">&#39;votes&#39;</span><span class="p">,</span>
<span class="s1">&#39;offset&#39;</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">number_of_results</span><span class="p">,</span>
<span class="s1">&#39;limit&#39;</span><span class="p">:</span> <span class="n">number_of_results</span><span class="p">,</span>
<span class="s1">&#39;hidebroken&#39;</span><span class="p">:</span> <span class="s1">&#39;true&#39;</span><span class="p">,</span>
<span class="s1">&#39;reverse&#39;</span><span class="p">:</span> <span class="s1">&#39;true&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="k">if</span> <span class="s1">&#39;language&#39;</span> <span class="ow">in</span> <span class="n">station_filters</span><span class="p">:</span>
<span class="n">lang</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">])</span> <span class="c1"># type: ignore</span>
<span class="k">if</span> <span class="n">lang</span><span class="p">:</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;language&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">lang</span>
<span class="k">if</span> <span class="s1">&#39;countrycode&#39;</span> <span class="ow">in</span> <span class="n">station_filters</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">))</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">countrycode</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
<span class="k">if</span> <span class="n">countrycode</span> <span class="ow">in</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;countrycodes&#39;</span><span class="p">]:</span> <span class="c1"># type: ignore</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;countrycode&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">countrycode</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">server</span><span class="si">}</span><span class="s2">/json/stations/search?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">json_resp</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_resp</span><span class="p">:</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;homepage&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">url</span><span class="p">:</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;url_resolved&#39;</span><span class="p">]</span>
<span class="n">content</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">tags</span> <span class="o">=</span> <span class="s1">&#39;, &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;tags&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">))</span>
<span class="k">if</span> <span class="n">tags</span><span class="p">:</span>
<span class="n">content</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tags</span><span class="p">)</span>
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;state&#39;</span><span class="p">,</span> <span class="s1">&#39;country&#39;</span><span class="p">]:</span>
<span class="n">v</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
<span class="k">if</span> <span class="n">v</span><span class="p">:</span>
<span class="n">v</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">v</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="n">content</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
<span class="n">metadata</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">codec</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;codec&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">codec</span> <span class="ow">and</span> <span class="n">codec</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">!=</span> <span class="s1">&#39;unknown&#39;</span><span class="p">:</span>
<span class="n">metadata</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">codec</span><span class="si">}</span><span class="s1"> &#39;</span> <span class="o">+</span> <span class="n">gettext</span><span class="p">(</span><span class="s1">&#39;radio&#39;</span><span class="p">))</span>
<span class="k">for</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="ow">in</span> <span class="p">[</span>
<span class="p">(</span><span class="n">gettext</span><span class="p">(</span><span class="s1">&#39;bitrate&#39;</span><span class="p">),</span> <span class="s1">&#39;bitrate&#39;</span><span class="p">),</span>
<span class="p">(</span><span class="n">gettext</span><span class="p">(</span><span class="s1">&#39;votes&#39;</span><span class="p">),</span> <span class="s1">&#39;votes&#39;</span><span class="p">),</span>
<span class="p">(</span><span class="n">gettext</span><span class="p">(</span><span class="s1">&#39;clicks&#39;</span><span class="p">),</span> <span class="s1">&#39;clickcount&#39;</span><span class="p">),</span>
<span class="p">]:</span>
<span class="n">v</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">y</span><span class="p">)</span>
<span class="k">if</span> <span class="n">v</span><span class="p">:</span>
<span class="n">v</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">v</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="n">metadata</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">x</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">v</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">],</span>
<span class="s1">&#39;thumbnail&#39;</span><span class="p">:</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;favicon&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;http://&quot;</span><span class="p">,</span> <span class="s2">&quot;https://&quot;</span><span class="p">),</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="s1">&#39; | &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">content</span><span class="p">),</span>
<span class="s1">&#39;metadata&#39;</span><span class="p">:</span> <span class="s1">&#39; | &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">metadata</span><span class="p">),</span>
<span class="s1">&#39;iframe_src&#39;</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;url_resolved&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;http://&quot;</span><span class="p">,</span> <span class="s2">&quot;https://&quot;</span><span class="p">),</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span>
<div class="viewcode-block" id="fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/online/radio_browser.html#searx.engines.radio_browser.fetch_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Fetch languages and countrycodes from RadioBrowser</span>
<span class="sd"> - ``traits.languages``: `list of languages API`_</span>
<span class="sd"> - ``traits.custom[&#39;countrycodes&#39;]``: `list of countries API`_</span>
<span class="sd"> .. _list of countries API: https://de1.api.radio-browser.info/#List_of_countries</span>
<span class="sd"> .. _list of languages API: https://de1.api.radio-browser.info/#List_of_languages</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=import-outside-toplevel</span>
<span class="n">init</span><span class="p">(</span><span class="kc">None</span><span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">babel.core</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_global</span>
<span class="n">babel_reg_list</span> <span class="o">=</span> <span class="n">get_global</span><span class="p">(</span><span class="s2">&quot;territory_languages&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>
<span class="n">server</span> <span class="o">=</span> <span class="n">server_list</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">language_list</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">server</span><span class="si">}</span><span class="s1">/json/languages&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">json</span><span class="p">()</span> <span class="c1"># type: ignore</span>
<span class="n">country_list</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">server</span><span class="si">}</span><span class="s1">/json/countries&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">json</span><span class="p">()</span> <span class="c1"># type: ignore</span>
<span class="k">for</span> <span class="n">lang</span> <span class="ow">in</span> <span class="n">language_list</span><span class="p">:</span>
<span class="n">babel_lang</span> <span class="o">=</span> <span class="n">lang</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;iso_639&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">babel_lang</span><span class="p">:</span>
<span class="c1"># the language doesn&#39;t have any iso code, and hence can&#39;t be parsed</span>
<span class="c1"># print(f&quot;ERROR: lang - no iso code in {lang}&quot;)</span>
<span class="k">continue</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">babel_lang</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot;-&quot;</span><span class="p">))</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="c1"># print(f&quot;ERROR: language tag {babel_lang} is unknown by babel&quot;)</span>
<span class="k">continue</span>
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">lang</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
<span class="n">countrycodes</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="k">for</span> <span class="n">region</span> <span class="ow">in</span> <span class="n">country_list</span><span class="p">:</span>
<span class="c1"># country_list contains duplicates that differ only in upper/lower case</span>
<span class="n">_reg</span> <span class="o">=</span> <span class="n">region</span><span class="p">[</span><span class="s1">&#39;iso_3166_1&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
<span class="k">if</span> <span class="n">_reg</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">babel_reg_list</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;ERROR: region tag </span><span class="si">{</span><span class="n">region</span><span class="p">[</span><span class="s1">&#39;iso_3166_1&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2"> is unknown by babel&quot;</span><span class="p">)</span>
<span class="k">continue</span>
<span class="n">countrycodes</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">_reg</span><span class="p">)</span>
<span class="n">countrycodes</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">countrycodes</span><span class="p">)</span>
<span class="n">countrycodes</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;countrycodes&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">countrycodes</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,267 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.recoll &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.recoll</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.recoll</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;.. sidebar:: info</span>
<span class="sd"> - `Recoll &lt;https://www.lesbonscomptes.com/recoll/&gt;`_</span>
<span class="sd"> - `recoll-webui &lt;https://framagit.org/medoc92/recollwebui.git&gt;`_</span>
<span class="sd"> - :origin:`searx/engines/recoll.py`</span>
<span class="sd">Recoll_ is a desktop full-text search tool based on Xapian. By itself Recoll_</span>
<span class="sd">does not offer WEB or API access, this can be achieved using recoll-webui_</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">You must configure the following settings:</span>
<span class="sd">- :py:obj:`base_url`</span>
<span class="sd">- :py:obj:`mount_prefix`</span>
<span class="sd">- :py:obj:`dl_prefix`</span>
<span class="sd">- :py:obj:`search_dir`</span>
<span class="sd">Example scenario:</span>
<span class="sd">#. Recoll indexes a local filesystem mounted in ``/export/documents/reference``,</span>
<span class="sd">#. the Recoll search interface can be reached at https://recoll.example.org/ and</span>
<span class="sd">#. the contents of this filesystem can be reached though https://download.example.org/reference</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> base_url: https://recoll.example.org</span>
<span class="sd"> mount_prefix: /export/documents</span>
<span class="sd"> dl_prefix: https://download.example.org</span>
<span class="sd"> search_dir: &quot;&quot;</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span><span class="p">,</span> <span class="n">timedelta</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">html_to_text</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s2">&quot;Q15735774&quot;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s2">&quot;https://www.lesbonscomptes.com/recoll/&quot;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s2">&quot;JSON&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">base_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Location where recoll-webui can be reached.&quot;&quot;&quot;</span>
<span class="n">mount_prefix</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Location where the file hierarchy is mounted on your *local* filesystem.&quot;&quot;&quot;</span>
<span class="n">dl_prefix</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Location where the file hierarchy as indexed by recoll can be reached.&quot;&quot;&quot;</span>
<span class="n">search_dir</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Part of the indexed file hierarchy to be search, if empty the full domain is</span>
<span class="sd">searched.&quot;&quot;&quot;</span>
<span class="n">_s2i</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;day&quot;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s2">&quot;week&quot;</span><span class="p">:</span> <span class="mi">7</span><span class="p">,</span> <span class="s2">&quot;month&quot;</span><span class="p">:</span> <span class="mi">30</span><span class="p">,</span> <span class="s2">&quot;year&quot;</span><span class="p">:</span> <span class="mi">365</span><span class="p">}</span>
<div class="viewcode-block" id="setup">
<a class="viewcode-back" href="../../../dev/engines/online/recoll.html#searx.engines.recoll.setup">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Initialization of the Recoll engine, checks if the mandatory values are</span>
<span class="sd"> configured.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">missing</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">cfg_name</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;base_url&quot;</span><span class="p">,</span> <span class="s2">&quot;mount_prefix&quot;</span><span class="p">,</span> <span class="s2">&quot;dl_prefix&quot;</span><span class="p">]:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cfg_name</span><span class="p">):</span>
<span class="n">missing</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">cfg_name</span><span class="p">)</span>
<span class="k">if</span> <span class="n">missing</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;missing recoll configuration: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">missing</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="k">if</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s2">&quot;base_url&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">):</span>
<span class="n">engine_settings</span><span class="p">[</span><span class="s2">&quot;base_url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s2">&quot;base_url&quot;</span><span class="p">][:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
<span class="k">return</span> <span class="kc">True</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">search_after</span><span class="p">(</span><span class="n">time_range</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="n">offset</span> <span class="o">=</span> <span class="n">_s2i</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">time_range</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">offset</span><span class="p">:</span>
<span class="k">return</span> <span class="s2">&quot;&quot;</span>
<span class="k">return</span> <span class="p">(</span><span class="n">date</span><span class="o">.</span><span class="n">today</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="n">offset</span><span class="p">))</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">&quot;OnlineParams&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;query&quot;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s2">&quot;page&quot;</span><span class="p">:</span> <span class="n">params</span><span class="p">[</span><span class="s2">&quot;pageno&quot;</span><span class="p">],</span>
<span class="s2">&quot;after&quot;</span><span class="p">:</span> <span class="n">search_after</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;time_range&quot;</span><span class="p">]),</span>
<span class="s2">&quot;dir&quot;</span><span class="p">:</span> <span class="n">search_dir</span><span class="p">,</span>
<span class="s2">&quot;highlight&quot;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">/json?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">json_data</span><span class="p">:</span>
<span class="k">return</span> <span class="n">res</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;results&quot;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;url&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;file://&quot;</span> <span class="o">+</span> <span class="n">mount_prefix</span><span class="p">,</span> <span class="n">dl_prefix</span><span class="p">)</span>
<span class="n">mtype</span> <span class="o">=</span> <span class="n">subtype</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;mtype&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">mtype</span><span class="p">:</span>
<span class="n">mtype</span><span class="p">,</span> <span class="n">subtype</span> <span class="o">=</span> <span class="p">(</span><span class="n">mtype</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="p">[</span><span class="s2">&quot;&quot;</span><span class="p">])[:</span><span class="mi">2</span><span class="p">]</span>
<span class="c1"># facilitate preview support for known mime types</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">embedded</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">if</span> <span class="n">mtype</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;audio&quot;</span><span class="p">,</span> <span class="s2">&quot;video&quot;</span><span class="p">]:</span>
<span class="n">embedded</span> <span class="o">=</span> <span class="n">url</span>
<span class="k">if</span> <span class="n">mtype</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">]</span> <span class="ow">and</span> <span class="n">subtype</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;bmp&quot;</span><span class="p">,</span> <span class="s2">&quot;gif&quot;</span><span class="p">,</span> <span class="s2">&quot;jpeg&quot;</span><span class="p">,</span> <span class="s2">&quot;png&quot;</span><span class="p">]:</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">url</span>
<span class="c1"># remove HTML from snippet</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">html_to_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;snippet&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">))</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">File</span><span class="p">(</span>
<span class="n">title</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;label&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
<span class="n">url</span><span class="o">=</span><span class="n">url</span><span class="p">,</span>
<span class="n">content</span><span class="o">=</span><span class="n">content</span><span class="p">,</span>
<span class="n">size</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;size&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
<span class="n">filename</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;filename&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
<span class="n">abstract</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;abstract&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
<span class="n">author</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;author&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
<span class="n">mtype</span><span class="o">=</span><span class="n">mtype</span><span class="p">,</span>
<span class="n">subtype</span><span class="o">=</span><span class="n">subtype</span><span class="p">,</span>
<span class="n">time</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;time&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">),</span>
<span class="n">embedded</span><span class="o">=</span><span class="n">embedded</span><span class="p">,</span>
<span class="n">thumbnail</span><span class="o">=</span><span class="n">thumbnail</span><span class="p">,</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">res</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,248 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.reuters &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.reuters</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.reuters</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Reuters_ (news) is an international news agency.</span>
<span class="sd">.. _Reuters: https://www.reuters.com</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">The engine has the following additional settings:</span>
<span class="sd">- :py:obj:`sort_order`</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: reuters</span>
<span class="sd"> engine: reuters</span>
<span class="sd"> shortcut: reu</span>
<span class="sd"> sort_order: &quot;relevance&quot;</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">json</span><span class="w"> </span><span class="kn">import</span> <span class="n">dumps</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">quote_plus</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil</span><span class="w"> </span><span class="kn">import</span> <span class="n">parser</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s2">&quot;https://www.reuters.com&quot;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s2">&quot;Q130879&quot;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s2">&quot;JSON&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;news&quot;</span><span class="p">]</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;https://www.reuters.com&quot;</span>
<span class="n">results_per_page</span> <span class="o">=</span> <span class="mi">20</span>
<span class="n">sort_order</span> <span class="o">=</span> <span class="s2">&quot;relevance&quot;</span>
<span class="sd">&quot;&quot;&quot;Sort order, one of ``relevance``, ``display_date:desc`` or ``display_data:asc``.&quot;&quot;&quot;</span>
<span class="n">time_range_duration_map</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;day&quot;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
<span class="s2">&quot;week&quot;</span><span class="p">:</span> <span class="mi">7</span><span class="p">,</span>
<span class="s2">&quot;month&quot;</span><span class="p">:</span> <span class="mi">30</span><span class="p">,</span>
<span class="s2">&quot;year&quot;</span><span class="p">:</span> <span class="mi">365</span><span class="p">,</span>
<span class="p">}</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;keyword&quot;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s2">&quot;offset&quot;</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;pageno&quot;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">results_per_page</span><span class="p">,</span>
<span class="s2">&quot;orderby&quot;</span><span class="p">:</span> <span class="n">sort_order</span><span class="p">,</span>
<span class="s2">&quot;size&quot;</span><span class="p">:</span> <span class="n">results_per_page</span><span class="p">,</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s2">&quot;reuters&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s2">&quot;time_range&quot;</span><span class="p">]:</span>
<span class="n">time_diff_days</span> <span class="o">=</span> <span class="n">time_range_duration_map</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;time_range&quot;</span><span class="p">]]</span>
<span class="n">start_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="n">time_diff_days</span><span class="p">)</span>
<span class="n">args</span><span class="p">[</span><span class="s2">&quot;start_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">start_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">/pf/api/v3/content/fetch/articles-by-search-v2?query=</span><span class="si">{</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">dumps</span><span class="p">(</span><span class="n">args</span><span class="p">))</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">return</span> <span class="n">params</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">resp_json</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;result&quot;</span><span class="p">):</span>
<span class="k">return</span> <span class="n">res</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">resp_json</span><span class="p">[</span><span class="s2">&quot;result&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;articles&quot;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">MainResult</span><span class="p">(</span>
<span class="n">url</span><span class="o">=</span><span class="n">base_url</span> <span class="o">+</span> <span class="n">result</span><span class="p">[</span><span class="s2">&quot;canonical_url&quot;</span><span class="p">],</span>
<span class="n">title</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;web&quot;</span><span class="p">],</span>
<span class="n">content</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;description&quot;</span><span class="p">],</span>
<span class="n">thumbnail</span><span class="o">=</span><span class="n">resize_url</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;thumbnail&quot;</span><span class="p">,</span> <span class="p">{}),</span> <span class="n">height</span><span class="o">=</span><span class="mi">80</span><span class="p">),</span>
<span class="n">metadata</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;kicker&quot;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;name&quot;</span><span class="p">),</span>
<span class="n">publishedDate</span><span class="o">=</span><span class="n">parser</span><span class="o">.</span><span class="n">isoparse</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">&quot;display_time&quot;</span><span class="p">]),</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">res</span>
<div class="viewcode-block" id="resize_url">
<a class="viewcode-back" href="../../../dev/engines/online/reuters.html#searx.engines.reuters.resize_url">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">resize_url</span><span class="p">(</span><span class="n">thumbnail</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="n">width</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span> <span class="n">height</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Generates a URL for Reuter&#39;s thumbnail with the dimensions *width* and</span>
<span class="sd"> *height*. If no URL can be generated from the *thumbnail data*, an empty</span>
<span class="sd"> string will be returned.</span>
<span class="sd"> width: default is *unset* (``0``)</span>
<span class="sd"> Image width in pixels (negative values are ignored). If only width is</span>
<span class="sd"> specified, the height matches the original aspect ratio.</span>
<span class="sd"> height: default is *unset* (``0``)</span>
<span class="sd"> Image height in pixels (negative values are ignored). If only height is</span>
<span class="sd"> specified, the width matches the original aspect ratio.</span>
<span class="sd"> The file size of a full-size image is usually several MB; when reduced to a</span>
<span class="sd"> height of, for example, 80 points, only a few KB remain!</span>
<span class="sd"> Fields of the *thumbnail data* (``result.articles.[&lt;int&gt;].thumbnail``):</span>
<span class="sd"> thumbnail.url:</span>
<span class="sd"> Is a full-size image (&gt;MB).</span>
<span class="sd"> thumbnail.width &amp; .height:</span>
<span class="sd"> Dimensions of the full-size image.</span>
<span class="sd"> thumbnail.resizer_url:</span>
<span class="sd"> Reuters has a *resizer* `REST-API for the images`_, this is the URL of the</span>
<span class="sd"> service. This URL includes the ``&amp;auth`` argument, other arguments are</span>
<span class="sd"> ``&amp;width=&lt;int&gt;`` and ``&amp;height=&lt;int&gt;``.</span>
<span class="sd"> .. _REST-API for the images:</span>
<span class="sd"> https://dev.arcxp.com/photo-center/image-resizer/resizer-v2-how-to-transform-images/#query-parameters</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">thumbnail</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;resizer_url&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">url</span><span class="p">:</span>
<span class="k">return</span> <span class="s2">&quot;&quot;</span>
<span class="k">if</span> <span class="nb">int</span><span class="p">(</span><span class="n">width</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">url</span> <span class="o">+=</span> <span class="sa">f</span><span class="s2">&quot;&amp;width=</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">width</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">if</span> <span class="nb">int</span><span class="p">(</span><span class="n">height</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">url</span> <span class="o">+=</span> <span class="sa">f</span><span class="s2">&quot;&amp;height=</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">height</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">return</span> <span class="n">url</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,186 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.sepiasearch &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.sepiasearch</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.sepiasearch</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;SepiaSearch uses the same languages as :py:obj:`Peertube</span>
<span class="sd">&lt;searx.engines.peertube&gt;` and the response is identical to the response from the</span>
<span class="sd">peertube engines.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.peertube</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="c1"># pylint: disable=unused-import</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.peertube</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="c1"># pylint: disable=unused-import</span>
<span class="n">video_response</span><span class="p">,</span>
<span class="n">safesearch_table</span><span class="p">,</span>
<span class="n">time_range_table</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="c1"># pylint: disable=line-too-long</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://sepiasearch.org&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://docs.joinpeertube.org/api-rest-reference.html#tag/Search/operation/searchVideos&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;JSON&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;videos&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;https://sepiasearch.org&#39;</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/peertube.html#searx.engines.sepiasearch.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Assemble request for the SepiaSearch API&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">query</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="c1"># eng_region = traits.get_region(params[&#39;searxng_locale&#39;], &#39;en_US&#39;)</span>
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="kc">None</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">base_url</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">)</span>
<span class="o">+</span> <span class="s2">&quot;/api/v1/search/videos?&quot;</span>
<span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;search&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;start&#39;</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span><span class="p">,</span>
<span class="s1">&#39;count&#39;</span><span class="p">:</span> <span class="mi">10</span><span class="p">,</span>
<span class="c1"># -createdAt: sort by date ascending / createdAt: date descending</span>
<span class="s1">&#39;sort&#39;</span><span class="p">:</span> <span class="s1">&#39;-match&#39;</span><span class="p">,</span> <span class="c1"># sort by *match descending*</span>
<span class="s1">&#39;nsfw&#39;</span><span class="p">:</span> <span class="n">safesearch_table</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]],</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">eng_lang</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">&#39;&amp;languageOneOf[]=&#39;</span> <span class="o">+</span> <span class="n">eng_lang</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">&#39;&amp;boostLanguages[]=&#39;</span> <span class="o">+</span> <span class="n">eng_lang</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_table</span><span class="p">:</span>
<span class="n">time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">date</span><span class="p">()</span> <span class="o">+</span> <span class="n">time_range_table</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">]]</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">&#39;&amp;startDate=&#39;</span> <span class="o">+</span> <span class="n">time</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
<span class="k">return</span> <span class="n">params</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="k">return</span> <span class="n">video_response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,286 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.springer &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.springer</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.springer</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;`Springer Nature`_ is a global publisher dedicated to providing service to</span>
<span class="sd">research community with official Springer-API_ (API-Playground_).</span>
<span class="sd">.. note::</span>
<span class="sd"> The Springer engine requires an API key, which can be obtained via the</span>
<span class="sd"> `Springer subscription`_.</span>
<span class="sd">Since the search term is passed 1:1 to the API, SearXNG users can use the</span>
<span class="sd">`Supported Query Parameters`_.</span>
<span class="sd">- ``!springer (doi:10.1007/s10948-025-07019-1 OR doi:10.1007/s10948-025-07035-1)``</span>
<span class="sd">- ``!springer keyword:ybco``</span>
<span class="sd">However, please note that the available options depend on the subscription type.</span>
<span class="sd">For example, the ``year:`` filter requires a *Premium Plan* subscription.</span>
<span class="sd">- ``!springer keyword:ybco year:2024``</span>
<span class="sd">The engine uses the REST Meta-API_ `v2` endpoint, but there is also a `Python</span>
<span class="sd">API Wrapper`_.</span>
<span class="sd">.. _Python API Wrapper: https://pypi.org/project/springernature-api-client/</span>
<span class="sd">.. _Springer Nature: https://www.springernature.com/</span>
<span class="sd">.. _Springer subscription: https://dev.springernature.com/subscription/</span>
<span class="sd">.. _Springer-API: https://dev.springernature.com/docs/introduction/</span>
<span class="sd">.. _API-Playground: https://dev.springernature.com/docs/live-documentation/</span>
<span class="sd">.. _Meta-API: https://dev.springernature.com/docs/api-endpoints/meta-api/</span>
<span class="sd">.. _Supported Query Parameters: https://dev.springernature.com/docs/supported-query-params/</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">The engine has the following additional settings:</span>
<span class="sd">- :py:obj:`api_key`</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: springer nature</span>
<span class="sd"> api_key: &quot;...&quot;</span>
<span class="sd"> inactive: false</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">raise_for_httperror</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s2">&quot;https://www.springernature.com/&quot;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s2">&quot;Q21096327&quot;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s2">&quot;https://dev.springernature.com/docs/live-documentation/&quot;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s2">&quot;JSON&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;science&quot;</span><span class="p">,</span> <span class="s2">&quot;scientific publications&quot;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">nb_per_page</span> <span class="o">=</span> <span class="mi">10</span>
<span class="sd">&quot;&quot;&quot;Number of results to return in the request, see `Pagination and Limits`_ for</span>
<span class="sd">more details.</span>
<span class="sd">.. _Pagination and Limits:</span>
<span class="sd"> https://dev.springernature.com/docs/advanced-querying/pagination-limits/</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">api_key</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Key used for the Meta-API_. Get your API key from: `Springer subscription`_&quot;&quot;&quot;</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;https://api.springernature.com/meta/v2/json&quot;</span>
<span class="sd">&quot;&quot;&quot;An enhanced endpoint with additional metadata fields and optimized queries</span>
<span class="sd">for more efficient and comprehensive retrieval (Meta-API_ `v2`).</span>
<span class="sd">&quot;&quot;&quot;</span>
<div class="viewcode-block" id="setup">
<a class="viewcode-back" href="../../../dev/engines/online/springer.html#searx.engines.springer.setup">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Initialization of the Springer engine, checks whether the</span>
<span class="sd"> :py:obj:`api_key` is set, otherwise the engine is inactive.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">key</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;api_key&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="c1"># Springer&#39;s API key is a hex value</span>
<span class="nb">int</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="mi">16</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Springer&#39;s API key is not set or invalid.&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">False</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">&quot;OnlineParams&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;api_key&quot;</span><span class="p">:</span> <span class="n">api_key</span><span class="p">,</span>
<span class="s2">&quot;q&quot;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s2">&quot;s&quot;</span><span class="p">:</span> <span class="n">nb_per_page</span> <span class="o">*</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;pageno&quot;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">),</span>
<span class="s2">&quot;p&quot;</span><span class="p">:</span> <span class="n">nb_per_page</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="c1"># For example, the ``year:`` filter requires a *Premium Plan* subscription.</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;raise_for_httperror&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">if</span> <span class="p">(</span>
<span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">403</span>
<span class="ow">and</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">&quot;status&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;fail&quot;</span>
<span class="ow">and</span> <span class="s2">&quot;premium feature&quot;</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">&quot;message&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="p">):</span>
<span class="k">return</span> <span class="n">res</span>
<span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">field</span><span class="p">(</span><span class="n">k</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">))</span>
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">&quot;records&quot;</span><span class="p">]:</span>
<span class="n">published</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;publicationDate&quot;</span><span class="p">],</span> <span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">authors</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">author</span><span class="p">[</span><span class="s2">&quot;creator&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;, &quot;</span><span class="p">)[::</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span> <span class="k">for</span> <span class="n">author</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;creators&quot;</span><span class="p">]]</span>
<span class="n">pdf_url</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="n">html_url</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="n">url_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">url_list</span><span class="p">:</span>
<span class="k">if</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;platform&quot;</span><span class="p">]</span> <span class="o">!=</span> <span class="s2">&quot;web&quot;</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">val</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;value&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;http://&quot;</span><span class="p">,</span> <span class="s2">&quot;https://&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="k">if</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;format&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;html&quot;</span><span class="p">:</span>
<span class="n">html_url</span> <span class="o">=</span> <span class="n">val</span>
<span class="k">elif</span> <span class="n">item</span><span class="p">[</span><span class="s2">&quot;format&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;pdf&quot;</span><span class="p">:</span>
<span class="n">pdf_url</span> <span class="o">=</span> <span class="n">val</span>
<span class="n">paper</span> <span class="o">=</span> <span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Paper</span><span class="p">(</span>
<span class="n">url</span><span class="o">=</span><span class="n">html_url</span><span class="p">,</span>
<span class="c1"># html_url=html_url,</span>
<span class="n">pdf_url</span><span class="o">=</span><span class="n">pdf_url</span><span class="p">,</span>
<span class="n">title</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">&quot;title&quot;</span><span class="p">),</span>
<span class="n">content</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">&quot;abstract&quot;</span><span class="p">),</span>
<span class="n">comments</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">&quot;publicationName&quot;</span><span class="p">),</span>
<span class="n">tags</span><span class="o">=</span><span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;keyword&quot;</span><span class="p">,</span> <span class="p">[]),</span>
<span class="n">publishedDate</span><span class="o">=</span><span class="n">published</span><span class="p">,</span>
<span class="nb">type</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">&quot;contentType&quot;</span><span class="p">),</span>
<span class="n">authors</span><span class="o">=</span><span class="n">authors</span><span class="p">,</span>
<span class="n">publisher</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">&quot;publisher&quot;</span><span class="p">),</span>
<span class="n">journal</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">&quot;publicationName&quot;</span><span class="p">),</span>
<span class="n">volume</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">&quot;volume&quot;</span><span class="p">),</span>
<span class="n">pages</span><span class="o">=</span><span class="s2">&quot;-&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span><span class="n">field</span><span class="p">(</span><span class="s2">&quot;startingPage&quot;</span><span class="p">),</span> <span class="n">field</span><span class="p">(</span><span class="s2">&quot;endingPage&quot;</span><span class="p">)]</span> <span class="k">if</span> <span class="n">x</span><span class="p">]),</span>
<span class="n">number</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">&quot;number&quot;</span><span class="p">),</span>
<span class="n">doi</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">&quot;doi&quot;</span><span class="p">),</span>
<span class="n">issn</span><span class="o">=</span><span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span><span class="n">field</span><span class="p">(</span><span class="s2">&quot;issn&quot;</span><span class="p">)]</span> <span class="k">if</span> <span class="n">x</span><span class="p">],</span>
<span class="n">isbn</span><span class="o">=</span><span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span><span class="n">field</span><span class="p">(</span><span class="s2">&quot;isbn&quot;</span><span class="p">)]</span> <span class="k">if</span> <span class="n">x</span><span class="p">],</span>
<span class="p">)</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">paper</span><span class="p">)</span>
<span class="k">return</span> <span class="n">res</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,230 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.sqlite &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.sqlite</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.sqlite</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;SQLite is a small, fast and reliable SQL database engine. It does not require</span>
<span class="sd">any extra dependency.</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">The engine has the following (additional) settings:</span>
<span class="sd">- :py:obj:`result_type`</span>
<span class="sd">Example</span>
<span class="sd">=======</span>
<span class="sd">.. _MediathekView: https://mediathekview.de/</span>
<span class="sd">To demonstrate the power of database engines, here is a more complex example</span>
<span class="sd">which reads from a MediathekView_ (DE) movie database. For this example of the</span>
<span class="sd">SQLite engine download the database:</span>
<span class="sd">- https://liste.mediathekview.de/filmliste-v2.db.bz2</span>
<span class="sd">and unpack into ``searx/data/filmliste-v2.db``. To search the database use e.g</span>
<span class="sd">Query to test: ``!mediathekview concert``</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: mediathekview</span>
<span class="sd"> engine: sqlite</span>
<span class="sd"> shortcut: mediathekview</span>
<span class="sd"> categories: [general, videos]</span>
<span class="sd"> result_type: MainResult</span>
<span class="sd"> database: searx/data/filmliste-v2.db</span>
<span class="sd"> query_str: &gt;-</span>
<span class="sd"> SELECT title || &#39; (&#39; || time(duration, &#39;unixepoch&#39;) || &#39;)&#39; AS title,</span>
<span class="sd"> COALESCE( NULLIF(url_video_hd,&#39;&#39;), NULLIF(url_video_sd,&#39;&#39;), url_video) AS url,</span>
<span class="sd"> description AS content</span>
<span class="sd"> FROM film</span>
<span class="sd"> WHERE title LIKE :wildcard OR description LIKE :wildcard</span>
<span class="sd"> ORDER BY duration DESC</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">sqlite3</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">contextlib</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">MainResult</span><span class="p">,</span> <span class="n">KeyValue</span>
<span class="n">engine_type</span> <span class="o">=</span> <span class="s2">&quot;offline&quot;</span>
<span class="n">database</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Filename of the SQLite DB.&quot;&quot;&quot;</span>
<span class="n">query_str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;SQL query that returns the result items.&quot;&quot;&quot;</span>
<span class="n">result_type</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;MainResult&quot;</span><span class="p">,</span> <span class="s2">&quot;KeyValue&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;KeyValue&quot;</span>
<span class="sd">&quot;&quot;&quot;The result type can be :py:obj:`MainResult` or :py:obj:`KeyValue`.&quot;&quot;&quot;</span>
<span class="n">limit</span> <span class="o">=</span> <span class="mi">10</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">):</span>
<span class="k">if</span> <span class="s1">&#39;query_str&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;query_str cannot be empty&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">&#39;query_str&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;select &#39;</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;only SELECT query is supported&#39;</span><span class="p">)</span>
<div class="viewcode-block" id="sqlite_cursor">
<a class="viewcode-back" href="../../../dev/engines/offline/sql-engines.html#searx.engines.sqlite.sqlite_cursor">[docs]</a>
<span class="nd">@contextlib</span><span class="o">.</span><span class="n">contextmanager</span>
<span class="k">def</span><span class="w"> </span><span class="nf">sqlite_cursor</span><span class="p">():</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Implements a :py:obj:`Context Manager &lt;contextlib.contextmanager&gt;` for a</span>
<span class="sd"> :py:obj:`sqlite3.Cursor`.</span>
<span class="sd"> Open database in read only mode: if the database doesn&#39;t exist. The default</span>
<span class="sd"> mode creates an empty file on the file system. See:</span>
<span class="sd"> * https://docs.python.org/3/library/sqlite3.html#sqlite3.connect</span>
<span class="sd"> * https://www.sqlite.org/uri.html</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">uri</span> <span class="o">=</span> <span class="s1">&#39;file:&#39;</span> <span class="o">+</span> <span class="n">database</span> <span class="o">+</span> <span class="s1">&#39;?mode=ro&#39;</span>
<span class="k">with</span> <span class="n">contextlib</span><span class="o">.</span><span class="n">closing</span><span class="p">(</span><span class="n">sqlite3</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">uri</span><span class="p">,</span> <span class="n">uri</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span> <span class="k">as</span> <span class="n">connect</span><span class="p">:</span>
<span class="n">connect</span><span class="o">.</span><span class="n">row_factory</span> <span class="o">=</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">Row</span>
<span class="k">with</span> <span class="n">contextlib</span><span class="o">.</span><span class="n">closing</span><span class="p">(</span><span class="n">connect</span><span class="o">.</span><span class="n">cursor</span><span class="p">())</span> <span class="k">as</span> <span class="n">cursor</span><span class="p">:</span>
<span class="k">yield</span> <span class="n">cursor</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">search</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">query_params</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;wildcard&#39;</span><span class="p">:</span> <span class="sa">r</span><span class="s1">&#39;%&#39;</span> <span class="o">+</span> <span class="n">query</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39; &#39;</span><span class="p">,</span> <span class="sa">r</span><span class="s1">&#39;%&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="sa">r</span><span class="s1">&#39;%&#39;</span><span class="p">,</span>
<span class="s1">&#39;limit&#39;</span><span class="p">:</span> <span class="n">limit</span><span class="p">,</span>
<span class="s1">&#39;offset&#39;</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">limit</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">query_to_run</span> <span class="o">=</span> <span class="n">query_str</span> <span class="o">+</span> <span class="s1">&#39; LIMIT :limit OFFSET :offset&#39;</span>
<span class="k">with</span> <span class="n">sqlite_cursor</span><span class="p">()</span> <span class="k">as</span> <span class="n">cur</span><span class="p">:</span>
<span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">query_to_run</span><span class="p">,</span> <span class="n">query_params</span><span class="p">)</span>
<span class="n">col_names</span> <span class="o">=</span> <span class="p">[</span><span class="n">cn</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">cn</span> <span class="ow">in</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span><span class="p">]</span>
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">cur</span><span class="o">.</span><span class="n">fetchall</span><span class="p">():</span>
<span class="n">kvmap</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">col_names</span><span class="p">,</span> <span class="nb">map</span><span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="n">row</span><span class="p">)))</span>
<span class="k">if</span> <span class="n">result_type</span> <span class="o">==</span> <span class="s2">&quot;MainResult&quot;</span><span class="p">:</span>
<span class="n">item</span> <span class="o">=</span> <span class="n">MainResult</span><span class="p">(</span><span class="o">**</span><span class="n">kvmap</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">item</span> <span class="o">=</span> <span class="n">KeyValue</span><span class="p">(</span><span class="n">kvmap</span><span class="o">=</span><span class="n">kvmap</span><span class="p">)</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">return</span> <span class="n">res</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,649 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.startpage &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.startpage</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.startpage</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Startpage&#39;s language &amp; region selectors are a mess ..</span>
<span class="sd">.. _startpage regions:</span>
<span class="sd">Startpage regions</span>
<span class="sd">=================</span>
<span class="sd">In the list of regions there are tags we need to map to common region tags::</span>
<span class="sd"> pt-BR_BR --&gt; pt_BR</span>
<span class="sd"> zh-CN_CN --&gt; zh_Hans_CN</span>
<span class="sd"> zh-TW_TW --&gt; zh_Hant_TW</span>
<span class="sd"> zh-TW_HK --&gt; zh_Hant_HK</span>
<span class="sd"> en-GB_GB --&gt; en_GB</span>
<span class="sd">and there is at least one tag with a three letter language tag (ISO 639-2)::</span>
<span class="sd"> fil_PH --&gt; fil_PH</span>
<span class="sd">The locale code ``no_NO`` from Startpage does not exists and is mapped to</span>
<span class="sd">``nb-NO``::</span>
<span class="sd"> babel.core.UnknownLocaleError: unknown locale &#39;no_NO&#39;</span>
<span class="sd">For reference see languages-subtag at iana; ``no`` is the macrolanguage [1]_ and</span>
<span class="sd">W3C recommends subtag over macrolanguage [2]_.</span>
<span class="sd">.. [1] `iana: language-subtag-registry</span>
<span class="sd"> &lt;https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry&gt;`_ ::</span>
<span class="sd"> type: language</span>
<span class="sd"> Subtag: nb</span>
<span class="sd"> Description: Norwegian Bokmål</span>
<span class="sd"> Added: 2005-10-16</span>
<span class="sd"> Suppress-Script: Latn</span>
<span class="sd"> Macrolanguage: no</span>
<span class="sd">.. [2]</span>
<span class="sd"> Use macrolanguages with care. Some language subtags have a Scope field set to</span>
<span class="sd"> macrolanguage, i.e. this primary language subtag encompasses a number of more</span>
<span class="sd"> specific primary language subtags in the registry. ... As we recommended for</span>
<span class="sd"> the collection subtags mentioned above, in most cases you should try to use</span>
<span class="sd"> the more specific subtags ... `W3: The primary language subtag</span>
<span class="sd"> &lt;https://www.w3.org/International/questions/qa-choosing-language-tags#langsubtag&gt;`_</span>
<span class="sd">.. _startpage languages:</span>
<span class="sd">Startpage languages</span>
<span class="sd">===================</span>
<span class="sd">:py:obj:`send_accept_language_header`:</span>
<span class="sd"> The displayed name in Startpage&#39;s settings page depend on the location of the</span>
<span class="sd"> IP when ``Accept-Language`` HTTP header is unset. In :py:obj:`fetch_traits`</span>
<span class="sd"> we use::</span>
<span class="sd"> &#39;Accept-Language&#39;: &quot;en-US,en;q=0.5&quot;,</span>
<span class="sd"> ..</span>
<span class="sd"> to get uniform names independent from the IP).</span>
<span class="sd">.. _startpage categories:</span>
<span class="sd">Startpage categories</span>
<span class="sd">====================</span>
<span class="sd">Startpage&#39;s category (for Web-search, News, Videos, ..) is set by</span>
<span class="sd">:py:obj:`startpage_categ` in settings.yml::</span>
<span class="sd"> - name: startpage</span>
<span class="sd"> engine: startpage</span>
<span class="sd"> startpage_categ: web</span>
<span class="sd"> ...</span>
<span class="sd">.. hint::</span>
<span class="sd"> Supported categories are ``web``, ``news`` and ``images``.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-many-statements</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">OrderedDict</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">unicodedata</span><span class="w"> </span><span class="kn">import</span> <span class="n">normalize</span><span class="p">,</span> <span class="n">combining</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">json</span><span class="w"> </span><span class="kn">import</span> <span class="n">loads</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">dateutil.parser</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">lxml.html</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.localedata</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extr</span><span class="p">,</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">gen_useragent</span><span class="p">,</span> <span class="n">html_to_text</span><span class="p">,</span> <span class="n">humanize_bytes</span><span class="p">,</span> <span class="n">remove_pua_from_str</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineCaptchaException</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">region_tag</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineCache</span>
<span class="c1"># about</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://startpage.com&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q2333295&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;HTML&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">startpage_categ</span> <span class="o">=</span> <span class="s1">&#39;web&#39;</span>
<span class="sd">&quot;&quot;&quot;Startpage&#39;s category, visit :ref:`startpage categories`.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
<span class="sd">&quot;&quot;&quot;Startpage tries to guess user&#39;s language and territory from the HTTP</span>
<span class="sd">``Accept-Language``. Optional the user can select a search-language (can be</span>
<span class="sd">different to the UI language) and a region filter.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;general&#39;</span><span class="p">,</span> <span class="s1">&#39;web&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">18</span>
<span class="sd">&quot;&quot;&quot;Tested 18 pages maximum (argument ``page``), to be save max is set to 20.&quot;&quot;&quot;</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">time_range_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;day&#39;</span><span class="p">:</span> <span class="s1">&#39;d&#39;</span><span class="p">,</span> <span class="s1">&#39;week&#39;</span><span class="p">:</span> <span class="s1">&#39;w&#39;</span><span class="p">,</span> <span class="s1">&#39;month&#39;</span><span class="p">:</span> <span class="s1">&#39;m&#39;</span><span class="p">,</span> <span class="s1">&#39;year&#39;</span><span class="p">:</span> <span class="s1">&#39;y&#39;</span><span class="p">}</span>
<span class="n">safesearch_dict</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">&#39;1&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">&#39;0&#39;</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">&#39;0&#39;</span><span class="p">}</span>
<span class="c1"># search-url</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;https://www.startpage.com&#39;</span>
<span class="n">search_url</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s1">&#39;/sp/search&#39;</span>
<span class="c1"># specific xpath variables</span>
<span class="c1"># ads xpath //div[@id=&quot;results&quot;]/div[@id=&quot;sponsored&quot;]//div[@class=&quot;result&quot;]</span>
<span class="c1"># not ads: div[@class=&quot;result&quot;] are the direct children of div[@id=&quot;results&quot;]</span>
<span class="n">search_form_xpath</span> <span class="o">=</span> <span class="s1">&#39;//form[@id=&quot;search&quot;]&#39;</span>
<span class="sd">&quot;&quot;&quot;XPath of Startpage&#39;s origin search form</span>
<span class="sd">.. code: html</span>
<span class="sd"> &lt;form action=&quot;/sp/search&quot; method=&quot;post&quot;&gt;</span>
<span class="sd"> &lt;input type=&quot;text&quot; name=&quot;query&quot; value=&quot;&quot; ..&gt;</span>
<span class="sd"> &lt;input type=&quot;hidden&quot; name=&quot;t&quot; value=&quot;device&quot;&gt;</span>
<span class="sd"> &lt;input type=&quot;hidden&quot; name=&quot;lui&quot; value=&quot;english&quot;&gt;</span>
<span class="sd"> &lt;input type=&quot;hidden&quot; name=&quot;sc&quot; value=&quot;Q7Mt5TRqowKB00&quot;&gt;</span>
<span class="sd"> &lt;input type=&quot;hidden&quot; name=&quot;cat&quot; value=&quot;web&quot;&gt;</span>
<span class="sd"> &lt;input type=&quot;hidden&quot; class=&quot;abp&quot; id=&quot;abp-input&quot; name=&quot;abp&quot; value=&quot;1&quot;&gt;</span>
<span class="sd"> &lt;/form&gt;</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">CACHE</span><span class="p">:</span> <span class="n">EngineCache</span>
<span class="sd">&quot;&quot;&quot;Persistent (SQLite) key/value cache that deletes its values after ``expire``</span>
<span class="sd">seconds.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">_</span><span class="p">):</span>
<span class="k">global</span> <span class="n">CACHE</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="c1"># hint: all three startpage engines (WEB, Images &amp; News) can/should use the</span>
<span class="c1"># same sc_code ..</span>
<span class="n">CACHE</span> <span class="o">=</span> <span class="n">EngineCache</span><span class="p">(</span><span class="s2">&quot;startpage&quot;</span><span class="p">)</span> <span class="c1"># type:ignore</span>
<span class="n">sc_code_cache_sec</span> <span class="o">=</span> <span class="mi">3600</span>
<span class="sd">&quot;&quot;&quot;Time in seconds the sc-code is cached in memory :py:obj:`get_sc_code`.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="get_sc_code">
<a class="viewcode-back" href="../../../dev/engines/online/startpage.html#searx.engines.startpage.get_sc_code">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_sc_code</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get an actual ``sc`` argument from Startpage&#39;s search form (HTML page).</span>
<span class="sd"> Startpage puts a ``sc`` argument on every HTML :py:obj:`search form</span>
<span class="sd"> &lt;search_form_xpath&gt;`. Without this argument Startpage considers the request</span>
<span class="sd"> is from a bot. We do not know what is encoded in the value of the ``sc``</span>
<span class="sd"> argument, but it seems to be a kind of a *timestamp*.</span>
<span class="sd"> Startpage&#39;s search form generates a new sc-code on each request. This</span>
<span class="sd"> function scrapes a new sc-code from Startpage&#39;s home page every</span>
<span class="sd"> :py:obj:`sc_code_cache_sec` seconds.&quot;&quot;&quot;</span>
<span class="n">sc_code</span> <span class="o">=</span> <span class="n">CACHE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;SC_CODE&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">sc_code</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;get_sc_code: using cached value: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">sc_code</span><span class="p">)</span>
<span class="k">return</span> <span class="n">sc_code</span>
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="o">**</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">]}</span>
<span class="c1"># add Accept-Language header</span>
<span class="k">if</span> <span class="n">searxng_locale</span> <span class="o">==</span> <span class="s1">&#39;all&#39;</span><span class="p">:</span>
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="s1">&#39;en-US&#39;</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;-&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">send_accept_language_header</span><span class="p">:</span>
<span class="n">ac_lang</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span>
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">:</span>
<span class="n">ac_lang</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">%s</span><span class="s2">-</span><span class="si">%s</span><span class="s2">,</span><span class="si">%s</span><span class="s2">;q=0.9,*;q=0.5&quot;</span> <span class="o">%</span> <span class="p">(</span>
<span class="n">locale</span><span class="o">.</span><span class="n">language</span><span class="p">,</span>
<span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">,</span>
<span class="n">locale</span><span class="o">.</span><span class="n">language</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">headers</span><span class="p">[</span><span class="s1">&#39;Accept-Language&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ac_lang</span>
<span class="n">get_sc_url</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s1">&#39;/&#39;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;get_sc_code: querying new sc timestamp @ </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">get_sc_url</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;get_sc_code: request headers: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">headers</span><span class="p">)</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">get_sc_url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
<span class="c1"># ?? x = network.get(&#39;https://www.startpage.com/sp/cdn/images/filter-chevron.svg&#39;, headers=headers)</span>
<span class="c1"># ?? https://www.startpage.com/sp/cdn/images/filter-chevron.svg</span>
<span class="c1"># ?? ping-back URL: https://www.startpage.com/sp/pb?sc=TLsB0oITjZ8F21</span>
<span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">url</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;https://www.startpage.com/sp/captcha&#39;</span><span class="p">):</span> <span class="c1"># type: ignore</span>
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">(</span>
<span class="n">message</span><span class="o">=</span><span class="s2">&quot;get_sc_code: got redirected to https://www.startpage.com/sp/captcha&quot;</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">sc_code</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">search_form_xpath</span> <span class="o">+</span> <span class="s1">&#39;//input[@name=&quot;sc&quot;]/@value&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">except</span> <span class="ne">IndexError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;suspend startpage API --&gt; https://github.com/searxng/searxng/pull/695&quot;</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">(</span>
<span class="n">message</span><span class="o">=</span><span class="s2">&quot;get_sc_code: [PR-695] querying new sc timestamp failed! (</span><span class="si">%s</span><span class="s2">)&quot;</span> <span class="o">%</span> <span class="n">resp</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="c1"># type: ignore</span>
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">exc</span>
<span class="n">sc_code</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">sc_code</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;get_sc_code: new value is: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">sc_code</span><span class="p">)</span>
<span class="n">CACHE</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="s2">&quot;SC_CODE&quot;</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">sc_code</span><span class="p">,</span> <span class="n">expire</span><span class="o">=</span><span class="n">sc_code_cache_sec</span><span class="p">)</span>
<span class="k">return</span> <span class="n">sc_code</span></div>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/startpage.html#searx.engines.startpage.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Assemble a Startpage request.</span>
<span class="sd"> To avoid CAPTCHAs we need to send a well formed HTTP POST request with a</span>
<span class="sd"> cookie. We need to form a request that is identical to the request built by</span>
<span class="sd"> Startpage&#39;s search form:</span>
<span class="sd"> - in the cookie the **region** is selected</span>
<span class="sd"> - in the HTTP POST data the **language** is selected</span>
<span class="sd"> Additionally the arguments form Startpage&#39;s search form needs to be set in</span>
<span class="sd"> HTML POST data / compare ``&lt;input&gt;`` elements: :py:obj:`search_form_xpath`.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">engine_region</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="s1">&#39;en-US&#39;</span><span class="p">)</span>
<span class="n">engine_language</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="s1">&#39;en&#39;</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Origin&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Referer&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s1">&#39;/&#39;</span>
<span class="c1"># Build form data</span>
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
<span class="s1">&#39;cat&#39;</span><span class="p">:</span> <span class="n">startpage_categ</span><span class="p">,</span>
<span class="s1">&#39;t&#39;</span><span class="p">:</span> <span class="s1">&#39;device&#39;</span><span class="p">,</span>
<span class="s1">&#39;sc&#39;</span><span class="p">:</span> <span class="n">get_sc_code</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="n">params</span><span class="p">),</span> <span class="c1"># hint: this func needs HTTP headers</span>
<span class="s1">&#39;with_date&#39;</span><span class="p">:</span> <span class="n">time_range_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">],</span> <span class="s1">&#39;&#39;</span><span class="p">),</span>
<span class="s1">&#39;abp&#39;</span><span class="p">:</span> <span class="s1">&#39;1&#39;</span><span class="p">,</span>
<span class="s1">&#39;abd&#39;</span><span class="p">:</span> <span class="s1">&#39;1&#39;</span><span class="p">,</span>
<span class="s1">&#39;abe&#39;</span><span class="p">:</span> <span class="s1">&#39;1&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="k">if</span> <span class="n">engine_language</span><span class="p">:</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;language&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_language</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;lui&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_language</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;page&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span>
<span class="n">args</span><span class="p">[</span><span class="s1">&#39;segment&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;startpage.udog&#39;</span>
<span class="c1"># Build cookie</span>
<span class="n">lang_homepage</span> <span class="o">=</span> <span class="s1">&#39;en&#39;</span>
<span class="n">cookie</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
<span class="n">cookie</span><span class="p">[</span><span class="s1">&#39;date_time&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;world&#39;</span>
<span class="n">cookie</span><span class="p">[</span><span class="s1">&#39;disable_family_filter&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">safesearch_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]]</span>
<span class="n">cookie</span><span class="p">[</span><span class="s1">&#39;disable_open_in_new_window&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;0&#39;</span>
<span class="n">cookie</span><span class="p">[</span><span class="s1">&#39;enable_post_method&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;1&#39;</span> <span class="c1"># hint: POST</span>
<span class="n">cookie</span><span class="p">[</span><span class="s1">&#39;enable_proxy_safety_suggest&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;1&#39;</span>
<span class="n">cookie</span><span class="p">[</span><span class="s1">&#39;enable_stay_control&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;1&#39;</span>
<span class="n">cookie</span><span class="p">[</span><span class="s1">&#39;instant_answers&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;1&#39;</span>
<span class="n">cookie</span><span class="p">[</span><span class="s1">&#39;lang_homepage&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;s/device/</span><span class="si">%s</span><span class="s1">/&#39;</span> <span class="o">%</span> <span class="n">lang_homepage</span>
<span class="n">cookie</span><span class="p">[</span><span class="s1">&#39;num_of_results&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;10&#39;</span>
<span class="n">cookie</span><span class="p">[</span><span class="s1">&#39;suggestions&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;1&#39;</span>
<span class="n">cookie</span><span class="p">[</span><span class="s1">&#39;wt_unit&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;celsius&#39;</span>
<span class="k">if</span> <span class="n">engine_language</span><span class="p">:</span>
<span class="n">cookie</span><span class="p">[</span><span class="s1">&#39;language&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_language</span>
<span class="n">cookie</span><span class="p">[</span><span class="s1">&#39;language_ui&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_language</span>
<span class="k">if</span> <span class="n">engine_region</span><span class="p">:</span>
<span class="n">cookie</span><span class="p">[</span><span class="s1">&#39;search_results_region&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_region</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">][</span><span class="s1">&#39;preferences&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;N1N&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="s2">&quot;</span><span class="si">%s</span><span class="s2">EEE</span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">cookie</span><span class="o">.</span><span class="n">items</span><span class="p">()])</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;cookie preferences: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">][</span><span class="s1">&#39;preferences&#39;</span><span class="p">])</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;data: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">args</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">args</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;method&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;POST&#39;</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span>
<span class="k">return</span> <span class="n">params</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_published_date</span><span class="p">(</span><span class="n">content</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">datetime</span> <span class="o">|</span> <span class="kc">None</span><span class="p">]:</span>
<span class="n">published_date</span> <span class="o">=</span> <span class="kc">None</span>
<span class="c1"># check if search result starts with something like: &quot;2 Sep 2014 ... &quot;</span>
<span class="k">if</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;^([1-9]|[1-2][0-9]|3[0-1]) [A-Z][a-z]</span><span class="si">{2}</span><span class="s2"> [0-9]</span><span class="si">{4}</span><span class="s2"> \.\.\. &quot;</span><span class="p">,</span> <span class="n">content</span><span class="p">):</span>
<span class="n">date_pos</span> <span class="o">=</span> <span class="n">content</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;...&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="mi">4</span>
<span class="n">date_string</span> <span class="o">=</span> <span class="n">content</span><span class="p">[</span><span class="mi">0</span> <span class="p">:</span> <span class="n">date_pos</span> <span class="o">-</span> <span class="mi">5</span><span class="p">]</span>
<span class="c1"># fix content string</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">content</span><span class="p">[</span><span class="n">date_pos</span><span class="p">:]</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">published_date</span> <span class="o">=</span> <span class="n">dateutil</span><span class="o">.</span><span class="n">parser</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">date_string</span><span class="p">,</span> <span class="n">dayfirst</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="k">pass</span>
<span class="c1"># check if search result starts with something like: &quot;5 days ago ... &quot;</span>
<span class="k">elif</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;^[0-9]+ days? ago \.\.\. &quot;</span><span class="p">,</span> <span class="n">content</span><span class="p">):</span>
<span class="n">date_pos</span> <span class="o">=</span> <span class="n">content</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;...&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="mi">4</span>
<span class="n">date_string</span> <span class="o">=</span> <span class="n">content</span><span class="p">[</span><span class="mi">0</span> <span class="p">:</span> <span class="n">date_pos</span> <span class="o">-</span> <span class="mi">5</span><span class="p">]</span>
<span class="c1"># calculate datetime</span>
<span class="n">published_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s1">&#39;\d+&#39;</span><span class="p">,</span> <span class="n">date_string</span><span class="p">)</span><span class="o">.</span><span class="n">group</span><span class="p">()))</span> <span class="c1"># type: ignore</span>
<span class="c1"># fix content string</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">content</span><span class="p">[</span><span class="n">date_pos</span><span class="p">:]</span>
<span class="k">return</span> <span class="n">content</span><span class="p">,</span> <span class="n">published_date</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_get_web_result</span><span class="p">(</span><span class="n">result</span><span class="p">):</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">html_to_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;description&#39;</span><span class="p">))</span>
<span class="n">content</span><span class="p">,</span> <span class="n">publishedDate</span> <span class="o">=</span> <span class="n">_parse_published_date</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
<span class="k">return</span> <span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;clickUrl&#39;</span><span class="p">],</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">html_to_text</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;title&#39;</span><span class="p">]),</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
<span class="s1">&#39;publishedDate&#39;</span><span class="p">:</span> <span class="n">publishedDate</span><span class="p">,</span>
<span class="p">}</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_get_news_result</span><span class="p">(</span><span class="n">result</span><span class="p">):</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">remove_pua_from_str</span><span class="p">(</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;title&#39;</span><span class="p">]))</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">remove_pua_from_str</span><span class="p">(</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;description&#39;</span><span class="p">)))</span>
<span class="n">publishedDate</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;date&#39;</span><span class="p">):</span>
<span class="n">publishedDate</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;date&#39;</span><span class="p">]</span> <span class="o">/</span> <span class="mi">1000</span><span class="p">)</span>
<span class="n">thumbnailUrl</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;thumbnailUrl&#39;</span><span class="p">):</span>
<span class="n">thumbnailUrl</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;thumbnailUrl&#39;</span><span class="p">]</span>
<span class="k">return</span> <span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;clickUrl&#39;</span><span class="p">],</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
<span class="s1">&#39;publishedDate&#39;</span><span class="p">:</span> <span class="n">publishedDate</span><span class="p">,</span>
<span class="s1">&#39;thumbnail&#39;</span><span class="p">:</span> <span class="n">thumbnailUrl</span><span class="p">,</span>
<span class="p">}</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_get_image_result</span><span class="p">(</span><span class="n">result</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;altClickUrl&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">url</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">thumbnailUrl</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;thumbnailUrl&#39;</span><span class="p">):</span>
<span class="n">thumbnailUrl</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;thumbnailUrl&#39;</span><span class="p">]</span>
<span class="n">resolution</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;width&#39;</span><span class="p">)</span> <span class="ow">and</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;height&#39;</span><span class="p">):</span>
<span class="n">resolution</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;width&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">x</span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;height&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">filesize</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;filesize&#39;</span><span class="p">):</span>
<span class="n">size_str</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="nb">str</span><span class="o">.</span><span class="n">isdigit</span><span class="p">,</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;filesize&#39;</span><span class="p">]))</span>
<span class="n">filesize</span> <span class="o">=</span> <span class="n">humanize_bytes</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">size_str</span><span class="p">))</span>
<span class="k">return</span> <span class="p">{</span>
<span class="s1">&#39;template&#39;</span><span class="p">:</span> <span class="s1">&#39;images.html&#39;</span><span class="p">,</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">html_to_text</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;title&#39;</span><span class="p">]),</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="s1">&#39;&#39;</span><span class="p">,</span>
<span class="s1">&#39;img_src&#39;</span><span class="p">:</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;rawImageUrl&#39;</span><span class="p">),</span>
<span class="s1">&#39;thumbnail_src&#39;</span><span class="p">:</span> <span class="n">thumbnailUrl</span><span class="p">,</span>
<span class="s1">&#39;resolution&#39;</span><span class="p">:</span> <span class="n">resolution</span><span class="p">,</span>
<span class="s1">&#39;img_format&#39;</span><span class="p">:</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;format&#39;</span><span class="p">),</span>
<span class="s1">&#39;filesize&#39;</span><span class="p">:</span> <span class="n">filesize</span><span class="p">,</span>
<span class="p">}</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="n">categ</span> <span class="o">=</span> <span class="n">startpage_categ</span><span class="o">.</span><span class="n">capitalize</span><span class="p">()</span>
<span class="n">results_raw</span> <span class="o">=</span> <span class="s1">&#39;{&#39;</span> <span class="o">+</span> <span class="n">extr</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="sa">f</span><span class="s2">&quot;React.createElement(UIStartpage.AppSerp</span><span class="si">{</span><span class="n">categ</span><span class="si">}</span><span class="s2">, </span><span class="se">{{</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s1">&#39;}})&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="s1">&#39;}}&#39;</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Location&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;https://www.startpage.com/sp/captcha&quot;</span><span class="p">):</span>
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">()</span>
<span class="n">results_json</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">results_raw</span><span class="p">)</span>
<span class="n">results_obj</span> <span class="o">=</span> <span class="n">results_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;render&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;presenter&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;regions&#39;</span><span class="p">,</span> <span class="p">{})</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">results_categ</span> <span class="ow">in</span> <span class="n">results_obj</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;mainline&#39;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">results_categ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;results&#39;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="k">if</span> <span class="n">results_categ</span><span class="p">[</span><span class="s1">&#39;display_type&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;web-google&#39;</span><span class="p">:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_get_web_result</span><span class="p">(</span><span class="n">item</span><span class="p">))</span>
<span class="k">elif</span> <span class="n">results_categ</span><span class="p">[</span><span class="s1">&#39;display_type&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;news-bing&#39;</span><span class="p">:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_get_news_result</span><span class="p">(</span><span class="n">item</span><span class="p">))</span>
<span class="k">elif</span> <span class="s1">&#39;images&#39;</span> <span class="ow">in</span> <span class="n">results_categ</span><span class="p">[</span><span class="s1">&#39;display_type&#39;</span><span class="p">]:</span>
<span class="n">item</span> <span class="o">=</span> <span class="n">_get_image_result</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">if</span> <span class="n">item</span><span class="p">:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">return</span> <span class="n">results</span>
<div class="viewcode-block" id="fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/online/startpage.html#searx.engines.startpage.fetch_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Fetch :ref:`languages &lt;startpage languages&gt;` and :ref:`regions &lt;startpage</span>
<span class="sd"> regions&gt;` from Startpage.&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-many-branches</span>
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;User-Agent&#39;</span><span class="p">:</span> <span class="n">gen_useragent</span><span class="p">(),</span>
<span class="s1">&#39;Accept-Language&#39;</span><span class="p">:</span> <span class="s2">&quot;en-US,en;q=0.5&quot;</span><span class="p">,</span> <span class="c1"># bing needs to set the English language</span>
<span class="p">}</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">&#39;https://www.startpage.com/do/settings&#39;</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: response from Startpage is not OK.&quot;</span><span class="p">)</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="c1"># regions</span>
<span class="n">sp_region_names</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">option</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;//form[@name=&quot;settings&quot;]//select[@name=&quot;search_results_region&quot;]/option&#39;</span><span class="p">):</span>
<span class="n">sp_region_names</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">option</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;value&#39;</span><span class="p">))</span>
<span class="k">for</span> <span class="n">eng_tag</span> <span class="ow">in</span> <span class="n">sp_region_names</span><span class="p">:</span>
<span class="k">if</span> <span class="n">eng_tag</span> <span class="o">==</span> <span class="s1">&#39;all&#39;</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">babel_region_tag</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;no_NO&#39;</span><span class="p">:</span> <span class="s1">&#39;nb_NO&#39;</span><span class="p">}</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">)</span> <span class="c1"># norway</span>
<span class="k">if</span> <span class="s1">&#39;-&#39;</span> <span class="ow">in</span> <span class="n">babel_region_tag</span><span class="p">:</span>
<span class="n">l</span><span class="p">,</span> <span class="n">r</span> <span class="o">=</span> <span class="n">babel_region_tag</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)</span>
<span class="n">r</span> <span class="o">=</span> <span class="n">r</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">l</span> <span class="o">+</span> <span class="s1">&#39;_&#39;</span> <span class="o">+</span> <span class="n">r</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;_&#39;</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">babel_region_tag</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;_&#39;</span><span class="p">))</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: can&#39;t determine babel locale of startpage&#39;s locale </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">eng_tag</span><span class="p">)</span>
<span class="k">continue</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
<span class="c1"># languages</span>
<span class="n">catalog_engine2code</span> <span class="o">=</span> <span class="p">{</span><span class="n">name</span><span class="o">.</span><span class="n">lower</span><span class="p">():</span> <span class="n">lang_code</span> <span class="k">for</span> <span class="n">lang_code</span><span class="p">,</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="p">(</span><span class="s1">&#39;en&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
<span class="c1"># get the native name of every language known by babel</span>
<span class="k">for</span> <span class="n">lang_code</span> <span class="ow">in</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">lang_code</span><span class="p">:</span> <span class="n">lang_code</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">babel</span><span class="o">.</span><span class="n">localedata</span><span class="o">.</span><span class="n">locale_identifiers</span><span class="p">()):</span>
<span class="n">native_name</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="p">(</span><span class="n">lang_code</span><span class="p">)</span><span class="o">.</span><span class="n">get_language_name</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">native_name</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;ERROR: language name of startpage&#39;s language </span><span class="si">{</span><span class="n">lang_code</span><span class="si">}</span><span class="s2"> is unknown by babel&quot;</span><span class="p">)</span>
<span class="k">continue</span>
<span class="n">native_name</span> <span class="o">=</span> <span class="n">native_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="c1"># add native name exactly as it is</span>
<span class="n">catalog_engine2code</span><span class="p">[</span><span class="n">native_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">lang_code</span>
<span class="c1"># add &quot;normalized&quot; language name (i.e. français becomes francais and español becomes espanol)</span>
<span class="n">unaccented_name</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">c</span><span class="p">:</span> <span class="ow">not</span> <span class="n">combining</span><span class="p">(</span><span class="n">c</span><span class="p">),</span> <span class="n">normalize</span><span class="p">(</span><span class="s1">&#39;NFKD&#39;</span><span class="p">,</span> <span class="n">native_name</span><span class="p">)))</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">unaccented_name</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">unaccented_name</span><span class="o">.</span><span class="n">encode</span><span class="p">()):</span>
<span class="c1"># add only if result is ascii (otherwise &quot;normalization&quot; didn&#39;t work)</span>
<span class="n">catalog_engine2code</span><span class="p">[</span><span class="n">unaccented_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">lang_code</span>
<span class="c1"># values that can&#39;t be determined by babel&#39;s languages names</span>
<span class="n">catalog_engine2code</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
<span class="p">{</span>
<span class="c1"># traditional chinese used in ..</span>
<span class="s1">&#39;fantizhengwen&#39;</span><span class="p">:</span> <span class="s1">&#39;zh_Hant&#39;</span><span class="p">,</span>
<span class="c1"># Korean alphabet</span>
<span class="s1">&#39;hangul&#39;</span><span class="p">:</span> <span class="s1">&#39;ko&#39;</span><span class="p">,</span>
<span class="c1"># Malayalam is one of 22 scheduled languages of India.</span>
<span class="s1">&#39;malayam&#39;</span><span class="p">:</span> <span class="s1">&#39;ml&#39;</span><span class="p">,</span>
<span class="s1">&#39;norsk&#39;</span><span class="p">:</span> <span class="s1">&#39;nb&#39;</span><span class="p">,</span>
<span class="s1">&#39;sinhalese&#39;</span><span class="p">:</span> <span class="s1">&#39;si&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="n">skip_eng_tags</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;english_uk&#39;</span><span class="p">,</span> <span class="c1"># SearXNG lang &#39;en&#39; already maps to &#39;english&#39;</span>
<span class="p">}</span>
<span class="k">for</span> <span class="n">option</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;//form[@name=&quot;settings&quot;]//select[@name=&quot;language&quot;]/option&#39;</span><span class="p">):</span>
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">option</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;value&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">eng_tag</span> <span class="ow">in</span> <span class="n">skip_eng_tags</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">name</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">option</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="c1"># type: ignore</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">catalog_engine2code</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">sxng_tag</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">catalog_engine2code</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,335 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.tineye &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.tineye</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.tineye</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;This engine implements *Tineye - reverse image search*</span>
<span class="sd">Using TinEye, you can search by image or perform what we call a reverse image</span>
<span class="sd">search. You can do that by uploading an image or searching by URL. You can also</span>
<span class="sd">simply drag and drop your images to start your search. TinEye constantly crawls</span>
<span class="sd">the web and adds images to its index. Today, the TinEye index is over 50.2</span>
<span class="sd">billion images `[tineye.com] &lt;https://tineye.com/how&gt;`_.</span>
<span class="sd">.. hint::</span>
<span class="sd"> This SearXNG engine only supports *&#39;searching by URL&#39;* and it does not use</span>
<span class="sd"> the official API `[api.tineye.com] &lt;https://api.tineye.com/python/docs/&gt;`_.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://tineye.com&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q2382535&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://api.tineye.com/python/docs/&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;JSON&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">engine_type</span> <span class="o">=</span> <span class="s1">&#39;online_url_search&#39;</span>
<span class="sd">&quot;&quot;&quot;:py:obj:`searx.search.processors.online_url_search`&quot;&quot;&quot;</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;general&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;https://tineye.com&#39;</span>
<span class="n">search_string</span> <span class="o">=</span> <span class="s1">&#39;/api/v1/result_json/?page=</span><span class="si">{page}</span><span class="s1">&amp;</span><span class="si">{query}</span><span class="s1">&#39;</span>
<span class="n">FORMAT_NOT_SUPPORTED</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span>
<span class="s2">&quot;Could not read that image url. This may be due to an unsupported file&quot;</span>
<span class="s2">&quot; format. TinEye only supports images that are JPEG, PNG, GIF, BMP, TIFF or WebP.&quot;</span>
<span class="p">)</span>
<span class="sd">&quot;&quot;&quot;TinEye error message&quot;&quot;&quot;</span>
<span class="n">NO_SIGNATURE_ERROR</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span>
<span class="s2">&quot;The image is too simple to find matches. TinEye requires a basic level of&quot;</span>
<span class="s2">&quot; visual detail to successfully identify matches.&quot;</span>
<span class="p">)</span>
<span class="sd">&quot;&quot;&quot;TinEye error message&quot;&quot;&quot;</span>
<span class="n">DOWNLOAD_ERROR</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;The image could not be downloaded.&quot;</span><span class="p">)</span>
<span class="sd">&quot;&quot;&quot;TinEye error message&quot;&quot;&quot;</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online_url_search/tineye.html#searx.engines.tineye.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Build TinEye HTTP request using ``search_urls`` of a :py:obj:`engine_type`.&quot;&quot;&quot;</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;raise_for_httperror&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;search_urls&#39;</span><span class="p">][</span><span class="s1">&#39;data:image&#39;</span><span class="p">]:</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;search_urls&#39;</span><span class="p">][</span><span class="s1">&#39;data:image&#39;</span><span class="p">]</span>
<span class="k">elif</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;search_urls&#39;</span><span class="p">][</span><span class="s1">&#39;http&#39;</span><span class="p">]:</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;search_urls&#39;</span><span class="p">][</span><span class="s1">&#39;http&#39;</span><span class="p">]</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;query URL: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">query</span><span class="p">)</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">})</span>
<span class="c1"># see https://github.com/TinEye/pytineye/blob/main/pytineye/api.py</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">search_string</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="n">query</span><span class="p">,</span> <span class="n">page</span><span class="o">=</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">])</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;Connection&#39;</span><span class="p">:</span> <span class="s1">&#39;keep-alive&#39;</span><span class="p">,</span>
<span class="s1">&#39;Host&#39;</span><span class="p">:</span> <span class="s1">&#39;tineye.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;DNT&#39;</span><span class="p">:</span> <span class="s1">&#39;1&#39;</span><span class="p">,</span>
<span class="s1">&#39;TE&#39;</span><span class="p">:</span> <span class="s1">&#39;trailers&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">params</span></div>
<div class="viewcode-block" id="parse_tineye_match">
<a class="viewcode-back" href="../../../dev/engines/online_url_search/tineye.html#searx.engines.tineye.parse_tineye_match">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">parse_tineye_match</span><span class="p">(</span><span class="n">match_json</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Takes parsed JSON from the API server and turns it into a :py:obj:`dict`</span>
<span class="sd"> object.</span>
<span class="sd"> Attributes `(class Match) &lt;https://github.com/TinEye/pytineye/blob/main/pytineye/api.py&gt;`__</span>
<span class="sd"> - `image_url`, link to the result image.</span>
<span class="sd"> - `domain`, domain this result was found on.</span>
<span class="sd"> - `score`, a number (0 to 100) that indicates how closely the images match.</span>
<span class="sd"> - `width`, image width in pixels.</span>
<span class="sd"> - `height`, image height in pixels.</span>
<span class="sd"> - `size`, image area in pixels.</span>
<span class="sd"> - `format`, image format.</span>
<span class="sd"> - `filesize`, image size in bytes.</span>
<span class="sd"> - `overlay`, overlay URL.</span>
<span class="sd"> - `tags`, whether this match belongs to a collection or stock domain.</span>
<span class="sd"> - `backlinks`, a list of Backlink objects pointing to the original websites</span>
<span class="sd"> and image URLs. List items are instances of :py:obj:`dict`, (`Backlink</span>
<span class="sd"> &lt;https://github.com/TinEye/pytineye/blob/main/pytineye/api.py&gt;`__):</span>
<span class="sd"> - `url`, the image URL to the image.</span>
<span class="sd"> - `backlink`, the original website URL.</span>
<span class="sd"> - `crawl_date`, the date the image was crawled.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># HINT: there exists an alternative backlink dict in the domains list / e.g.::</span>
<span class="c1">#</span>
<span class="c1"># match_json[&#39;domains&#39;][0][&#39;backlinks&#39;]</span>
<span class="n">backlinks</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="s2">&quot;backlinks&quot;</span> <span class="ow">in</span> <span class="n">match_json</span><span class="p">:</span>
<span class="k">for</span> <span class="n">backlink_json</span> <span class="ow">in</span> <span class="n">match_json</span><span class="p">[</span><span class="s2">&quot;backlinks&quot;</span><span class="p">]:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">backlink_json</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">continue</span>
<span class="n">crawl_date</span> <span class="o">=</span> <span class="n">backlink_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;crawl_date&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">crawl_date</span><span class="p">:</span>
<span class="n">crawl_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">crawl_date</span><span class="p">,</span> <span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">crawl_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">min</span>
<span class="n">backlinks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">backlink_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;url&quot;</span><span class="p">),</span>
<span class="s1">&#39;backlink&#39;</span><span class="p">:</span> <span class="n">backlink_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;backlink&quot;</span><span class="p">),</span>
<span class="s1">&#39;crawl_date&#39;</span><span class="p">:</span> <span class="n">crawl_date</span><span class="p">,</span>
<span class="s1">&#39;image_name&#39;</span><span class="p">:</span> <span class="n">backlink_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;image_name&quot;</span><span class="p">),</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="p">{</span>
<span class="s1">&#39;image_url&#39;</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;image_url&quot;</span><span class="p">),</span>
<span class="s1">&#39;domain&#39;</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;domain&quot;</span><span class="p">),</span>
<span class="s1">&#39;score&#39;</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;score&quot;</span><span class="p">),</span>
<span class="s1">&#39;width&#39;</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;width&quot;</span><span class="p">),</span>
<span class="s1">&#39;height&#39;</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;height&quot;</span><span class="p">),</span>
<span class="s1">&#39;size&#39;</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;size&quot;</span><span class="p">),</span>
<span class="s1">&#39;image_format&#39;</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;format&quot;</span><span class="p">),</span>
<span class="s1">&#39;filesize&#39;</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;filesize&quot;</span><span class="p">),</span>
<span class="s1">&#39;overlay&#39;</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;overlay&quot;</span><span class="p">),</span>
<span class="s1">&#39;tags&#39;</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;tags&quot;</span><span class="p">),</span>
<span class="s1">&#39;backlinks&#39;</span><span class="p">:</span> <span class="n">backlinks</span><span class="p">,</span>
<span class="p">}</span></div>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/online_url_search/tineye.html#searx.engines.tineye.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Parse HTTP response from TinEye.&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="c1"># handle the 422 client side errors, and the possible 400 status code error</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="ow">in</span> <span class="p">(</span><span class="mi">400</span><span class="p">,</span> <span class="mi">422</span><span class="p">):</span>
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="n">suggestions</span> <span class="o">=</span> <span class="n">json_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;suggestions&#39;</span><span class="p">,</span> <span class="p">{})</span>
<span class="n">message</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;HTTP Status Code: </span><span class="si">{</span><span class="n">resp</span><span class="o">.</span><span class="n">status_code</span><span class="si">}</span><span class="s1">&#39;</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">422</span><span class="p">:</span>
<span class="n">s_key</span> <span class="o">=</span> <span class="n">suggestions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;key&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">s_key</span> <span class="o">==</span> <span class="s2">&quot;Invalid image URL&quot;</span><span class="p">:</span>
<span class="c1"># test https://docs.searxng.org/_static/searxng-wordmark.svg</span>
<span class="n">message</span> <span class="o">=</span> <span class="n">FORMAT_NOT_SUPPORTED</span>
<span class="k">elif</span> <span class="n">s_key</span> <span class="o">==</span> <span class="s1">&#39;NO_SIGNATURE_ERROR&#39;</span><span class="p">:</span>
<span class="c1"># test https://pngimg.com/uploads/dot/dot_PNG4.png</span>
<span class="n">message</span> <span class="o">=</span> <span class="n">NO_SIGNATURE_ERROR</span>
<span class="k">elif</span> <span class="n">s_key</span> <span class="o">==</span> <span class="s1">&#39;Download Error&#39;</span><span class="p">:</span>
<span class="c1"># test https://notexists</span>
<span class="n">message</span> <span class="o">=</span> <span class="n">DOWNLOAD_ERROR</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;Unknown suggestion key encountered: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">s_key</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span> <span class="c1"># 400</span>
<span class="n">description</span> <span class="o">=</span> <span class="n">suggestions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;description&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">description</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
<span class="n">message</span> <span class="o">=</span> <span class="s1">&#39;,&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">description</span><span class="p">)</span>
<span class="c1"># see https://github.com/searxng/searxng/pull/1456#issuecomment-1193105023</span>
<span class="c1"># results.add(results.types.Answer(answer=message))</span>
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
<span class="k">return</span> <span class="n">results</span>
<span class="c1"># Raise for all other responses</span>
<span class="n">resp</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">for</span> <span class="n">match_json</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s1">&#39;matches&#39;</span><span class="p">]:</span>
<span class="n">tineye_match</span> <span class="o">=</span> <span class="n">parse_tineye_match</span><span class="p">(</span><span class="n">match_json</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">&#39;backlinks&#39;</span><span class="p">]:</span>
<span class="k">continue</span>
<span class="n">backlink</span> <span class="o">=</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">&#39;backlinks&#39;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;template&#39;</span><span class="p">:</span> <span class="s1">&#39;images.html&#39;</span><span class="p">,</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">&#39;backlink&#39;</span><span class="p">],</span>
<span class="s1">&#39;thumbnail_src&#39;</span><span class="p">:</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">&#39;image_url&#39;</span><span class="p">],</span>
<span class="s1">&#39;source&#39;</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">],</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">&#39;image_name&#39;</span><span class="p">],</span>
<span class="s1">&#39;img_src&#39;</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">],</span>
<span class="s1">&#39;format&#39;</span><span class="p">:</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">&#39;image_format&#39;</span><span class="p">],</span>
<span class="s1">&#39;width&#39;</span><span class="p">:</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">&#39;width&#39;</span><span class="p">],</span>
<span class="s1">&#39;height&#39;</span><span class="p">:</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">&#39;height&#39;</span><span class="p">],</span>
<span class="s1">&#39;publishedDate&#39;</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">&#39;crawl_date&#39;</span><span class="p">],</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="c1"># append number of results</span>
<span class="n">number_of_results</span> <span class="o">=</span> <span class="n">json_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;num_matches&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">number_of_results</span><span class="p">:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;number_of_results&#39;</span><span class="p">:</span> <span class="n">number_of_results</span><span class="p">})</span>
<span class="k">return</span> <span class="n">results</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,363 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.torznab &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.torznab</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.torznab</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Torznab_ is an API specification that provides a standardized way to query</span>
<span class="sd">torrent site for content. It is used by a number of torrent applications,</span>
<span class="sd">including Prowlarr_ and Jackett_.</span>
<span class="sd">Using this engine together with Prowlarr_ or Jackett_ allows you to search</span>
<span class="sd">a huge number of torrent sites which are not directly supported.</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">The engine has the following settings:</span>
<span class="sd">``base_url``:</span>
<span class="sd"> Torznab endpoint URL.</span>
<span class="sd">``api_key``:</span>
<span class="sd"> The API key to use for authentication.</span>
<span class="sd">``torznab_categories``:</span>
<span class="sd"> The categories to use for searching. This is a list of category IDs. See</span>
<span class="sd"> Prowlarr-categories_ or Jackett-categories_ for more information.</span>
<span class="sd">``show_torrent_files``:</span>
<span class="sd"> Whether to show the torrent file in the search results. Be careful as using</span>
<span class="sd"> this with Prowlarr_ or Jackett_ leaks the API key. This should be used only</span>
<span class="sd"> if you are querying a Torznab endpoint without authentication or if the</span>
<span class="sd"> instance is private. Be aware that private trackers may ban you if you share</span>
<span class="sd"> the torrent file. Defaults to ``false``.</span>
<span class="sd">``show_magnet_links``:</span>
<span class="sd"> Whether to show the magnet link in the search results. Be aware that private</span>
<span class="sd"> trackers may ban you if you share the magnet link. Defaults to ``true``.</span>
<span class="sd">.. _Torznab:</span>
<span class="sd"> https://torznab.github.io/spec-1.3-draft/index.html</span>
<span class="sd">.. _Prowlarr:</span>
<span class="sd"> https://github.com/Prowlarr/Prowlarr</span>
<span class="sd">.. _Jackett:</span>
<span class="sd"> https://github.com/Jackett/Jackett</span>
<span class="sd">.. _Prowlarr-categories:</span>
<span class="sd"> https://wiki.servarr.com/en/prowlarr/cardigann-yml-definition#categories</span>
<span class="sd">.. _Jackett-categories:</span>
<span class="sd"> https://github.com/Jackett/Jackett/wiki/Jackett-Categories</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">quote</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">etree</span> <span class="c1"># type: ignore</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineAPIException</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">humanize_bytes</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="c1"># engine settings</span>
<span class="n">about</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s2">&quot;https://torznab.github.io/spec-1.3-draft&quot;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;XML&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">categories</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;files&#39;</span><span class="p">]</span>
<span class="n">paging</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">time_range_support</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
<span class="c1"># defined in settings.yml</span>
<span class="c1"># example (Jackett): &quot;http://localhost:9117/api/v2.0/indexers/all/results/torznab&quot;</span>
<span class="n">base_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="n">api_key</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="c1"># https://newznab.readthedocs.io/en/latest/misc/api/#predefined-categories</span>
<span class="n">torznab_categories</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">show_torrent_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">show_magnet_links</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
<div class="viewcode-block" id="init">
<a class="viewcode-back" href="../../../dev/engines/online/torznab.html#searx.engines.torznab.init">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> <span class="c1"># pylint: disable=unused-argument</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Initialize the engine.&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">base_url</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;missing torznab base_url&#39;</span><span class="p">)</span></div>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/torznab.html#searx.engines.torznab.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Build the request params.&quot;&quot;&quot;</span>
<span class="n">search_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s1">&#39;?t=search&amp;q=</span><span class="si">{search_query}</span><span class="s1">&#39;</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">api_key</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">search_url</span> <span class="o">+=</span> <span class="s1">&#39;&amp;apikey=</span><span class="si">{api_key}</span><span class="s1">&#39;</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">torznab_categories</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">search_url</span> <span class="o">+=</span> <span class="s1">&#39;&amp;cat=</span><span class="si">{torznab_categories}</span><span class="s1">&#39;</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">search_query</span><span class="o">=</span><span class="n">quote</span><span class="p">(</span><span class="n">query</span><span class="p">),</span> <span class="n">api_key</span><span class="o">=</span><span class="n">api_key</span><span class="p">,</span> <span class="n">torznab_categories</span><span class="o">=</span><span class="s2">&quot;,&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="nb">str</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">torznab_categories</span><span class="p">])</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">params</span></div>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/online/torznab.html#searx.engines.torznab.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Parse the XML response and return a list of results.&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">search_results</span> <span class="o">=</span> <span class="n">etree</span><span class="o">.</span><span class="n">XML</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
<span class="c1"># handle errors: https://newznab.readthedocs.io/en/latest/misc/api/#newznab-error-codes</span>
<span class="k">if</span> <span class="n">search_results</span><span class="o">.</span><span class="n">tag</span> <span class="o">==</span> <span class="s2">&quot;error&quot;</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">SearxEngineAPIException</span><span class="p">(</span><span class="n">search_results</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;description&quot;</span><span class="p">))</span>
<span class="n">channel</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span> <span class="o">=</span> <span class="n">search_results</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">item</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">channel</span><span class="o">.</span><span class="n">iterfind</span><span class="p">(</span><span class="s1">&#39;item&#39;</span><span class="p">):</span>
<span class="n">result</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">build_result</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
<span class="k">return</span> <span class="n">results</span></div>
<div class="viewcode-block" id="build_result">
<a class="viewcode-back" href="../../../dev/engines/online/torznab.html#searx.engines.torznab.build_result">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">build_result</span><span class="p">(</span><span class="n">item</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Build a result from a XML item.&quot;&quot;&quot;</span>
<span class="c1"># extract attributes from XML</span>
<span class="c1"># see https://torznab.github.io/spec-1.3-draft/torznab/Specification-v1.3.html#predefined-attributes</span>
<span class="n">enclosure</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;enclosure&#39;</span><span class="p">)</span>
<span class="n">enclosure_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">enclosure</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">enclosure_url</span> <span class="o">=</span> <span class="n">enclosure</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;url&#39;</span><span class="p">)</span>
<span class="n">filesize</span> <span class="o">=</span> <span class="n">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;size&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">filesize</span> <span class="ow">and</span> <span class="n">enclosure</span><span class="p">:</span>
<span class="n">filesize</span> <span class="o">=</span> <span class="n">enclosure</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;length&#39;</span><span class="p">)</span>
<span class="n">guid</span> <span class="o">=</span> <span class="n">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;guid&#39;</span><span class="p">)</span>
<span class="n">comments</span> <span class="o">=</span> <span class="n">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;comments&#39;</span><span class="p">)</span>
<span class="n">pubDate</span> <span class="o">=</span> <span class="n">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;pubDate&#39;</span><span class="p">)</span>
<span class="n">seeders</span> <span class="o">=</span> <span class="n">get_torznab_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;seeders&#39;</span><span class="p">)</span>
<span class="n">leechers</span> <span class="o">=</span> <span class="n">get_torznab_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;leechers&#39;</span><span class="p">)</span>
<span class="n">peers</span> <span class="o">=</span> <span class="n">get_torznab_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;peers&#39;</span><span class="p">)</span>
<span class="c1"># map attributes to SearXNG result</span>
<span class="n">result</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;template&#39;</span><span class="p">:</span> <span class="s1">&#39;torrent.html&#39;</span><span class="p">,</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;title&#39;</span><span class="p">),</span>
<span class="s1">&#39;filesize&#39;</span><span class="p">:</span> <span class="n">humanize_bytes</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">filesize</span><span class="p">))</span> <span class="k">if</span> <span class="n">filesize</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
<span class="s1">&#39;files&#39;</span><span class="p">:</span> <span class="n">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;files&#39;</span><span class="p">),</span>
<span class="s1">&#39;seed&#39;</span><span class="p">:</span> <span class="n">seeders</span><span class="p">,</span>
<span class="s1">&#39;leech&#39;</span><span class="p">:</span> <span class="n">_map_leechers</span><span class="p">(</span><span class="n">leechers</span><span class="p">,</span> <span class="n">seeders</span><span class="p">,</span> <span class="n">peers</span><span class="p">),</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">_map_result_url</span><span class="p">(</span><span class="n">guid</span><span class="p">,</span> <span class="n">comments</span><span class="p">),</span>
<span class="s1">&#39;publishedDate&#39;</span><span class="p">:</span> <span class="n">_map_published_date</span><span class="p">(</span><span class="n">pubDate</span><span class="p">),</span>
<span class="s1">&#39;torrentfile&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s1">&#39;magnetlink&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">link</span> <span class="o">=</span> <span class="n">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;link&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">show_torrent_files</span><span class="p">:</span>
<span class="n">result</span><span class="p">[</span><span class="s1">&#39;torrentfile&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">_map_torrent_file</span><span class="p">(</span><span class="n">link</span><span class="p">,</span> <span class="n">enclosure_url</span><span class="p">)</span>
<span class="k">if</span> <span class="n">show_magnet_links</span><span class="p">:</span>
<span class="n">magneturl</span> <span class="o">=</span> <span class="n">get_torznab_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;magneturl&#39;</span><span class="p">)</span>
<span class="n">result</span><span class="p">[</span><span class="s1">&#39;magnetlink&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">_map_magnet_link</span><span class="p">(</span><span class="n">magneturl</span><span class="p">,</span> <span class="n">guid</span><span class="p">,</span> <span class="n">enclosure_url</span><span class="p">,</span> <span class="n">link</span><span class="p">)</span>
<span class="k">return</span> <span class="n">result</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">_map_result_url</span><span class="p">(</span><span class="n">guid</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">comments</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="n">guid</span> <span class="ow">and</span> <span class="n">guid</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;http&#39;</span><span class="p">):</span>
<span class="k">return</span> <span class="n">guid</span>
<span class="k">if</span> <span class="n">comments</span> <span class="ow">and</span> <span class="n">comments</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;http&#39;</span><span class="p">):</span>
<span class="k">return</span> <span class="n">comments</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_map_leechers</span><span class="p">(</span><span class="n">leechers</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">seeders</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">peers</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="n">leechers</span><span class="p">:</span>
<span class="k">return</span> <span class="n">leechers</span>
<span class="k">if</span> <span class="n">seeders</span> <span class="ow">and</span> <span class="n">peers</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">peers</span><span class="p">)</span> <span class="o">-</span> <span class="nb">int</span><span class="p">(</span><span class="n">seeders</span><span class="p">))</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_map_published_date</span><span class="p">(</span><span class="n">pubDate</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">datetime</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="n">pubDate</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">pubDate</span><span class="p">,</span> <span class="s1">&#39;</span><span class="si">%a</span><span class="s1">, </span><span class="si">%d</span><span class="s1"> %b %Y %H:%M:%S %z&#39;</span><span class="p">)</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;ignore exception (publishedDate): </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_map_torrent_file</span><span class="p">(</span><span class="n">link</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">enclosure_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="n">link</span> <span class="ow">and</span> <span class="n">link</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;http&#39;</span><span class="p">):</span>
<span class="k">return</span> <span class="n">link</span>
<span class="k">if</span> <span class="n">enclosure_url</span> <span class="ow">and</span> <span class="n">enclosure_url</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;http&#39;</span><span class="p">):</span>
<span class="k">return</span> <span class="n">enclosure_url</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_map_magnet_link</span><span class="p">(</span>
<span class="n">magneturl</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span>
<span class="n">guid</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span>
<span class="n">enclosure_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span>
<span class="n">link</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="n">magneturl</span> <span class="ow">and</span> <span class="n">magneturl</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;magnet&#39;</span><span class="p">):</span>
<span class="k">return</span> <span class="n">magneturl</span>
<span class="k">if</span> <span class="n">guid</span> <span class="ow">and</span> <span class="n">guid</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;magnet&#39;</span><span class="p">):</span>
<span class="k">return</span> <span class="n">guid</span>
<span class="k">if</span> <span class="n">enclosure_url</span> <span class="ow">and</span> <span class="n">enclosure_url</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;magnet&#39;</span><span class="p">):</span>
<span class="k">return</span> <span class="n">enclosure_url</span>
<span class="k">if</span> <span class="n">link</span> <span class="ow">and</span> <span class="n">link</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;magnet&#39;</span><span class="p">):</span>
<span class="k">return</span> <span class="n">link</span>
<span class="k">return</span> <span class="kc">None</span>
<div class="viewcode-block" id="get_attribute">
<a class="viewcode-back" href="../../../dev/engines/online/torznab.html#searx.engines.torznab.get_attribute">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span><span class="p">,</span> <span class="n">property_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get attribute from item.&quot;&quot;&quot;</span>
<span class="n">property_element</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">property_name</span><span class="p">)</span>
<span class="k">if</span> <span class="n">property_element</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">property_element</span><span class="o">.</span><span class="n">text</span>
<span class="k">return</span> <span class="kc">None</span></div>
<div class="viewcode-block" id="get_torznab_attribute">
<a class="viewcode-back" href="../../../dev/engines/online/torznab.html#searx.engines.torznab.get_torznab_attribute">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_torznab_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span><span class="p">,</span> <span class="n">attribute_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get torznab special attribute from item.&quot;&quot;&quot;</span>
<span class="n">element</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">find</span><span class="p">(</span>
<span class="s1">&#39;.//torznab:attr[@name=&quot;</span><span class="si">{attribute_name}</span><span class="s1">&quot;]&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">attribute_name</span><span class="o">=</span><span class="n">attribute_name</span><span class="p">),</span>
<span class="p">{</span><span class="s1">&#39;torznab&#39;</span><span class="p">:</span> <span class="s1">&#39;http://torznab.com/schemas/2015/feed&#39;</span><span class="p">},</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">element</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">element</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">None</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,297 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.tubearchivist &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.tubearchivist</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.tubearchivist</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;`Tube Archivist`_ - *Your self hosted YouTube media server.*</span>
<span class="sd">.. _Tube Archivist: https://www.tubearchivist.com</span>
<span class="sd">This engine connects with a self-hosted instance of `Tube Archivist`_ to allow</span>
<span class="sd">searching for your hosted videos.</span>
<span class="sd">`Tube Archivist`_ (TA) requires authentication for all image loads via cookie</span>
<span class="sd">authentication. What this means is that by default, SearXNG will have no way to</span>
<span class="sd">pull images from TA (as there is no way to pass cookies in a URL string only).</span>
<span class="sd">In the meantime while work is done on the TA side, this can be worked around by</span>
<span class="sd">bypassing auth for images in TA by altering the default TA nginx file.</span>
<span class="sd">This is located in the main tubearchivist docker container at::</span>
<span class="sd"> /etc/nginx/sites-available/default</span>
<span class="sd">It is **strongly** recommended first setting up the intial connection and</span>
<span class="sd">verying searching works first with broken images, and then attempting this</span>
<span class="sd">change. This will limit any debugging to only images, rather than</span>
<span class="sd">tokens/networking.</span>
<span class="sd">Steps to enable **unauthenticated** metadata access for channels and videos:</span>
<span class="sd">#. Perform any backups of TA before editing core configurations.</span>
<span class="sd">#. Copy the contents of the file ``/etc/nginx/sites-available/default`` in the</span>
<span class="sd"> TA docker container</span>
<span class="sd">#. Edit ``location /cache/videos`` and ``location /cache/channels``. Comment</span>
<span class="sd"> out the line ``auth_request /api/ping/;`` to ``# auth_request /api/ping/;``.</span>
<span class="sd">#. Save the file to wherever you normally store your docker configuration.</span>
<span class="sd">#. Mount this new configuration over the default configuration. With ``docker</span>
<span class="sd"> run``, this would be::</span>
<span class="sd"> -v ./your-new-config.yml:/etc/nginx/sites-available/default</span>
<span class="sd"> With ``docker compose``, this would be::</span>
<span class="sd"> - &quot;./your-new-config.yml:/etc/nginx/sites-available/default:ro&quot;</span>
<span class="sd">#. Start the TA container.</span>
<span class="sd">After these steps, double check that TA works as normal (nothing should be</span>
<span class="sd">different on the TA side). Searching again should now show images.</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">The engine has the following required settings:</span>
<span class="sd">- :py:obj:`base_url`</span>
<span class="sd">- :py:obj:`ta_token`</span>
<span class="sd">Optional settings:</span>
<span class="sd">- :py:obj:`ta_link_to_mp4`</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: tubearchivist</span>
<span class="sd"> engine: tubearchivist</span>
<span class="sd"> shortcut: tuba</span>
<span class="sd"> base_url:</span>
<span class="sd"> ta_token:</span>
<span class="sd"> ta_link_to_mp4: true</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil.parser</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">html_to_text</span><span class="p">,</span> <span class="n">humanize_number</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="c1"># pylint: disable=line-too-long</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.tubearchivist.com&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://docs.tubearchivist.com/api/introduction/&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;JSON&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;videos&quot;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Base URL of the Tube Archivist instance. Fill this in with your own</span>
<span class="sd">Tube Archivist URL (``http://your-instance:port``).&quot;&quot;&quot;</span>
<span class="n">ta_token</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;The API key to use for Authorization_ header. Can be found under:</span>
<span class="sd"> :menuselection:`Settings --&gt; User --&gt; Admin Interface`.</span>
<span class="sd">.. _Authorization: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Authorization</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">ta_link_to_mp4</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&quot;&quot;&quot;Optional, if true SearXNG will link directly to the mp4 of the video to play</span>
<span class="sd">in the browser. The default behavior is to link into TubeArchivist&#39;s interface</span>
<span class="sd">directly.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">absolute_url</span><span class="p">(</span><span class="n">relative_url</span><span class="p">):</span>
<span class="k">return</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">base_url</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">)</span><span class="si">}{</span><span class="n">relative_url</span><span class="si">}</span><span class="s1">&#39;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">_</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">base_url</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;tubearchivist engine: base_url is unset&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">ta_token</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;tubearchivist engine: ta_token is unset&#39;</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">query</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">}</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">base_url</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span><span class="si">}</span><span class="s2">/api/search?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">][</span><span class="s1">&#39;Authorization&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;Token </span><span class="si">{</span><span class="n">ta_token</span><span class="si">}</span><span class="s1">&#39;</span>
<span class="k">return</span> <span class="n">params</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">video_response</span><span class="p">(</span><span class="n">resp</span><span class="p">,</span> <span class="n">results</span><span class="p">)</span>
<span class="k">return</span> <span class="n">results</span>
<div class="viewcode-block" id="video_response">
<a class="viewcode-back" href="../../../dev/engines/online/tubearchivist.html#searx.engines.tubearchivist.video_response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">video_response</span><span class="p">(</span><span class="n">resp</span><span class="p">,</span> <span class="n">results</span><span class="p">:</span> <span class="n">EngineResults</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Parse video response from Tubearchivist instances.&quot;&quot;&quot;</span>
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">if</span> <span class="s1">&#39;results&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">:</span>
<span class="k">return</span>
<span class="k">for</span> <span class="n">channel_result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s1">&#39;results&#39;</span><span class="p">][</span><span class="s1">&#39;channel_results&#39;</span><span class="p">]:</span>
<span class="n">channel_url</span> <span class="o">=</span> <span class="n">absolute_url</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;/channel/</span><span class="si">{</span><span class="n">channel_result</span><span class="p">[</span><span class="s2">&quot;channel_id&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">MainResult</span><span class="p">(</span>
<span class="n">url</span><span class="o">=</span><span class="n">channel_url</span><span class="p">,</span>
<span class="n">title</span><span class="o">=</span><span class="n">channel_result</span><span class="p">[</span><span class="s1">&#39;channel_name&#39;</span><span class="p">],</span>
<span class="n">content</span><span class="o">=</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">channel_result</span><span class="p">[</span><span class="s1">&#39;channel_description&#39;</span><span class="p">]),</span>
<span class="n">author</span><span class="o">=</span><span class="n">channel_result</span><span class="p">[</span><span class="s1">&#39;channel_name&#39;</span><span class="p">],</span>
<span class="n">views</span><span class="o">=</span><span class="n">humanize_number</span><span class="p">(</span><span class="n">channel_result</span><span class="p">[</span><span class="s1">&#39;channel_subs&#39;</span><span class="p">]),</span>
<span class="n">thumbnail</span><span class="o">=</span><span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">absolute_url</span><span class="p">(</span><span class="n">channel_result</span><span class="p">[</span><span class="s2">&quot;channel_thumb_url&quot;</span><span class="p">])</span><span class="si">}</span><span class="s1">?auth=</span><span class="si">{</span><span class="n">ta_token</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">result</span><span class="o">=</span><span class="n">res</span><span class="p">)</span>
<span class="k">for</span> <span class="n">video_result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s1">&#39;results&#39;</span><span class="p">][</span><span class="s1">&#39;video_results&#39;</span><span class="p">]:</span>
<span class="n">metadata</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="p">[</span><span class="n">video_result</span><span class="p">[</span><span class="s1">&#39;channel&#39;</span><span class="p">][</span><span class="s1">&#39;channel_name&#39;</span><span class="p">],</span> <span class="o">*</span><span class="n">video_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;tags&#39;</span><span class="p">,</span> <span class="p">[])]))[:</span><span class="mi">5</span><span class="p">]</span>
<span class="k">if</span> <span class="n">ta_link_to_mp4</span><span class="p">:</span>
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">base_url</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">)</span><span class="si">}{</span><span class="n">video_result</span><span class="p">[</span><span class="s2">&quot;media_url&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">&#39;</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">base_url</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">)</span><span class="si">}</span><span class="s1">/?videoId=</span><span class="si">{</span><span class="n">video_result</span><span class="p">[</span><span class="s2">&quot;youtube_id&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">&#39;</span>
<span class="c1"># a type for the video.html template is not yet implemented</span>
<span class="c1"># --&gt; using LegacyResult</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;template&#39;</span><span class="p">:</span> <span class="s1">&#39;videos.html&#39;</span><span class="p">,</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">video_result</span><span class="p">[</span><span class="s1">&#39;title&#39;</span><span class="p">],</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">html_to_text</span><span class="p">(</span><span class="n">video_result</span><span class="p">[</span><span class="s1">&#39;description&#39;</span><span class="p">]),</span>
<span class="s1">&#39;author&#39;</span><span class="p">:</span> <span class="n">video_result</span><span class="p">[</span><span class="s1">&#39;channel&#39;</span><span class="p">][</span><span class="s1">&#39;channel_name&#39;</span><span class="p">],</span>
<span class="s1">&#39;length&#39;</span><span class="p">:</span> <span class="n">video_result</span><span class="p">[</span><span class="s1">&#39;player&#39;</span><span class="p">][</span><span class="s1">&#39;duration_str&#39;</span><span class="p">],</span>
<span class="s1">&#39;views&#39;</span><span class="p">:</span> <span class="n">humanize_number</span><span class="p">(</span><span class="n">video_result</span><span class="p">[</span><span class="s1">&#39;stats&#39;</span><span class="p">][</span><span class="s1">&#39;view_count&#39;</span><span class="p">]),</span>
<span class="s1">&#39;publishedDate&#39;</span><span class="p">:</span> <span class="n">parse</span><span class="p">(</span><span class="n">video_result</span><span class="p">[</span><span class="s1">&#39;published&#39;</span><span class="p">]),</span>
<span class="s1">&#39;thumbnail&#39;</span><span class="p">:</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">absolute_url</span><span class="p">(</span><span class="n">video_result</span><span class="p">[</span><span class="s2">&quot;vid_thumb_url&quot;</span><span class="p">])</span><span class="si">}</span><span class="s1">?auth=</span><span class="si">{</span><span class="n">ta_token</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="s1">&#39;metadata&#39;</span><span class="p">:</span> <span class="s1">&#39; | &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">metadata</span><span class="p">),</span>
<span class="p">}</span>
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">))</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,201 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.voidlinux &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.voidlinux</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.voidlinux</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;SearXNG engine for `Void Linux binary packages`_. Void is a general purpose</span>
<span class="sd">operating system, based on the monolithic Linux kernel. Its package system</span>
<span class="sd">allows you to quickly install, update and remove software; software is provided</span>
<span class="sd">in binary packages or can be built directly from sources with the help of the</span>
<span class="sd">XBPS source packages collection.</span>
<span class="sd">.. _Void Linux binary packages: https://voidlinux.org/packages/</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">quote_plus</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">humanize_bytes</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;website&#39;</span><span class="p">:</span> <span class="s1">&#39;https://voidlinux.org/packages/&#39;</span><span class="p">,</span>
<span class="s1">&#39;wikidata_id&#39;</span><span class="p">:</span> <span class="s1">&#39;Q19310966&#39;</span><span class="p">,</span>
<span class="s1">&#39;use_official_api&#39;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s1">&#39;official_api_documentation&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s1">&#39;require_api_key&#39;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s1">&#39;results&#39;</span><span class="p">:</span> <span class="s1">&#39;JSON&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;packages&#39;</span><span class="p">,</span> <span class="s1">&#39;it&#39;</span><span class="p">]</span>
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;https://xq-api.voidlinux.org&quot;</span>
<span class="n">pkg_repo_url</span> <span class="o">=</span> <span class="s2">&quot;https://github.com/void-linux/void-packages&quot;</span>
<span class="n">void_arch</span> <span class="o">=</span> <span class="s1">&#39;x86_64&#39;</span>
<span class="sd">&quot;&quot;&quot;Default architecture to search for. For valid values see :py:obj:`ARCH_RE`&quot;&quot;&quot;</span>
<span class="n">ARCH_RE</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="s1">&#39;aarch64-musl|armv6l-musl|armv7l-musl|x86_64-musl|aarch64|armv6l|armv7l|i686|x86_64&#39;</span><span class="p">)</span>
<span class="sd">&quot;&quot;&quot;Regular expression that match a architecture in the query string.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="n">arch_path</span> <span class="o">=</span> <span class="n">ARCH_RE</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">query</span><span class="p">)</span>
<span class="k">if</span> <span class="n">arch_path</span><span class="p">:</span>
<span class="n">arch_path</span> <span class="o">=</span> <span class="n">arch_path</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">arch_path</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">arch_path</span> <span class="o">=</span> <span class="n">void_arch</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">/v1/query/</span><span class="si">{</span><span class="n">arch_path</span><span class="si">}</span><span class="s2">?q=</span><span class="si">{</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">query</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">return</span> <span class="n">params</span>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/online/void.html#searx.engines.voidlinux.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> At Void Linux, several packages sometimes share the same source code</span>
<span class="sd"> (template) and therefore also have the same URL. Results with identical</span>
<span class="sd"> URLs are merged as one result for SearXNG.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">packages</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()[</span><span class="s1">&#39;data&#39;</span><span class="p">]:</span>
<span class="c1"># 32bit and dbg packages don&#39;t have their own package templates</span>
<span class="n">github_slug</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;-(32bit|dbg)$&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">])</span>
<span class="n">pkg_url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">pkg_repo_url</span><span class="si">}</span><span class="s2">/tree/master/srcpkgs/</span><span class="si">{</span><span class="n">github_slug</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">pkg_list</span> <span class="o">=</span> <span class="n">packages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">pkg_url</span><span class="p">,</span> <span class="p">[])</span>
<span class="n">pkg_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">],</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;short_desc&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2"> - </span><span class="si">{</span><span class="n">humanize_bytes</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;filename_size&#39;</span><span class="p">])</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
<span class="s1">&#39;package_name&#39;</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">],</span>
<span class="s1">&#39;version&#39;</span><span class="p">:</span> <span class="sa">f</span><span class="s2">&quot;v</span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;version&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">&#39;revision&#39;</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
<span class="s1">&#39;tags&#39;</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;repository&#39;</span><span class="p">],</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="n">packages</span><span class="p">[</span><span class="n">pkg_url</span><span class="p">]</span> <span class="o">=</span> <span class="n">pkg_list</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">pkg_url</span><span class="p">,</span> <span class="n">pkg_list</span> <span class="ow">in</span> <span class="n">packages</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">pkg_url</span><span class="p">,</span>
<span class="s1">&#39;template&#39;</span><span class="p">:</span> <span class="s1">&#39;packages.html&#39;</span><span class="p">,</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="s1">&#39; | &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="s1">&#39;title&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">pkg_list</span><span class="p">),</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">pkg_list</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;content&#39;</span><span class="p">],</span>
<span class="s1">&#39;package_name&#39;</span><span class="p">:</span> <span class="s1">&#39; | &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="s1">&#39;package_name&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">pkg_list</span><span class="p">),</span>
<span class="s1">&#39;version&#39;</span><span class="p">:</span> <span class="n">pkg_list</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;version&#39;</span><span class="p">],</span>
<span class="s1">&#39;tags&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">x</span><span class="p">[</span><span class="s1">&#39;tags&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">pkg_list</span><span class="p">],</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,938 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.wikidata &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.wikidata</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.wikidata</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;This module implements the Wikidata engine. Some implementations are shared</span>
<span class="sd">from :ref:`wikipedia engine`.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=missing-class-docstring</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">hashlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">md5</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span><span class="p">,</span> <span class="n">unquote</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">json</span><span class="w"> </span><span class="kn">import</span> <span class="n">loads</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil.parser</span><span class="w"> </span><span class="kn">import</span> <span class="n">isoparse</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">babel.dates</span><span class="w"> </span><span class="kn">import</span> <span class="n">format_datetime</span><span class="p">,</span> <span class="n">format_date</span><span class="p">,</span> <span class="n">format_time</span><span class="p">,</span> <span class="n">get_datetime_format</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.data</span><span class="w"> </span><span class="kn">import</span> <span class="n">WIKIDATA_UNITS</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">post</span><span class="p">,</span> <span class="n">get</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">searxng_useragent</span><span class="p">,</span> <span class="n">get_string_replaces_function</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.external_urls</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_external_url</span><span class="p">,</span> <span class="n">get_earth_coordinates_url</span><span class="p">,</span> <span class="n">area_to_osm_zoom</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.wikipedia</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">fetch_wikimedia_traits</span><span class="p">,</span>
<span class="n">get_wiki_params</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="c1"># about</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://wikidata.org/&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q2013&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://query.wikidata.org/&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;JSON&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">display_type</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;infobox&quot;</span><span class="p">]</span>
<span class="sd">&quot;&quot;&quot;A list of display types composed from ``infobox`` and ``list``. The latter</span>
<span class="sd">one will add a hit to the result list. The first one will show a hit in the</span>
<span class="sd">info box. Both values can be set, or one of the two can be set.&quot;&quot;&quot;</span>
<span class="c1"># SPARQL</span>
<span class="n">SPARQL_ENDPOINT_URL</span> <span class="o">=</span> <span class="s1">&#39;https://query.wikidata.org/sparql&#39;</span>
<span class="n">SPARQL_EXPLAIN_URL</span> <span class="o">=</span> <span class="s1">&#39;https://query.wikidata.org/bigdata/namespace/wdq/sparql?explain&#39;</span>
<span class="n">WIKIDATA_PROPERTIES</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;P434&#39;</span><span class="p">:</span> <span class="s1">&#39;MusicBrainz&#39;</span><span class="p">,</span>
<span class="s1">&#39;P435&#39;</span><span class="p">:</span> <span class="s1">&#39;MusicBrainz&#39;</span><span class="p">,</span>
<span class="s1">&#39;P436&#39;</span><span class="p">:</span> <span class="s1">&#39;MusicBrainz&#39;</span><span class="p">,</span>
<span class="s1">&#39;P966&#39;</span><span class="p">:</span> <span class="s1">&#39;MusicBrainz&#39;</span><span class="p">,</span>
<span class="s1">&#39;P345&#39;</span><span class="p">:</span> <span class="s1">&#39;IMDb&#39;</span><span class="p">,</span>
<span class="s1">&#39;P2397&#39;</span><span class="p">:</span> <span class="s1">&#39;YouTube&#39;</span><span class="p">,</span>
<span class="s1">&#39;P1651&#39;</span><span class="p">:</span> <span class="s1">&#39;YouTube&#39;</span><span class="p">,</span>
<span class="s1">&#39;P2002&#39;</span><span class="p">:</span> <span class="s1">&#39;Twitter&#39;</span><span class="p">,</span>
<span class="s1">&#39;P2013&#39;</span><span class="p">:</span> <span class="s1">&#39;Facebook&#39;</span><span class="p">,</span>
<span class="s1">&#39;P2003&#39;</span><span class="p">:</span> <span class="s1">&#39;Instagram&#39;</span><span class="p">,</span>
<span class="s1">&#39;P4033&#39;</span><span class="p">:</span> <span class="s1">&#39;Mastodon&#39;</span><span class="p">,</span>
<span class="s1">&#39;P11947&#39;</span><span class="p">:</span> <span class="s1">&#39;Lemmy&#39;</span><span class="p">,</span>
<span class="s1">&#39;P12622&#39;</span><span class="p">:</span> <span class="s1">&#39;PeerTube&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># SERVICE wikibase:mwapi : https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual/MWAPI</span>
<span class="c1"># SERVICE wikibase:label: https://en.wikibooks.org/wiki/SPARQL/SERVICE_-_Label#Manual_Label_SERVICE</span>
<span class="c1"># https://en.wikibooks.org/wiki/SPARQL/WIKIDATA_Precision,_Units_and_Coordinates</span>
<span class="c1"># https://www.mediawiki.org/wiki/Wikibase/Indexing/RDF_Dump_Format#Data_model</span>
<span class="c1"># optimization:</span>
<span class="c1"># * https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/query_optimization</span>
<span class="c1"># * https://github.com/blazegraph/database/wiki/QueryHints</span>
<span class="n">QUERY_TEMPLATE</span> <span class="o">=</span> <span class="s2">&quot;&quot;&quot;</span>
<span class="s2">SELECT ?item ?itemLabel ?itemDescription ?lat ?long %SELECT%</span>
<span class="s2">WHERE</span>
<span class="s2">{</span>
<span class="s2"> SERVICE wikibase:mwapi {</span>
<span class="s2"> bd:serviceParam wikibase:endpoint &quot;www.wikidata.org&quot;;</span>
<span class="s2"> wikibase:api &quot;EntitySearch&quot;;</span>
<span class="s2"> wikibase:limit 1;</span>
<span class="s2"> mwapi:search &quot;%QUERY%&quot;;</span>
<span class="s2"> mwapi:language &quot;%LANGUAGE%&quot;.</span>
<span class="s2"> ?item wikibase:apiOutputItem mwapi:item.</span>
<span class="s2"> }</span>
<span class="s2"> hint:Prior hint:runFirst &quot;true&quot;.</span>
<span class="s2"> %WHERE%</span>
<span class="s2"> SERVICE wikibase:label {</span>
<span class="s2"> bd:serviceParam wikibase:language &quot;%LANGUAGE%,en&quot;.</span>
<span class="s2"> ?item rdfs:label ?itemLabel .</span>
<span class="s2"> ?item schema:description ?itemDescription .</span>
<span class="s2"> %WIKIBASE_LABELS%</span>
<span class="s2"> }</span>
<span class="s2">}</span>
<span class="s2">GROUP BY ?item ?itemLabel ?itemDescription ?lat ?long </span><span class="si">%G</span><span class="s2">ROUP_BY%</span>
<span class="s2">&quot;&quot;&quot;</span>
<span class="c1"># Get the calendar names and the property names</span>
<span class="n">QUERY_PROPERTY_NAMES</span> <span class="o">=</span> <span class="s2">&quot;&quot;&quot;</span>
<span class="s2">SELECT ?item ?name</span>
<span class="s2">WHERE {</span>
<span class="s2"> {</span>
<span class="s2"> SELECT ?item</span>
<span class="s2"> WHERE { ?item wdt:P279* wd:Q12132 }</span>
<span class="s2"> } UNION {</span>
<span class="s2"> VALUES ?item { %ATTRIBUTES% }</span>
<span class="s2"> }</span>
<span class="s2"> OPTIONAL { ?item rdfs:label ?name. }</span>
<span class="s2">}</span>
<span class="s2">&quot;&quot;&quot;</span>
<span class="c1"># see the property &quot;dummy value&quot; of https://www.wikidata.org/wiki/Q2013 (Wikidata)</span>
<span class="c1"># hard coded here to avoid to an additional SPARQL request when the server starts</span>
<span class="n">DUMMY_ENTITY_URLS</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span>
<span class="s2">&quot;http://www.wikidata.org/entity/&quot;</span> <span class="o">+</span> <span class="n">wid</span> <span class="k">for</span> <span class="n">wid</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">&quot;Q4115189&quot;</span><span class="p">,</span> <span class="s2">&quot;Q13406268&quot;</span><span class="p">,</span> <span class="s2">&quot;Q15397819&quot;</span><span class="p">,</span> <span class="s2">&quot;Q17339402&quot;</span><span class="p">)</span>
<span class="p">)</span>
<span class="c1"># https://www.w3.org/TR/sparql11-query/#rSTRING_LITERAL1</span>
<span class="c1"># https://lists.w3.org/Archives/Public/public-rdf-dawg/2011OctDec/0175.html</span>
<span class="n">sparql_string_escape</span> <span class="o">=</span> <span class="n">get_string_replaces_function</span><span class="p">(</span>
<span class="c1"># fmt: off</span>
<span class="p">{</span>
<span class="s1">&#39;</span><span class="se">\t</span><span class="s1">&#39;</span><span class="p">:</span> <span class="s1">&#39;</span><span class="se">\\\t</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">:</span> <span class="s1">&#39;</span><span class="se">\\\n</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="s1">&#39;</span><span class="se">\r</span><span class="s1">&#39;</span><span class="p">:</span> <span class="s1">&#39;</span><span class="se">\\\r</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="s1">&#39;</span><span class="se">\b</span><span class="s1">&#39;</span><span class="p">:</span> <span class="s1">&#39;</span><span class="se">\\\b</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="s1">&#39;</span><span class="se">\f</span><span class="s1">&#39;</span><span class="p">:</span> <span class="s1">&#39;</span><span class="se">\\\f</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="s1">&#39;</span><span class="se">\&quot;</span><span class="s1">&#39;</span><span class="p">:</span> <span class="s1">&#39;</span><span class="se">\\\&quot;</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="s1">&#39;</span><span class="se">\&#39;</span><span class="s1">&#39;</span><span class="p">:</span> <span class="s1">&#39;</span><span class="se">\\\&#39;</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="s1">&#39;</span><span class="se">\\</span><span class="s1">&#39;</span><span class="p">:</span> <span class="s1">&#39;</span><span class="se">\\\\</span><span class="s1">&#39;</span>
<span class="p">}</span>
<span class="c1"># fmt: on</span>
<span class="p">)</span>
<span class="n">replace_http_by_https</span> <span class="o">=</span> <span class="n">get_string_replaces_function</span><span class="p">({</span><span class="s1">&#39;http:&#39;</span><span class="p">:</span> <span class="s1">&#39;https:&#39;</span><span class="p">})</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_headers</span><span class="p">():</span>
<span class="c1"># user agent: https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Query_limits</span>
<span class="k">return</span> <span class="p">{</span><span class="s1">&#39;Accept&#39;</span><span class="p">:</span> <span class="s1">&#39;application/sparql-results+json&#39;</span><span class="p">,</span> <span class="s1">&#39;User-Agent&#39;</span><span class="p">:</span> <span class="n">searxng_useragent</span><span class="p">()}</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_label_for_entity</span><span class="p">(</span><span class="n">entity_id</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="n">WIKIDATA_PROPERTIES</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">entity_id</span><span class="p">)</span>
<span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">name</span> <span class="o">=</span> <span class="n">WIKIDATA_PROPERTIES</span><span class="o">.</span><span class="n">get</span><span class="p">((</span><span class="n">entity_id</span><span class="p">,</span> <span class="n">language</span><span class="p">))</span>
<span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">name</span> <span class="o">=</span> <span class="n">WIKIDATA_PROPERTIES</span><span class="o">.</span><span class="n">get</span><span class="p">((</span><span class="n">entity_id</span><span class="p">,</span> <span class="n">language</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]))</span>
<span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">name</span> <span class="o">=</span> <span class="n">WIKIDATA_PROPERTIES</span><span class="o">.</span><span class="n">get</span><span class="p">((</span><span class="n">entity_id</span><span class="p">,</span> <span class="s1">&#39;en&#39;</span><span class="p">))</span>
<span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">name</span> <span class="o">=</span> <span class="n">entity_id</span>
<span class="k">return</span> <span class="n">name</span>
<span class="k">def</span><span class="w"> </span><span class="nf">send_wikidata_query</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s1">&#39;GET&#39;</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="k">if</span> <span class="n">method</span> <span class="o">==</span> <span class="s1">&#39;GET&#39;</span><span class="p">:</span>
<span class="c1"># query will be cached by wikidata</span>
<span class="n">http_response</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">SPARQL_ENDPOINT_URL</span> <span class="o">+</span> <span class="s1">&#39;?&#39;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">}),</span> <span class="n">headers</span><span class="o">=</span><span class="n">get_headers</span><span class="p">(),</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># query won&#39;t be cached by wikidata</span>
<span class="n">http_response</span> <span class="o">=</span> <span class="n">post</span><span class="p">(</span><span class="n">SPARQL_ENDPOINT_URL</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">},</span> <span class="n">headers</span><span class="o">=</span><span class="n">get_headers</span><span class="p">(),</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="k">if</span> <span class="n">http_response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">!=</span> <span class="mi">200</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;SPARQL endpoint error </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">http_response</span><span class="o">.</span><span class="n">content</span><span class="o">.</span><span class="n">decode</span><span class="p">())</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;request time </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">http_response</span><span class="o">.</span><span class="n">elapsed</span><span class="p">))</span>
<span class="n">http_response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
<span class="k">return</span> <span class="n">loads</span><span class="p">(</span><span class="n">http_response</span><span class="o">.</span><span class="n">content</span><span class="o">.</span><span class="n">decode</span><span class="p">())</span>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="n">eng_tag</span><span class="p">,</span> <span class="n">_wiki_netloc</span> <span class="o">=</span> <span class="n">get_wiki_params</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="n">traits</span><span class="p">)</span>
<span class="n">query</span><span class="p">,</span> <span class="n">attributes</span> <span class="o">=</span> <span class="n">get_query</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;request --&gt; language </span><span class="si">%s</span><span class="s2"> // len(attributes): </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">attributes</span><span class="p">))</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;method&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;POST&#39;</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">SPARQL_ENDPOINT_URL</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">}</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_headers</span><span class="p">()</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;language&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;attributes&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">attributes</span>
<span class="k">return</span> <span class="n">params</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">jsonresponse</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">content</span><span class="o">.</span><span class="n">decode</span><span class="p">())</span>
<span class="n">language</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="p">[</span><span class="s1">&#39;language&#39;</span><span class="p">]</span>
<span class="n">attributes</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="p">[</span><span class="s1">&#39;attributes&#39;</span><span class="p">]</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;request --&gt; language </span><span class="si">%s</span><span class="s2"> // len(attributes): </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">language</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">attributes</span><span class="p">))</span>
<span class="n">seen_entities</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">jsonresponse</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;results&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;bindings&#39;</span><span class="p">,</span> <span class="p">[]):</span>
<span class="n">attribute_result</span> <span class="o">=</span> <span class="p">{</span><span class="n">key</span><span class="p">:</span> <span class="n">value</span><span class="p">[</span><span class="s1">&#39;value&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">result</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
<span class="n">entity_url</span> <span class="o">=</span> <span class="n">attribute_result</span><span class="p">[</span><span class="s1">&#39;item&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">entity_url</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">seen_entities</span> <span class="ow">and</span> <span class="n">entity_url</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">DUMMY_ENTITY_URLS</span><span class="p">:</span>
<span class="n">seen_entities</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">entity_url</span><span class="p">)</span>
<span class="n">results</span> <span class="o">+=</span> <span class="n">get_results</span><span class="p">(</span><span class="n">attribute_result</span><span class="p">,</span> <span class="n">attributes</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;The SPARQL request returns duplicate entities: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">attribute_result</span><span class="p">))</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">_IMG_SRC_DEFAULT_URL_PREFIX</span> <span class="o">=</span> <span class="s2">&quot;https://commons.wikimedia.org/wiki/Special:FilePath/&quot;</span>
<span class="n">_IMG_SRC_NEW_URL_PREFIX</span> <span class="o">=</span> <span class="s2">&quot;https://upload.wikimedia.org/wikipedia/commons/thumb/&quot;</span>
<div class="viewcode-block" id="get_thumbnail">
<a class="viewcode-back" href="../../../dev/engines/online/wikipedia.html#searx.engines.wikidata.get_thumbnail">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_thumbnail</span><span class="p">(</span><span class="n">img_src</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Get Thumbnail image from wikimedia commons</span>
<span class="sd"> Images from commons.wikimedia.org are (HTTP) redirected to</span>
<span class="sd"> upload.wikimedia.org. The redirected URL can be calculated by this</span>
<span class="sd"> function.</span>
<span class="sd"> - https://stackoverflow.com/a/33691240</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;get_thumbnail(): </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">img_src</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">img_src</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">_IMG_SRC_DEFAULT_URL_PREFIX</span> <span class="ow">in</span> <span class="n">img_src</span><span class="o">.</span><span class="n">split</span><span class="p">()[</span><span class="mi">0</span><span class="p">]:</span>
<span class="n">img_src_name</span> <span class="o">=</span> <span class="n">unquote</span><span class="p">(</span><span class="n">img_src</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">_IMG_SRC_DEFAULT_URL_PREFIX</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;?&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;%20&quot;</span><span class="p">,</span> <span class="s2">&quot;_&quot;</span><span class="p">))</span>
<span class="n">img_src_name_first</span> <span class="o">=</span> <span class="n">img_src_name</span>
<span class="n">img_src_name_second</span> <span class="o">=</span> <span class="n">img_src_name</span>
<span class="k">if</span> <span class="s2">&quot;.svg&quot;</span> <span class="ow">in</span> <span class="n">img_src_name</span><span class="o">.</span><span class="n">split</span><span class="p">()[</span><span class="mi">0</span><span class="p">]:</span>
<span class="n">img_src_name_second</span> <span class="o">=</span> <span class="n">img_src_name</span> <span class="o">+</span> <span class="s2">&quot;.png&quot;</span>
<span class="n">img_src_size</span> <span class="o">=</span> <span class="n">img_src</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">_IMG_SRC_DEFAULT_URL_PREFIX</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;?&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>
<span class="n">img_src_size</span> <span class="o">=</span> <span class="n">img_src_size</span><span class="p">[</span><span class="n">img_src_size</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s2">&quot;=&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span> <span class="p">:</span> <span class="n">img_src_size</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s2">&quot;&amp;&quot;</span><span class="p">)]</span>
<span class="n">img_src_name_md5</span> <span class="o">=</span> <span class="n">md5</span><span class="p">(</span><span class="n">img_src_name</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s2">&quot;utf-8&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
<span class="n">img_src</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">_IMG_SRC_NEW_URL_PREFIX</span>
<span class="o">+</span> <span class="n">img_src_name_md5</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="o">+</span> <span class="s2">&quot;/&quot;</span>
<span class="o">+</span> <span class="n">img_src_name_md5</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="mi">2</span><span class="p">]</span>
<span class="o">+</span> <span class="s2">&quot;/&quot;</span>
<span class="o">+</span> <span class="n">img_src_name_first</span>
<span class="o">+</span> <span class="s2">&quot;/&quot;</span>
<span class="o">+</span> <span class="n">img_src_size</span>
<span class="o">+</span> <span class="s2">&quot;px-&quot;</span>
<span class="o">+</span> <span class="n">img_src_name_second</span>
<span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;get_thumbnail() redirected: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">img_src</span><span class="p">)</span>
<span class="k">return</span> <span class="n">img_src</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">get_results</span><span class="p">(</span><span class="n">attribute_result</span><span class="p">,</span> <span class="n">attributes</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
<span class="c1"># pylint: disable=too-many-branches</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">infobox_title</span> <span class="o">=</span> <span class="n">attribute_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;itemLabel&#39;</span><span class="p">)</span>
<span class="n">infobox_id</span> <span class="o">=</span> <span class="n">attribute_result</span><span class="p">[</span><span class="s1">&#39;item&#39;</span><span class="p">]</span>
<span class="n">infobox_id_lang</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">infobox_urls</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">infobox_attributes</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">infobox_content</span> <span class="o">=</span> <span class="n">attribute_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;itemDescription&#39;</span><span class="p">,</span> <span class="p">[])</span>
<span class="n">img_src</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">img_src_priority</span> <span class="o">=</span> <span class="mi">0</span>
<span class="k">for</span> <span class="n">attribute</span> <span class="ow">in</span> <span class="n">attributes</span><span class="p">:</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">attribute</span><span class="o">.</span><span class="n">get_str</span><span class="p">(</span><span class="n">attribute_result</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
<span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">value</span> <span class="o">!=</span> <span class="s1">&#39;&#39;</span><span class="p">:</span>
<span class="n">attribute_type</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">attribute</span><span class="p">)</span>
<span class="k">if</span> <span class="n">attribute_type</span> <span class="ow">in</span> <span class="p">(</span><span class="n">WDURLAttribute</span><span class="p">,</span> <span class="n">WDArticle</span><span class="p">):</span>
<span class="c1"># get_select() method : there is group_concat(distinct ...;separator=&quot;, &quot;)</span>
<span class="c1"># split the value here</span>
<span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">value</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;, &#39;</span><span class="p">):</span>
<span class="n">infobox_urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">attribute</span><span class="o">.</span><span class="n">get_label</span><span class="p">(</span><span class="n">language</span><span class="p">),</span> <span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">attribute</span><span class="o">.</span><span class="n">kwargs</span><span class="p">})</span>
<span class="c1"># &quot;normal&quot; results (not infobox) include official website and Wikipedia links.</span>
<span class="k">if</span> <span class="s2">&quot;list&quot;</span> <span class="ow">in</span> <span class="n">display_type</span> <span class="ow">and</span> <span class="p">(</span><span class="n">attribute</span><span class="o">.</span><span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;official&#39;</span><span class="p">)</span> <span class="ow">or</span> <span class="n">attribute_type</span> <span class="o">==</span> <span class="n">WDArticle</span><span class="p">):</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">infobox_title</span><span class="p">,</span> <span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s2">&quot;content&quot;</span><span class="p">:</span> <span class="n">infobox_content</span><span class="p">})</span>
<span class="c1"># update the infobox_id with the wikipedia URL</span>
<span class="c1"># first the local wikipedia URL, and as fallback the english wikipedia URL</span>
<span class="k">if</span> <span class="n">attribute_type</span> <span class="o">==</span> <span class="n">WDArticle</span> <span class="ow">and</span> <span class="p">(</span>
<span class="p">(</span><span class="n">attribute</span><span class="o">.</span><span class="n">language</span> <span class="o">==</span> <span class="s1">&#39;en&#39;</span> <span class="ow">and</span> <span class="n">infobox_id_lang</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">or</span> <span class="n">attribute</span><span class="o">.</span><span class="n">language</span> <span class="o">!=</span> <span class="s1">&#39;en&#39;</span>
<span class="p">):</span>
<span class="n">infobox_id_lang</span> <span class="o">=</span> <span class="n">attribute</span><span class="o">.</span><span class="n">language</span>
<span class="n">infobox_id</span> <span class="o">=</span> <span class="n">url</span>
<span class="k">elif</span> <span class="n">attribute_type</span> <span class="o">==</span> <span class="n">WDImageAttribute</span><span class="p">:</span>
<span class="c1"># this attribute is an image.</span>
<span class="c1"># replace the current image only the priority is lower</span>
<span class="c1"># (the infobox contain only one image).</span>
<span class="k">if</span> <span class="n">attribute</span><span class="o">.</span><span class="n">priority</span> <span class="o">&gt;</span> <span class="n">img_src_priority</span><span class="p">:</span>
<span class="n">img_src</span> <span class="o">=</span> <span class="n">get_thumbnail</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="n">img_src_priority</span> <span class="o">=</span> <span class="n">attribute</span><span class="o">.</span><span class="n">priority</span>
<span class="k">elif</span> <span class="n">attribute_type</span> <span class="o">==</span> <span class="n">WDGeoAttribute</span><span class="p">:</span>
<span class="c1"># geocoordinate link</span>
<span class="c1"># use the area to get the OSM zoom</span>
<span class="c1"># Note: ignore the unit (must be km² otherwise the calculation is wrong)</span>
<span class="c1"># Should use normalized value p:P2046/psn:P2046/wikibase:quantityAmount</span>
<span class="n">area</span> <span class="o">=</span> <span class="n">attribute_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;P2046&#39;</span><span class="p">)</span>
<span class="n">osm_zoom</span> <span class="o">=</span> <span class="n">area_to_osm_zoom</span><span class="p">(</span><span class="n">area</span><span class="p">)</span> <span class="k">if</span> <span class="n">area</span> <span class="k">else</span> <span class="mi">19</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">attribute</span><span class="o">.</span><span class="n">get_geo_url</span><span class="p">(</span><span class="n">attribute_result</span><span class="p">,</span> <span class="n">osm_zoom</span><span class="o">=</span><span class="n">osm_zoom</span><span class="p">)</span>
<span class="k">if</span> <span class="n">url</span><span class="p">:</span>
<span class="n">infobox_urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">attribute</span><span class="o">.</span><span class="n">get_label</span><span class="p">(</span><span class="n">language</span><span class="p">),</span> <span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">&#39;entity&#39;</span><span class="p">:</span> <span class="n">attribute</span><span class="o">.</span><span class="n">name</span><span class="p">})</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">infobox_attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span><span class="s1">&#39;label&#39;</span><span class="p">:</span> <span class="n">attribute</span><span class="o">.</span><span class="n">get_label</span><span class="p">(</span><span class="n">language</span><span class="p">),</span> <span class="s1">&#39;value&#39;</span><span class="p">:</span> <span class="n">value</span><span class="p">,</span> <span class="s1">&#39;entity&#39;</span><span class="p">:</span> <span class="n">attribute</span><span class="o">.</span><span class="n">name</span><span class="p">}</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">infobox_id</span><span class="p">:</span>
<span class="n">infobox_id</span> <span class="o">=</span> <span class="n">replace_http_by_https</span><span class="p">(</span><span class="n">infobox_id</span><span class="p">)</span>
<span class="c1"># add the wikidata URL at the end</span>
<span class="n">infobox_urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="s1">&#39;Wikidata&#39;</span><span class="p">,</span> <span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">attribute_result</span><span class="p">[</span><span class="s1">&#39;item&#39;</span><span class="p">]})</span>
<span class="k">if</span> <span class="p">(</span>
<span class="s2">&quot;list&quot;</span> <span class="ow">in</span> <span class="n">display_type</span>
<span class="ow">and</span> <span class="n">img_src</span> <span class="ow">is</span> <span class="kc">None</span>
<span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">infobox_attributes</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span>
<span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">infobox_urls</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span>
<span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">infobox_content</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span>
<span class="p">):</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">infobox_urls</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;url&#39;</span><span class="p">],</span> <span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">infobox_title</span><span class="p">,</span> <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">infobox_content</span><span class="p">})</span>
<span class="k">elif</span> <span class="s2">&quot;infobox&quot;</span> <span class="ow">in</span> <span class="n">display_type</span><span class="p">:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;infobox&#39;</span><span class="p">:</span> <span class="n">infobox_title</span><span class="p">,</span>
<span class="s1">&#39;id&#39;</span><span class="p">:</span> <span class="n">infobox_id</span><span class="p">,</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">infobox_content</span><span class="p">,</span>
<span class="s1">&#39;img_src&#39;</span><span class="p">:</span> <span class="n">img_src</span><span class="p">,</span>
<span class="s1">&#39;urls&#39;</span><span class="p">:</span> <span class="n">infobox_urls</span><span class="p">,</span>
<span class="s1">&#39;attributes&#39;</span><span class="p">:</span> <span class="n">infobox_attributes</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_query</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
<span class="n">attributes</span> <span class="o">=</span> <span class="n">get_attributes</span><span class="p">(</span><span class="n">language</span><span class="p">)</span>
<span class="n">select</span> <span class="o">=</span> <span class="p">[</span><span class="n">a</span><span class="o">.</span><span class="n">get_select</span><span class="p">()</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">attributes</span><span class="p">]</span>
<span class="n">where</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">s</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">s</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">,</span> <span class="p">[</span><span class="n">a</span><span class="o">.</span><span class="n">get_where</span><span class="p">()</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">attributes</span><span class="p">]))</span>
<span class="n">wikibase_label</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">s</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">s</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">,</span> <span class="p">[</span><span class="n">a</span><span class="o">.</span><span class="n">get_wikibase_label</span><span class="p">()</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">attributes</span><span class="p">]))</span>
<span class="n">group_by</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">s</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">s</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">,</span> <span class="p">[</span><span class="n">a</span><span class="o">.</span><span class="n">get_group_by</span><span class="p">()</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">attributes</span><span class="p">]))</span>
<span class="n">query</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">QUERY_TEMPLATE</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;%QUERY%&#39;</span><span class="p">,</span> <span class="n">sparql_string_escape</span><span class="p">(</span><span class="n">query</span><span class="p">))</span>
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;%SELECT%&#39;</span><span class="p">,</span> <span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">select</span><span class="p">))</span>
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;%WHERE%&#39;</span><span class="p">,</span> <span class="s1">&#39;</span><span class="se">\n</span><span class="s1"> &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">where</span><span class="p">))</span>
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;%WIKIBASE_LABELS%&#39;</span><span class="p">,</span> <span class="s1">&#39;</span><span class="se">\n</span><span class="s1"> &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">wikibase_label</span><span class="p">))</span>
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">%G</span><span class="s1">ROUP_BY%&#39;</span><span class="p">,</span> <span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">group_by</span><span class="p">))</span>
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;%LANGUAGE%&#39;</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">query</span><span class="p">,</span> <span class="n">attributes</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_attributes</span><span class="p">(</span><span class="n">language</span><span class="p">):</span>
<span class="c1"># pylint: disable=too-many-statements</span>
<span class="n">attributes</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">def</span><span class="w"> </span><span class="nf">add_value</span><span class="p">(</span><span class="n">name</span><span class="p">):</span>
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDAttribute</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
<span class="k">def</span><span class="w"> </span><span class="nf">add_amount</span><span class="p">(</span><span class="n">name</span><span class="p">):</span>
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDAmountAttribute</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
<span class="k">def</span><span class="w"> </span><span class="nf">add_label</span><span class="p">(</span><span class="n">name</span><span class="p">):</span>
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDLabelAttribute</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
<span class="k">def</span><span class="w"> </span><span class="nf">add_url</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">url_path_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDURLAttribute</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url_id</span><span class="p">,</span> <span class="n">url_path_prefix</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">))</span>
<span class="k">def</span><span class="w"> </span><span class="nf">add_image</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">1</span><span class="p">):</span>
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDImageAttribute</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url_id</span><span class="p">,</span> <span class="n">priority</span><span class="p">))</span>
<span class="k">def</span><span class="w"> </span><span class="nf">add_date</span><span class="p">(</span><span class="n">name</span><span class="p">):</span>
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDDateAttribute</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
<span class="c1"># Dates</span>
<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="p">[</span>
<span class="s1">&#39;P571&#39;</span><span class="p">,</span> <span class="c1"># inception date</span>
<span class="s1">&#39;P576&#39;</span><span class="p">,</span> <span class="c1"># dissolution date</span>
<span class="s1">&#39;P580&#39;</span><span class="p">,</span> <span class="c1"># start date</span>
<span class="s1">&#39;P582&#39;</span><span class="p">,</span> <span class="c1"># end date</span>
<span class="s1">&#39;P569&#39;</span><span class="p">,</span> <span class="c1"># date of birth</span>
<span class="s1">&#39;P570&#39;</span><span class="p">,</span> <span class="c1"># date of death</span>
<span class="s1">&#39;P619&#39;</span><span class="p">,</span> <span class="c1"># date of spacecraft launch</span>
<span class="s1">&#39;P620&#39;</span><span class="p">,</span>
<span class="p">]:</span> <span class="c1"># date of spacecraft landing</span>
<span class="n">add_date</span><span class="p">(</span><span class="n">p</span><span class="p">)</span>
<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="p">[</span>
<span class="s1">&#39;P27&#39;</span><span class="p">,</span> <span class="c1"># country of citizenship</span>
<span class="s1">&#39;P495&#39;</span><span class="p">,</span> <span class="c1"># country of origin</span>
<span class="s1">&#39;P17&#39;</span><span class="p">,</span> <span class="c1"># country</span>
<span class="s1">&#39;P159&#39;</span><span class="p">,</span>
<span class="p">]:</span> <span class="c1"># headquarters location</span>
<span class="n">add_label</span><span class="p">(</span><span class="n">p</span><span class="p">)</span>
<span class="c1"># Places</span>
<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="p">[</span>
<span class="s1">&#39;P36&#39;</span><span class="p">,</span> <span class="c1"># capital</span>
<span class="s1">&#39;P35&#39;</span><span class="p">,</span> <span class="c1"># head of state</span>
<span class="s1">&#39;P6&#39;</span><span class="p">,</span> <span class="c1"># head of government</span>
<span class="s1">&#39;P122&#39;</span><span class="p">,</span> <span class="c1"># basic form of government</span>
<span class="s1">&#39;P37&#39;</span><span class="p">,</span>
<span class="p">]:</span> <span class="c1"># official language</span>
<span class="n">add_label</span><span class="p">(</span><span class="n">p</span><span class="p">)</span>
<span class="n">add_value</span><span class="p">(</span><span class="s1">&#39;P1082&#39;</span><span class="p">)</span> <span class="c1"># population</span>
<span class="n">add_amount</span><span class="p">(</span><span class="s1">&#39;P2046&#39;</span><span class="p">)</span> <span class="c1"># area</span>
<span class="n">add_amount</span><span class="p">(</span><span class="s1">&#39;P281&#39;</span><span class="p">)</span> <span class="c1"># postal code</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P38&#39;</span><span class="p">)</span> <span class="c1"># currency</span>
<span class="n">add_amount</span><span class="p">(</span><span class="s1">&#39;P2048&#39;</span><span class="p">)</span> <span class="c1"># height (building)</span>
<span class="c1"># Media</span>
<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="p">[</span>
<span class="s1">&#39;P400&#39;</span><span class="p">,</span> <span class="c1"># platform (videogames, computing)</span>
<span class="s1">&#39;P50&#39;</span><span class="p">,</span> <span class="c1"># author</span>
<span class="s1">&#39;P170&#39;</span><span class="p">,</span> <span class="c1"># creator</span>
<span class="s1">&#39;P57&#39;</span><span class="p">,</span> <span class="c1"># director</span>
<span class="s1">&#39;P175&#39;</span><span class="p">,</span> <span class="c1"># performer</span>
<span class="s1">&#39;P178&#39;</span><span class="p">,</span> <span class="c1"># developer</span>
<span class="s1">&#39;P162&#39;</span><span class="p">,</span> <span class="c1"># producer</span>
<span class="s1">&#39;P176&#39;</span><span class="p">,</span> <span class="c1"># manufacturer</span>
<span class="s1">&#39;P58&#39;</span><span class="p">,</span> <span class="c1"># screenwriter</span>
<span class="s1">&#39;P272&#39;</span><span class="p">,</span> <span class="c1"># production company</span>
<span class="s1">&#39;P264&#39;</span><span class="p">,</span> <span class="c1"># record label</span>
<span class="s1">&#39;P123&#39;</span><span class="p">,</span> <span class="c1"># publisher</span>
<span class="s1">&#39;P449&#39;</span><span class="p">,</span> <span class="c1"># original network</span>
<span class="s1">&#39;P750&#39;</span><span class="p">,</span> <span class="c1"># distributed by</span>
<span class="s1">&#39;P86&#39;</span><span class="p">,</span>
<span class="p">]:</span> <span class="c1"># composer</span>
<span class="n">add_label</span><span class="p">(</span><span class="n">p</span><span class="p">)</span>
<span class="n">add_date</span><span class="p">(</span><span class="s1">&#39;P577&#39;</span><span class="p">)</span> <span class="c1"># publication date</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P136&#39;</span><span class="p">)</span> <span class="c1"># genre (music, film, artistic...)</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P364&#39;</span><span class="p">)</span> <span class="c1"># original language</span>
<span class="n">add_value</span><span class="p">(</span><span class="s1">&#39;P212&#39;</span><span class="p">)</span> <span class="c1"># ISBN-13</span>
<span class="n">add_value</span><span class="p">(</span><span class="s1">&#39;P957&#39;</span><span class="p">)</span> <span class="c1"># ISBN-10</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P275&#39;</span><span class="p">)</span> <span class="c1"># copyright license</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P277&#39;</span><span class="p">)</span> <span class="c1"># programming language</span>
<span class="n">add_value</span><span class="p">(</span><span class="s1">&#39;P348&#39;</span><span class="p">)</span> <span class="c1"># version</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P840&#39;</span><span class="p">)</span> <span class="c1"># narrative location</span>
<span class="c1"># Languages</span>
<span class="n">add_value</span><span class="p">(</span><span class="s1">&#39;P1098&#39;</span><span class="p">)</span> <span class="c1"># number of speakers</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P282&#39;</span><span class="p">)</span> <span class="c1"># writing system</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P1018&#39;</span><span class="p">)</span> <span class="c1"># language regulatory body</span>
<span class="n">add_value</span><span class="p">(</span><span class="s1">&#39;P218&#39;</span><span class="p">)</span> <span class="c1"># language code (ISO 639-1)</span>
<span class="c1"># Other</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P169&#39;</span><span class="p">)</span> <span class="c1"># ceo</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P112&#39;</span><span class="p">)</span> <span class="c1"># founded by</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P1454&#39;</span><span class="p">)</span> <span class="c1"># legal form (company, organization)</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P137&#39;</span><span class="p">)</span> <span class="c1"># operator (service, facility, ...)</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P1029&#39;</span><span class="p">)</span> <span class="c1"># crew members (tripulation)</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P225&#39;</span><span class="p">)</span> <span class="c1"># taxon name</span>
<span class="n">add_value</span><span class="p">(</span><span class="s1">&#39;P274&#39;</span><span class="p">)</span> <span class="c1"># chemical formula</span>
<span class="n">add_label</span><span class="p">(</span><span class="s1">&#39;P1346&#39;</span><span class="p">)</span> <span class="c1"># winner (sports, contests, ...)</span>
<span class="n">add_value</span><span class="p">(</span><span class="s1">&#39;P1120&#39;</span><span class="p">)</span> <span class="c1"># number of deaths</span>
<span class="n">add_value</span><span class="p">(</span><span class="s1">&#39;P498&#39;</span><span class="p">)</span> <span class="c1"># currency code (ISO 4217)</span>
<span class="c1"># URL</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P856&#39;</span><span class="p">,</span> <span class="n">official</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> <span class="c1"># official website</span>
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDArticle</span><span class="p">(</span><span class="n">language</span><span class="p">))</span> <span class="c1"># wikipedia (user language)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">language</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;en&#39;</span><span class="p">):</span>
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDArticle</span><span class="p">(</span><span class="s1">&#39;en&#39;</span><span class="p">))</span> <span class="c1"># wikipedia (english)</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P1324&#39;</span><span class="p">)</span> <span class="c1"># source code repository</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P1581&#39;</span><span class="p">)</span> <span class="c1"># blog</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P434&#39;</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;musicbrainz_artist&#39;</span><span class="p">)</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P435&#39;</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;musicbrainz_work&#39;</span><span class="p">)</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P436&#39;</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;musicbrainz_release_group&#39;</span><span class="p">)</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P966&#39;</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;musicbrainz_label&#39;</span><span class="p">)</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P345&#39;</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;imdb_id&#39;</span><span class="p">)</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P2397&#39;</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;youtube_channel&#39;</span><span class="p">)</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P1651&#39;</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;youtube_video&#39;</span><span class="p">)</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P2002&#39;</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;twitter_profile&#39;</span><span class="p">)</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P2013&#39;</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;facebook_profile&#39;</span><span class="p">)</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P2003&#39;</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;instagram_profile&#39;</span><span class="p">)</span>
<span class="c1"># Fediverse</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P4033&#39;</span><span class="p">,</span> <span class="n">url_path_prefix</span><span class="o">=</span><span class="s1">&#39;/@&#39;</span><span class="p">)</span> <span class="c1"># Mastodon user</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P11947&#39;</span><span class="p">,</span> <span class="n">url_path_prefix</span><span class="o">=</span><span class="s1">&#39;/c/&#39;</span><span class="p">)</span> <span class="c1"># Lemmy community</span>
<span class="n">add_url</span><span class="p">(</span><span class="s1">&#39;P12622&#39;</span><span class="p">,</span> <span class="n">url_path_prefix</span><span class="o">=</span><span class="s1">&#39;/c/&#39;</span><span class="p">)</span> <span class="c1"># PeerTube channel</span>
<span class="c1"># Map</span>
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDGeoAttribute</span><span class="p">(</span><span class="s1">&#39;P625&#39;</span><span class="p">))</span>
<span class="c1"># Image</span>
<span class="n">add_image</span><span class="p">(</span><span class="s1">&#39;P15&#39;</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;wikimedia_image&#39;</span><span class="p">)</span> <span class="c1"># route map</span>
<span class="n">add_image</span><span class="p">(</span><span class="s1">&#39;P242&#39;</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;wikimedia_image&#39;</span><span class="p">)</span> <span class="c1"># locator map</span>
<span class="n">add_image</span><span class="p">(</span><span class="s1">&#39;P154&#39;</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;wikimedia_image&#39;</span><span class="p">)</span> <span class="c1"># logo</span>
<span class="n">add_image</span><span class="p">(</span><span class="s1">&#39;P18&#39;</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;wikimedia_image&#39;</span><span class="p">)</span> <span class="c1"># image</span>
<span class="n">add_image</span><span class="p">(</span><span class="s1">&#39;P41&#39;</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;wikimedia_image&#39;</span><span class="p">)</span> <span class="c1"># flag</span>
<span class="n">add_image</span><span class="p">(</span><span class="s1">&#39;P2716&#39;</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">6</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;wikimedia_image&#39;</span><span class="p">)</span> <span class="c1"># collage</span>
<span class="n">add_image</span><span class="p">(</span><span class="s1">&#39;P2910&#39;</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">7</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">&#39;wikimedia_image&#39;</span><span class="p">)</span> <span class="c1"># icon</span>
<span class="k">return</span> <span class="n">attributes</span>
<span class="k">class</span><span class="w"> </span><span class="nc">WDAttribute</span><span class="p">:</span>
<span class="vm">__slots__</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;name&#39;</span><span class="p">,)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">name</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_select</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s1">&#39;(group_concat(distinct ?</span><span class="si">{name}</span><span class="s1">;separator=&quot;, &quot;) as ?</span><span class="si">{name}</span><span class="s1">s)&#39;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{name}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_label</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
<span class="k">return</span> <span class="n">get_label_for_entity</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_where</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;OPTIONAL { ?item wdt:</span><span class="si">{name}</span><span class="s2"> ?</span><span class="si">{name}</span><span class="s2"> . }&quot;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{name}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_wikibase_label</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_group_by</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_str</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span> <span class="c1"># pylint: disable=unused-argument</span>
<span class="k">return</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">&#39;s&#39;</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s1">&#39;&lt;&#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)</span> <span class="o">+</span> <span class="s1">&#39;:&#39;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">&#39;&gt;&#39;</span>
<span class="k">class</span><span class="w"> </span><span class="nc">WDAmountAttribute</span><span class="p">(</span><span class="n">WDAttribute</span><span class="p">):</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_select</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s1">&#39;?</span><span class="si">{name}</span><span class="s1"> ?</span><span class="si">{name}</span><span class="s1">Unit&#39;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{name}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_where</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;&quot;&quot; OPTIONAL { ?item p:</span><span class="si">{name}</span><span class="s2"> ?</span><span class="si">{name}</span><span class="s2">Node .</span>
<span class="s2"> ?</span><span class="si">{name}</span><span class="s2">Node rdf:type wikibase:BestRank ; ps:</span><span class="si">{name}</span><span class="s2"> ?</span><span class="si">{name}</span><span class="s2"> .</span>
<span class="s2"> OPTIONAL { ?</span><span class="si">{name}</span><span class="s2">Node psv:</span><span class="si">{name}</span><span class="s2">/wikibase:quantityUnit ?</span><span class="si">{name}</span><span class="s2">Unit. } }&quot;&quot;&quot;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span>
<span class="s1">&#39;</span><span class="si">{name}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span>
<span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_group_by</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_select</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_str</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="n">unit</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s2">&quot;Unit&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">unit</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">unit</span> <span class="o">=</span> <span class="n">unit</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;http://www.wikidata.org/entity/&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">value</span> <span class="o">+</span> <span class="s2">&quot; &quot;</span> <span class="o">+</span> <span class="n">get_label_for_entity</span><span class="p">(</span><span class="n">unit</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
<span class="k">return</span> <span class="n">value</span>
<span class="k">class</span><span class="w"> </span><span class="nc">WDArticle</span><span class="p">(</span><span class="n">WDAttribute</span><span class="p">):</span>
<span class="vm">__slots__</span> <span class="o">=</span> <span class="s1">&#39;language&#39;</span><span class="p">,</span> <span class="s1">&#39;kwargs&#39;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">language</span><span class="p">,</span> <span class="n">kwargs</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="s1">&#39;wikipedia&#39;</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">language</span> <span class="o">=</span> <span class="n">language</span>
<span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span> <span class="ow">or</span> <span class="p">{}</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_label</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
<span class="c1"># language parameter is ignored</span>
<span class="k">return</span> <span class="s2">&quot;Wikipedia (</span><span class="si">{language}</span><span class="s2">)&quot;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{language}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">language</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_select</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;?article</span><span class="si">{language}</span><span class="s2"> ?articleName</span><span class="si">{language}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{language}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">language</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_where</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;&quot;&quot;OPTIONAL { ?article</span><span class="si">{language}</span><span class="s2"> schema:about ?item ;</span>
<span class="s2"> schema:inLanguage &quot;</span><span class="si">{language}</span><span class="s2">&quot; ;</span>
<span class="s2"> schema:isPartOf &lt;https://</span><span class="si">{language}</span><span class="s2">.wikipedia.org/&gt; ;</span>
<span class="s2"> schema:name ?articleName</span><span class="si">{language}</span><span class="s2"> . }&quot;&quot;&quot;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span>
<span class="s1">&#39;</span><span class="si">{language}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">language</span>
<span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_group_by</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_select</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_str</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
<span class="n">key</span> <span class="o">=</span> <span class="s1">&#39;article</span><span class="si">{language}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{language}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">language</span><span class="p">)</span>
<span class="k">return</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
<span class="k">class</span><span class="w"> </span><span class="nc">WDLabelAttribute</span><span class="p">(</span><span class="n">WDAttribute</span><span class="p">):</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_select</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s1">&#39;(group_concat(distinct ?</span><span class="si">{name}</span><span class="s1">Label;separator=&quot;, &quot;) as ?</span><span class="si">{name}</span><span class="s1">Labels)&#39;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{name}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_where</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;OPTIONAL { ?item wdt:</span><span class="si">{name}</span><span class="s2"> ?</span><span class="si">{name}</span><span class="s2"> . }&quot;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{name}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_wikibase_label</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;?</span><span class="si">{name}</span><span class="s2"> rdfs:label ?</span><span class="si">{name}</span><span class="s2">Label .&quot;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{name}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_str</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
<span class="k">return</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">&#39;Labels&#39;</span><span class="p">)</span>
<span class="k">class</span><span class="w"> </span><span class="nc">WDURLAttribute</span><span class="p">(</span><span class="n">WDAttribute</span><span class="p">):</span>
<span class="n">HTTP_WIKIMEDIA_IMAGE</span> <span class="o">=</span> <span class="s1">&#39;http://commons.wikimedia.org/wiki/Special:FilePath/&#39;</span>
<span class="vm">__slots__</span> <span class="o">=</span> <span class="s1">&#39;url_id&#39;</span><span class="p">,</span> <span class="s1">&#39;url_path_prefix&#39;</span><span class="p">,</span> <span class="s1">&#39;kwargs&#39;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">url_path_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">kwargs</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> :param url_id: ID matching one key in ``external_urls.json`` for</span>
<span class="sd"> converting IDs to full URLs.</span>
<span class="sd"> :param url_path_prefix: Path prefix if the values are of format</span>
<span class="sd"> ``account@domain``. If provided, value are rewritten to</span>
<span class="sd"> ``https://&lt;domain&gt;&lt;url_path_prefix&gt;&lt;account&gt;``. For example::</span>
<span class="sd"> WDURLAttribute(&#39;P4033&#39;, url_path_prefix=&#39;/@&#39;)</span>
<span class="sd"> Adds Property `P4033 &lt;https://www.wikidata.org/wiki/Property:P4033&gt;`_</span>
<span class="sd"> to the wikidata query. This field might return for example</span>
<span class="sd"> ``libreoffice@fosstodon.org`` and the URL built from this is then:</span>
<span class="sd"> - account: ``libreoffice``</span>
<span class="sd"> - domain: ``fosstodon.org``</span>
<span class="sd"> - result url: https://fosstodon.org/@libreoffice</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">url_id</span> <span class="o">=</span> <span class="n">url_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">url_path_prefix</span> <span class="o">=</span> <span class="n">url_path_prefix</span>
<span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_str</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">&#39;s&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">value</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">value</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">url_id</span><span class="p">:</span>
<span class="n">url_id</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">url_id</span>
<span class="k">if</span> <span class="n">value</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">WDURLAttribute</span><span class="o">.</span><span class="n">HTTP_WIKIMEDIA_IMAGE</span><span class="p">):</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">value</span><span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="n">WDURLAttribute</span><span class="o">.</span><span class="n">HTTP_WIKIMEDIA_IMAGE</span><span class="p">)</span> <span class="p">:]</span>
<span class="n">url_id</span> <span class="o">=</span> <span class="s1">&#39;wikimedia_image&#39;</span>
<span class="k">return</span> <span class="n">get_external_url</span><span class="p">(</span><span class="n">url_id</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">url_path_prefix</span><span class="p">:</span>
<span class="p">[</span><span class="n">account</span><span class="p">,</span> <span class="n">domain</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">&quot;@ &quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">value</span><span class="o">.</span><span class="n">rsplit</span><span class="p">(</span><span class="s1">&#39;@&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)]</span>
<span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;https://</span><span class="si">{</span><span class="n">domain</span><span class="si">}{</span><span class="bp">self</span><span class="o">.</span><span class="n">url_path_prefix</span><span class="si">}{</span><span class="n">account</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">return</span> <span class="n">value</span>
<span class="k">class</span><span class="w"> </span><span class="nc">WDGeoAttribute</span><span class="p">(</span><span class="n">WDAttribute</span><span class="p">):</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_label</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;OpenStreetMap&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_select</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;?</span><span class="si">{name}</span><span class="s2">Lat ?</span><span class="si">{name}</span><span class="s2">Long&quot;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{name}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_where</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;&quot;&quot;OPTIONAL { ?item p:</span><span class="si">{name}</span><span class="s2">/psv:</span><span class="si">{name}</span><span class="s2"> [</span>
<span class="s2"> wikibase:geoLatitude ?</span><span class="si">{name}</span><span class="s2">Lat ;</span>
<span class="s2"> wikibase:geoLongitude ?</span><span class="si">{name}</span><span class="s2">Long ] }&quot;&quot;&quot;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span>
<span class="s1">&#39;</span><span class="si">{name}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span>
<span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_group_by</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_select</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_str</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
<span class="n">latitude</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">&#39;Lat&#39;</span><span class="p">)</span>
<span class="n">longitude</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">&#39;Long&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">latitude</span> <span class="ow">and</span> <span class="n">longitude</span><span class="p">:</span>
<span class="k">return</span> <span class="n">latitude</span> <span class="o">+</span> <span class="s1">&#39; &#39;</span> <span class="o">+</span> <span class="n">longitude</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_geo_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">osm_zoom</span><span class="o">=</span><span class="mi">19</span><span class="p">):</span>
<span class="n">latitude</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">&#39;Lat&#39;</span><span class="p">)</span>
<span class="n">longitude</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">&#39;Long&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">latitude</span> <span class="ow">and</span> <span class="n">longitude</span><span class="p">:</span>
<span class="k">return</span> <span class="n">get_earth_coordinates_url</span><span class="p">(</span><span class="n">latitude</span><span class="p">,</span> <span class="n">longitude</span><span class="p">,</span> <span class="n">osm_zoom</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">class</span><span class="w"> </span><span class="nc">WDImageAttribute</span><span class="p">(</span><span class="n">WDURLAttribute</span><span class="p">):</span>
<span class="vm">__slots__</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;priority&#39;</span><span class="p">,)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">100</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">priority</span> <span class="o">=</span> <span class="n">priority</span>
<span class="k">class</span><span class="w"> </span><span class="nc">WDDateAttribute</span><span class="p">(</span><span class="n">WDAttribute</span><span class="p">):</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_select</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s1">&#39;?</span><span class="si">{name}</span><span class="s1"> ?</span><span class="si">{name}</span><span class="s1">timePrecision ?</span><span class="si">{name}</span><span class="s1">timeZone ?</span><span class="si">{name}</span><span class="s1">timeCalendar&#39;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{name}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_where</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="c1"># To remove duplicate, add</span>
<span class="c1"># FILTER NOT EXISTS { ?item p:{name}/psv:{name}/wikibase:timeValue ?{name}bis FILTER (?{name}bis &lt; ?{name}) }</span>
<span class="c1"># this filter is too slow, so the response function ignore duplicate results</span>
<span class="c1"># (see the seen_entities variable)</span>
<span class="k">return</span> <span class="s2">&quot;&quot;&quot;OPTIONAL { ?item p:</span><span class="si">{name}</span><span class="s2">/psv:</span><span class="si">{name}</span><span class="s2"> [</span>
<span class="s2"> wikibase:timeValue ?</span><span class="si">{name}</span><span class="s2"> ;</span>
<span class="s2"> wikibase:timePrecision ?</span><span class="si">{name}</span><span class="s2">timePrecision ;</span>
<span class="s2"> wikibase:timeTimezone ?</span><span class="si">{name}</span><span class="s2">timeZone ;</span>
<span class="s2"> wikibase:timeCalendarModel ?</span><span class="si">{name}</span><span class="s2">timeCalendar ] . }</span>
<span class="s2"> hint:Prior hint:rangeSafe true;&quot;&quot;&quot;</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span>
<span class="s1">&#39;</span><span class="si">{name}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span>
<span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_group_by</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_select</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">format_8</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="p">):</span> <span class="c1"># pylint: disable=unused-argument</span>
<span class="c1"># precision: less than a year</span>
<span class="k">return</span> <span class="n">value</span>
<span class="k">def</span><span class="w"> </span><span class="nf">format_9</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="p">):</span>
<span class="n">year</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="c1"># precision: year</span>
<span class="k">if</span> <span class="n">year</span> <span class="o">&lt;</span> <span class="mi">1584</span><span class="p">:</span>
<span class="k">if</span> <span class="n">year</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">year</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">year</span><span class="p">)</span>
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">isoparse</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="k">return</span> <span class="n">format_date</span><span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="nb">format</span><span class="o">=</span><span class="s1">&#39;yyyy&#39;</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">locale</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">format_10</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="p">):</span>
<span class="c1"># precision: month</span>
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">isoparse</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="k">return</span> <span class="n">format_date</span><span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="nb">format</span><span class="o">=</span><span class="s1">&#39;MMMM y&#39;</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">locale</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">format_11</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="p">):</span>
<span class="c1"># precision: day</span>
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">isoparse</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="k">return</span> <span class="n">format_date</span><span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="nb">format</span><span class="o">=</span><span class="s1">&#39;full&#39;</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">locale</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">format_13</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="p">):</span>
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">isoparse</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="c1"># precision: minute</span>
<span class="k">return</span> <span class="p">(</span>
<span class="n">get_datetime_format</span><span class="p">(</span><span class="nb">format</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">locale</span><span class="p">)</span>
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;&#39;&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{0}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">format_time</span><span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="s1">&#39;full&#39;</span><span class="p">,</span> <span class="n">tzinfo</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">locale</span><span class="p">))</span>
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{1}</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">format_date</span><span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="s1">&#39;short&#39;</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">locale</span><span class="p">))</span>
<span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">format_14</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="p">):</span>
<span class="c1"># precision: second.</span>
<span class="k">return</span> <span class="n">format_datetime</span><span class="p">(</span><span class="n">isoparse</span><span class="p">(</span><span class="n">value</span><span class="p">),</span> <span class="nb">format</span><span class="o">=</span><span class="s1">&#39;full&#39;</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">locale</span><span class="p">)</span>
<span class="n">DATE_FORMAT</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;0&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_8&#39;</span><span class="p">,</span> <span class="mi">1000000000</span><span class="p">),</span>
<span class="s1">&#39;1&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_8&#39;</span><span class="p">,</span> <span class="mi">100000000</span><span class="p">),</span>
<span class="s1">&#39;2&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_8&#39;</span><span class="p">,</span> <span class="mi">10000000</span><span class="p">),</span>
<span class="s1">&#39;3&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_8&#39;</span><span class="p">,</span> <span class="mi">1000000</span><span class="p">),</span>
<span class="s1">&#39;4&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_8&#39;</span><span class="p">,</span> <span class="mi">100000</span><span class="p">),</span>
<span class="s1">&#39;5&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_8&#39;</span><span class="p">,</span> <span class="mi">10000</span><span class="p">),</span>
<span class="s1">&#39;6&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_8&#39;</span><span class="p">,</span> <span class="mi">1000</span><span class="p">),</span>
<span class="s1">&#39;7&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_8&#39;</span><span class="p">,</span> <span class="mi">100</span><span class="p">),</span>
<span class="s1">&#39;8&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_8&#39;</span><span class="p">,</span> <span class="mi">10</span><span class="p">),</span>
<span class="s1">&#39;9&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_9&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="c1"># year</span>
<span class="s1">&#39;10&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_10&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="c1"># month</span>
<span class="s1">&#39;11&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_11&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> <span class="c1"># day</span>
<span class="s1">&#39;12&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_13&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> <span class="c1"># hour (not supported by babel, display minute)</span>
<span class="s1">&#39;13&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_13&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> <span class="c1"># minute</span>
<span class="s1">&#39;14&#39;</span><span class="p">:</span> <span class="p">(</span><span class="s1">&#39;format_14&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> <span class="c1"># second</span>
<span class="p">}</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_str</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="k">if</span> <span class="n">value</span> <span class="o">==</span> <span class="s1">&#39;&#39;</span> <span class="ow">or</span> <span class="n">value</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">precision</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">&#39;timePrecision&#39;</span><span class="p">)</span>
<span class="n">date_format</span> <span class="o">=</span> <span class="n">WDDateAttribute</span><span class="o">.</span><span class="n">DATE_FORMAT</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">precision</span><span class="p">)</span>
<span class="k">if</span> <span class="n">date_format</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">format_method</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">date_format</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
<span class="n">precision</span> <span class="o">=</span> <span class="n">date_format</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">if</span> <span class="n">precision</span> <span class="o">&gt;=</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">t</span> <span class="o">=</span> <span class="n">value</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">value</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">):</span>
<span class="n">value</span> <span class="o">=</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="n">t</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">t</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">return</span> <span class="n">format_method</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
<span class="k">return</span> <span class="n">value</span>
<span class="k">return</span> <span class="n">value</span>
<span class="k">def</span><span class="w"> </span><span class="nf">debug_explain_wikidata_query</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s1">&#39;GET&#39;</span><span class="p">):</span>
<span class="k">if</span> <span class="n">method</span> <span class="o">==</span> <span class="s1">&#39;GET&#39;</span><span class="p">:</span>
<span class="n">http_response</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">SPARQL_EXPLAIN_URL</span> <span class="o">+</span> <span class="s1">&#39;&amp;&#39;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">}),</span> <span class="n">headers</span><span class="o">=</span><span class="n">get_headers</span><span class="p">())</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">http_response</span> <span class="o">=</span> <span class="n">post</span><span class="p">(</span><span class="n">SPARQL_EXPLAIN_URL</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">},</span> <span class="n">headers</span><span class="o">=</span><span class="n">get_headers</span><span class="p">())</span>
<span class="n">http_response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
<span class="k">return</span> <span class="n">http_response</span><span class="o">.</span><span class="n">content</span>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> <span class="c1"># pylint: disable=unused-argument</span>
<span class="c1"># WIKIDATA_PROPERTIES : add unit symbols</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">WIKIDATA_UNITS</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">WIKIDATA_PROPERTIES</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">v</span><span class="p">[</span><span class="s1">&#39;symbol&#39;</span><span class="p">]</span>
<span class="c1"># WIKIDATA_PROPERTIES : add property labels</span>
<span class="n">wikidata_property_names</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">attribute</span> <span class="ow">in</span> <span class="n">get_attributes</span><span class="p">(</span><span class="s1">&#39;en&#39;</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">attribute</span><span class="p">)</span> <span class="ow">in</span> <span class="p">(</span><span class="n">WDAttribute</span><span class="p">,</span> <span class="n">WDAmountAttribute</span><span class="p">,</span> <span class="n">WDURLAttribute</span><span class="p">,</span> <span class="n">WDDateAttribute</span><span class="p">,</span> <span class="n">WDLabelAttribute</span><span class="p">):</span>
<span class="k">if</span> <span class="n">attribute</span><span class="o">.</span><span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">WIKIDATA_PROPERTIES</span><span class="p">:</span>
<span class="n">wikidata_property_names</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;wd:&quot;</span> <span class="o">+</span> <span class="n">attribute</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">QUERY_PROPERTY_NAMES</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;%ATTRIBUTES%&#39;</span><span class="p">,</span> <span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">wikidata_property_names</span><span class="p">))</span>
<span class="n">jsonresponse</span> <span class="o">=</span> <span class="n">send_wikidata_query</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">20</span><span class="p">)</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">jsonresponse</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;results&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;bindings&#39;</span><span class="p">,</span> <span class="p">{}):</span>
<span class="n">name</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">][</span><span class="s1">&#39;value&#39;</span><span class="p">]</span>
<span class="n">lang</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">][</span><span class="s1">&#39;xml:lang&#39;</span><span class="p">]</span>
<span class="n">entity_id</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">&#39;item&#39;</span><span class="p">][</span><span class="s1">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;http://www.wikidata.org/entity/&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">WIKIDATA_PROPERTIES</span><span class="p">[(</span><span class="n">entity_id</span><span class="p">,</span> <span class="n">lang</span><span class="p">)]</span> <span class="o">=</span> <span class="n">name</span><span class="o">.</span><span class="n">capitalize</span><span class="p">()</span>
<div class="viewcode-block" id="fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/online/wikipedia.html#searx.engines.wikidata.fetch_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Uses languages evaluated from :py:obj:`wikipedia.fetch_wikimedia_traits</span>
<span class="sd"> &lt;searx.engines.wikipedia.fetch_wikimedia_traits&gt;` and removes</span>
<span class="sd"> - ``traits.custom[&#39;wiki_netloc&#39;]``: wikidata does not have net-locations for</span>
<span class="sd"> the languages and the list of all</span>
<span class="sd"> - ``traits.custom[&#39;WIKIPEDIA_LANGUAGES&#39;]``: not used in the wikipedia engine</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">fetch_wikimedia_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">)</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;wiki_netloc&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;WIKIPEDIA_LANGUAGES&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,441 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.wikipedia &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.wikipedia</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.wikipedia</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;This module implements the Wikipedia engine. Some of this implementations</span>
<span class="sd">are shared by other engines:</span>
<span class="sd">- :ref:`wikidata engine`</span>
<span class="sd">The list of supported languages is :py:obj:`fetched &lt;fetch_wikimedia_traits&gt;` from</span>
<span class="sd">the article linked by :py:obj:`list_of_wikipedias`.</span>
<span class="sd">Unlike traditional search engines, wikipedia does not support one Wikipedia for</span>
<span class="sd">all languages, but there is one Wikipedia for each supported language. Some of</span>
<span class="sd">these Wikipedias have a LanguageConverter_ enabled</span>
<span class="sd">(:py:obj:`rest_v1_summary_url`).</span>
<span class="sd">A LanguageConverter_ (LC) is a system based on language variants that</span>
<span class="sd">automatically converts the content of a page into a different variant. A variant</span>
<span class="sd">is mostly the same language in a different script.</span>
<span class="sd">- `Wikipedias in multiple writing systems`_</span>
<span class="sd">- `Automatic conversion between traditional and simplified Chinese characters`_</span>
<span class="sd">PR-2554_:</span>
<span class="sd"> The Wikipedia link returned by the API is still the same in all cases</span>
<span class="sd"> (`https://zh.wikipedia.org/wiki/出租車`_) but if your browser&#39;s</span>
<span class="sd"> ``Accept-Language`` is set to any of ``zh``, ``zh-CN``, ``zh-TW``, ``zh-HK``</span>
<span class="sd"> or .. Wikipedia&#39;s LC automatically returns the desired script in their</span>
<span class="sd"> web-page.</span>
<span class="sd"> - You can test the API here: https://reqbin.com/gesg2kvx</span>
<span class="sd">.. _https://zh.wikipedia.org/wiki/出租車:</span>
<span class="sd"> https://zh.wikipedia.org/wiki/%E5%87%BA%E7%A7%9F%E8%BB%8A</span>
<span class="sd">To support Wikipedia&#39;s LanguageConverter_, a SearXNG request to Wikipedia uses</span>
<span class="sd">:py:obj:`get_wiki_params` and :py:obj:`wiki_lc_locale_variants&#39; in the</span>
<span class="sd">:py:obj:`fetch_wikimedia_traits` function.</span>
<span class="sd">To test in SearXNG, query for ``!wp 出租車`` with each of the available Chinese</span>
<span class="sd">options:</span>
<span class="sd">- ``!wp 出租車 :zh`` should show 出租車</span>
<span class="sd">- ``!wp 出租車 :zh-CN`` should show 出租车</span>
<span class="sd">- ``!wp 出租車 :zh-TW`` should show 計程車</span>
<span class="sd">- ``!wp 出租車 :zh-HK`` should show 的士</span>
<span class="sd">- ``!wp 出租車 :zh-SG`` should show 德士</span>
<span class="sd">.. _LanguageConverter:</span>
<span class="sd"> https://www.mediawiki.org/wiki/Writing_systems#LanguageConverter</span>
<span class="sd">.. _Wikipedias in multiple writing systems:</span>
<span class="sd"> https://meta.wikimedia.org/wiki/Wikipedias_in_multiple_writing_systems</span>
<span class="sd">.. _Automatic conversion between traditional and simplified Chinese characters:</span>
<span class="sd"> https://en.wikipedia.org/wiki/Chinese_Wikipedia#Automatic_conversion_between_traditional_and_simplified_Chinese_characters</span>
<span class="sd">.. _PR-2554: https://github.com/searx/searx/pull/2554</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">urllib.parse</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">utils</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">network</span> <span class="k">as</span> <span class="n">_network</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">locales</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="c1"># about</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://www.wikipedia.org/&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s1">&#39;Q52&#39;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://en.wikipedia.org/api/&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;JSON&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">display_type</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;infobox&quot;</span><span class="p">]</span>
<span class="sd">&quot;&quot;&quot;A list of display types composed from ``infobox`` and ``list``. The latter</span>
<span class="sd">one will add a hit to the result list. The first one will show a hit in the</span>
<span class="sd">info box. Both values can be set, or one of the two can be set.&quot;&quot;&quot;</span>
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
<span class="sd">&quot;&quot;&quot;The HTTP ``Accept-Language`` header is needed for wikis where</span>
<span class="sd">LanguageConverter_ is enabled.&quot;&quot;&quot;</span>
<span class="n">list_of_wikipedias</span> <span class="o">=</span> <span class="s1">&#39;https://meta.wikimedia.org/wiki/List_of_Wikipedias&#39;</span>
<span class="sd">&quot;&quot;&quot;`List of all wikipedias &lt;https://meta.wikimedia.org/wiki/List_of_Wikipedias&gt;`_</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">wikipedia_article_depth</span> <span class="o">=</span> <span class="s1">&#39;https://meta.wikimedia.org/wiki/Wikipedia_article_depth&#39;</span>
<span class="sd">&quot;&quot;&quot;The *editing depth* of Wikipedia is one of several possible rough indicators</span>
<span class="sd">of the encyclopedia&#39;s collaborative quality, showing how frequently its articles</span>
<span class="sd">are updated. The measurement of depth was introduced after some limitations of</span>
<span class="sd">the classic measurement of article count were realized.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">rest_v1_summary_url</span> <span class="o">=</span> <span class="s1">&#39;https://</span><span class="si">{wiki_netloc}</span><span class="s1">/api/rest_v1/page/summary/</span><span class="si">{title}</span><span class="s1">&#39;</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">`wikipedia rest_v1 summary API`_:</span>
<span class="sd"> The summary response includes an extract of the first paragraph of the page in</span>
<span class="sd"> plain text and HTML as well as the type of page. This is useful for page</span>
<span class="sd"> previews (fka. Hovercards, aka. Popups) on the web and link previews in the</span>
<span class="sd"> apps.</span>
<span class="sd">HTTP ``Accept-Language`` header (:py:obj:`send_accept_language_header`):</span>
<span class="sd"> The desired language variant code for wikis where LanguageConverter_ is</span>
<span class="sd"> enabled.</span>
<span class="sd">.. _wikipedia rest_v1 summary API:</span>
<span class="sd"> https://en.wikipedia.org/api/rest_v1/#/Page%20content/get_page_summary__title_</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">wiki_lc_locale_variants</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;zh&quot;</span><span class="p">:</span> <span class="p">(</span>
<span class="s2">&quot;zh-CN&quot;</span><span class="p">,</span>
<span class="s2">&quot;zh-HK&quot;</span><span class="p">,</span>
<span class="s2">&quot;zh-MO&quot;</span><span class="p">,</span>
<span class="s2">&quot;zh-MY&quot;</span><span class="p">,</span>
<span class="s2">&quot;zh-SG&quot;</span><span class="p">,</span>
<span class="s2">&quot;zh-TW&quot;</span><span class="p">,</span>
<span class="p">),</span>
<span class="s2">&quot;zh-classical&quot;</span><span class="p">:</span> <span class="p">(</span><span class="s2">&quot;zh-classical&quot;</span><span class="p">,),</span>
<span class="p">}</span>
<span class="sd">&quot;&quot;&quot;Mapping rule of the LanguageConverter_ to map a language and its variants to</span>
<span class="sd">a Locale (used in the HTTP ``Accept-Language`` header). For example see `LC</span>
<span class="sd">Chinese`_.</span>
<span class="sd">.. _LC Chinese:</span>
<span class="sd"> https://meta.wikimedia.org/wiki/Wikipedias_in_multiple_writing_systems#Chinese</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">wikipedia_script_variants</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;zh&quot;</span><span class="p">:</span> <span class="p">(</span>
<span class="s2">&quot;zh_Hant&quot;</span><span class="p">,</span>
<span class="s2">&quot;zh_Hans&quot;</span><span class="p">,</span>
<span class="p">)</span>
<span class="p">}</span>
<div class="viewcode-block" id="get_wiki_params">
<a class="viewcode-back" href="../../../dev/engines/online/wikipedia.html#searx.engines.wikipedia.get_wiki_params">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_wiki_params</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">eng_traits</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns the Wikipedia language tag and the netloc that fits to the</span>
<span class="sd"> ``sxng_locale``. To support LanguageConverter_ this function rates a locale</span>
<span class="sd"> (region) higher than a language (compare :py:obj:`wiki_lc_locale_variants`).</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="s1">&#39;en&#39;</span><span class="p">))</span>
<span class="n">wiki_netloc</span> <span class="o">=</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;wiki_netloc&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">,</span> <span class="s1">&#39;en.wikipedia.org&#39;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">eng_tag</span><span class="p">,</span> <span class="n">wiki_netloc</span></div>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/wikipedia.html#searx.engines.wikipedia.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Assemble a request (`wikipedia rest_v1 summary API`_).&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">query</span><span class="o">.</span><span class="n">islower</span><span class="p">():</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">title</span><span class="p">()</span>
<span class="n">_eng_tag</span><span class="p">,</span> <span class="n">wiki_netloc</span> <span class="o">=</span> <span class="n">get_wiki_params</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;searxng_locale&#39;</span><span class="p">],</span> <span class="n">traits</span><span class="p">)</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">quote</span><span class="p">(</span><span class="n">query</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">rest_v1_summary_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">wiki_netloc</span><span class="o">=</span><span class="n">wiki_netloc</span><span class="p">,</span> <span class="n">title</span><span class="o">=</span><span class="n">title</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;raise_for_httperror&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;soft_max_redirects&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">2</span>
<span class="k">return</span> <span class="n">params</span></div>
<span class="c1"># get response from search-request</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">404</span><span class="p">:</span>
<span class="k">return</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">400</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">api_result</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
<span class="k">pass</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="p">(</span>
<span class="n">api_result</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;https://mediawiki.org/wiki/HyperSwitch/errors/bad_request&#39;</span>
<span class="ow">and</span> <span class="n">api_result</span><span class="p">[</span><span class="s1">&#39;detail&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;title-invalid-characters&#39;</span>
<span class="p">):</span>
<span class="k">return</span> <span class="p">[]</span>
<span class="n">_network</span><span class="o">.</span><span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<span class="n">api_result</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;titles&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;display&#39;</span><span class="p">)</span> <span class="ow">or</span> <span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;title&#39;</span><span class="p">))</span>
<span class="n">wikipedia_link</span> <span class="o">=</span> <span class="n">api_result</span><span class="p">[</span><span class="s1">&#39;content_urls&#39;</span><span class="p">][</span><span class="s1">&#39;desktop&#39;</span><span class="p">][</span><span class="s1">&#39;page&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;list&quot;</span> <span class="ow">in</span> <span class="n">display_type</span> <span class="ow">or</span> <span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;type&#39;</span><span class="p">)</span> <span class="o">!=</span> <span class="s1">&#39;standard&#39;</span><span class="p">:</span>
<span class="c1"># show item in the result list if &#39;list&#39; is in the display options or it</span>
<span class="c1"># is a item that can&#39;t be displayed in a infobox.</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">wikipedia_link</span><span class="p">,</span> <span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span> <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;description&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)})</span>
<span class="k">if</span> <span class="s2">&quot;infobox&quot;</span> <span class="ow">in</span> <span class="n">display_type</span><span class="p">:</span>
<span class="k">if</span> <span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;type&#39;</span><span class="p">)</span> <span class="o">==</span> <span class="s1">&#39;standard&#39;</span><span class="p">:</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;infobox&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
<span class="s1">&#39;id&#39;</span><span class="p">:</span> <span class="n">wikipedia_link</span><span class="p">,</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;extract&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">),</span>
<span class="s1">&#39;img_src&#39;</span><span class="p">:</span> <span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;thumbnail&#39;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;source&#39;</span><span class="p">),</span>
<span class="s1">&#39;urls&#39;</span><span class="p">:</span> <span class="p">[{</span><span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="s1">&#39;Wikipedia&#39;</span><span class="p">,</span> <span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">wikipedia_link</span><span class="p">}],</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span>
<span class="c1"># Nonstandard language codes</span>
<span class="c1">#</span>
<span class="c1"># These Wikipedias use language codes that do not conform to the ISO 639</span>
<span class="c1"># standard (which is how wiki subdomains are chosen nowadays).</span>
<span class="n">lang_map</span> <span class="o">=</span> <span class="n">locales</span><span class="o">.</span><span class="n">LOCALE_BEST_MATCH</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">lang_map</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;be-tarask&#39;</span><span class="p">:</span> <span class="s1">&#39;bel&#39;</span><span class="p">,</span>
<span class="s1">&#39;ak&#39;</span><span class="p">:</span> <span class="s1">&#39;aka&#39;</span><span class="p">,</span>
<span class="s1">&#39;als&#39;</span><span class="p">:</span> <span class="s1">&#39;gsw&#39;</span><span class="p">,</span>
<span class="s1">&#39;bat-smg&#39;</span><span class="p">:</span> <span class="s1">&#39;sgs&#39;</span><span class="p">,</span>
<span class="s1">&#39;cbk-zam&#39;</span><span class="p">:</span> <span class="s1">&#39;cbk&#39;</span><span class="p">,</span>
<span class="s1">&#39;fiu-vro&#39;</span><span class="p">:</span> <span class="s1">&#39;vro&#39;</span><span class="p">,</span>
<span class="s1">&#39;map-bms&#39;</span><span class="p">:</span> <span class="s1">&#39;map&#39;</span><span class="p">,</span>
<span class="s1">&#39;no&#39;</span><span class="p">:</span> <span class="s1">&#39;nb-NO&#39;</span><span class="p">,</span>
<span class="s1">&#39;nrm&#39;</span><span class="p">:</span> <span class="s1">&#39;nrf&#39;</span><span class="p">,</span>
<span class="s1">&#39;roa-rup&#39;</span><span class="p">:</span> <span class="s1">&#39;rup&#39;</span><span class="p">,</span>
<span class="s1">&#39;nds-nl&#39;</span><span class="p">:</span> <span class="s1">&#39;nds&#39;</span><span class="p">,</span>
<span class="c1">#&#39;simple: invented code used for the Simple English Wikipedia (not the official IETF code en-simple)</span>
<span class="s1">&#39;zh-min-nan&#39;</span><span class="p">:</span> <span class="s1">&#39;nan&#39;</span><span class="p">,</span>
<span class="s1">&#39;zh-yue&#39;</span><span class="p">:</span> <span class="s1">&#39;yue&#39;</span><span class="p">,</span>
<span class="s1">&#39;an&#39;</span><span class="p">:</span> <span class="s1">&#39;arg&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="n">fetch_wikimedia_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;WIKIPEDIA_LANGUAGES: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="nb">len</span><span class="p">(</span><span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;WIKIPEDIA_LANGUAGES&#39;</span><span class="p">]))</span>
<div class="viewcode-block" id="fetch_wikimedia_traits">
<a class="viewcode-back" href="../../../dev/engines/online/wikipedia.html#searx.engines.wikipedia.fetch_wikimedia_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_wikimedia_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Fetch languages from Wikipedia. Not all languages from the</span>
<span class="sd"> :py:obj:`list_of_wikipedias` are supported by SearXNG locales, only those</span>
<span class="sd"> known from :py:obj:`searx.locales.LOCALE_NAMES` or those with a minimal</span>
<span class="sd"> :py:obj:`editing depth &lt;wikipedia_article_depth&gt;`.</span>
<span class="sd"> The location of the Wikipedia address of a language is mapped in a</span>
<span class="sd"> :py:obj:`custom field &lt;searx.enginelib.traits.EngineTraits.custom&gt;`</span>
<span class="sd"> (``wiki_netloc``). Here is a reduced example:</span>
<span class="sd"> .. code:: python</span>
<span class="sd"> traits.custom[&#39;wiki_netloc&#39;] = {</span>
<span class="sd"> &quot;en&quot;: &quot;en.wikipedia.org&quot;,</span>
<span class="sd"> ..</span>
<span class="sd"> &quot;gsw&quot;: &quot;als.wikipedia.org&quot;,</span>
<span class="sd"> ..</span>
<span class="sd"> &quot;zh&quot;: &quot;zh.wikipedia.org&quot;,</span>
<span class="sd"> &quot;zh-classical&quot;: &quot;zh-classical.wikipedia.org&quot;</span>
<span class="sd"> }</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-many-branches</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;wiki_netloc&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;WIKIPEDIA_LANGUAGES&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="c1"># insert alias to map from a script or region to a wikipedia variant</span>
<span class="k">for</span> <span class="n">eng_tag</span><span class="p">,</span> <span class="n">sxng_tag_list</span> <span class="ow">in</span> <span class="n">wikipedia_script_variants</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">for</span> <span class="n">sxng_tag</span> <span class="ow">in</span> <span class="n">sxng_tag_list</span><span class="p">:</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
<span class="k">for</span> <span class="n">eng_tag</span><span class="p">,</span> <span class="n">sxng_tag_list</span> <span class="ow">in</span> <span class="n">wiki_lc_locale_variants</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">for</span> <span class="n">sxng_tag</span> <span class="ow">in</span> <span class="n">sxng_tag_list</span><span class="p">:</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">_network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">list_of_wikipedias</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;ERROR: response from Wikipedia is not OK.&quot;</span><span class="p">)</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;//table[contains(@class,&quot;sortable&quot;)]//tbody/tr&#39;</span><span class="p">):</span>
<span class="n">cols</span> <span class="o">=</span> <span class="n">row</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;./td&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">cols</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">cols</span> <span class="o">=</span> <span class="p">[</span><span class="n">c</span><span class="o">.</span><span class="n">text_content</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">cols</span><span class="p">]</span>
<span class="n">depth</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">cols</span><span class="p">[</span><span class="mi">11</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">,</span> <span class="s1">&#39;0&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">))</span>
<span class="n">articles</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">cols</span><span class="p">[</span><span class="mi">4</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">))</span>
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">cols</span><span class="p">[</span><span class="mi">3</span><span class="p">]</span>
<span class="n">wiki_url</span> <span class="o">=</span> <span class="n">row</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;./td[4]/a/@href&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">wiki_url</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">wiki_url</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">locales</span><span class="o">.</span><span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">lang_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">),</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;-&#39;</span><span class="p">))</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="c1"># print(&quot;ERROR: %s [%s] is unknown by babel&quot; % (cols[0], eng_tag))</span>
<span class="k">continue</span>
<span class="k">finally</span><span class="p">:</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;WIKIPEDIA_LANGUAGES&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">sxng_tag</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">locales</span><span class="o">.</span><span class="n">LOCALE_NAMES</span><span class="p">:</span>
<span class="k">if</span> <span class="n">articles</span> <span class="o">&lt;</span> <span class="mi">10000</span><span class="p">:</span>
<span class="c1"># exclude languages with too few articles</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="nb">int</span><span class="p">(</span><span class="n">depth</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">20</span><span class="p">:</span>
<span class="c1"># Rough indicator of a Wikipedias quality, showing how</span>
<span class="c1"># frequently its articles are updated.</span>
<span class="k">continue</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;wiki_netloc&#39;</span><span class="p">][</span><span class="n">eng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">wiki_url</span><span class="o">.</span><span class="n">netloc</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">&#39;WIKIPEDIA_LANGUAGES&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,447 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.xpath &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.xpath</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.xpath</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;The XPath engine is a *generic* engine with which it is possible to configure</span>
<span class="sd">engines in the settings.</span>
<span class="sd">.. _XPath selector: https://quickref.me/xpath.html#xpath-selectors</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">Request:</span>
<span class="sd">- :py:obj:`search_url`</span>
<span class="sd">- :py:obj:`lang_all`</span>
<span class="sd">- :py:obj:`soft_max_redirects`</span>
<span class="sd">- :py:obj:`method`</span>
<span class="sd">- :py:obj:`request_body`</span>
<span class="sd">- :py:obj:`cookies`</span>
<span class="sd">- :py:obj:`headers`</span>
<span class="sd">Paging:</span>
<span class="sd">- :py:obj:`paging`</span>
<span class="sd">- :py:obj:`page_size`</span>
<span class="sd">- :py:obj:`first_page_num`</span>
<span class="sd">Time Range:</span>
<span class="sd">- :py:obj:`time_range_support`</span>
<span class="sd">- :py:obj:`time_range_url`</span>
<span class="sd">- :py:obj:`time_range_map`</span>
<span class="sd">Safe-Search:</span>
<span class="sd">- :py:obj:`safe_search_support`</span>
<span class="sd">- :py:obj:`safe_search_map`</span>
<span class="sd">Response:</span>
<span class="sd">- :py:obj:`no_result_for_http_status`</span>
<span class="sd">`XPath selector`_:</span>
<span class="sd">- :py:obj:`results_xpath`</span>
<span class="sd">- :py:obj:`url_xpath`</span>
<span class="sd">- :py:obj:`title_xpath`</span>
<span class="sd">- :py:obj:`content_xpath`</span>
<span class="sd">- :py:obj:`thumbnail_xpath`</span>
<span class="sd">- :py:obj:`suggestion_xpath`</span>
<span class="sd">Example</span>
<span class="sd">=======</span>
<span class="sd">Here is a simple example of a XPath engine configured in the :ref:`settings</span>
<span class="sd">engines` section, further read :ref:`engines-dev`.</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name : bitbucket</span>
<span class="sd"> engine : xpath</span>
<span class="sd"> paging : True</span>
<span class="sd"> search_url : https://bitbucket.org/repo/all/{pageno}?name={query}</span>
<span class="sd"> url_xpath : //article[@class=&quot;repo-summary&quot;]//a[@class=&quot;repo-link&quot;]/@href</span>
<span class="sd"> title_xpath : //article[@class=&quot;repo-summary&quot;]//a[@class=&quot;repo-link&quot;]</span>
<span class="sd"> content_xpath : //article[@class=&quot;repo-summary&quot;]/p</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">extract_url</span><span class="p">,</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">eval_xpath_list</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">raise_for_httperror</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="n">search_url</span> <span class="o">=</span> <span class="kc">None</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">Search URL of the engine. Example::</span>
<span class="sd"> https://example.org/?search={query}&amp;page={pageno}{time_range}{safe_search}</span>
<span class="sd">Replacements are:</span>
<span class="sd">``{query}``:</span>
<span class="sd"> Search terms from user.</span>
<span class="sd">``{pageno}``:</span>
<span class="sd"> Page number if engine supports paging :py:obj:`paging`</span>
<span class="sd">``{lang}``:</span>
<span class="sd"> ISO 639-1 language code (en, de, fr ..)</span>
<span class="sd">``{time_range}``:</span>
<span class="sd"> :py:obj:`URL parameter &lt;time_range_url&gt;` if engine :py:obj:`supports time</span>
<span class="sd"> range &lt;time_range_support&gt;`. The value for the parameter is taken from</span>
<span class="sd"> :py:obj:`time_range_map`.</span>
<span class="sd">``{safe_search}``:</span>
<span class="sd"> Safe-search :py:obj:`URL parameter &lt;safe_search_map&gt;` if engine</span>
<span class="sd"> :py:obj:`supports safe-search &lt;safe_search_support&gt;`. The ``{safe_search}``</span>
<span class="sd"> replacement is taken from the :py:obj:`safes_search_map`. Filter results::</span>
<span class="sd"> 0: none, 1: moderate, 2:strict</span>
<span class="sd"> If not supported, the URL parameter is an empty string.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">lang_all</span> <span class="o">=</span> <span class="s1">&#39;en&#39;</span>
<span class="sd">&#39;&#39;&#39;Replacement ``{lang}`` in :py:obj:`search_url` if language ``all`` is</span>
<span class="sd">selected.</span>
<span class="sd">&#39;&#39;&#39;</span>
<span class="n">no_result_for_http_status</span> <span class="o">=</span> <span class="p">[]</span>
<span class="sd">&#39;&#39;&#39;Return empty result for these HTTP status codes instead of throwing an error.</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> no_result_for_http_status: []</span>
<span class="sd">&#39;&#39;&#39;</span>
<span class="n">soft_max_redirects</span> <span class="o">=</span> <span class="mi">0</span>
<span class="sd">&#39;&#39;&#39;Maximum redirects, soft limit. Record an error but don&#39;t stop the engine&#39;&#39;&#39;</span>
<span class="n">results_xpath</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="sd">&#39;&#39;&#39;`XPath selector`_ for the list of result items&#39;&#39;&#39;</span>
<span class="n">url_xpath</span> <span class="o">=</span> <span class="kc">None</span>
<span class="sd">&#39;&#39;&#39;`XPath selector`_ of result&#39;s ``url``.&#39;&#39;&#39;</span>
<span class="n">content_xpath</span> <span class="o">=</span> <span class="kc">None</span>
<span class="sd">&#39;&#39;&#39;`XPath selector`_ of result&#39;s ``content``.&#39;&#39;&#39;</span>
<span class="n">title_xpath</span> <span class="o">=</span> <span class="kc">None</span>
<span class="sd">&#39;&#39;&#39;`XPath selector`_ of result&#39;s ``title``.&#39;&#39;&#39;</span>
<span class="n">thumbnail_xpath</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&#39;&#39;&#39;`XPath selector`_ of result&#39;s ``thumbnail``.&#39;&#39;&#39;</span>
<span class="n">suggestion_xpath</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="sd">&#39;&#39;&#39;`XPath selector`_ of result&#39;s ``suggestion``.&#39;&#39;&#39;</span>
<span class="n">cached_xpath</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="n">cached_url</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="n">cookies</span> <span class="o">=</span> <span class="p">{}</span>
<span class="sd">&#39;&#39;&#39;Some engines might offer different result based on cookies.</span>
<span class="sd">Possible use-case: To set safesearch cookie.&#39;&#39;&#39;</span>
<span class="n">headers</span> <span class="o">=</span> <span class="p">{}</span>
<span class="sd">&#39;&#39;&#39;Some engines might offer different result based headers. Possible use-case:</span>
<span class="sd">To set header to moderate.&#39;&#39;&#39;</span>
<span class="n">method</span> <span class="o">=</span> <span class="s1">&#39;GET&#39;</span>
<span class="sd">&#39;&#39;&#39;Some engines might require to do POST requests for search.&#39;&#39;&#39;</span>
<span class="n">request_body</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="sd">&#39;&#39;&#39;The body of the request. This can only be used if different :py:obj:`method`</span>
<span class="sd">is set, e.g. ``POST``. For formatting see the documentation of :py:obj:`search_url`::</span>
<span class="sd"> search={query}&amp;page={pageno}{time_range}{safe_search}</span>
<span class="sd">&#39;&#39;&#39;</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&#39;&#39;&#39;Engine supports paging [True or False].&#39;&#39;&#39;</span>
<span class="n">page_size</span> <span class="o">=</span> <span class="mi">1</span>
<span class="sd">&#39;&#39;&#39;Number of results on each page. Only needed if the site requires not a page</span>
<span class="sd">number, but an offset.&#39;&#39;&#39;</span>
<span class="n">first_page_num</span> <span class="o">=</span> <span class="mi">1</span>
<span class="sd">&#39;&#39;&#39;Number of the first page (usually 0 or 1).&#39;&#39;&#39;</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&#39;&#39;&#39;Engine supports search time range.&#39;&#39;&#39;</span>
<span class="n">time_range_url</span> <span class="o">=</span> <span class="s1">&#39;&amp;hours=</span><span class="si">{time_range_val}</span><span class="s1">&#39;</span>
<span class="sd">&#39;&#39;&#39;Time range URL parameter in the in :py:obj:`search_url`. If no time range is</span>
<span class="sd">requested by the user, the URL parameter is an empty string. The</span>
<span class="sd">``{time_range_val}`` replacement is taken from the :py:obj:`time_range_map`.</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> time_range_url : &#39;&amp;days={time_range_val}&#39;</span>
<span class="sd">&#39;&#39;&#39;</span>
<span class="n">time_range_map</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;day&#39;</span><span class="p">:</span> <span class="mi">24</span><span class="p">,</span>
<span class="s1">&#39;week&#39;</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span><span class="p">,</span>
<span class="s1">&#39;month&#39;</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">30</span><span class="p">,</span>
<span class="s1">&#39;year&#39;</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">365</span><span class="p">,</span>
<span class="p">}</span>
<span class="sd">&#39;&#39;&#39;Maps time range value from user to ``{time_range_val}`` in</span>
<span class="sd">:py:obj:`time_range_url`.</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> time_range_map:</span>
<span class="sd"> day: 1</span>
<span class="sd"> week: 7</span>
<span class="sd"> month: 30</span>
<span class="sd"> year: 365</span>
<span class="sd">&#39;&#39;&#39;</span>
<span class="n">safe_search_support</span> <span class="o">=</span> <span class="kc">False</span>
<span class="sd">&#39;&#39;&#39;Engine supports safe-search.&#39;&#39;&#39;</span>
<span class="n">safe_search_map</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">&#39;&amp;filter=none&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">&#39;&amp;filter=moderate&#39;</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">&#39;&amp;filter=strict&#39;</span><span class="p">}</span>
<span class="sd">&#39;&#39;&#39;Maps safe-search value to ``{safe_search}`` in :py:obj:`search_url`.</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> safesearch: true</span>
<span class="sd"> safes_search_map:</span>
<span class="sd"> 0: &#39;&amp;filter=none&#39;</span>
<span class="sd"> 1: &#39;&amp;filter=moderate&#39;</span>
<span class="sd"> 2: &#39;&amp;filter=strict&#39;</span>
<span class="sd">&#39;&#39;&#39;</span>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/xpath.html#searx.engines.xpath.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&#39;&#39;&#39;Build request parameters (see :ref:`engine request`).&#39;&#39;&#39;</span>
<span class="n">lang</span> <span class="o">=</span> <span class="n">lang_all</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;language&#39;</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">&#39;all&#39;</span><span class="p">:</span>
<span class="n">lang</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;language&#39;</span><span class="p">][:</span><span class="mi">2</span><span class="p">]</span>
<span class="n">time_range</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="k">if</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;time_range&#39;</span><span class="p">):</span>
<span class="n">time_range_val</span> <span class="o">=</span> <span class="n">time_range_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;time_range&#39;</span><span class="p">))</span>
<span class="n">time_range</span> <span class="o">=</span> <span class="n">time_range_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">time_range_val</span><span class="o">=</span><span class="n">time_range_val</span><span class="p">)</span>
<span class="n">safe_search</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]:</span>
<span class="n">safe_search</span> <span class="o">=</span> <span class="n">safe_search_map</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]]</span>
<span class="n">fargs</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;query&#39;</span><span class="p">:</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">})[</span><span class="mi">2</span><span class="p">:],</span>
<span class="s1">&#39;lang&#39;</span><span class="p">:</span> <span class="n">lang</span><span class="p">,</span>
<span class="s1">&#39;pageno&#39;</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">page_size</span> <span class="o">+</span> <span class="n">first_page_num</span><span class="p">,</span>
<span class="s1">&#39;time_range&#39;</span><span class="p">:</span> <span class="n">time_range</span><span class="p">,</span>
<span class="s1">&#39;safe_search&#39;</span><span class="p">:</span> <span class="n">safe_search</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">cookies</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;headers&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">headers</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">fargs</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;method&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">method</span>
<span class="k">if</span> <span class="n">request_body</span><span class="p">:</span>
<span class="c1"># don&#39;t url-encode the query if it&#39;s in the request body</span>
<span class="n">fargs</span><span class="p">[</span><span class="s1">&#39;query&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">query</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">request_body</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">fargs</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;soft_max_redirects&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">soft_max_redirects</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;raise_for_httperror&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">return</span> <span class="n">params</span></div>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/xpath.html#searx.engines.xpath.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span> <span class="c1"># pylint: disable=too-many-branches</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Scrap *results* from the response (see :ref:`result types`).&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="k">if</span> <span class="n">no_result_for_http_status</span> <span class="ow">and</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="ow">in</span> <span class="n">no_result_for_http_status</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="n">is_onion</span> <span class="o">=</span> <span class="s1">&#39;onions&#39;</span> <span class="ow">in</span> <span class="n">categories</span>
<span class="k">if</span> <span class="n">results_xpath</span><span class="p">:</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">results_xpath</span><span class="p">):</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">extract_url</span><span class="p">(</span><span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">url_xpath</span><span class="p">,</span> <span class="n">min_len</span><span class="o">=</span><span class="mi">1</span><span class="p">),</span> <span class="n">search_url</span><span class="p">)</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">title_xpath</span><span class="p">,</span> <span class="n">min_len</span><span class="o">=</span><span class="mi">1</span><span class="p">))</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">content_xpath</span><span class="p">))</span>
<span class="n">tmp_result</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span> <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">}</span>
<span class="c1"># add thumbnail if available</span>
<span class="k">if</span> <span class="n">thumbnail_xpath</span><span class="p">:</span>
<span class="n">thumbnail_xpath_result</span> <span class="o">=</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">thumbnail_xpath</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">thumbnail_xpath_result</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">&#39;thumbnail&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">extract_url</span><span class="p">(</span><span class="n">thumbnail_xpath_result</span><span class="p">,</span> <span class="n">search_url</span><span class="p">)</span>
<span class="c1"># add alternative cached url if available</span>
<span class="k">if</span> <span class="n">cached_xpath</span><span class="p">:</span>
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">&#39;cached_url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">cached_url</span> <span class="o">+</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">cached_xpath</span><span class="p">,</span> <span class="n">min_len</span><span class="o">=</span><span class="mi">1</span><span class="p">))</span>
<span class="k">if</span> <span class="n">is_onion</span><span class="p">:</span>
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">&#39;is_onion&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tmp_result</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="n">cached_xpath</span><span class="p">:</span>
<span class="k">for</span> <span class="n">url</span><span class="p">,</span> <span class="n">title</span><span class="p">,</span> <span class="n">content</span><span class="p">,</span> <span class="n">cached</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span>
<span class="p">(</span><span class="n">extract_url</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">search_url</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">url_xpath</span><span class="p">)),</span>
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">title_xpath</span><span class="p">)),</span>
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">content_xpath</span><span class="p">)),</span>
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">cached_xpath</span><span class="p">)),</span>
<span class="p">):</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
<span class="s1">&#39;cached_url&#39;</span><span class="p">:</span> <span class="n">cached_url</span> <span class="o">+</span> <span class="n">cached</span><span class="p">,</span>
<span class="s1">&#39;is_onion&#39;</span><span class="p">:</span> <span class="n">is_onion</span><span class="p">,</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">for</span> <span class="n">url</span><span class="p">,</span> <span class="n">title</span><span class="p">,</span> <span class="n">content</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span>
<span class="p">(</span><span class="n">extract_url</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">search_url</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">url_xpath</span><span class="p">)),</span>
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">title_xpath</span><span class="p">)),</span>
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">content_xpath</span><span class="p">)),</span>
<span class="p">):</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span> <span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span> <span class="s1">&#39;is_onion&#39;</span><span class="p">:</span> <span class="n">is_onion</span><span class="p">})</span>
<span class="k">if</span> <span class="n">suggestion_xpath</span><span class="p">:</span>
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">suggestion_xpath</span><span class="p">):</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;suggestion&#39;</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)})</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;found </span><span class="si">%s</span><span class="s2"> results&quot;</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">results</span><span class="p">))</span>
<span class="k">return</span> <span class="n">results</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,377 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.yahoo &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.yahoo</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.yahoo</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Yahoo Search (Web)</span>
<span class="sd">Languages are supported by mapping the language to a domain. If domain is not</span>
<span class="sd">found in :py:obj:`lang2domain` URL ``&lt;lang&gt;.search.yahoo.com`` is used.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">unquote</span><span class="p">,</span>
<span class="n">urlencode</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
<span class="n">eval_xpath_list</span><span class="p">,</span>
<span class="n">extract_text</span><span class="p">,</span>
<span class="n">html_to_text</span><span class="p">,</span>
<span class="p">)</span>
<span class="c1"># about</span>
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s1">&#39;https://search.yahoo.com/&#39;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="s1">&#39;https://developer.yahoo.com/api/&#39;</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s1">&#39;HTML&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="c1"># engine dependent config</span>
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;general&#39;</span><span class="p">,</span> <span class="s1">&#39;web&#39;</span><span class="p">]</span>
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
<span class="c1"># send_accept_language_header = True</span>
<span class="n">time_range_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;day&#39;</span><span class="p">:</span> <span class="s1">&#39;d&#39;</span><span class="p">,</span> <span class="s1">&#39;week&#39;</span><span class="p">:</span> <span class="s1">&#39;w&#39;</span><span class="p">,</span> <span class="s1">&#39;month&#39;</span><span class="p">:</span> <span class="s1">&#39;m&#39;</span><span class="p">}</span>
<span class="n">safesearch_dict</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">&#39;p&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">&#39;i&#39;</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">&#39;r&#39;</span><span class="p">}</span>
<span class="n">region2domain</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;CO&quot;</span><span class="p">:</span> <span class="s2">&quot;co.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Colombia</span>
<span class="s2">&quot;TH&quot;</span><span class="p">:</span> <span class="s2">&quot;th.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Thailand</span>
<span class="s2">&quot;VE&quot;</span><span class="p">:</span> <span class="s2">&quot;ve.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Venezuela</span>
<span class="s2">&quot;CL&quot;</span><span class="p">:</span> <span class="s2">&quot;cl.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Chile</span>
<span class="s2">&quot;HK&quot;</span><span class="p">:</span> <span class="s2">&quot;hk.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Hong Kong</span>
<span class="s2">&quot;PE&quot;</span><span class="p">:</span> <span class="s2">&quot;pe.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Peru</span>
<span class="s2">&quot;CA&quot;</span><span class="p">:</span> <span class="s2">&quot;ca.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Canada</span>
<span class="s2">&quot;DE&quot;</span><span class="p">:</span> <span class="s2">&quot;de.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Germany</span>
<span class="s2">&quot;FR&quot;</span><span class="p">:</span> <span class="s2">&quot;fr.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># France</span>
<span class="s2">&quot;TW&quot;</span><span class="p">:</span> <span class="s2">&quot;tw.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Taiwan</span>
<span class="s2">&quot;GB&quot;</span><span class="p">:</span> <span class="s2">&quot;uk.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># United Kingdom</span>
<span class="s2">&quot;UK&quot;</span><span class="p">:</span> <span class="s2">&quot;uk.search.yahoo.com&quot;</span><span class="p">,</span>
<span class="s2">&quot;BR&quot;</span><span class="p">:</span> <span class="s2">&quot;br.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Brazil</span>
<span class="s2">&quot;IN&quot;</span><span class="p">:</span> <span class="s2">&quot;in.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># India</span>
<span class="s2">&quot;ES&quot;</span><span class="p">:</span> <span class="s2">&quot;espanol.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Espanol</span>
<span class="s2">&quot;PH&quot;</span><span class="p">:</span> <span class="s2">&quot;ph.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Philippines</span>
<span class="s2">&quot;AR&quot;</span><span class="p">:</span> <span class="s2">&quot;ar.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Argentina</span>
<span class="s2">&quot;MX&quot;</span><span class="p">:</span> <span class="s2">&quot;mx.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Mexico</span>
<span class="s2">&quot;SG&quot;</span><span class="p">:</span> <span class="s2">&quot;sg.search.yahoo.com&quot;</span><span class="p">,</span> <span class="c1"># Singapore</span>
<span class="p">}</span>
<span class="sd">&quot;&quot;&quot;Map regions to domain&quot;&quot;&quot;</span>
<span class="n">lang2domain</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;zh_chs&#39;</span><span class="p">:</span> <span class="s1">&#39;hk.search.yahoo.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;zh_cht&#39;</span><span class="p">:</span> <span class="s1">&#39;tw.search.yahoo.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;any&#39;</span><span class="p">:</span> <span class="s1">&#39;search.yahoo.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;en&#39;</span><span class="p">:</span> <span class="s1">&#39;search.yahoo.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;bg&#39;</span><span class="p">:</span> <span class="s1">&#39;search.yahoo.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;cs&#39;</span><span class="p">:</span> <span class="s1">&#39;search.yahoo.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;da&#39;</span><span class="p">:</span> <span class="s1">&#39;search.yahoo.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;el&#39;</span><span class="p">:</span> <span class="s1">&#39;search.yahoo.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;et&#39;</span><span class="p">:</span> <span class="s1">&#39;search.yahoo.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;he&#39;</span><span class="p">:</span> <span class="s1">&#39;search.yahoo.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;hr&#39;</span><span class="p">:</span> <span class="s1">&#39;search.yahoo.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;ja&#39;</span><span class="p">:</span> <span class="s1">&#39;search.yahoo.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;ko&#39;</span><span class="p">:</span> <span class="s1">&#39;search.yahoo.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;sk&#39;</span><span class="p">:</span> <span class="s1">&#39;search.yahoo.com&#39;</span><span class="p">,</span>
<span class="s1">&#39;sl&#39;</span><span class="p">:</span> <span class="s1">&#39;search.yahoo.com&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="sd">&quot;&quot;&quot;Map language to domain&quot;&quot;&quot;</span>
<span class="n">yahoo_languages</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;all&quot;</span><span class="p">:</span> <span class="s2">&quot;any&quot;</span><span class="p">,</span>
<span class="s2">&quot;ar&quot;</span><span class="p">:</span> <span class="s2">&quot;ar&quot;</span><span class="p">,</span> <span class="c1"># Arabic</span>
<span class="s2">&quot;bg&quot;</span><span class="p">:</span> <span class="s2">&quot;bg&quot;</span><span class="p">,</span> <span class="c1"># Bulgarian</span>
<span class="s2">&quot;cs&quot;</span><span class="p">:</span> <span class="s2">&quot;cs&quot;</span><span class="p">,</span> <span class="c1"># Czech</span>
<span class="s2">&quot;da&quot;</span><span class="p">:</span> <span class="s2">&quot;da&quot;</span><span class="p">,</span> <span class="c1"># Danish</span>
<span class="s2">&quot;de&quot;</span><span class="p">:</span> <span class="s2">&quot;de&quot;</span><span class="p">,</span> <span class="c1"># German</span>
<span class="s2">&quot;el&quot;</span><span class="p">:</span> <span class="s2">&quot;el&quot;</span><span class="p">,</span> <span class="c1"># Greek</span>
<span class="s2">&quot;en&quot;</span><span class="p">:</span> <span class="s2">&quot;en&quot;</span><span class="p">,</span> <span class="c1"># English</span>
<span class="s2">&quot;es&quot;</span><span class="p">:</span> <span class="s2">&quot;es&quot;</span><span class="p">,</span> <span class="c1"># Spanish</span>
<span class="s2">&quot;et&quot;</span><span class="p">:</span> <span class="s2">&quot;et&quot;</span><span class="p">,</span> <span class="c1"># Estonian</span>
<span class="s2">&quot;fi&quot;</span><span class="p">:</span> <span class="s2">&quot;fi&quot;</span><span class="p">,</span> <span class="c1"># Finnish</span>
<span class="s2">&quot;fr&quot;</span><span class="p">:</span> <span class="s2">&quot;fr&quot;</span><span class="p">,</span> <span class="c1"># French</span>
<span class="s2">&quot;he&quot;</span><span class="p">:</span> <span class="s2">&quot;he&quot;</span><span class="p">,</span> <span class="c1"># Hebrew</span>
<span class="s2">&quot;hr&quot;</span><span class="p">:</span> <span class="s2">&quot;hr&quot;</span><span class="p">,</span> <span class="c1"># Croatian</span>
<span class="s2">&quot;hu&quot;</span><span class="p">:</span> <span class="s2">&quot;hu&quot;</span><span class="p">,</span> <span class="c1"># Hungarian</span>
<span class="s2">&quot;it&quot;</span><span class="p">:</span> <span class="s2">&quot;it&quot;</span><span class="p">,</span> <span class="c1"># Italian</span>
<span class="s2">&quot;ja&quot;</span><span class="p">:</span> <span class="s2">&quot;ja&quot;</span><span class="p">,</span> <span class="c1"># Japanese</span>
<span class="s2">&quot;ko&quot;</span><span class="p">:</span> <span class="s2">&quot;ko&quot;</span><span class="p">,</span> <span class="c1"># Korean</span>
<span class="s2">&quot;lt&quot;</span><span class="p">:</span> <span class="s2">&quot;lt&quot;</span><span class="p">,</span> <span class="c1"># Lithuanian</span>
<span class="s2">&quot;lv&quot;</span><span class="p">:</span> <span class="s2">&quot;lv&quot;</span><span class="p">,</span> <span class="c1"># Latvian</span>
<span class="s2">&quot;nl&quot;</span><span class="p">:</span> <span class="s2">&quot;nl&quot;</span><span class="p">,</span> <span class="c1"># Dutch</span>
<span class="s2">&quot;no&quot;</span><span class="p">:</span> <span class="s2">&quot;no&quot;</span><span class="p">,</span> <span class="c1"># Norwegian</span>
<span class="s2">&quot;pl&quot;</span><span class="p">:</span> <span class="s2">&quot;pl&quot;</span><span class="p">,</span> <span class="c1"># Polish</span>
<span class="s2">&quot;pt&quot;</span><span class="p">:</span> <span class="s2">&quot;pt&quot;</span><span class="p">,</span> <span class="c1"># Portuguese</span>
<span class="s2">&quot;ro&quot;</span><span class="p">:</span> <span class="s2">&quot;ro&quot;</span><span class="p">,</span> <span class="c1"># Romanian</span>
<span class="s2">&quot;ru&quot;</span><span class="p">:</span> <span class="s2">&quot;ru&quot;</span><span class="p">,</span> <span class="c1"># Russian</span>
<span class="s2">&quot;sk&quot;</span><span class="p">:</span> <span class="s2">&quot;sk&quot;</span><span class="p">,</span> <span class="c1"># Slovak</span>
<span class="s2">&quot;sl&quot;</span><span class="p">:</span> <span class="s2">&quot;sl&quot;</span><span class="p">,</span> <span class="c1"># Slovenian</span>
<span class="s2">&quot;sv&quot;</span><span class="p">:</span> <span class="s2">&quot;sv&quot;</span><span class="p">,</span> <span class="c1"># Swedish</span>
<span class="s2">&quot;th&quot;</span><span class="p">:</span> <span class="s2">&quot;th&quot;</span><span class="p">,</span> <span class="c1"># Thai</span>
<span class="s2">&quot;tr&quot;</span><span class="p">:</span> <span class="s2">&quot;tr&quot;</span><span class="p">,</span> <span class="c1"># Turkish</span>
<span class="s2">&quot;zh&quot;</span><span class="p">:</span> <span class="s2">&quot;zh_chs&quot;</span><span class="p">,</span> <span class="c1"># Chinese (Simplified)</span>
<span class="s2">&quot;zh_Hans&quot;</span><span class="p">:</span> <span class="s2">&quot;zh_chs&quot;</span><span class="p">,</span>
<span class="s1">&#39;zh-CN&#39;</span><span class="p">:</span> <span class="s2">&quot;zh_chs&quot;</span><span class="p">,</span>
<span class="s2">&quot;zh_Hant&quot;</span><span class="p">:</span> <span class="s2">&quot;zh_cht&quot;</span><span class="p">,</span> <span class="c1"># Chinese (Traditional)</span>
<span class="s2">&quot;zh-HK&quot;</span><span class="p">:</span> <span class="s2">&quot;zh_cht&quot;</span><span class="p">,</span>
<span class="s1">&#39;zh-TW&#39;</span><span class="p">:</span> <span class="s2">&quot;zh_cht&quot;</span><span class="p">,</span>
<span class="p">}</span>
<div class="viewcode-block" id="build_sb_cookie">
<a class="viewcode-back" href="../../../dev/engines/online/yahoo.html#searx.engines.yahoo.build_sb_cookie">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">build_sb_cookie</span><span class="p">(</span><span class="n">cookie_params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Build sB cookie parameter from provided parameters.</span>
<span class="sd"> :param cookie_params: Dictionary of cookie parameters</span>
<span class="sd"> :type cookie_params: dict</span>
<span class="sd"> :returns: Formatted cookie string</span>
<span class="sd"> :rtype: str</span>
<span class="sd"> Example:</span>
<span class="sd"> &gt;&gt;&gt; cookie_params = {&#39;v&#39;: &#39;1&#39;, &#39;vm&#39;: &#39;p&#39;, &#39;fl&#39;: &#39;1&#39;, &#39;vl&#39;: &#39;lang_fr&#39;}</span>
<span class="sd"> &gt;&gt;&gt; build_sb_cookie(cookie_params)</span>
<span class="sd"> &#39;v=1&amp;vm=p&amp;fl=1&amp;vl=lang_fr&#39;</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">cookie_parts</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">cookie_params</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">cookie_parts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">=</span><span class="si">{</span><span class="n">value</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="s2">&quot;&amp;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">cookie_parts</span><span class="p">)</span></div>
<div class="viewcode-block" id="request">
<a class="viewcode-back" href="../../../dev/engines/online/yahoo.html#searx.engines.yahoo.request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Build Yahoo search request.&quot;&quot;&quot;</span>
<span class="n">lang</span><span class="p">,</span> <span class="n">region</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;language&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;-&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="p">[</span><span class="kc">None</span><span class="p">])[:</span><span class="mi">2</span><span class="p">]</span>
<span class="n">lang</span> <span class="o">=</span> <span class="n">yahoo_languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">lang</span><span class="p">,</span> <span class="s2">&quot;any&quot;</span><span class="p">)</span>
<span class="c1"># Build URL parameters</span>
<span class="c1"># - p (str): Search query string</span>
<span class="c1"># - btf (str): Time filter, maps to values like &#39;d&#39; (day), &#39;w&#39; (week), &#39;m&#39; (month)</span>
<span class="c1"># - iscqry (str): Empty string, necessary for results to appear properly on first page</span>
<span class="c1"># - b (int): Search offset for pagination</span>
<span class="c1"># - pz (str): Amount of results expected for the page</span>
<span class="n">url_params</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;p&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">}</span>
<span class="n">btf</span> <span class="o">=</span> <span class="n">time_range_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;time_range&#39;</span><span class="p">])</span>
<span class="k">if</span> <span class="n">btf</span><span class="p">:</span>
<span class="n">url_params</span><span class="p">[</span><span class="s1">&#39;btf&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">btf</span>
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">url_params</span><span class="p">[</span><span class="s1">&#39;iscqry&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="k">elif</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">&gt;=</span> <span class="mi">2</span><span class="p">:</span>
<span class="n">url_params</span><span class="p">[</span><span class="s1">&#39;b&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">]</span> <span class="o">*</span> <span class="mi">7</span> <span class="o">+</span> <span class="mi">1</span> <span class="c1"># 8, 15, 21, etc.</span>
<span class="n">url_params</span><span class="p">[</span><span class="s1">&#39;pz&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">7</span>
<span class="n">url_params</span><span class="p">[</span><span class="s1">&#39;bct&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
<span class="n">url_params</span><span class="p">[</span><span class="s1">&#39;xargs&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
<span class="c1"># Build sB cookie (for filters)</span>
<span class="c1"># - vm (str): SafeSearch filter, maps to values like &#39;p&#39; (None), &#39;i&#39; (Moderate), &#39;r&#39; (Strict)</span>
<span class="c1"># - fl (bool): Indicates if a search language is used or not</span>
<span class="c1"># - vl (str): The search language to use (e.g. lang_fr)</span>
<span class="n">sbcookie_params</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;v&#39;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
<span class="s1">&#39;vm&#39;</span><span class="p">:</span> <span class="n">safesearch_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;safesearch&#39;</span><span class="p">]],</span>
<span class="s1">&#39;fl&#39;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
<span class="s1">&#39;vl&#39;</span><span class="p">:</span> <span class="sa">f</span><span class="s1">&#39;lang_</span><span class="si">{</span><span class="n">lang</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="s1">&#39;pn&#39;</span><span class="p">:</span> <span class="mi">10</span><span class="p">,</span>
<span class="s1">&#39;rw&#39;</span><span class="p">:</span> <span class="s1">&#39;new&#39;</span><span class="p">,</span>
<span class="s1">&#39;userset&#39;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;cookies&#39;</span><span class="p">][</span><span class="s1">&#39;sB&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">build_sb_cookie</span><span class="p">(</span><span class="n">sbcookie_params</span><span class="p">)</span>
<span class="c1"># Search region/language</span>
<span class="n">domain</span> <span class="o">=</span> <span class="n">region2domain</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">region</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">domain</span><span class="p">:</span>
<span class="n">domain</span> <span class="o">=</span> <span class="n">lang2domain</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">lang</span><span class="p">,</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">lang</span><span class="si">}</span><span class="s1">.search.yahoo.com&#39;</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;domain selected: </span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;cookies: </span><span class="si">{</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;cookies&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">&#39;https://</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s1">/search?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">url_params</span><span class="p">)</span><span class="si">}</span><span class="s1">&#39;</span>
<span class="n">params</span><span class="p">[</span><span class="s1">&#39;domain&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">domain</span></div>
<div class="viewcode-block" id="parse_url">
<a class="viewcode-back" href="../../../dev/engines/online/yahoo.html#searx.engines.yahoo.parse_url">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">parse_url</span><span class="p">(</span><span class="n">url_string</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;remove yahoo-specific tracking-url&quot;&quot;&quot;</span>
<span class="n">endings</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;/RS&#39;</span><span class="p">,</span> <span class="s1">&#39;/RK&#39;</span><span class="p">]</span>
<span class="n">endpositions</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">start</span> <span class="o">=</span> <span class="n">url_string</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;http&#39;</span><span class="p">,</span> <span class="n">url_string</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;/RU=&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
<span class="k">for</span> <span class="n">ending</span> <span class="ow">in</span> <span class="n">endings</span><span class="p">:</span>
<span class="n">endpos</span> <span class="o">=</span> <span class="n">url_string</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="n">ending</span><span class="p">)</span>
<span class="k">if</span> <span class="n">endpos</span> <span class="o">&gt;</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
<span class="n">endpositions</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">endpos</span><span class="p">)</span>
<span class="k">if</span> <span class="n">start</span> <span class="o">==</span> <span class="mi">0</span> <span class="ow">or</span> <span class="nb">len</span><span class="p">(</span><span class="n">endpositions</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">return</span> <span class="n">url_string</span>
<span class="n">end</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">endpositions</span><span class="p">)</span>
<span class="k">return</span> <span class="n">unquote</span><span class="p">(</span><span class="n">url_string</span><span class="p">[</span><span class="n">start</span><span class="p">:</span><span class="n">end</span><span class="p">])</span></div>
<div class="viewcode-block" id="response">
<a class="viewcode-back" href="../../../dev/engines/online/yahoo.html#searx.engines.yahoo.response">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;parse response&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="n">url_xpath</span> <span class="o">=</span> <span class="s1">&#39;.//div[contains(@class,&quot;compTitle&quot;)]/h3/a/@href&#39;</span>
<span class="n">title_xpath</span> <span class="o">=</span> <span class="s1">&#39;.//h3//a/@aria-label&#39;</span>
<span class="n">domain</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="p">[</span><span class="s1">&#39;domain&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">domain</span> <span class="o">==</span> <span class="s2">&quot;search.yahoo.com&quot;</span><span class="p">:</span>
<span class="n">url_xpath</span> <span class="o">=</span> <span class="s1">&#39;.//div[contains(@class,&quot;compTitle&quot;)]/a/@href&#39;</span>
<span class="n">title_xpath</span> <span class="o">=</span> <span class="s1">&#39;.//div[contains(@class,&quot;compTitle&quot;)]/a/h3/span&#39;</span>
<span class="c1"># parse results</span>
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">&#39;//div[contains(@class,&quot;algo-sr&quot;)]&#39;</span><span class="p">):</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">url_xpath</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">url</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">parse_url</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
<span class="n">title</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">title_xpath</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">title</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">title</span><span class="p">)</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">&#39;.//div[contains(@class, &quot;compText&quot;)]&#39;</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">content</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="c1"># append result</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="p">{</span>
<span class="s1">&#39;url&#39;</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
<span class="c1"># title sometimes contains HTML tags / see</span>
<span class="c1"># https://github.com/searxng/searxng/issues/3790</span>
<span class="s1">&#39;title&#39;</span><span class="p">:</span> <span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">title</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">()),</span>
<span class="s1">&#39;content&#39;</span><span class="p">:</span> <span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">content</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">()),</span>
<span class="p">}</span>
<span class="p">)</span>
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">&#39;//div[contains(@class, &quot;AlsoTry&quot;)]//table//a&#39;</span><span class="p">):</span>
<span class="c1"># append suggestion</span>
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">&#39;suggestion&#39;</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)})</span>
<span class="k">return</span> <span class="n">results</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,370 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.engines.zlibrary &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.engines.zlibrary</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.engines.zlibrary</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;`Z-Library`_ (abbreviated as z-lib, formerly BookFinder) is a shadow library</span>
<span class="sd">project for file-sharing access to scholarly journal articles, academic texts</span>
<span class="sd">and general-interest books. It began as a mirror of Library Genesis, from which</span>
<span class="sd">most of its books originate.</span>
<span class="sd">.. _Z-Library: https://zlibrary-global.se/</span>
<span class="sd">Configuration</span>
<span class="sd">=============</span>
<span class="sd">The engine has the following additional settings:</span>
<span class="sd">- :py:obj:`zlib_year_from`</span>
<span class="sd">- :py:obj:`zlib_year_to`</span>
<span class="sd">- :py:obj:`zlib_ext`</span>
<span class="sd">With this options a SearXNG maintainer is able to configure **additional**</span>
<span class="sd">engines for specific searches in Z-Library. For example a engine to search</span>
<span class="sd">only for EPUB from 2010 to 2020.</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> - name: z-library 2010s epub</span>
<span class="sd"> engine: zlibrary</span>
<span class="sd"> shortcut: zlib2010s</span>
<span class="sd"> zlib_year_from: &#39;2010&#39;</span>
<span class="sd"> zlib_year_to: &#39;2020&#39;</span>
<span class="sd"> zlib_ext: &#39;EPUB&#39;</span>
<span class="sd">Implementations</span>
<span class="sd">===============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">quote</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span> <span class="c1"># pyright: ignore[reportUnknownVariableType]</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">,</span> <span class="n">ElementType</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.data</span><span class="w"> </span><span class="kn">import</span> <span class="n">ENGINE_TRAITS</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxException</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
<span class="n">about</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;website&quot;</span><span class="p">:</span> <span class="s2">&quot;https://zlibrary-global.se&quot;</span><span class="p">,</span>
<span class="s2">&quot;wikidata_id&quot;</span><span class="p">:</span> <span class="s2">&quot;Q104863992&quot;</span><span class="p">,</span>
<span class="s2">&quot;official_api_documentation&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<span class="s2">&quot;use_official_api&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;require_api_key&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;results&quot;</span><span class="p">:</span> <span class="s2">&quot;HTML&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">categories</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;files&quot;</span><span class="p">,</span> <span class="s2">&quot;books&quot;</span><span class="p">]</span>
<span class="n">paging</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">base_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;https://zlibrary-global.se&quot;</span>
<span class="n">zlib_year_from</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Filter z-library&#39;s results by year from. E.g &#39;2010&#39;.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">zlib_year_to</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Filter z-library&#39;s results by year to. E.g. &#39;2010&#39;.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">zlib_ext</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Filter z-library&#39;s results by a file ending. Common filters for example are</span>
<span class="sd">``PDF`` and ``EPUB``.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">i18n_language</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Language&quot;</span><span class="p">)</span>
<span class="n">i18n_book_rating</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Book rating&quot;</span><span class="p">)</span>
<span class="n">i18n_file_quality</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;File quality&quot;</span><span class="p">)</span>
<div class="viewcode-block" id="setup">
<a class="viewcode-back" href="../../../dev/engines/online/zlibrary.html#searx.engines.zlibrary.setup">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Check of engine&#39;s settings.&quot;&quot;&quot;</span>
<span class="n">traits</span><span class="p">:</span> <span class="n">EngineTraits</span> <span class="o">=</span> <span class="n">EngineTraits</span><span class="p">(</span><span class="o">**</span><span class="n">ENGINE_TRAITS</span><span class="p">[</span><span class="s2">&quot;z-library&quot;</span><span class="p">])</span>
<span class="k">if</span> <span class="n">zlib_ext</span> <span class="ow">and</span> <span class="n">zlib_ext</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;ext&quot;</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;invalid setting ext: </span><span class="si">{</span><span class="n">zlib_ext</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">zlib_year_from</span> <span class="ow">and</span> <span class="n">zlib_year_from</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;year_from&quot;</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;invalid setting year_from: </span><span class="si">{</span><span class="n">zlib_year_from</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">zlib_year_to</span> <span class="ow">and</span> <span class="n">zlib_year_to</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;year_to&quot;</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;invalid setting year_to: </span><span class="si">{</span><span class="n">zlib_year_to</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">&quot;OnlineParams&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">lang</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;searxng_locale&quot;</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span>
<span class="n">search_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">base_url</span>
<span class="o">+</span> <span class="s2">&quot;/s/</span><span class="si">{search_query}</span><span class="s2">/?page=</span><span class="si">{pageno}</span><span class="s2">&quot;</span>
<span class="o">+</span> <span class="s2">&quot;&amp;yearFrom=</span><span class="si">{zlib_year_from}</span><span class="s2">&quot;</span>
<span class="o">+</span> <span class="s2">&quot;&amp;yearTo=</span><span class="si">{zlib_year_to}</span><span class="s2">&quot;</span>
<span class="o">+</span> <span class="s2">&quot;&amp;languages[]=</span><span class="si">{lang}</span><span class="s2">&quot;</span>
<span class="o">+</span> <span class="s2">&quot;&amp;extensions[]=</span><span class="si">{zlib_ext}</span><span class="s2">&quot;</span>
<span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">search_query</span><span class="o">=</span><span class="n">quote</span><span class="p">(</span><span class="n">query</span><span class="p">),</span>
<span class="n">pageno</span><span class="o">=</span><span class="n">params</span><span class="p">[</span><span class="s2">&quot;pageno&quot;</span><span class="p">],</span>
<span class="n">lang</span><span class="o">=</span><span class="n">lang</span><span class="p">,</span>
<span class="n">zlib_year_from</span><span class="o">=</span><span class="n">zlib_year_from</span><span class="p">,</span>
<span class="n">zlib_year_to</span><span class="o">=</span><span class="n">zlib_year_to</span><span class="p">,</span>
<span class="n">zlib_ext</span><span class="o">=</span><span class="n">zlib_ext</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">params</span><span class="p">[</span><span class="s2">&quot;verify&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">if</span> <span class="n">domain_is_seized</span><span class="p">(</span><span class="n">dom</span><span class="p">):</span>
<span class="k">raise</span> <span class="n">SearxException</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;zlibrary domain is seized: </span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;//div[@id=&quot;searchResultBox&quot;]//div[contains(@class, &quot;resItemBox&quot;)]&#39;</span><span class="p">):</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="n">_parse_result</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Paper</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">))</span>
<span class="k">return</span> <span class="n">res</span>
<span class="k">def</span><span class="w"> </span><span class="nf">domain_is_seized</span><span class="p">(</span><span class="n">dom</span><span class="p">:</span> <span class="n">ElementType</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;//title&#39;</span><span class="p">)</span> <span class="ow">and</span> <span class="s2">&quot;seized&quot;</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;//title&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_text</span><span class="p">(</span><span class="n">item</span><span class="p">:</span> <span class="n">ElementType</span><span class="p">,</span> <span class="n">selector</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="n">selector</span><span class="p">))</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_result</span><span class="p">(</span><span class="n">item</span><span class="p">:</span> <span class="n">ElementType</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]:</span>
<span class="n">author_elements</span> <span class="o">=</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;.//div[@class=&quot;authors&quot;]//a[@itemprop=&quot;author&quot;]&#39;</span><span class="p">)</span>
<span class="n">result</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;url&quot;</span><span class="p">:</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">item</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">&#39;(.//a[starts-with(@href, &quot;/book/&quot;)])[1]/@href&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">],</span>
<span class="s2">&quot;title&quot;</span><span class="p">:</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;.//*[@itemprop=&quot;name&quot;]&#39;</span><span class="p">),</span>
<span class="s2">&quot;authors&quot;</span><span class="p">:</span> <span class="p">[</span><span class="n">extract_text</span><span class="p">(</span><span class="n">author</span><span class="p">)</span> <span class="k">for</span> <span class="n">author</span> <span class="ow">in</span> <span class="n">author_elements</span><span class="p">],</span>
<span class="s2">&quot;publisher&quot;</span><span class="p">:</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;.//a[@title=&quot;Publisher&quot;]&#39;</span><span class="p">),</span>
<span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;.//div[contains(@class, &quot;property__file&quot;)]//div[contains(@class, &quot;property_value&quot;)]&#39;</span><span class="p">),</span>
<span class="p">}</span>
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;.//img[contains(@class, &quot;cover&quot;)]/@data-src&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">thumbnail</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">thumbnail</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">):</span>
<span class="n">result</span><span class="p">[</span><span class="s2">&quot;thumbnail&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">thumbnail</span>
<span class="n">year</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;.//div[contains(@class, &quot;property_year&quot;)]//div[contains(@class, &quot;property_value&quot;)]&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">year</span><span class="p">:</span>
<span class="n">result</span><span class="p">[</span><span class="s2">&quot;publishedDate&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="s1">&#39;%Y&#39;</span><span class="p">)</span>
<span class="n">content</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">language</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;.//div[contains(@class, &quot;property_language&quot;)]//div[contains(@class, &quot;property_value&quot;)]&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">language</span><span class="p">:</span>
<span class="n">content</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">i18n_language</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">language</span><span class="o">.</span><span class="n">capitalize</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">book_rating</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;.//span[contains(@class, &quot;book-rating-interest-score&quot;)]&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">book_rating</span> <span class="ow">and</span> <span class="nb">float</span><span class="p">(</span><span class="n">book_rating</span><span class="p">):</span>
<span class="n">content</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">i18n_book_rating</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">book_rating</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">file_quality</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">&#39;.//span[contains(@class, &quot;book-rating-quality-score&quot;)]&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">file_quality</span> <span class="ow">and</span> <span class="nb">float</span><span class="p">(</span><span class="n">file_quality</span><span class="p">):</span>
<span class="n">content</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">i18n_file_quality</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">file_quality</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">result</span><span class="p">[</span><span class="s2">&quot;content&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot; | &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
<span class="k">return</span> <span class="n">result</span>
<div class="viewcode-block" id="fetch_traits">
<a class="viewcode-back" href="../../../dev/engines/online/zlibrary.html#searx.engines.zlibrary.fetch_traits">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Fetch languages and other search arguments from zlibrary&#39;s search form.&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=import-outside-toplevel, too-many-branches, too-many-statements</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.core</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">httpx</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_use_old_values</span><span class="p">():</span>
<span class="c1"># don&#39;t change anything, re-use the existing values</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="n">ENGINE_TRAITS</span><span class="p">[</span><span class="s2">&quot;z-library&quot;</span><span class="p">][</span><span class="s2">&quot;all_locale&quot;</span><span class="p">]</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span> <span class="o">=</span> <span class="n">ENGINE_TRAITS</span><span class="p">[</span><span class="s2">&quot;z-library&quot;</span><span class="p">][</span><span class="s2">&quot;custom&quot;</span><span class="p">]</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span> <span class="o">=</span> <span class="n">ENGINE_TRAITS</span><span class="p">[</span><span class="s2">&quot;z-library&quot;</span><span class="p">][</span><span class="s2">&quot;languages&quot;</span><span class="p">]</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">base_url</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="k">except</span> <span class="p">(</span><span class="n">SearxException</span><span class="p">,</span> <span class="n">httpx</span><span class="o">.</span><span class="n">HTTPError</span><span class="p">)</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;ERROR: zlibrary domain &#39;</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">&#39; is seized?&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot; --&gt; </span><span class="si">{</span><span class="n">exc</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">_use_old_values</span><span class="p">()</span>
<span class="k">return</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;Response from zlibrary&#39;s search page is not OK.&quot;</span><span class="p">)</span>
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">if</span> <span class="n">domain_is_seized</span><span class="p">(</span><span class="n">dom</span><span class="p">):</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;ERROR: zlibrary domain is seized: </span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">_use_old_values</span><span class="p">()</span>
<span class="k">return</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;ext&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">l</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
<span class="c1"># years_from</span>
<span class="n">l</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">year</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//div[@id=&#39;advSearch-noJS&#39;]//select[@id=&#39;sf_yearFrom&#39;]/option&quot;</span><span class="p">):</span>
<span class="n">l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">year</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;year_from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">l</span>
<span class="c1"># years_to</span>
<span class="n">l</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">year</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//div[@id=&#39;advSearch-noJS&#39;]//select[@id=&#39;sf_yearTo&#39;]/option&quot;</span><span class="p">):</span>
<span class="n">l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">year</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;year_to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">l</span>
<span class="c1"># ext (file extensions)</span>
<span class="n">l</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">ext</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//div[@id=&#39;advSearch-noJS&#39;]//select[@id=&#39;sf_extensions&#39;]/option&quot;</span><span class="p">):</span>
<span class="n">l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ext</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">&quot;ext&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">l</span>
<span class="c1"># Handle languages</span>
<span class="c1"># Z-library uses English names for languages, so we need to map them to their respective locales</span>
<span class="n">language_name_locale_map</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">for</span> <span class="n">locale</span> <span class="ow">in</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">localedata</span><span class="o">.</span><span class="n">locale_identifiers</span><span class="p">():</span>
<span class="c1"># Create a Locale object for the current locale</span>
<span class="n">loc</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
<span class="k">if</span> <span class="n">loc</span><span class="o">.</span><span class="n">english_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">language_name_locale_map</span><span class="p">[</span><span class="n">loc</span><span class="o">.</span><span class="n">english_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">loc</span>
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">&quot;//div[@id=&#39;advSearch-noJS&#39;]//select[@id=&#39;sf_languages&#39;]/option&quot;</span><span class="p">):</span>
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">eng_lang</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">continue</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">language_name_locale_map</span><span class="p">[</span><span class="n">eng_lang</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span>
<span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
<span class="c1"># silently ignore unknown languages</span>
<span class="c1"># print(&quot;ERROR: %s is unknown by babel&quot; % (eng_lang))</span>
<span class="k">continue</span>
<span class="n">sxng_lang</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_lang</span><span class="p">)</span>
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_lang</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --&gt; </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_lang</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">))</span>
<span class="k">continue</span>
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_lang</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_lang</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../engines.html">searx.engines</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,257 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.exceptions &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.exceptions</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.exceptions</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Exception types raised by SearXNG modules.&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml.etree</span><span class="w"> </span><span class="kn">import</span> <span class="n">XPath</span>
<div class="viewcode-block" id="SearxException">
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxException">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SearxException</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Base SearXNG exception.&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="SearxParameterException">
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxParameterException">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SearxParameterException</span><span class="p">(</span><span class="n">SearxException</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Raised when query miss a required parameter&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">):</span>
<span class="k">if</span> <span class="n">value</span> <span class="o">==</span> <span class="s1">&#39;&#39;</span> <span class="ow">or</span> <span class="n">value</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">message</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;Empty </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2"> parameter&quot;</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">message</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;Invalid value </span><span class="si">{</span><span class="n">value</span><span class="si">}</span><span class="s2"> for parameter </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">message</span>
<span class="bp">self</span><span class="o">.</span><span class="n">parameter_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">name</span>
<span class="bp">self</span><span class="o">.</span><span class="n">parameter_value</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="n">value</span></div>
<div class="viewcode-block" id="SearxSettingsException">
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxSettingsException">[docs]</a>
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">SearxSettingsException</span><span class="p">(</span><span class="n">SearxException</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Error while loading the settings&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="ne">Exception</span><span class="p">,</span> <span class="n">filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">message</span> <span class="o">=</span> <span class="n">message</span>
<span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="o">=</span> <span class="n">filename</span></div>
<div class="viewcode-block" id="SearxEngineException">
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxEngineException">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SearxEngineException</span><span class="p">(</span><span class="n">SearxException</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Error inside an engine&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="SearxXPathSyntaxException">
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxXPathSyntaxException">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SearxXPathSyntaxException</span><span class="p">(</span><span class="n">SearxEngineException</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Syntax error in a XPATH&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">xpath_spec</span><span class="p">:</span> <span class="s2">&quot;str | XPath&quot;</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot; &quot;</span> <span class="o">+</span> <span class="n">message</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">message</span>
<span class="c1"># str(xpath_spec) to deal with str and XPath instance</span>
<span class="bp">self</span><span class="o">.</span><span class="n">xpath_str</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">)</span></div>
<div class="viewcode-block" id="SearxEngineResponseException">
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxEngineResponseException">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SearxEngineResponseException</span><span class="p">(</span><span class="n">SearxEngineException</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Impossible to parse the result of an engine&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="SearxEngineAPIException">
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxEngineAPIException">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SearxEngineAPIException</span><span class="p">(</span><span class="n">SearxEngineResponseException</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The website has returned an application error&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="SearxEngineAccessDeniedException">
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxEngineAccessDeniedException">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SearxEngineAccessDeniedException</span><span class="p">(</span><span class="n">SearxEngineResponseException</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The website is blocking the access&quot;&quot;&quot;</span>
<span class="n">SUSPEND_TIME_SETTING</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;search.suspended_times.SearxEngineAccessDenied&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;This settings contains the default suspended time (default 86400 sec / 1</span>
<span class="sd"> day).&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">suspended_time</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;Access denied&#39;</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Generic exception to raise when an engine denies access to the results.</span>
<span class="sd"> :param suspended_time: How long the engine is going to be suspended in</span>
<span class="sd"> second. Defaults to None.</span>
<span class="sd"> :type suspended_time: int, None</span>
<span class="sd"> :param message: Internal message. Defaults to ``Access denied``</span>
<span class="sd"> :type message: str</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">suspended_time</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">suspended_time</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_default_suspended_time</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> (suspended_time=</span><span class="si">{</span><span class="n">suspended_time</span><span class="si">}</span><span class="s2">)&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">suspended_time</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">suspended_time</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">message</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_get_default_suspended_time</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_setting</span> <span class="c1"># pylint: disable=C0415</span>
<span class="k">return</span> <span class="n">get_setting</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SUSPEND_TIME_SETTING</span><span class="p">)</span></div>
<div class="viewcode-block" id="SearxEngineCaptchaException">
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxEngineCaptchaException">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SearxEngineCaptchaException</span><span class="p">(</span><span class="n">SearxEngineAccessDeniedException</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The website has returned a CAPTCHA.&quot;&quot;&quot;</span>
<span class="n">SUSPEND_TIME_SETTING</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;search.suspended_times.SearxEngineCaptcha&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;This settings contains the default suspended time (default 86400 sec / 1</span>
<span class="sd"> day).&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">suspended_time</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;CAPTCHA&#39;</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span> <span class="n">suspended_time</span><span class="o">=</span><span class="n">suspended_time</span><span class="p">)</span></div>
<div class="viewcode-block" id="SearxEngineTooManyRequestsException">
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxEngineTooManyRequestsException">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SearxEngineTooManyRequestsException</span><span class="p">(</span><span class="n">SearxEngineAccessDeniedException</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The website has returned a Too Many Request status code</span>
<span class="sd"> By default, SearXNG stops sending requests to this engine for 1 hour.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">SUSPEND_TIME_SETTING</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;search.suspended_times.SearxEngineTooManyRequests&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;This settings contains the default suspended time (default 3660 sec / 1</span>
<span class="sd"> hour).&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">suspended_time</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;Too many request&#39;</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span> <span class="n">suspended_time</span><span class="o">=</span><span class="n">suspended_time</span><span class="p">)</span></div>
<div class="viewcode-block" id="SearxEngineXPathException">
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxEngineXPathException">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SearxEngineXPathException</span><span class="p">(</span><span class="n">SearxEngineResponseException</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Error while getting the result of an XPath expression&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">xpath_spec</span><span class="p">:</span> <span class="s2">&quot;str | XPath&quot;</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot; &quot;</span> <span class="o">+</span> <span class="n">message</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">message</span>
<span class="c1"># str(xpath_spec) to deal with str and XPath instance</span>
<span class="bp">self</span><span class="o">.</span><span class="n">xpath_str</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">)</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../index.html">
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../index.html">Overview</a>
<ul>
<li><a href="../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,196 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.extended_types &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.extended_types</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.extended_types</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;This module implements the type extensions applied by SearXNG.</span>
<span class="sd">- :py:obj:`flask.request` is replaced by :py:obj:`sxng_request`</span>
<span class="sd">- :py:obj:`flask.Request` is replaced by :py:obj:`SXNG_Request`</span>
<span class="sd">- :py:obj:`httpx.response` is replaced by :py:obj:`SXNG_Response`</span>
<span class="sd">----</span>
<span class="sd">.. py:attribute:: sxng_request</span>
<span class="sd"> :type: SXNG_Request</span>
<span class="sd"> A replacement for :py:obj:`flask.request` with type cast :py:obj:`SXNG_Request`.</span>
<span class="sd">.. autoclass:: SXNG_Request</span>
<span class="sd"> :members:</span>
<span class="sd">.. autoclass:: SXNG_Response</span>
<span class="sd"> :members:</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=invalid-name</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;SXNG_Request&quot;</span><span class="p">,</span> <span class="s2">&quot;sxng_request&quot;</span><span class="p">,</span> <span class="s2">&quot;SXNG_Response&quot;</span><span class="p">]</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">httpx</span>
<span class="k">if</span> <span class="n">typing</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">searx.preferences</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">searx.results</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParamTypes</span>
<div class="viewcode-block" id="SXNG_Request">
<a class="viewcode-back" href="../../dev/extended_types.html#searx.extended_types.SXNG_Request">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SXNG_Request</span><span class="p">(</span><span class="n">flask</span><span class="o">.</span><span class="n">Request</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;SearXNG extends the class :py:obj:`flask.Request` with properties from</span>
<span class="sd"> *this* class definition, see type cast :py:obj:`sxng_request`.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">user_plugins</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;list of searx.plugins.Plugin.id (the id of the plugins)&quot;&quot;&quot;</span>
<span class="n">preferences</span><span class="p">:</span> <span class="s2">&quot;searx.preferences.Preferences&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The preferences of the request.&quot;&quot;&quot;</span>
<span class="n">errors</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A list of errors (translated text) added by :py:obj:`searx.webapp` in</span>
<span class="sd"> case of errors.&quot;&quot;&quot;</span>
<span class="c1"># request.form is of type werkzeug.datastructures.ImmutableMultiDict</span>
<span class="c1"># form: dict[str, str]</span>
<span class="n">start_time</span><span class="p">:</span> <span class="nb">float</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Start time of the request, :py:obj:`timeit.default_timer` added by</span>
<span class="sd"> :py:obj:`searx.webapp` to calculate the total time of the request.&quot;&quot;&quot;</span>
<span class="n">render_time</span><span class="p">:</span> <span class="nb">float</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Duration of the rendering, calculated and added by</span>
<span class="sd"> :py:obj:`searx.webapp`.&quot;&quot;&quot;</span>
<span class="n">timings</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="s2">&quot;searx.results.Timing&quot;</span><span class="p">]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A list of :py:obj:`searx.results.Timing` of the engines, calculatid in</span>
<span class="sd"> and hold by :py:obj:`searx.results.ResultContainer.timings`.&quot;&quot;&quot;</span>
<span class="n">remote_addr</span><span class="p">:</span> <span class="nb">str</span></div>
<span class="c1">#: A replacement for :py:obj:`flask.request` with type cast :py:`SXNG_Request`.</span>
<span class="n">sxng_request</span> <span class="o">=</span> <span class="n">typing</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">flask</span><span class="o">.</span><span class="n">request</span><span class="p">)</span>
<div class="viewcode-block" id="SXNG_Response">
<a class="viewcode-back" href="../../dev/extended_types.html#searx.extended_types.SXNG_Response">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SXNG_Response</span><span class="p">(</span><span class="n">httpx</span><span class="o">.</span><span class="n">Response</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;SearXNG extends the class :py:obj:`httpx.Response` with properties from</span>
<span class="sd"> *this* class (type cast of :py:obj:`httpx.Response`).</span>
<span class="sd"> .. code:: python</span>
<span class="sd"> response = httpx.get(&quot;https://example.org&quot;)</span>
<span class="sd"> response = typing.cast(SXNG_Response, response)</span>
<span class="sd"> if response.ok:</span>
<span class="sd"> ...</span>
<span class="sd"> query_was = search_params[&quot;query&quot;]</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">ok</span><span class="p">:</span> <span class="nb">bool</span>
<span class="n">search_params</span><span class="p">:</span> <span class="s2">&quot;OnlineParamTypes&quot;</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../index.html">
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../index.html">Overview</a>
<ul>
<li><a href="../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,663 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.favicons.cache &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.favicons.cache</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.favicons.cache</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Implementations for caching favicons.</span>
<span class="sd">:py:obj:`FaviconCacheConfig`:</span>
<span class="sd"> Configuration of the favicon cache</span>
<span class="sd">:py:obj:`FaviconCache`:</span>
<span class="sd"> Abstract base class for the implementation of a favicon cache.</span>
<span class="sd">:py:obj:`FaviconCacheSQLite`:</span>
<span class="sd"> Favicon cache that manages the favicon BLOBs in a SQLite DB.</span>
<span class="sd">:py:obj:`FaviconCacheNull`:</span>
<span class="sd"> Fallback solution if the configured cache cannot be used for system reasons.</span>
<span class="sd">----</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">abc</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">dataclasses</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">hashlib</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">sqlite3</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">tempfile</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typer</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">msgspec</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">sqlitedb</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">humanize_bytes</span><span class="p">,</span> <span class="n">humanize_number</span>
<span class="n">CACHE</span><span class="p">:</span> <span class="s2">&quot;FaviconCache&quot;</span>
<span class="n">FALLBACK_ICON</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">&quot;FALLBACK_ICON&quot;</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">&#39;favicons.cache&#39;</span><span class="p">)</span>
<span class="n">app</span> <span class="o">=</span> <span class="n">typer</span><span class="o">.</span><span class="n">Typer</span><span class="p">()</span>
<div class="viewcode-block" id="state">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.state">[docs]</a>
<span class="nd">@app</span><span class="o">.</span><span class="n">command</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">():</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;show state of the cache&quot;&quot;&quot;</span>
<span class="nb">print</span><span class="p">(</span><span class="n">CACHE</span><span class="o">.</span><span class="n">state</span><span class="p">()</span><span class="o">.</span><span class="n">report</span><span class="p">())</span></div>
<div class="viewcode-block" id="maintenance">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.maintenance">[docs]</a>
<span class="nd">@app</span><span class="o">.</span><span class="n">command</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> <span class="n">debug</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;perform maintenance of the cache&quot;&quot;&quot;</span>
<span class="n">root_log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">()</span>
<span class="k">if</span> <span class="n">debug</span><span class="p">:</span>
<span class="n">root_log</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">DEBUG</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">root_log</span><span class="o">.</span><span class="n">handlers</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">handler</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">StreamHandler</span><span class="p">()</span>
<span class="n">handler</span><span class="o">.</span><span class="n">setFormatter</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">Formatter</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%(message)s</span><span class="s2">&quot;</span><span class="p">))</span>
<span class="n">logger</span><span class="o">.</span><span class="n">addHandler</span><span class="p">(</span><span class="n">handler</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">DEBUG</span><span class="p">)</span>
<span class="n">state_t0</span> <span class="o">=</span> <span class="n">CACHE</span><span class="o">.</span><span class="n">state</span><span class="p">()</span>
<span class="n">CACHE</span><span class="o">.</span><span class="n">maintenance</span><span class="p">(</span><span class="n">force</span><span class="o">=</span><span class="n">force</span><span class="p">)</span>
<span class="n">state_t1</span> <span class="o">=</span> <span class="n">CACHE</span><span class="o">.</span><span class="n">state</span><span class="p">()</span>
<span class="n">state_delta</span> <span class="o">=</span> <span class="n">state_t0</span> <span class="o">-</span> <span class="n">state_t1</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;The cache has been reduced by:&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">state_delta</span><span class="o">.</span><span class="n">report</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">- </span><span class="si">{descr}</span><span class="s2">: </span><span class="si">{val}</span><span class="s2">&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">))</span></div>
<div class="viewcode-block" id="init">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.init">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">cfg</span><span class="p">:</span> <span class="s2">&quot;FaviconCacheConfig&quot;</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Initialization of a global ``CACHE``&quot;&quot;&quot;</span>
<span class="k">global</span> <span class="n">CACHE</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="k">if</span> <span class="n">cfg</span><span class="o">.</span><span class="n">db_type</span> <span class="o">==</span> <span class="s2">&quot;sqlite&quot;</span><span class="p">:</span>
<span class="k">if</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">sqlite_version_info</span> <span class="o">&lt;=</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">35</span><span class="p">):</span>
<span class="n">logger</span><span class="o">.</span><span class="n">critical</span><span class="p">(</span>
<span class="s2">&quot;Disable favicon caching completely: SQLite library (</span><span class="si">%s</span><span class="s2">) is too old! (require &gt;= 3.35)&quot;</span><span class="p">,</span>
<span class="n">sqlite3</span><span class="o">.</span><span class="n">sqlite_version</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">CACHE</span> <span class="o">=</span> <span class="n">FaviconCacheNull</span><span class="p">(</span><span class="n">cfg</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">CACHE</span> <span class="o">=</span> <span class="n">FaviconCacheSQLite</span><span class="p">(</span><span class="n">cfg</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">cfg</span><span class="o">.</span><span class="n">db_type</span> <span class="o">==</span> <span class="s2">&quot;mem&quot;</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Favicons are cached in memory, don&#39;t use this in production!&quot;</span><span class="p">)</span>
<span class="n">CACHE</span> <span class="o">=</span> <span class="n">FaviconCacheMEM</span><span class="p">(</span><span class="n">cfg</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;favicons db_type &#39;</span><span class="si">{</span><span class="n">cfg</span><span class="o">.</span><span class="n">db_type</span><span class="si">}</span><span class="s2">&#39; is unknown&quot;</span><span class="p">)</span></div>
<div class="viewcode-block" id="FaviconCacheConfig">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheConfig">[docs]</a>
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconCacheConfig</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">):</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Configuration of the favicon cache.&quot;&quot;&quot;</span>
<span class="n">db_type</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;sqlite&quot;</span><span class="p">,</span> <span class="s2">&quot;mem&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;sqlite&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Type of the database:</span>
<span class="sd"> ``sqlite``:</span>
<span class="sd"> :py:obj:`.cache.FaviconCacheSQLite`</span>
<span class="sd"> ``mem``:</span>
<span class="sd"> :py:obj:`.cache.FaviconCacheMEM` (not recommended)</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">db_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">gettempdir</span><span class="p">()</span> <span class="o">+</span> <span class="n">os</span><span class="o">.</span><span class="n">sep</span> <span class="o">+</span> <span class="s2">&quot;faviconcache.db&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;URL of the SQLite DB, the path to the database file.&quot;&quot;&quot;</span>
<span class="n">HOLD_TIME</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">30</span> <span class="c1"># 30 days</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Hold time (default in sec.), after which a BLOB is removed from the cache.&quot;&quot;&quot;</span>
<span class="n">LIMIT_TOTAL_BYTES</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">50</span> <span class="c1"># 50 MB</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Maximum of bytes (default) stored in the cache of all blobs. Note: The</span>
<span class="sd"> limit is only reached at each maintenance interval after which the oldest</span>
<span class="sd"> BLOBs are deleted; the limit is exceeded during the maintenance period. If</span>
<span class="sd"> the maintenance period is *too long* or maintenance is switched off</span>
<span class="sd"> completely, the cache grows uncontrollably.&quot;&quot;&quot;</span>
<span class="n">BLOB_MAX_BYTES</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">20</span> <span class="c1"># 20 KB</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The maximum BLOB size in bytes that a favicon may have so that it can be</span>
<span class="sd"> saved in the cache. If the favicon is larger, it is not saved in the cache</span>
<span class="sd"> and must be requested by the client via the proxy.&quot;&quot;&quot;</span>
<span class="n">MAINTENANCE_PERIOD</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Maintenance period in seconds / when :py:obj:`MAINTENANCE_MODE` is set to</span>
<span class="sd"> ``auto``.&quot;&quot;&quot;</span>
<span class="n">MAINTENANCE_MODE</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;auto&quot;</span><span class="p">,</span> <span class="s2">&quot;off&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;auto&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Type of maintenance mode</span>
<span class="sd"> ``auto``:</span>
<span class="sd"> Maintenance is carried out automatically as part of the maintenance</span>
<span class="sd"> intervals (:py:obj:`MAINTENANCE_PERIOD`); no external process is required.</span>
<span class="sd"> ``off``:</span>
<span class="sd"> Maintenance is switched off and must be carried out by an external process</span>
<span class="sd"> if required.</span>
<span class="sd"> &quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="FaviconCacheStats">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheStats">[docs]</a>
<span class="nd">@dataclasses</span><span class="o">.</span><span class="n">dataclass</span>
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconCacheStats</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Dataclass which provides information on the status of the cache.&quot;&quot;&quot;</span>
<span class="n">favicons</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="nb">bytes</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">domains</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">resolvers</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">field_descr</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Callable</span><span class="p">[[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="nb">type</span><span class="p">],</span> <span class="o">...</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
<span class="p">(</span><span class="s2">&quot;favicons&quot;</span><span class="p">,</span> <span class="s2">&quot;number of favicons in cache&quot;</span><span class="p">,</span> <span class="n">humanize_number</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;bytes&quot;</span><span class="p">,</span> <span class="s2">&quot;total size (approx. bytes) of cache&quot;</span><span class="p">,</span> <span class="n">humanize_bytes</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;domains&quot;</span><span class="p">,</span> <span class="s2">&quot;total number of domains in cache&quot;</span><span class="p">,</span> <span class="n">humanize_number</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;resolvers&quot;</span><span class="p">,</span> <span class="s2">&quot;number of resolvers&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">),</span>
<span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__sub__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="s2">&quot;FaviconCacheStats&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;FaviconCacheStats&quot;</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;unsupported operand type(s) for +: &#39;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="si">}</span><span class="s2">&#39; and &#39;</span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span><span class="si">}</span><span class="s2">&#39;&quot;</span><span class="p">)</span>
<span class="n">kwargs</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">for</span> <span class="n">field</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">field_descr</span><span class="p">:</span>
<span class="n">self_val</span><span class="p">,</span> <span class="n">other_val</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field</span><span class="p">),</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">field</span><span class="p">)</span>
<span class="k">if</span> <span class="kc">None</span> <span class="ow">in</span> <span class="p">(</span><span class="n">self_val</span><span class="p">,</span> <span class="n">other_val</span><span class="p">):</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">self_val</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
<span class="n">kwargs</span><span class="p">[</span><span class="n">field</span><span class="p">]</span> <span class="o">=</span> <span class="n">self_val</span> <span class="o">-</span> <span class="n">other_val</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">kwargs</span><span class="p">[</span><span class="n">field</span><span class="p">]</span> <span class="o">=</span> <span class="n">self_val</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="k">def</span><span class="w"> </span><span class="nf">report</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fmt</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{descr}</span><span class="s2">: </span><span class="si">{val}</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">):</span>
<span class="n">s</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">field</span><span class="p">,</span> <span class="n">descr</span><span class="p">,</span> <span class="n">cast</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">field_descr</span><span class="p">:</span>
<span class="n">val</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field</span><span class="p">)</span>
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">val</span> <span class="o">=</span> <span class="s2">&quot;--&quot;</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">val</span> <span class="o">=</span> <span class="n">cast</span><span class="p">(</span><span class="n">val</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">s</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">fmt</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">descr</span><span class="o">=</span><span class="n">descr</span><span class="p">,</span> <span class="n">val</span><span class="o">=</span><span class="n">val</span><span class="p">))</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
<span class="k">return</span> <span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">s</span><span class="p">)</span></div>
<div class="viewcode-block" id="FaviconCache">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCache">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconCache</span><span class="p">(</span><span class="n">abc</span><span class="o">.</span><span class="n">ABC</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Abstract base class for the implementation of a favicon cache.&quot;&quot;&quot;</span>
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">FaviconCacheConfig</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;An instance of the favicon cache is build up from the configuration.&quot;&quot;&quot;</span>
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns ``None`` or the tuple of ``(data, mime)`` that has been</span>
<span class="sd"> registered in the cache. The ``None`` indicates that there was no entry</span>
<span class="sd"> in the cache.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="FaviconCache.set">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCache.set">[docs]</a>
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">mime</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Set data and mime-type in the cache. If data is None, the</span>
<span class="sd"> :py:obj:`FALLBACK_ICON` is registered. in the cache.&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="FaviconCache.state">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCache.state">[docs]</a>
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">FaviconCacheStats</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns a :py:obj:`FaviconCacheStats` (key/values) with information</span>
<span class="sd"> on the state of the cache.&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="FaviconCache.maintenance">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCache.maintenance">[docs]</a>
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Performs maintenance on the cache&quot;&quot;&quot;</span></div>
</div>
<div class="viewcode-block" id="FaviconCacheNull">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheNull">[docs]</a>
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconCacheNull</span><span class="p">(</span><span class="n">FaviconCache</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A dummy favicon cache that caches nothing / a fallback solution. The</span>
<span class="sd"> NullCache is used when more efficient caches such as the</span>
<span class="sd"> :py:obj:`FaviconCacheSQLite` cannot be used because, for example, the SQLite</span>
<span class="sd"> library is only available in an old version and does not meet the</span>
<span class="sd"> requirements.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">FaviconCacheConfig</span><span class="p">):</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
<span class="k">return</span> <span class="kc">None</span>
<div class="viewcode-block" id="FaviconCacheNull.set">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheNull.set">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">mime</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">False</span></div>
<div class="viewcode-block" id="FaviconCacheNull.state">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheNull.state">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">FaviconCacheStats</span><span class="p">(</span><span class="n">favicons</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span></div>
<div class="viewcode-block" id="FaviconCacheNull.maintenance">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheNull.maintenance">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
<span class="k">pass</span></div>
</div>
<div class="viewcode-block" id="FaviconCacheSQLite">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheSQLite">[docs]</a>
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconCacheSQLite</span><span class="p">(</span><span class="n">sqlitedb</span><span class="o">.</span><span class="n">SQLiteAppl</span><span class="p">,</span> <span class="n">FaviconCache</span><span class="p">):</span> <span class="c1"># pyright: ignore[reportUnsafeMultipleInheritance]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Favicon cache that manages the favicon BLOBs in a SQLite DB. The DB</span>
<span class="sd"> model in the SQLite DB is implemented using the abstract class</span>
<span class="sd"> :py:obj:`sqlitedb.SQLiteAppl`.</span>
<span class="sd"> For introspection of the DB, jump into developer environment and run command</span>
<span class="sd"> to show cache state::</span>
<span class="sd"> $ ./manage pyenv.cmd bash --norc --noprofile</span>
<span class="sd"> (py3) python -m searx.favicons cache state</span>
<span class="sd"> The following configurations are required / supported:</span>
<span class="sd"> - :py:obj:`FaviconCacheConfig.db_url`</span>
<span class="sd"> - :py:obj:`FaviconCacheConfig.HOLD_TIME`</span>
<span class="sd"> - :py:obj:`FaviconCacheConfig.LIMIT_TOTAL_BYTES`</span>
<span class="sd"> - :py:obj:`FaviconCacheConfig.BLOB_MAX_BYTES`</span>
<span class="sd"> - :py:obj:`MAINTENANCE_PERIOD`</span>
<span class="sd"> - :py:obj:`MAINTENANCE_MODE`</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">DB_SCHEMA</span> <span class="o">=</span> <span class="mi">1</span>
<span class="n">DDL_BLOBS</span> <span class="o">=</span> <span class="s2">&quot;&quot;&quot;</span><span class="se">\</span>
<span class="s2">CREATE TABLE IF NOT EXISTS blobs (</span>
<span class="s2"> sha256 TEXT,</span>
<span class="s2"> bytes_c INTEGER,</span>
<span class="s2"> mime TEXT NOT NULL,</span>
<span class="s2"> data BLOB NOT NULL,</span>
<span class="s2"> PRIMARY KEY (sha256))&quot;&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Table to store BLOB objects by their sha256 hash values.&quot;&quot;&quot;</span>
<span class="n">DDL_BLOB_MAP</span> <span class="o">=</span> <span class="s2">&quot;&quot;&quot;</span><span class="se">\</span>
<span class="s2">CREATE TABLE IF NOT EXISTS blob_map (</span>
<span class="s2"> m_time INTEGER DEFAULT (strftime(&#39;</span><span class="si">%s</span><span class="s2">&#39;, &#39;now&#39;)), -- last modified (unix epoch) time in sec.</span>
<span class="s2"> sha256 TEXT,</span>
<span class="s2"> resolver TEXT,</span>
<span class="s2"> authority TEXT,</span>
<span class="s2"> PRIMARY KEY (resolver, authority))&quot;&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Table to map from (resolver, authority) to sha256 hash values.&quot;&quot;&quot;</span>
<span class="n">DDL_CREATE_TABLES</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;blobs&quot;</span><span class="p">:</span> <span class="n">DDL_BLOBS</span><span class="p">,</span>
<span class="s2">&quot;blob_map&quot;</span><span class="p">:</span> <span class="n">DDL_BLOB_MAP</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">SQL_DROP_LEFTOVER_BLOBS</span> <span class="o">=</span> <span class="p">(</span>
<span class="s2">&quot;DELETE FROM blobs WHERE sha256 IN (&quot;</span>
<span class="s2">&quot; SELECT b.sha256&quot;</span>
<span class="s2">&quot; FROM blobs b&quot;</span>
<span class="s2">&quot; LEFT JOIN blob_map bm&quot;</span>
<span class="s2">&quot; ON b.sha256 = bm.sha256&quot;</span>
<span class="s2">&quot; WHERE bm.sha256 IS NULL)&quot;</span>
<span class="p">)</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Delete blobs.sha256 (BLOBs) no longer in blob_map.sha256.&quot;&quot;&quot;</span>
<span class="n">SQL_ITER_BLOBS_SHA256_BYTES_C</span> <span class="o">=</span> <span class="p">(</span>
<span class="s2">&quot;SELECT b.sha256, b.bytes_c FROM blobs b&quot;</span>
<span class="s2">&quot; JOIN blob_map bm &quot;</span>
<span class="s2">&quot; ON b.sha256 = bm.sha256&quot;</span>
<span class="s2">&quot; ORDER BY bm.m_time ASC&quot;</span>
<span class="p">)</span>
<span class="n">SQL_INSERT_BLOBS</span> <span class="o">=</span> <span class="p">(</span>
<span class="s2">&quot;INSERT INTO blobs (sha256, bytes_c, mime, data) VALUES (?, ?, ?, ?)&quot;</span>
<span class="s2">&quot; ON CONFLICT (sha256) DO NOTHING&quot;</span>
<span class="p">)</span> <span class="c1"># fmt: skip</span>
<span class="n">SQL_INSERT_BLOB_MAP</span> <span class="o">=</span> <span class="p">(</span>
<span class="s2">&quot;INSERT INTO blob_map (sha256, resolver, authority) VALUES (?, ?, ?)&quot;</span>
<span class="s2">&quot; ON CONFLICT DO UPDATE &quot;</span>
<span class="s2">&quot; SET sha256=excluded.sha256, m_time=strftime(&#39;</span><span class="si">%s</span><span class="s2">&#39;, &#39;now&#39;)&quot;</span>
<span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">FaviconCacheConfig</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;An instance of the favicon cache is build up from the configuration.&quot;&quot;&quot;</span> <span class="c1">#</span>
<span class="k">if</span> <span class="n">cfg</span><span class="o">.</span><span class="n">db_url</span> <span class="o">==</span> <span class="s2">&quot;:memory:&quot;</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">critical</span><span class="p">(</span><span class="s2">&quot;don&#39;t use SQLite DB in :memory: in production!!&quot;</span><span class="p">)</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">cfg</span><span class="o">.</span><span class="n">db_url</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">cfg</span> <span class="o">=</span> <span class="n">cfg</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
<span class="n">sql</span> <span class="o">=</span> <span class="s2">&quot;SELECT sha256 FROM blob_map WHERE resolver = ? AND authority = ?&quot;</span>
<span class="n">res</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">))</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
<span class="k">if</span> <span class="n">res</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="n">sha256</span> <span class="o">=</span> <span class="n">res</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">if</span> <span class="n">sha256</span> <span class="o">==</span> <span class="n">FALLBACK_ICON</span><span class="p">:</span>
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span>
<span class="n">sql</span> <span class="o">=</span> <span class="s2">&quot;SELECT data, mime FROM blobs WHERE sha256 = ?&quot;</span>
<span class="n">res</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="p">(</span><span class="n">sha256</span><span class="p">,))</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
<span class="k">if</span> <span class="n">res</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="n">res</span>
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span>
<div class="viewcode-block" id="FaviconCacheSQLite.set">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheSQLite.set">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">mime</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAINTENANCE_MODE</span> <span class="o">==</span> <span class="s2">&quot;auto&quot;</span> <span class="ow">and</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_maintenance_time</span><span class="p">:</span>
<span class="c1"># Should automatic maintenance be moved to a new thread?</span>
<span class="bp">self</span><span class="o">.</span><span class="n">maintenance</span><span class="p">()</span>
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">mime</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
<span class="s2">&quot;favicon resolver </span><span class="si">%s</span><span class="s2"> tries to cache mime-type None for authority </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span>
<span class="n">resolver</span><span class="p">,</span>
<span class="n">authority</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="n">bytes_c</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span> <span class="ow">or</span> <span class="sa">b</span><span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">bytes_c</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">BLOB_MAX_BYTES</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s2">&quot;favicon of resolver: </span><span class="si">%s</span><span class="s2"> / authority: </span><span class="si">%s</span><span class="s2"> to big to cache (bytes: </span><span class="si">%s</span><span class="s2">) &quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">,</span> <span class="n">bytes_c</span><span class="p">)</span>
<span class="p">)</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">sha256</span> <span class="o">=</span> <span class="n">FALLBACK_ICON</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">sha256</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
<span class="k">if</span> <span class="n">sha256</span> <span class="o">!=</span> <span class="n">FALLBACK_ICON</span><span class="p">:</span>
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SQL_INSERT_BLOBS</span><span class="p">,</span> <span class="p">(</span><span class="n">sha256</span><span class="p">,</span> <span class="n">bytes_c</span><span class="p">,</span> <span class="n">mime</span><span class="p">,</span> <span class="n">data</span><span class="p">))</span>
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SQL_INSERT_BLOB_MAP</span><span class="p">,</span> <span class="p">(</span><span class="n">sha256</span><span class="p">,</span> <span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">))</span>
<span class="c1"># hint: the with context of the connection object closes the transaction</span>
<span class="c1"># but not the DB connection. The connection has to be closed by the</span>
<span class="c1"># caller of self.connect()!</span>
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">return</span> <span class="kc">True</span></div>
<span class="nd">@property</span>
<span class="k">def</span><span class="w"> </span><span class="nf">next_maintenance_time</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns (unix epoch) time of the next maintenance.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAINTENANCE_PERIOD</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">m_time</span><span class="p">(</span><span class="s2">&quot;LAST_MAINTENANCE&quot;</span><span class="p">)</span>
<div class="viewcode-block" id="FaviconCacheSQLite.maintenance">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheSQLite.maintenance">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
<span class="c1"># Prevent parallel DB maintenance cycles from other DB connections</span>
<span class="c1"># (e.g. in multi thread or process environments).</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">force</span> <span class="ow">and</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_maintenance_time</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;no maintenance required yet, next maintenance interval is in the future&quot;</span><span class="p">)</span>
<span class="k">return</span>
<span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s2">&quot;LAST_MAINTENANCE&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span> <span class="c1"># hint: this (also) sets the m_time of the property!</span>
<span class="c1"># Do maintenance tasks. This can be take a little more time, to avoid</span>
<span class="c1"># DB locks, establish a new DB connection.</span>
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
<span class="c1"># drop items not in HOLD time</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;DELETE FROM blob_map&quot;</span>
<span class="sa">f</span><span class="s2">&quot; WHERE cast(m_time as integer) &lt; cast(strftime(&#39;%s&#39;, &#39;now&#39;) as integer) - </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">HOLD_TIME</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;dropped </span><span class="si">%s</span><span class="s2"> obsolete blob_map items from db&quot;</span><span class="p">,</span> <span class="n">res</span><span class="o">.</span><span class="n">rowcount</span><span class="p">)</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SQL_DROP_LEFTOVER_BLOBS</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;dropped </span><span class="si">%s</span><span class="s2"> obsolete BLOBS from db&quot;</span><span class="p">,</span> <span class="n">res</span><span class="o">.</span><span class="n">rowcount</span><span class="p">)</span>
<span class="c1"># drop old items to be in LIMIT_TOTAL_BYTES</span>
<span class="n">total_bytes</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">&quot;SELECT SUM(bytes_c) FROM blobs&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span> <span class="ow">or</span> <span class="mi">0</span>
<span class="k">if</span> <span class="n">total_bytes</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">LIMIT_TOTAL_BYTES</span><span class="p">:</span>
<span class="n">x</span> <span class="o">=</span> <span class="n">total_bytes</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">LIMIT_TOTAL_BYTES</span>
<span class="n">c</span> <span class="o">=</span> <span class="mi">0</span>
<span class="n">sha_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SQL_ITER_BLOBS_SHA256_BYTES_C</span><span class="p">):</span>
<span class="n">sha256</span><span class="p">,</span> <span class="n">bytes_c</span> <span class="o">=</span> <span class="n">row</span>
<span class="n">sha_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">sha256</span><span class="p">)</span>
<span class="n">c</span> <span class="o">+=</span> <span class="n">bytes_c</span>
<span class="k">if</span> <span class="n">c</span> <span class="o">&gt;</span> <span class="n">x</span><span class="p">:</span>
<span class="k">break</span>
<span class="k">if</span> <span class="n">sha_list</span><span class="p">:</span>
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">&quot;DELETE FROM blobs WHERE sha256 IN (&#39;</span><span class="si">%s</span><span class="s2">&#39;)&quot;</span> <span class="o">%</span> <span class="s2">&quot;&#39;,&#39;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">sha_list</span><span class="p">))</span>
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">&quot;DELETE FROM blob_map WHERE sha256 IN (&#39;</span><span class="si">%s</span><span class="s2">&#39;)&quot;</span> <span class="o">%</span> <span class="s2">&quot;&#39;,&#39;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">sha_list</span><span class="p">))</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;dropped </span><span class="si">%s</span><span class="s2"> blobs with total size of </span><span class="si">%s</span><span class="s2"> bytes&quot;</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">sha_list</span><span class="p">),</span> <span class="n">c</span><span class="p">)</span>
<span class="c1"># Vacuuming the WALs</span>
<span class="c1"># https://www.theunterminatedstring.com/sqlite-vacuuming/</span>
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">&quot;PRAGMA wal_checkpoint(TRUNCATE)&quot;</span><span class="p">)</span>
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">_query_val</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
<span class="n">val</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">val</span> <span class="o">=</span> <span class="n">val</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">val</span> <span class="o">=</span> <span class="n">default</span>
<span class="k">return</span> <span class="n">val</span>
<div class="viewcode-block" id="FaviconCacheSQLite.state">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheSQLite.state">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">FaviconCacheStats</span><span class="p">:</span>
<span class="k">return</span> <span class="n">FaviconCacheStats</span><span class="p">(</span>
<span class="n">favicons</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_query_val</span><span class="p">(</span><span class="s2">&quot;SELECT count(*) FROM blobs&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span>
<span class="nb">bytes</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_query_val</span><span class="p">(</span><span class="s2">&quot;SELECT SUM(bytes_c) FROM blobs&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span>
<span class="n">domains</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_query_val</span><span class="p">(</span><span class="s2">&quot;SELECT count(*) FROM (SELECT authority FROM blob_map GROUP BY authority)&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span>
<span class="n">resolvers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_query_val</span><span class="p">(</span><span class="s2">&quot;SELECT count(*) FROM (SELECT resolver FROM blob_map GROUP BY resolver)&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span>
<span class="p">)</span></div>
</div>
<div class="viewcode-block" id="FaviconCacheMEM">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheMEM">[docs]</a>
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconCacheMEM</span><span class="p">(</span><span class="n">FaviconCache</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Favicon cache in process&#39; memory. Its just a POC that stores the</span>
<span class="sd"> favicons in the memory of the process.</span>
<span class="sd"> .. attention::</span>
<span class="sd"> Don&#39;t use it in production, it will blow up your memory!!</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">FaviconCacheConfig</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">cfg</span> <span class="o">=</span> <span class="n">cfg</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_sha_mime</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">]:</span>
<span class="n">sha</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sha_mime</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">resolver</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">authority</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span>
<span class="k">if</span> <span class="n">sha</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sha</span><span class="p">)</span>
<span class="k">if</span> <span class="n">data</span> <span class="o">==</span> <span class="n">FALLBACK_ICON</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span>
<div class="viewcode-block" id="FaviconCacheMEM.set">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheMEM.set">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">mime</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">FALLBACK_ICON</span>
<span class="n">mime</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">elif</span> <span class="n">mime</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
<span class="s2">&quot;favicon resolver </span><span class="si">%s</span><span class="s2"> tries to cache mime-type None for authority </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span>
<span class="n">resolver</span><span class="p">,</span>
<span class="n">authority</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="n">digest</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_data</span><span class="p">[</span><span class="n">digest</span><span class="p">]</span> <span class="o">=</span> <span class="n">data</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_sha_mime</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">resolver</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">authority</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">digest</span><span class="p">,</span> <span class="n">mime</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span></div>
<div class="viewcode-block" id="FaviconCacheMEM.state">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheMEM.state">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">FaviconCacheStats</span><span class="p">(</span><span class="n">favicons</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_data</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span></div>
<div class="viewcode-block" id="FaviconCacheMEM.maintenance">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheMEM.maintenance">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
<span class="k">pass</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,174 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.favicons.config &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.favicons.config</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.favicons.config</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="c1"># pylint: disable=missing-module-docstring</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">pathlib</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">msgspec</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.cache</span><span class="w"> </span><span class="kn">import</span> <span class="n">FaviconCacheConfig</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.proxy</span><span class="w"> </span><span class="kn">import</span> <span class="n">FaviconProxyConfig</span>
<span class="n">CONFIG_SCHEMA</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span>
<span class="sd">&quot;&quot;&quot;Version of the configuration schema.&quot;&quot;&quot;</span>
<span class="n">TOML_CACHE_CFG</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="s2">&quot;FaviconConfig&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="sd">&quot;&quot;&quot;Cache config objects by TOML&#39;s filename.&quot;&quot;&quot;</span>
<span class="n">DEFAULT_CFG_TOML_PATH</span> <span class="o">=</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)</span><span class="o">.</span><span class="n">parent</span> <span class="o">/</span> <span class="s2">&quot;favicons.toml&quot;</span>
<div class="viewcode-block" id="FaviconConfig">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.config.FaviconConfig">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconConfig</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">):</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The class aggregates configurations of the favicon tools&quot;&quot;&quot;</span>
<span class="n">cfg_schema</span><span class="p">:</span> <span class="nb">int</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Config&#39;s schema version. The specification of the version of the schema</span>
<span class="sd"> is mandatory, currently only version :py:obj:`CONFIG_SCHEMA` is supported.</span>
<span class="sd"> By specifying a version, it is possible to ensure downward compatibility in</span>
<span class="sd"> the event of future changes to the configuration schema&quot;&quot;&quot;</span>
<span class="n">cache</span><span class="p">:</span> <span class="n">FaviconCacheConfig</span> <span class="o">=</span> <span class="n">msgspec</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="n">FaviconCacheConfig</span><span class="p">)</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Setup of the :py:obj:`.cache.FaviconCacheConfig`.&quot;&quot;&quot;</span>
<span class="n">proxy</span><span class="p">:</span> <span class="n">FaviconProxyConfig</span> <span class="o">=</span> <span class="n">msgspec</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="n">FaviconProxyConfig</span><span class="p">)</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Setup of the :py:obj:`.proxy.FaviconProxyConfig`.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="FaviconConfig.from_toml_file">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.config.FaviconConfig.from_toml_file">[docs]</a>
<span class="nd">@classmethod</span>
<span class="k">def</span><span class="w"> </span><span class="nf">from_toml_file</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">cfg_file</span><span class="p">:</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">,</span> <span class="n">use_cache</span><span class="p">:</span> <span class="nb">bool</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;FaviconConfig&quot;</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Create a config object from a TOML file, the ``use_cache`` argument</span>
<span class="sd"> specifies whether a cache should be used.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">cached</span> <span class="o">=</span> <span class="n">TOML_CACHE_CFG</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">cfg_file</span><span class="p">))</span>
<span class="k">if</span> <span class="n">use_cache</span> <span class="ow">and</span> <span class="n">cached</span><span class="p">:</span>
<span class="k">return</span> <span class="n">cached</span>
<span class="k">with</span> <span class="n">cfg_file</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="s2">&quot;rb&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
<span class="n">cfg</span> <span class="o">=</span> <span class="n">msgspec</span><span class="o">.</span><span class="n">toml</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="n">_FaviconConfig</span><span class="p">)</span>
<span class="n">schema</span> <span class="o">=</span> <span class="n">cfg</span><span class="o">.</span><span class="n">favicons</span><span class="o">.</span><span class="n">cfg_schema</span>
<span class="k">if</span> <span class="n">schema</span> <span class="o">!=</span> <span class="n">CONFIG_SCHEMA</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;config schema version </span><span class="si">{</span><span class="n">CONFIG_SCHEMA</span><span class="si">}</span><span class="s2"> is needed, version </span><span class="si">{</span><span class="n">schema</span><span class="si">}</span><span class="s2"> is given in </span><span class="si">{</span><span class="n">cfg_file</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="p">)</span>
<span class="n">cfg</span> <span class="o">=</span> <span class="n">cfg</span><span class="o">.</span><span class="n">favicons</span>
<span class="k">if</span> <span class="n">use_cache</span> <span class="ow">and</span> <span class="n">cached</span><span class="p">:</span>
<span class="n">TOML_CACHE_CFG</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">cfg_file</span><span class="o">.</span><span class="n">resolve</span><span class="p">())]</span> <span class="o">=</span> <span class="n">cfg</span>
<span class="k">return</span> <span class="n">cfg</span></div>
</div>
<span class="k">class</span><span class="w"> </span><span class="nc">_FaviconConfig</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">):</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
<span class="c1"># wrapper struct for root object &quot;favicons.&quot;</span>
<span class="n">favicons</span><span class="p">:</span> <span class="n">FaviconConfig</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,362 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.favicons.proxy &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.favicons.proxy</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.favicons.proxy</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Implementations for a favicon proxy&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Callable</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">importlib</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">base64</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">pathlib</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">urllib.parse</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">httpx</span><span class="w"> </span><span class="kn">import</span> <span class="n">HTTPError</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">msgspec</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_setting</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.webutils</span><span class="w"> </span><span class="kn">import</span> <span class="n">new_hmac</span><span class="p">,</span> <span class="n">is_hmac_of</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineResponseException</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">sxng_request</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.resolvers</span><span class="w"> </span><span class="kn">import</span> <span class="n">DEFAULT_RESOLVER_MAP</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">cache</span>
<span class="n">DEFAULT_FAVICON_URL</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">CFG</span><span class="p">:</span> <span class="s2">&quot;FaviconProxyConfig&quot;</span> <span class="o">=</span> <span class="kc">None</span> <span class="c1"># type: ignore</span>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">cfg</span><span class="p">:</span> <span class="s2">&quot;FaviconProxyConfig&quot;</span><span class="p">):</span>
<span class="k">global</span> <span class="n">CFG</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="n">CFG</span> <span class="o">=</span> <span class="n">cfg</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_initial_resolver_map</span><span class="p">():</span>
<span class="n">d</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">get_setting</span><span class="p">(</span><span class="s2">&quot;search.favicon_resolver&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="k">if</span> <span class="n">name</span><span class="p">:</span>
<span class="n">func</span> <span class="o">=</span> <span class="n">DEFAULT_RESOLVER_MAP</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
<span class="k">if</span> <span class="n">func</span><span class="p">:</span>
<span class="n">d</span> <span class="o">=</span> <span class="p">{</span><span class="n">name</span><span class="p">:</span> <span class="sa">f</span><span class="s2">&quot;searx.favicons.resolvers.</span><span class="si">{</span><span class="n">func</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">}</span>
<span class="k">return</span> <span class="n">d</span>
<div class="viewcode-block" id="FaviconProxyConfig">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.FaviconProxyConfig">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconProxyConfig</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Configuration of the favicon proxy.&quot;&quot;&quot;</span>
<span class="n">max_age</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span> <span class="c1"># seven days</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;HTTP header Cache-Control_ ``max-age``</span>
<span class="sd"> .. _Cache-Control: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">secret_key</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">get_setting</span><span class="p">(</span><span class="s2">&quot;server.secret_key&quot;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;By default, the value from :ref:`server.secret_key &lt;settings server&gt;`</span>
<span class="sd"> setting is used.&quot;&quot;&quot;</span>
<span class="n">resolver_timeout</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">get_setting</span><span class="p">(</span><span class="s2">&quot;outgoing.request_timeout&quot;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Timeout which the resolvers should not exceed, is usually passed to the</span>
<span class="sd"> outgoing request of the resolver. By default, the value from</span>
<span class="sd"> :ref:`outgoing.request_timeout &lt;settings outgoing&gt;` setting is used.&quot;&quot;&quot;</span>
<span class="n">resolver_map</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">msgspec</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="n">_initial_resolver_map</span><span class="p">)</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The resolver_map is a key / value dictionary where the key is the name of</span>
<span class="sd"> the resolver and the value is the fully qualifying name (fqn) of resolver&#39;s</span>
<span class="sd"> function (the callable). The resolvers from the python module</span>
<span class="sd"> :py:obj:`searx.favicons.resolver` are available by default.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="FaviconProxyConfig.get_resolver">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.FaviconProxyConfig.get_resolver">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_resolver</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Callable</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns the callable object (function) of the resolver with the</span>
<span class="sd"> ``name``. If no resolver is registered for the ``name``, ``None`` is</span>
<span class="sd"> returned.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">fqn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">resolver_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
<span class="k">if</span> <span class="n">fqn</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">mod_name</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">func_name</span> <span class="o">=</span> <span class="n">fqn</span><span class="o">.</span><span class="n">rpartition</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)</span>
<span class="n">mod</span> <span class="o">=</span> <span class="n">importlib</span><span class="o">.</span><span class="n">import_module</span><span class="p">(</span><span class="n">mod_name</span><span class="p">)</span>
<span class="n">func</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">mod</span><span class="p">,</span> <span class="n">func_name</span><span class="p">)</span>
<span class="k">if</span> <span class="n">func</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;resolver </span><span class="si">{</span><span class="n">fqn</span><span class="si">}</span><span class="s2"> is not implemented&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">func</span></div>
<span class="n">favicon_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">get_setting</span><span class="p">(</span><span class="s2">&quot;ui.static_path&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;/themes/</span><span class="si">{theme}</span><span class="s2">/img/empty_favicon.svg&quot;</span> <span class="c1"># type: ignore</span>
<span class="n">favicon_mime_type</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;image/svg+xml&quot;</span>
<div class="viewcode-block" id="FaviconProxyConfig.favicon">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.FaviconProxyConfig.favicon">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">favicon</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">replacements</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns pathname and mimetype of the default favicon.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="p">(</span>
<span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">favicon_path</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">replacements</span><span class="p">)),</span>
<span class="bp">self</span><span class="o">.</span><span class="n">favicon_mime_type</span><span class="p">,</span>
<span class="p">)</span></div>
<div class="viewcode-block" id="FaviconProxyConfig.favicon_data_url">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.FaviconProxyConfig.favicon_data_url">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">favicon_data_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">replacements</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns data image URL of the default favicon.&quot;&quot;&quot;</span>
<span class="n">cache_key</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">x</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">replacements</span><span class="p">[</span><span class="n">x</span><span class="p">]</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">replacements</span><span class="o">.</span><span class="n">keys</span><span class="p">()),</span> <span class="n">key</span><span class="o">=</span><span class="nb">str</span><span class="p">))</span>
<span class="n">data_url</span> <span class="o">=</span> <span class="n">DEFAULT_FAVICON_URL</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cache_key</span><span class="p">)</span>
<span class="k">if</span> <span class="n">data_url</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">data_url</span>
<span class="n">fav</span><span class="p">,</span> <span class="n">mimetype</span> <span class="o">=</span> <span class="n">CFG</span><span class="o">.</span><span class="n">favicon</span><span class="p">(</span><span class="o">**</span><span class="n">replacements</span><span class="p">)</span>
<span class="c1"># hint: encoding utf-8 limits favicons to be a SVG image</span>
<span class="k">with</span> <span class="n">fav</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="s2">&quot;r&quot;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">&quot;utf-8&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="n">data_url</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
<span class="n">data_url</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">quote</span><span class="p">(</span><span class="n">data_url</span><span class="p">)</span>
<span class="n">data_url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;data:</span><span class="si">{</span><span class="n">mimetype</span><span class="si">}</span><span class="s2">;utf8,</span><span class="si">{</span><span class="n">data_url</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">DEFAULT_FAVICON_URL</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">data_url</span>
<span class="k">return</span> <span class="n">data_url</span></div>
</div>
<div class="viewcode-block" id="favicon_proxy">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.favicon_proxy">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">favicon_proxy</span><span class="p">():</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;REST API of SearXNG&#39;s favicon proxy service</span>
<span class="sd"> ::</span>
<span class="sd"> /favicon_proxy?authority=&lt;...&gt;&amp;h=&lt;...&gt;</span>
<span class="sd"> ``authority``:</span>
<span class="sd"> Domain name :rfc:`3986` / see :py:obj:`favicon_url`</span>
<span class="sd"> ``h``:</span>
<span class="sd"> HMAC :rfc:`2104`, build up from the :ref:`server.secret_key &lt;settings</span>
<span class="sd"> server&gt;` setting.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">authority</span> <span class="o">=</span> <span class="n">sxng_request</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;authority&#39;</span><span class="p">)</span>
<span class="c1"># malformed request or RFC 3986 authority</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">authority</span> <span class="ow">or</span> <span class="s2">&quot;/&quot;</span> <span class="ow">in</span> <span class="n">authority</span><span class="p">:</span>
<span class="k">return</span> <span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="mi">400</span>
<span class="c1"># malformed request / does not have authorisation</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_hmac_of</span><span class="p">(</span>
<span class="n">CFG</span><span class="o">.</span><span class="n">secret_key</span><span class="p">,</span>
<span class="n">authority</span><span class="o">.</span><span class="n">encode</span><span class="p">(),</span>
<span class="n">sxng_request</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;h&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">),</span>
<span class="p">):</span>
<span class="k">return</span> <span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="mi">400</span>
<span class="n">resolver</span> <span class="o">=</span> <span class="n">sxng_request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s1">&#39;favicon_resolver&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="c1"># if resolver is empty or not valid, just return HTTP 400.</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resolver</span> <span class="ow">or</span> <span class="n">resolver</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">CFG</span><span class="o">.</span><span class="n">resolver_map</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
<span class="k">return</span> <span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="mi">400</span>
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="n">search_favicon</span><span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">)</span>
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">mime</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">flask</span><span class="o">.</span><span class="n">Response</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">mimetype</span><span class="o">=</span><span class="n">mime</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">resp</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">&#39;Cache-Control&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;max-age=</span><span class="si">{</span><span class="n">CFG</span><span class="o">.</span><span class="n">max_age</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">return</span> <span class="n">resp</span>
<span class="c1"># return default favicon from static path</span>
<span class="n">theme</span> <span class="o">=</span> <span class="n">sxng_request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s2">&quot;theme&quot;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="n">fav</span><span class="p">,</span> <span class="n">mimetype</span> <span class="o">=</span> <span class="n">CFG</span><span class="o">.</span><span class="n">favicon</span><span class="p">(</span><span class="n">theme</span><span class="o">=</span><span class="n">theme</span><span class="p">)</span>
<span class="k">return</span> <span class="n">flask</span><span class="o">.</span><span class="n">send_from_directory</span><span class="p">(</span><span class="n">fav</span><span class="o">.</span><span class="n">parent</span><span class="p">,</span> <span class="n">fav</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">mimetype</span><span class="o">=</span><span class="n">mimetype</span><span class="p">)</span></div>
<div class="viewcode-block" id="search_favicon">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.search_favicon">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">search_favicon</span><span class="p">(</span><span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Sends the request to the favicon resolver and returns a tuple for the</span>
<span class="sd"> favicon. The tuple consists of ``(data, mime)``, if the resolver has not</span>
<span class="sd"> determined a favicon, both values are ``None``.</span>
<span class="sd"> ``data``:</span>
<span class="sd"> Binary data of the favicon.</span>
<span class="sd"> ``mime``:</span>
<span class="sd"> Mime type of the favicon.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="n">func</span> <span class="o">=</span> <span class="n">CFG</span><span class="o">.</span><span class="n">get_resolver</span><span class="p">(</span><span class="n">resolver</span><span class="p">)</span>
<span class="k">if</span> <span class="n">func</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span>
<span class="c1"># to avoid superfluous requests to the resolver, first look in the cache</span>
<span class="n">data_mime</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">CACHE</span><span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">)</span>
<span class="k">if</span> <span class="n">data_mime</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">data_mime</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="n">func</span><span class="p">(</span><span class="n">authority</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">CFG</span><span class="o">.</span><span class="n">resolver_timeout</span><span class="p">)</span>
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">mime</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">except</span> <span class="p">(</span><span class="n">HTTPError</span><span class="p">,</span> <span class="n">SearxEngineResponseException</span><span class="p">):</span>
<span class="k">pass</span>
<span class="n">cache</span><span class="o">.</span><span class="n">CACHE</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">,</span> <span class="n">mime</span><span class="p">,</span> <span class="n">data</span><span class="p">)</span>
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div>
<div class="viewcode-block" id="favicon_url">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.favicon_url">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">favicon_url</span><span class="p">(</span><span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Function to generate the image URL used for favicons in SearXNG&#39;s result</span>
<span class="sd"> lists. The ``authority`` argument (aka netloc / :rfc:`3986`) is usually a</span>
<span class="sd"> (sub-) domain name. This function is used in the HTML (jinja) templates.</span>
<span class="sd"> .. code:: html</span>
<span class="sd"> &lt;div class=&quot;favicon&quot;&gt;</span>
<span class="sd"> &lt;img src=&quot;{{ favicon_url(result.parsed_url.netloc) }}&quot;&gt;</span>
<span class="sd"> &lt;/div&gt;</span>
<span class="sd"> The returned URL is a route to :py:obj:`favicon_proxy` REST API.</span>
<span class="sd"> If the favicon is already in the cache, the returned URL is a `data URL`_</span>
<span class="sd"> (something like ``data:image/png;base64,...``). By generating a data url from</span>
<span class="sd"> the :py:obj:`.cache.FaviconCache`, additional HTTP roundtripps via the</span>
<span class="sd"> :py:obj:`favicon_proxy` are saved. However, it must also be borne in mind</span>
<span class="sd"> that data urls are not cached in the client (web browser).</span>
<span class="sd"> .. _data URL: https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">resolver</span> <span class="o">=</span> <span class="n">sxng_request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s1">&#39;favicon_resolver&#39;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="c1"># if resolver is empty or not valid, just return nothing.</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">resolver</span> <span class="ow">or</span> <span class="n">resolver</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">CFG</span><span class="o">.</span><span class="n">resolver_map</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
<span class="k">return</span> <span class="s2">&quot;&quot;</span>
<span class="n">data_mime</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">CACHE</span><span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">)</span>
<span class="k">if</span> <span class="n">data_mime</span> <span class="o">==</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span>
<span class="c1"># we have already checked, the resolver does not have a favicon</span>
<span class="n">theme</span> <span class="o">=</span> <span class="n">sxng_request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s2">&quot;theme&quot;</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="k">return</span> <span class="n">CFG</span><span class="o">.</span><span class="n">favicon_data_url</span><span class="p">(</span><span class="n">theme</span><span class="o">=</span><span class="n">theme</span><span class="p">)</span>
<span class="k">if</span> <span class="n">data_mime</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="n">data_mime</span>
<span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;data:</span><span class="si">{</span><span class="n">mime</span><span class="si">}</span><span class="s2">;base64,</span><span class="si">{</span><span class="nb">str</span><span class="p">(</span><span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="n">data</span><span class="p">),</span><span class="w"> </span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span> <span class="c1"># type: ignore</span>
<span class="n">h</span> <span class="o">=</span> <span class="n">new_hmac</span><span class="p">(</span><span class="n">CFG</span><span class="o">.</span><span class="n">secret_key</span><span class="p">,</span> <span class="n">authority</span><span class="o">.</span><span class="n">encode</span><span class="p">())</span>
<span class="n">proxy_url</span> <span class="o">=</span> <span class="n">flask</span><span class="o">.</span><span class="n">url_for</span><span class="p">(</span><span class="s1">&#39;favicon_proxy&#39;</span><span class="p">)</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlencode</span><span class="p">({</span><span class="s2">&quot;authority&quot;</span><span class="p">:</span> <span class="n">authority</span><span class="p">,</span> <span class="s2">&quot;h&quot;</span><span class="p">:</span> <span class="n">h</span><span class="p">})</span>
<span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">proxy_url</span><span class="si">}</span><span class="s2">?</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s2">&quot;</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,215 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.favicons.resolvers &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.favicons.resolvers</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.favicons.resolvers</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Implementations of the favicon *resolvers* that are available in the favicon</span>
<span class="sd">proxy by default. A *resolver* is a function that obtains the favicon from an</span>
<span class="sd">external source. The *resolver* function receives two arguments (``domain,</span>
<span class="sd">timeout``) and returns a tuple ``(data, mime)``.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;DEFAULT_RESOLVER_MAP&quot;</span><span class="p">,</span> <span class="s2">&quot;allesedv&quot;</span><span class="p">,</span> <span class="s2">&quot;duckduckgo&quot;</span><span class="p">,</span> <span class="s2">&quot;google&quot;</span><span class="p">,</span> <span class="s2">&quot;yandex&quot;</span><span class="p">]</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Callable</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">network</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
<span class="n">DEFAULT_RESOLVER_MAP</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Callable</span><span class="p">]</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">&#39;favicons.resolvers&#39;</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_req_args</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="c1"># add the request arguments from the searx.network</span>
<span class="n">d</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;raise_for_httperror&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">}</span>
<span class="n">d</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>
<span class="k">return</span> <span class="n">d</span>
<div class="viewcode-block" id="allesedv">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.resolvers.allesedv">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">allesedv</span><span class="p">(</span><span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">timeout</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Favicon Resolver from allesedv.com / https://favicon.allesedv.com/&quot;&quot;&quot;</span>
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;https://f1.allesedv.com/32/</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;fetch favicon from: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
<span class="c1"># will just return a 200 regardless of the favicon existing or not</span>
<span class="c1"># sometimes will be correct size, sometimes not</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">_req_args</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">))</span>
<span class="k">if</span> <span class="n">response</span> <span class="ow">and</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
<span class="n">mime</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">&#39;Content-Type&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">mime</span> <span class="o">!=</span> <span class="s1">&#39;image/gif&#39;</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">content</span>
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div>
<div class="viewcode-block" id="duckduckgo">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.resolvers.duckduckgo">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">duckduckgo</span><span class="p">(</span><span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">timeout</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Favicon Resolver from duckduckgo.com / https://blog.jim-nielsen.com/2021/displaying-favicons-for-any-domain/&quot;&quot;&quot;</span>
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;https://icons.duckduckgo.com/ip2/</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2">.ico&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;fetch favicon from: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
<span class="c1"># will return a 404 if the favicon does not exist and a 200 if it does,</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">_req_args</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">))</span>
<span class="k">if</span> <span class="n">response</span> <span class="ow">and</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
<span class="c1"># api will respond with a 32x32 png image</span>
<span class="n">mime</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">&#39;Content-Type&#39;</span><span class="p">]</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">content</span>
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div>
<div class="viewcode-block" id="google">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.resolvers.google">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">google</span><span class="p">(</span><span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">timeout</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Favicon Resolver from google.com&quot;&quot;&quot;</span>
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="c1"># URL https://www.google.com/s2/favicons?sz=32&amp;domain={domain}&quot; will be</span>
<span class="c1"># redirected (HTTP 301 Moved Permanently) to t1.gstatic.com/faviconV2:</span>
<span class="n">url</span> <span class="o">=</span> <span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;https://t1.gstatic.com/faviconV2?client=SOCIAL&amp;type=FAVICON&amp;fallback_opts=TYPE,SIZE,URL&quot;</span>
<span class="sa">f</span><span class="s2">&quot;&amp;url=https://</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2">&amp;size=32&quot;</span>
<span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;fetch favicon from: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
<span class="c1"># will return a 404 if the favicon does not exist and a 200 if it does,</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">_req_args</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">))</span>
<span class="k">if</span> <span class="n">response</span> <span class="ow">and</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
<span class="c1"># api will respond with a 32x32 png image</span>
<span class="n">mime</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">&#39;Content-Type&#39;</span><span class="p">]</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">content</span>
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div>
<div class="viewcode-block" id="yandex">
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.resolvers.yandex">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">yandex</span><span class="p">(</span><span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">timeout</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Favicon Resolver from yandex.com&quot;&quot;&quot;</span>
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;https://favicon.yandex.net/favicon/</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;fetch favicon from: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
<span class="c1"># api will respond with a 16x16 png image, if it doesn&#39;t exist, it will be a</span>
<span class="c1"># 1x1 png image (70 bytes)</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">_req_args</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">))</span>
<span class="k">if</span> <span class="n">response</span> <span class="ow">and</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">70</span><span class="p">:</span>
<span class="n">mime</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">&#39;Content-Type&#39;</span><span class="p">]</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">content</span>
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div>
<span class="n">DEFAULT_RESOLVER_MAP</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;allesedv&quot;</span><span class="p">:</span> <span class="n">allesedv</span><span class="p">,</span>
<span class="s2">&quot;duckduckgo&quot;</span><span class="p">:</span> <span class="n">duckduckgo</span><span class="p">,</span>
<span class="s2">&quot;google&quot;</span><span class="p">:</span> <span class="n">google</span><span class="p">,</span>
<span class="s2">&quot;yandex&quot;</span><span class="p">:</span> <span class="n">yandex</span><span class="p">,</span>
<span class="p">}</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,326 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.infopage &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.infopage</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.infopage</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Render SearXNG instance documentation.</span>
<span class="sd">Usage in a Flask app route:</span>
<span class="sd">.. code:: python</span>
<span class="sd"> from searx import infopage</span>
<span class="sd"> from searx.extended_types import sxng_request</span>
<span class="sd"> _INFO_PAGES = infopage.InfoPageSet(infopage.MistletoePage)</span>
<span class="sd"> @app.route(&#39;/info/&lt;pagename&gt;&#39;, methods=[&#39;GET&#39;])</span>
<span class="sd"> def info(pagename):</span>
<span class="sd"> locale = sxng_request.preferences.get_value(&#39;locale&#39;)</span>
<span class="sd"> page = _INFO_PAGES.get_page(pagename, locale)</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;InfoPage&#39;</span><span class="p">,</span> <span class="s1">&#39;InfoPageSet&#39;</span><span class="p">]</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">os.path</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">urllib.parse</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">functools</span><span class="w"> </span><span class="kn">import</span> <span class="n">cached_property</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">jinja2</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask.helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">url_for</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">markdown_it</span><span class="w"> </span><span class="kn">import</span> <span class="n">MarkdownIt</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">..</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_setting</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">..version</span><span class="w"> </span><span class="kn">import</span> <span class="n">GIT_URL</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">..locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">LOCALE_NAMES</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s1">&#39;searx.infopage&#39;</span><span class="p">)</span>
<span class="n">_INFO_FOLDER</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="vm">__file__</span><span class="p">))</span>
<span class="n">INFO_PAGES</span><span class="p">:</span> <span class="s1">&#39;InfoPageSet&#39;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__getattr__</span><span class="p">(</span><span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="k">if</span> <span class="n">name</span> <span class="o">==</span> <span class="s1">&#39;INFO_PAGES&#39;</span><span class="p">:</span>
<span class="k">global</span> <span class="n">INFO_PAGES</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="n">INFO_PAGES</span> <span class="o">=</span> <span class="n">InfoPageSet</span><span class="p">()</span>
<span class="k">return</span> <span class="n">INFO_PAGES</span>
<span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;module </span><span class="si">{</span><span class="vm">__name__</span><span class="si">!r}</span><span class="s2"> has no attribute </span><span class="si">{</span><span class="n">name</span><span class="si">!r}</span><span class="s2">&quot;</span><span class="p">)</span>
<div class="viewcode-block" id="InfoPage">
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPage">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">InfoPage</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A page of the :py:obj:`online documentation &lt;InfoPageSet&gt;`.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fname</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">fname</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">fname</span>
<div class="viewcode-block" id="InfoPage.raw_content">
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPage.raw_content">[docs]</a>
<span class="nd">@cached_property</span>
<span class="k">def</span><span class="w"> </span><span class="nf">raw_content</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Raw content of the page (without any jinja rendering)&quot;&quot;&quot;</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fname</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="k">return</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span></div>
<div class="viewcode-block" id="InfoPage.content">
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPage.content">[docs]</a>
<span class="nd">@cached_property</span>
<span class="k">def</span><span class="w"> </span><span class="nf">content</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Content of the page (rendered in a Jinja context)&quot;&quot;&quot;</span>
<span class="n">ctx</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_ctx</span><span class="p">()</span>
<span class="n">template</span> <span class="o">=</span> <span class="n">jinja2</span><span class="o">.</span><span class="n">Environment</span><span class="p">()</span><span class="o">.</span><span class="n">from_string</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">raw_content</span><span class="p">)</span>
<span class="k">return</span> <span class="n">template</span><span class="o">.</span><span class="n">render</span><span class="p">(</span><span class="o">**</span><span class="n">ctx</span><span class="p">)</span></div>
<div class="viewcode-block" id="InfoPage.title">
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPage.title">[docs]</a>
<span class="nd">@cached_property</span>
<span class="k">def</span><span class="w"> </span><span class="nf">title</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Title of the content (without any markup)&quot;&quot;&quot;</span>
<span class="n">_t</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">raw_content</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">):</span>
<span class="k">if</span> <span class="n">l</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">&#39;# &#39;</span><span class="p">):</span>
<span class="n">_t</span> <span class="o">=</span> <span class="n">l</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;# &#39;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">_t</span></div>
<div class="viewcode-block" id="InfoPage.html">
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPage.html">[docs]</a>
<span class="nd">@cached_property</span>
<span class="k">def</span><span class="w"> </span><span class="nf">html</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Render Markdown (CommonMark_) to HTML by using markdown-it-py_.</span>
<span class="sd"> .. _CommonMark: https://commonmark.org/</span>
<span class="sd"> .. _markdown-it-py: https://github.com/executablebooks/markdown-it-py</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="p">(</span>
<span class="n">MarkdownIt</span><span class="p">(</span><span class="s2">&quot;commonmark&quot;</span><span class="p">,</span> <span class="p">{</span><span class="s2">&quot;typographer&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">})</span><span class="o">.</span><span class="n">enable</span><span class="p">([</span><span class="s2">&quot;replacements&quot;</span><span class="p">,</span> <span class="s2">&quot;smartquotes&quot;</span><span class="p">])</span><span class="o">.</span><span class="n">render</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
<span class="p">)</span></div>
<div class="viewcode-block" id="InfoPage.get_ctx">
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPage.get_ctx">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_ctx</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Jinja context to render :py:obj:`InfoPage.content`&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_md_link</span><span class="p">(</span><span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">url</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="n">url</span> <span class="o">=</span> <span class="n">url_for</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">_external</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">return</span> <span class="s2">&quot;[</span><span class="si">%s</span><span class="s2">](</span><span class="si">%s</span><span class="s2">)&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_md_search</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
<span class="n">url</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="si">%s</span><span class="s1">?q=</span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">url_for</span><span class="p">(</span><span class="s1">&#39;search&#39;</span><span class="p">,</span> <span class="n">_external</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">quote</span><span class="p">(</span><span class="n">query</span><span class="p">))</span>
<span class="k">return</span> <span class="s1">&#39;[</span><span class="si">%s</span><span class="s1">](</span><span class="si">%s</span><span class="s1">)&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
<span class="n">ctx</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">ctx</span><span class="p">[</span><span class="s1">&#39;GIT_URL&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">GIT_URL</span>
<span class="n">ctx</span><span class="p">[</span><span class="s1">&#39;get_setting&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_setting</span>
<span class="n">ctx</span><span class="p">[</span><span class="s1">&#39;link&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">_md_link</span>
<span class="n">ctx</span><span class="p">[</span><span class="s1">&#39;search&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">_md_search</span>
<span class="k">return</span> <span class="n">ctx</span></div>
<span class="k">def</span><span class="w"> </span><span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="sa">f</span><span class="s1">&#39;&lt;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s1"> fname=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">fname</span><span class="si">!r}</span><span class="s1">&gt;&#39;</span></div>
<div class="viewcode-block" id="InfoPageSet">
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPageSet">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">InfoPageSet</span><span class="p">:</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Cached rendering of the online documentation a SearXNG instance has.</span>
<span class="sd"> :param page_class: render online documentation by :py:obj:`InfoPage` parser.</span>
<span class="sd"> :type page_class: :py:obj:`InfoPage`</span>
<span class="sd"> :param info_folder: information directory</span>
<span class="sd"> :type info_folder: str</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">page_class</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">InfoPage</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">info_folder</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">page_class</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">InfoPage</span><span class="p">]</span> <span class="o">=</span> <span class="n">page_class</span> <span class="ow">or</span> <span class="n">InfoPage</span>
<span class="bp">self</span><span class="o">.</span><span class="n">folder</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">info_folder</span> <span class="ow">or</span> <span class="n">_INFO_FOLDER</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;location of the Markdown files&quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="n">InfoPage</span> <span class="o">|</span> <span class="kc">None</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;en&#39;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;default language&quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">locales</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
<span class="n">locale</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">,</span> <span class="s1">&#39;-&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">locale</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">_INFO_FOLDER</span><span class="p">)</span> <span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">,</span> <span class="s1">&#39;-&#39;</span><span class="p">)</span> <span class="ow">in</span> <span class="n">LOCALE_NAMES</span>
<span class="p">]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;list of supported languages (aka locales)&quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">toc</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
<span class="s1">&#39;search-syntax&#39;</span><span class="p">,</span>
<span class="s1">&#39;about&#39;</span><span class="p">,</span>
<span class="s1">&#39;donate&#39;</span><span class="p">,</span>
<span class="p">]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;list of articles in the online documentation&quot;&quot;&quot;</span>
<div class="viewcode-block" id="InfoPageSet.get_page">
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPageSet.get_page">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_page</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pagename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">locale</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Return ``pagename`` instance of :py:obj:`InfoPage`</span>
<span class="sd"> :param pagename: name of the page, a value from :py:obj:`InfoPageSet.toc`</span>
<span class="sd"> :type pagename: str</span>
<span class="sd"> :param locale: language of the page, e.g. ``en``, ``zh_Hans_CN``</span>
<span class="sd"> (default: :py:obj:`InfoPageSet.i18n_origin`)</span>
<span class="sd"> :type locale: str</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">locale</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span>
<span class="k">if</span> <span class="n">pagename</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">toc</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">locale</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">locales</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">cache_key</span> <span class="o">=</span> <span class="p">(</span><span class="n">pagename</span><span class="p">,</span> <span class="n">locale</span><span class="p">)</span>
<span class="k">if</span> <span class="n">cache_key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span>
<span class="c1"># not yet instantiated</span>
<span class="n">fname</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">folder</span><span class="p">,</span> <span class="n">locale</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">,</span> <span class="s1">&#39;_&#39;</span><span class="p">),</span> <span class="n">pagename</span><span class="p">)</span> <span class="o">+</span> <span class="s1">&#39;.md&#39;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">fname</span><span class="p">):</span>
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;file </span><span class="si">%s</span><span class="s1"> does not exists&#39;</span><span class="p">,</span> <span class="n">fname</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">page</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">page_class</span><span class="p">(</span><span class="n">fname</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">page</span>
<span class="k">return</span> <span class="n">page</span></div>
<div class="viewcode-block" id="InfoPageSet.iter_pages">
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPageSet.iter_pages">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">iter_pages</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">locale</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">fallback_to_default</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Iterate over all pages of the TOC&quot;&quot;&quot;</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">locale</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span>
<span class="k">for</span> <span class="n">page_name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">toc</span><span class="p">:</span>
<span class="n">page_locale</span> <span class="o">=</span> <span class="n">locale</span>
<span class="n">page</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_page</span><span class="p">(</span><span class="n">page_name</span><span class="p">,</span> <span class="n">locale</span><span class="p">)</span>
<span class="k">if</span> <span class="n">fallback_to_default</span> <span class="ow">and</span> <span class="n">page</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">page_locale</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span>
<span class="n">page</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_page</span><span class="p">(</span><span class="n">page_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span><span class="p">)</span>
<span class="k">if</span> <span class="n">page</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="c1"># page is None if the page was deleted by the administrator</span>
<span class="k">yield</span> <span class="n">page_name</span><span class="p">,</span> <span class="n">page_locale</span><span class="p">,</span> <span class="n">page</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../index.html">
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../index.html">Overview</a>
<ul>
<li><a href="../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

367
_modules/searx/limiter.html Normal file
View File

@@ -0,0 +1,367 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.limiter &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.limiter</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.limiter</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Bot protection / IP rate limitation. The intention of rate limitation is to</span>
<span class="sd">limit suspicious requests from an IP. The motivation behind this is the fact</span>
<span class="sd">that SearXNG passes through requests from bots and is thus classified as a bot</span>
<span class="sd">itself. As a result, the SearXNG engine then receives a CAPTCHA or is blocked</span>
<span class="sd">by the search engine (the origin) in some other way.</span>
<span class="sd">To avoid blocking, the requests from bots to SearXNG must also be blocked, this</span>
<span class="sd">is the task of the limiter. To perform this task, the limiter uses the methods</span>
<span class="sd">from the :ref:`botdetection`:</span>
<span class="sd">- Analysis of the HTTP header in the request / :ref:`botdetection probe headers`</span>
<span class="sd"> can be easily bypassed.</span>
<span class="sd">- Block and pass lists in which IPs are listed / :ref:`botdetection ip_lists`</span>
<span class="sd"> are hard to maintain, since the IPs of bots are not all known and change over</span>
<span class="sd"> the time.</span>
<span class="sd">- Detection &amp; dynamically :ref:`botdetection rate limit` of bots based on the</span>
<span class="sd"> behavior of the requests. For dynamically changeable IP lists a Valkey</span>
<span class="sd"> database is needed.</span>
<span class="sd">The prerequisite for IP based methods is the correct determination of the IP of</span>
<span class="sd">the client. The IP of the client is determined via the X-Forwarded-For_ HTTP</span>
<span class="sd">header.</span>
<span class="sd">.. attention::</span>
<span class="sd"> A correct setup of the HTTP request headers ``X-Forwarded-For`` and</span>
<span class="sd"> ``X-Real-IP`` is essential to be able to assign a request to an IP correctly:</span>
<span class="sd"> - `NGINX RequestHeader`_</span>
<span class="sd"> - `Apache RequestHeader`_</span>
<span class="sd">.. _X-Forwarded-For:</span>
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For</span>
<span class="sd">.. _NGINX RequestHeader:</span>
<span class="sd"> https://docs.searxng.org/admin/installation-nginx.html#nginx-s-searxng-site</span>
<span class="sd">.. _Apache RequestHeader:</span>
<span class="sd"> https://docs.searxng.org/admin/installation-apache.html#apache-s-searxng-site</span>
<span class="sd">Enable Limiter</span>
<span class="sd">==============</span>
<span class="sd">To enable the limiter activate:</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> server:</span>
<span class="sd"> ...</span>
<span class="sd"> limiter: true # rate limit the number of request on the instance, block some bots</span>
<span class="sd">and set the valkey-url connection. Check the value, it depends on your valkey DB</span>
<span class="sd">(see :ref:`settings valkey`), by example:</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> valkey:</span>
<span class="sd"> url: valkey://localhost:6379/0</span>
<span class="sd">Configure Limiter</span>
<span class="sd">=================</span>
<span class="sd">The methods of :ref:`botdetection` the limiter uses are configured in a local</span>
<span class="sd">file ``/etc/searxng/limiter.toml``. The defaults are shown in limiter.toml_ /</span>
<span class="sd">Don&#39;t copy all values to your local configuration, just enable what you need by</span>
<span class="sd">overwriting the defaults. For instance to activate the ``link_token`` method in</span>
<span class="sd">the :ref:`botdetection.ip_limit` you only need to set this option to ``true``:</span>
<span class="sd">.. code:: toml</span>
<span class="sd"> [botdetection.ip_limit]</span>
<span class="sd"> link_token = true</span>
<span class="sd">.. _limiter.toml:</span>
<span class="sd">``limiter.toml``</span>
<span class="sd">================</span>
<span class="sd">In this file the limiter finds the configuration of the :ref:`botdetection`:</span>
<span class="sd">- :ref:`botdetection ip_lists`</span>
<span class="sd">- :ref:`botdetection rate limit`</span>
<span class="sd">- :ref:`botdetection probe headers`</span>
<span class="sd">.. kernel-include:: $SOURCEDIR/limiter.toml</span>
<span class="sd"> :code: toml</span>
<span class="sd">Implementation</span>
<span class="sd">==============</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="n">ip_address</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">sys</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pathlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Path</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">werkzeug</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">searx.compat</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">logger</span><span class="p">,</span>
<span class="n">valkeydb</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">botdetection</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">sxng_request</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.botdetection</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">config</span><span class="p">,</span>
<span class="n">http_accept</span><span class="p">,</span>
<span class="n">http_accept_encoding</span><span class="p">,</span>
<span class="n">http_accept_language</span><span class="p">,</span>
<span class="n">http_user_agent</span><span class="p">,</span>
<span class="n">http_sec_fetch</span><span class="p">,</span>
<span class="n">ip_limit</span><span class="p">,</span>
<span class="n">ip_lists</span><span class="p">,</span>
<span class="n">get_network</span><span class="p">,</span>
<span class="n">dump_request</span><span class="p">,</span>
<span class="p">)</span>
<span class="c1"># the configuration are limiter.toml and &quot;limiter&quot; in settings.yml so, for</span>
<span class="c1"># coherency, the logger is &quot;limiter&quot;</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">&#39;limiter&#39;</span><span class="p">)</span>
<span class="n">CFG</span><span class="p">:</span> <span class="n">config</span><span class="o">.</span><span class="n">Config</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">_INSTALLED</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">LIMITER_CFG_SCHEMA</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)</span><span class="o">.</span><span class="n">parent</span> <span class="o">/</span> <span class="s2">&quot;limiter.toml&quot;</span>
<span class="sd">&quot;&quot;&quot;Base configuration (schema) of the botdetection.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="get_cfg">
<a class="viewcode-back" href="../../admin/searx.limiter.html#searx.limiter.get_cfg">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_cfg</span><span class="p">()</span> <span class="o">-&gt;</span> <span class="n">config</span><span class="o">.</span><span class="n">Config</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns SearXNG&#39;s global limiter configuration.&quot;&quot;&quot;</span>
<span class="k">global</span> <span class="n">CFG</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="k">if</span> <span class="n">CFG</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">settings_loader</span> <span class="c1"># pylint: disable=import-outside-toplevel</span>
<span class="n">cfg_file</span> <span class="o">=</span> <span class="p">(</span><span class="n">settings_loader</span><span class="o">.</span><span class="n">get_user_cfg_folder</span><span class="p">()</span> <span class="ow">or</span> <span class="n">Path</span><span class="p">(</span><span class="s2">&quot;/etc/searxng&quot;</span><span class="p">))</span> <span class="o">/</span> <span class="s2">&quot;limiter.toml&quot;</span>
<span class="n">CFG</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">Config</span><span class="o">.</span><span class="n">from_toml</span><span class="p">(</span><span class="n">LIMITER_CFG_SCHEMA</span><span class="p">,</span> <span class="n">cfg_file</span><span class="p">,</span> <span class="n">searx</span><span class="o">.</span><span class="n">compat</span><span class="o">.</span><span class="n">LIMITER_CFG_DEPRECATED</span><span class="p">)</span>
<span class="n">searx</span><span class="o">.</span><span class="n">compat</span><span class="o">.</span><span class="n">limiter_fix_cfg</span><span class="p">(</span><span class="n">CFG</span><span class="p">,</span> <span class="n">cfg_file</span><span class="p">)</span>
<span class="k">return</span> <span class="n">CFG</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">filter_request</span><span class="p">(</span><span class="n">request</span><span class="p">:</span> <span class="n">SXNG_Request</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">werkzeug</span><span class="o">.</span><span class="n">Response</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="c1"># pylint: disable=too-many-return-statements</span>
<span class="n">cfg</span> <span class="o">=</span> <span class="n">get_cfg</span><span class="p">()</span>
<span class="n">real_ip</span> <span class="o">=</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">remote_addr</span><span class="p">)</span>
<span class="n">network</span> <span class="o">=</span> <span class="n">get_network</span><span class="p">(</span><span class="n">real_ip</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span>
<span class="k">if</span> <span class="n">request</span><span class="o">.</span><span class="n">path</span> <span class="o">==</span> <span class="s1">&#39;/healthz&#39;</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="c1"># link-local</span>
<span class="k">if</span> <span class="n">network</span><span class="o">.</span><span class="n">is_link_local</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="c1"># block- &amp; pass- lists</span>
<span class="c1">#</span>
<span class="c1"># 1. The IP of the request is first checked against the pass-list; if the IP</span>
<span class="c1"># matches an entry in the list, the request is not blocked.</span>
<span class="c1"># 2. If no matching entry is found in the pass-list, then a check is made against</span>
<span class="c1"># the block list; if the IP matches an entry in the list, the request is</span>
<span class="c1"># blocked.</span>
<span class="c1"># 3. If the IP is not in either list, the request is not blocked.</span>
<span class="n">match</span><span class="p">,</span> <span class="n">msg</span> <span class="o">=</span> <span class="n">ip_lists</span><span class="o">.</span><span class="n">pass_ip</span><span class="p">(</span><span class="n">real_ip</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span>
<span class="k">if</span> <span class="n">match</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;PASS </span><span class="si">%s</span><span class="s2">: matched PASSLIST - </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">network</span><span class="o">.</span><span class="n">compressed</span><span class="p">,</span> <span class="n">msg</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">match</span><span class="p">,</span> <span class="n">msg</span> <span class="o">=</span> <span class="n">ip_lists</span><span class="o">.</span><span class="n">block_ip</span><span class="p">(</span><span class="n">real_ip</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span>
<span class="k">if</span> <span class="n">match</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;BLOCK </span><span class="si">%s</span><span class="s2">: matched BLOCKLIST - </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">network</span><span class="o">.</span><span class="n">compressed</span><span class="p">,</span> <span class="n">msg</span><span class="p">)</span>
<span class="k">return</span> <span class="n">flask</span><span class="o">.</span><span class="n">make_response</span><span class="p">((</span><span class="s1">&#39;IP is on BLOCKLIST - </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">msg</span><span class="p">,</span> <span class="mi">429</span><span class="p">))</span>
<span class="c1"># methods applied on all requests</span>
<span class="k">for</span> <span class="n">func</span> <span class="ow">in</span> <span class="p">[</span>
<span class="n">http_user_agent</span><span class="p">,</span>
<span class="p">]:</span>
<span class="n">val</span> <span class="o">=</span> <span class="n">func</span><span class="o">.</span><span class="n">filter_request</span><span class="p">(</span><span class="n">network</span><span class="p">,</span> <span class="n">request</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span>
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;NOT OK (</span><span class="si">{</span><span class="n">func</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">): </span><span class="si">{</span><span class="n">network</span><span class="si">}</span><span class="s2">: %s&quot;</span><span class="p">,</span> <span class="n">dump_request</span><span class="p">(</span><span class="n">sxng_request</span><span class="p">))</span>
<span class="k">return</span> <span class="n">val</span>
<span class="c1"># methods applied on /search requests</span>
<span class="k">if</span> <span class="n">request</span><span class="o">.</span><span class="n">path</span> <span class="o">==</span> <span class="s1">&#39;/search&#39;</span><span class="p">:</span>
<span class="k">for</span> <span class="n">func</span> <span class="ow">in</span> <span class="p">[</span>
<span class="n">http_accept</span><span class="p">,</span>
<span class="n">http_accept_encoding</span><span class="p">,</span>
<span class="n">http_accept_language</span><span class="p">,</span>
<span class="n">http_user_agent</span><span class="p">,</span>
<span class="n">http_sec_fetch</span><span class="p">,</span>
<span class="n">ip_limit</span><span class="p">,</span>
<span class="p">]:</span>
<span class="n">val</span> <span class="o">=</span> <span class="n">func</span><span class="o">.</span><span class="n">filter_request</span><span class="p">(</span><span class="n">network</span><span class="p">,</span> <span class="n">request</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span>
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;NOT OK (</span><span class="si">{</span><span class="n">func</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">): </span><span class="si">{</span><span class="n">network</span><span class="si">}</span><span class="s2">: %s&quot;</span><span class="p">,</span> <span class="n">dump_request</span><span class="p">(</span><span class="n">sxng_request</span><span class="p">))</span>
<span class="k">return</span> <span class="n">val</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;OK </span><span class="si">{</span><span class="n">network</span><span class="si">}</span><span class="s2">: %s&quot;</span><span class="p">,</span> <span class="n">dump_request</span><span class="p">(</span><span class="n">sxng_request</span><span class="p">))</span>
<span class="k">return</span> <span class="kc">None</span>
<div class="viewcode-block" id="pre_request">
<a class="viewcode-back" href="../../admin/searx.limiter.html#searx.limiter.pre_request">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">pre_request</span><span class="p">():</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;See :py:obj:`flask.Flask.before_request`&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">filter_request</span><span class="p">(</span><span class="n">sxng_request</span><span class="p">)</span></div>
<div class="viewcode-block" id="is_installed">
<a class="viewcode-back" href="../../admin/searx.limiter.html#searx.limiter.is_installed">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">is_installed</span><span class="p">():</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns ``True`` if limiter is active and a valkey DB is available.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">_INSTALLED</span></div>
<div class="viewcode-block" id="initialize">
<a class="viewcode-back" href="../../admin/searx.limiter.html#searx.limiter.initialize">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">initialize</span><span class="p">(</span><span class="n">app</span><span class="p">:</span> <span class="n">flask</span><span class="o">.</span><span class="n">Flask</span><span class="p">,</span> <span class="n">settings</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Install the limiter&quot;&quot;&quot;</span>
<span class="k">global</span> <span class="n">_INSTALLED</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="c1"># even if the limiter is not activated, the botdetection must be activated</span>
<span class="c1"># (e.g. the self_info plugin uses the botdetection to get client IP)</span>
<span class="n">cfg</span> <span class="o">=</span> <span class="n">get_cfg</span><span class="p">()</span>
<span class="n">valkey_client</span> <span class="o">=</span> <span class="n">valkeydb</span><span class="o">.</span><span class="n">client</span><span class="p">()</span>
<span class="n">botdetection</span><span class="o">.</span><span class="n">init</span><span class="p">(</span><span class="n">cfg</span><span class="p">,</span> <span class="n">valkey_client</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">settings</span><span class="p">[</span><span class="s1">&#39;server&#39;</span><span class="p">][</span><span class="s1">&#39;limiter&#39;</span><span class="p">]</span> <span class="ow">or</span> <span class="n">settings</span><span class="p">[</span><span class="s1">&#39;server&#39;</span><span class="p">][</span><span class="s1">&#39;public_instance&#39;</span><span class="p">]):</span>
<span class="k">return</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">valkey_client</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
<span class="s2">&quot;The limiter requires Valkey, please consult the documentation: &quot;</span>
<span class="s2">&quot;https://docs.searxng.org/admin/searx.limiter.html&quot;</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">settings</span><span class="p">[</span><span class="s1">&#39;server&#39;</span><span class="p">][</span><span class="s1">&#39;public_instance&#39;</span><span class="p">]:</span>
<span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
<span class="k">return</span>
<span class="n">_INSTALLED</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">if</span> <span class="n">settings</span><span class="p">[</span><span class="s1">&#39;server&#39;</span><span class="p">][</span><span class="s1">&#39;public_instance&#39;</span><span class="p">]:</span>
<span class="c1"># overwrite limiter.toml setting</span>
<span class="n">cfg</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">&#39;botdetection.ip_limit.link_token&#39;</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
<span class="n">app</span><span class="o">.</span><span class="n">before_request</span><span class="p">(</span><span class="n">pre_request</span><span class="p">)</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../index.html">
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../index.html">Overview</a>
<ul>
<li><a href="../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

598
_modules/searx/locales.html Normal file
View File

@@ -0,0 +1,598 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.locales &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.locales</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.locales</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">SearXNGs locale data</span>
<span class="sd">=====================</span>
<span class="sd">The variables :py:obj:`RTL_LOCALES` and :py:obj:`LOCALE_NAMES` are loaded from</span>
<span class="sd">:origin:`searx/data/locales.json` / see :py:obj:`locales_initialize` and</span>
<span class="sd">:ref:`update_locales.py`.</span>
<span class="sd">.. hint::</span>
<span class="sd"> Whenever the value of :py:obj:`ADDITIONAL_TRANSLATIONS` or</span>
<span class="sd"> :py:obj:`LOCALE_BEST_MATCH` is modified, the</span>
<span class="sd"> :origin:`searx/data/locales.json` needs to be rebuild::</span>
<span class="sd"> ./manage data.locales</span>
<span class="sd">SearXNG&#39;s locale codes</span>
<span class="sd">======================</span>
<span class="sd">.. automodule:: searx.sxng_locales</span>
<span class="sd"> :members:</span>
<span class="sd">SearXNGs locale implementations</span>
<span class="sd">================================</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pathlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Path</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">babel.support</span><span class="w"> </span><span class="kn">import</span> <span class="n">Translations</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.languages</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.core</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">flask_babel</span> <span class="c1"># pyright: ignore[reportMissingTypeStubs]</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask.ctx</span><span class="w"> </span><span class="kn">import</span> <span class="n">has_request_context</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
<span class="n">data</span><span class="p">,</span>
<span class="n">logger</span><span class="p">,</span>
<span class="n">searx_dir</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">sxng_request</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">&#39;locales&#39;</span><span class="p">)</span>
<span class="c1"># safe before monkey patching flask_babel.get_translations</span>
<span class="n">_flask_babel_get_translations</span> <span class="o">=</span> <span class="n">flask_babel</span><span class="o">.</span><span class="n">get_translations</span>
<span class="n">LOCALE_NAMES</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="sd">&quot;&quot;&quot;Mapping of locales and their description. Locales e.g. &#39;fr&#39; or &#39;pt-BR&#39; (see</span>
<span class="sd">:py:obj:`locales_initialize`).</span>
<span class="sd">:meta hide-value:</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">RTL_LOCALES</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="sd">&quot;&quot;&quot;List of *Right-To-Left* locales e.g. &#39;he&#39; or &#39;fa-IR&#39; (see</span>
<span class="sd">:py:obj:`locales_initialize`).&quot;&quot;&quot;</span>
<span class="n">ADDITIONAL_TRANSLATIONS</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;dv&quot;</span><span class="p">:</span> <span class="s2">&quot;ދިވެހި (Dhivehi)&quot;</span><span class="p">,</span>
<span class="s2">&quot;oc&quot;</span><span class="p">:</span> <span class="s2">&quot;Occitan&quot;</span><span class="p">,</span>
<span class="s2">&quot;szl&quot;</span><span class="p">:</span> <span class="s2">&quot;Ślōnski (Silesian)&quot;</span><span class="p">,</span>
<span class="s2">&quot;pap&quot;</span><span class="p">:</span> <span class="s2">&quot;Papiamento&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="sd">&quot;&quot;&quot;Additional languages SearXNG has translations for but not supported by</span>
<span class="sd">python-babel (see :py:obj:`locales_initialize`).&quot;&quot;&quot;</span>
<span class="n">LOCALE_BEST_MATCH</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;dv&quot;</span><span class="p">:</span> <span class="s2">&quot;si&quot;</span><span class="p">,</span>
<span class="s2">&quot;oc&quot;</span><span class="p">:</span> <span class="s1">&#39;fr-FR&#39;</span><span class="p">,</span>
<span class="s2">&quot;szl&quot;</span><span class="p">:</span> <span class="s2">&quot;pl&quot;</span><span class="p">,</span>
<span class="s2">&quot;nl-BE&quot;</span><span class="p">:</span> <span class="s2">&quot;nl&quot;</span><span class="p">,</span>
<span class="s2">&quot;zh-HK&quot;</span><span class="p">:</span> <span class="s2">&quot;zh-Hant-TW&quot;</span><span class="p">,</span>
<span class="s2">&quot;pap&quot;</span><span class="p">:</span> <span class="s2">&quot;pt-BR&quot;</span><span class="p">,</span>
<span class="p">}</span>
<span class="sd">&quot;&quot;&quot;Map a locale we do not have a translations for to a locale we have a</span>
<span class="sd">translation for. By example: use Taiwan version of the translation for Hong</span>
<span class="sd">Kong.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">localeselector</span><span class="p">():</span>
<span class="n">locale</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">&#39;en&#39;</span>
<span class="k">if</span> <span class="n">has_request_context</span><span class="p">():</span>
<span class="n">value</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">sxng_request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s1">&#39;locale&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">value</span><span class="p">:</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">value</span>
<span class="c1"># first, set the language that is not supported by babel</span>
<span class="k">if</span> <span class="n">locale</span> <span class="ow">in</span> <span class="n">ADDITIONAL_TRANSLATIONS</span><span class="p">:</span>
<span class="n">sxng_request</span><span class="o">.</span><span class="n">form</span><span class="p">[</span><span class="s1">&#39;use-translation&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">locale</span>
<span class="c1"># second, map locale to a value python-babel supports</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">LOCALE_BEST_MATCH</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">locale</span><span class="p">,</span> <span class="n">locale</span><span class="p">)</span>
<span class="k">if</span> <span class="n">locale</span> <span class="o">==</span> <span class="s1">&#39;&#39;</span><span class="p">:</span>
<span class="c1"># if there is an error loading the preferences</span>
<span class="c1"># the locale is going to be &#39;&#39;</span>
<span class="n">locale</span> <span class="o">=</span> <span class="s1">&#39;en&#39;</span>
<span class="c1"># babel uses underscore instead of hyphen.</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">,</span> <span class="s1">&#39;_&#39;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">locale</span>
<div class="viewcode-block" id="get_translations">
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.get_translations">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_translations</span><span class="p">():</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Monkey patch of :py:obj:`flask_babel.get_translations`&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">has_request_context</span><span class="p">():</span>
<span class="n">use_translation</span> <span class="o">=</span> <span class="n">sxng_request</span><span class="o">.</span><span class="n">form</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;use-translation&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">use_translation</span> <span class="ow">in</span> <span class="n">ADDITIONAL_TRANSLATIONS</span><span class="p">:</span>
<span class="n">babel_ext</span> <span class="o">=</span> <span class="n">flask_babel</span><span class="o">.</span><span class="n">current_app</span><span class="o">.</span><span class="n">extensions</span><span class="p">[</span><span class="s1">&#39;babel&#39;</span><span class="p">]</span>
<span class="k">return</span> <span class="n">Translations</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">babel_ext</span><span class="o">.</span><span class="n">translation_directories</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">use_translation</span><span class="p">)</span>
<span class="k">return</span> <span class="n">_flask_babel_get_translations</span><span class="p">()</span></div>
<span class="n">_TR_LOCALES</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<div class="viewcode-block" id="get_translation_locales">
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.get_translation_locales">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_translation_locales</span><span class="p">()</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns the list of translation locales (*underscore*). The list is</span>
<span class="sd"> generated from the translation folders in :origin:`searx/translations`&quot;&quot;&quot;</span>
<span class="k">global</span> <span class="n">_TR_LOCALES</span> <span class="c1"># pylint:disable=global-statement</span>
<span class="k">if</span> <span class="n">_TR_LOCALES</span><span class="p">:</span>
<span class="k">return</span> <span class="n">_TR_LOCALES</span>
<span class="n">tr_locales</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">folder</span> <span class="ow">in</span> <span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="n">searx_dir</span><span class="p">)</span> <span class="o">/</span> <span class="s1">&#39;translations&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">iterdir</span><span class="p">():</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">folder</span><span class="o">.</span><span class="n">is_dir</span><span class="p">():</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">folder</span> <span class="o">/</span> <span class="s1">&#39;LC_MESSAGES&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">is_dir</span><span class="p">():</span>
<span class="k">continue</span>
<span class="n">tr_locales</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">folder</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="n">_TR_LOCALES</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">tr_locales</span><span class="p">)</span>
<span class="k">return</span> <span class="n">_TR_LOCALES</span></div>
<div class="viewcode-block" id="locales_initialize">
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.locales_initialize">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">locales_initialize</span><span class="p">():</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Initialize locales environment of the SearXNG session.</span>
<span class="sd"> - monkey patch :py:obj:`flask_babel.get_translations` by :py:obj:`get_translations`</span>
<span class="sd"> - init global names :py:obj:`LOCALE_NAMES`, :py:obj:`RTL_LOCALES`</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">flask_babel</span><span class="o">.</span><span class="n">get_translations</span> <span class="o">=</span> <span class="n">get_translations</span>
<span class="n">LOCALE_NAMES</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">LOCALES</span><span class="p">[</span><span class="s2">&quot;LOCALE_NAMES&quot;</span><span class="p">])</span>
<span class="n">RTL_LOCALES</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">LOCALES</span><span class="p">[</span><span class="s2">&quot;RTL_LOCALES&quot;</span><span class="p">])</span></div>
<div class="viewcode-block" id="region_tag">
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.region_tag">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">region_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">:</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns SearXNG&#39;s region tag from the locale (e.g. zh-TW , en-US).&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;babel.Locale </span><span class="si">%s</span><span class="s1">: missed a territory&#39;</span> <span class="o">%</span> <span class="n">locale</span><span class="p">)</span>
<span class="k">return</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span></div>
<div class="viewcode-block" id="language_tag">
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.language_tag">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">:</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns SearXNG&#39;s language tag from the locale and if exits, the tag</span>
<span class="sd"> includes the script name (e.g. en, zh_Hant).</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">sxng_lang</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span>
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">script</span><span class="p">:</span>
<span class="n">sxng_lang</span> <span class="o">+=</span> <span class="s1">&#39;_&#39;</span> <span class="o">+</span> <span class="n">locale</span><span class="o">.</span><span class="n">script</span>
<span class="k">return</span> <span class="n">sxng_lang</span></div>
<div class="viewcode-block" id="get_locale">
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.get_locale">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_locale</span><span class="p">(</span><span class="n">locale_tag</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns a :py:obj:`babel.Locale` object parsed from argument</span>
<span class="sd"> ``locale_tag``&quot;&quot;&quot;</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">locale_tag</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;-&#39;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">locale</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span></div>
<div class="viewcode-block" id="get_official_locales">
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.get_official_locales">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_official_locales</span><span class="p">(</span>
<span class="n">territory</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">languages</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">regional</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">de_facto</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">set</span><span class="p">[</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns a list of :py:obj:`babel.Locale` with languages from</span>
<span class="sd"> :py:obj:`babel.languages.get_official_languages`.</span>
<span class="sd"> :param territory: The territory (country or region) code.</span>
<span class="sd"> :param languages: A list of language codes the languages from</span>
<span class="sd"> :py:obj:`babel.languages.get_official_languages` should be in</span>
<span class="sd"> (intersection). If this argument is ``None``, all official languages in</span>
<span class="sd"> this territory are used.</span>
<span class="sd"> :param regional: If the regional flag is set, then languages which are</span>
<span class="sd"> regionally official are also returned.</span>
<span class="sd"> :param de_facto: If the de_facto flag is set to `False`, then languages</span>
<span class="sd"> which are “de facto” official are not returned.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">ret_val</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="n">o_languages</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get_official_languages</span><span class="p">(</span><span class="n">territory</span><span class="p">,</span> <span class="n">regional</span><span class="o">=</span><span class="n">regional</span><span class="p">,</span> <span class="n">de_facto</span><span class="o">=</span><span class="n">de_facto</span><span class="p">)</span>
<span class="k">if</span> <span class="n">languages</span><span class="p">:</span>
<span class="n">languages</span> <span class="o">=</span> <span class="p">[</span><span class="n">l</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">languages</span><span class="p">]</span>
<span class="n">o_languages</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">l</span> <span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">o_languages</span> <span class="k">if</span> <span class="n">l</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">in</span> <span class="n">languages</span><span class="p">)</span>
<span class="k">for</span> <span class="n">lang</span> <span class="ow">in</span> <span class="n">o_languages</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">lang</span> <span class="o">+</span> <span class="s1">&#39;_&#39;</span> <span class="o">+</span> <span class="n">territory</span><span class="p">)</span>
<span class="n">ret_val</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="k">continue</span>
<span class="k">return</span> <span class="n">ret_val</span></div>
<div class="viewcode-block" id="get_engine_locale">
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.get_engine_locale">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">get_engine_locale</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">engine_locales</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="n">default</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Return engine&#39;s language (aka locale) string that best fits to argument</span>
<span class="sd"> ``searxng_locale``.</span>
<span class="sd"> Argument ``engine_locales`` is a python dict that maps *SearXNG locales* to</span>
<span class="sd"> corresponding *engine locales*::</span>
<span class="sd"> &lt;engine&gt;: {</span>
<span class="sd"> # SearXNG string : engine-string</span>
<span class="sd"> &#39;ca-ES&#39; : &#39;ca_ES&#39;,</span>
<span class="sd"> &#39;fr-BE&#39; : &#39;fr_BE&#39;,</span>
<span class="sd"> &#39;fr-CA&#39; : &#39;fr_CA&#39;,</span>
<span class="sd"> &#39;fr-CH&#39; : &#39;fr_CH&#39;,</span>
<span class="sd"> &#39;fr&#39; : &#39;fr_FR&#39;,</span>
<span class="sd"> ...</span>
<span class="sd"> &#39;pl-PL&#39; : &#39;pl_PL&#39;,</span>
<span class="sd"> &#39;pt-PT&#39; : &#39;pt_PT&#39;</span>
<span class="sd"> ..</span>
<span class="sd"> &#39;zh&#39; : &#39;zh&#39;</span>
<span class="sd"> &#39;zh_Hans&#39; : &#39;zh&#39;</span>
<span class="sd"> &#39;zh_Hant&#39; : &#39;zh_TW&#39;</span>
<span class="sd"> }</span>
<span class="sd"> .. hint::</span>
<span class="sd"> The *SearXNG locale* string has to be known by babel!</span>
<span class="sd"> If there is no direct 1:1 mapping, this functions tries to narrow down</span>
<span class="sd"> engine&#39;s language (locale). If no value can be determined by these</span>
<span class="sd"> approximation attempts the ``default`` value is returned.</span>
<span class="sd"> Assumptions:</span>
<span class="sd"> A. When user select a language the results should be optimized according to</span>
<span class="sd"> the selected language.</span>
<span class="sd"> B. When user select a language and a territory the results should be</span>
<span class="sd"> optimized with first priority on territory and second on language.</span>
<span class="sd"> First approximation rule (*by territory*):</span>
<span class="sd"> When the user selects a locale with territory (and a language), the</span>
<span class="sd"> territory has priority over the language. If any of the official languages</span>
<span class="sd"> in the territory is supported by the engine (``engine_locales``) it will</span>
<span class="sd"> be used.</span>
<span class="sd"> Second approximation rule (*by language*):</span>
<span class="sd"> If &quot;First approximation rule&quot; brings no result or the user selects only a</span>
<span class="sd"> language without a territory. Check in which territories the language</span>
<span class="sd"> has an official status and if one of these territories is supported by the</span>
<span class="sd"> engine.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-many-branches, too-many-return-statements</span>
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">engine_locales</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span>
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="c1"># There was a 1:1 mapping (e.g. a region &quot;fr-BE --&gt; fr_BE&quot; or a language</span>
<span class="c1"># &quot;zh --&gt; zh&quot;), no need to narrow language-script nor territory.</span>
<span class="k">return</span> <span class="n">engine_locale</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;-&#39;</span><span class="p">)</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">searxng_locale</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
<span class="k">return</span> <span class="n">default</span>
<span class="n">searxng_lang</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">engine_locales</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_lang</span><span class="p">)</span>
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="c1"># There was a 1:1 mapping (e.g. &quot;zh-HK --&gt; zh_Hant&quot; or &quot;zh-CN --&gt; zh_Hans&quot;)</span>
<span class="k">return</span> <span class="n">engine_locale</span>
<span class="c1"># SearXNG&#39;s selected locale is not supported by the engine ..</span>
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">:</span>
<span class="c1"># Try to narrow by *official* languages in the territory (??-XX).</span>
<span class="k">for</span> <span class="n">official_language</span> <span class="ow">in</span> <span class="n">babel</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get_official_languages</span><span class="p">(</span><span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">,</span> <span class="n">de_facto</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="n">official_language</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span>
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">engine_locales</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span>
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">engine_locale</span>
<span class="c1"># Engine does not support one of the official languages in the territory or</span>
<span class="c1"># there is only a language selected without a territory.</span>
<span class="c1"># Now lets have a look if the searxng_lang (the language selected by the</span>
<span class="c1"># user) is a official language in other territories. If so, check if</span>
<span class="c1"># engine does support the searxng_lang in this other territory.</span>
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span><span class="p">:</span>
<span class="n">terr_lang_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">territory</span><span class="p">:</span> <span class="nb">str</span>
<span class="n">langs</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]</span>
<span class="k">for</span> <span class="n">territory</span><span class="p">,</span> <span class="n">langs</span> <span class="ow">in</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">get_global</span><span class="p">(</span><span class="s2">&quot;territory_languages&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">_lang</span> <span class="o">=</span> <span class="n">langs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_lang</span><span class="p">)</span>
<span class="k">if</span> <span class="n">_lang</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">_lang</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;official_status&#39;</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">terr_lang_dict</span><span class="p">[</span><span class="n">territory</span><span class="p">]</span> <span class="o">=</span> <span class="n">_lang</span>
<span class="c1"># first: check fr-FR, de-DE .. is supported by the engine</span>
<span class="c1"># exception: &#39;en&#39; --&gt; &#39;en-US&#39;</span>
<span class="n">territory</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
<span class="k">if</span> <span class="n">territory</span> <span class="o">==</span> <span class="s1">&#39;EN&#39;</span><span class="p">:</span>
<span class="n">territory</span> <span class="o">=</span> <span class="s1">&#39;US&#39;</span>
<span class="k">if</span> <span class="n">terr_lang_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">territory</span><span class="p">):</span>
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="n">territory</span>
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">engine_locales</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span>
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">engine_locale</span>
<span class="c1"># second: sort by population_percent and take first match</span>
<span class="c1"># drawback of &quot;population percent&quot;: if there is a territory with a</span>
<span class="c1"># small number of people (e.g 100) but the majority speaks the</span>
<span class="c1"># language, then the percentage might be 100% (--&gt; 100 people) but in</span>
<span class="c1"># a different territory with more people (e.g. 10.000) where only 10%</span>
<span class="c1"># speak the language the total amount of speaker is higher (--&gt; 200</span>
<span class="c1"># people).</span>
<span class="c1">#</span>
<span class="c1"># By example: The population of Saint-Martin is 33.000, of which 100%</span>
<span class="c1"># speak French, but this is less than the 30% of the approximately 2.5</span>
<span class="c1"># million Belgian citizens</span>
<span class="c1">#</span>
<span class="c1"># - &#39;fr-MF&#39;, &#39;population_percent&#39;: 100.0, &#39;official_status&#39;: &#39;official&#39;</span>
<span class="c1"># - &#39;fr-BE&#39;, &#39;population_percent&#39;: 38.0, &#39;official_status&#39;: &#39;official&#39;</span>
<span class="n">terr_lang_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">terr_lang_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">terr_lang_list</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">))</span>
<span class="k">for</span> <span class="n">territory</span><span class="p">,</span> <span class="n">_lang</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">terr_lang_list</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">item</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="mi">1</span><span class="p">][</span><span class="s1">&#39;population_percent&#39;</span><span class="p">],</span> <span class="n">reverse</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="n">territory</span>
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">engine_locales</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span>
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">engine_locale</span>
<span class="c1"># No luck: narrow by &quot;language from territory&quot; and &quot;territory from language&quot;</span>
<span class="c1"># does not fit to a locale supported by the engine.</span>
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">default</span>
<span class="k">return</span> <span class="n">default</span></div>
<div class="viewcode-block" id="match_locale">
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.match_locale">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">match_locale</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">locale_tag_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">fallback</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Return tag from ``locale_tag_list`` that best fits to ``searxng_locale``.</span>
<span class="sd"> :param str searxng_locale: SearXNG&#39;s internal representation of locale (de,</span>
<span class="sd"> de-DE, fr-BE, zh, zh-CN, zh-TW ..).</span>
<span class="sd"> :param list locale_tag_list: The list of locale tags to select from</span>
<span class="sd"> :param str fallback: fallback locale tag (if unset --&gt; ``None``)</span>
<span class="sd"> The rules to find a match are implemented in :py:obj:`get_engine_locale`,</span>
<span class="sd"> the ``engine_locales`` is build up by :py:obj:`build_engine_locales`.</span>
<span class="sd"> .. hint::</span>
<span class="sd"> The *SearXNG locale* string and the members of ``locale_tag_list`` has to</span>
<span class="sd"> be known by babel! The :py:obj:`ADDITIONAL_TRANSLATIONS` are used in the</span>
<span class="sd"> UI and are not known by babel --&gt; will be ignored.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># searxng_locale = &#39;es&#39;</span>
<span class="c1"># locale_tag_list = [&#39;es-AR&#39;, &#39;es-ES&#39;, &#39;es-MX&#39;]</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">searxng_locale</span><span class="p">:</span>
<span class="k">return</span> <span class="n">fallback</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">get_locale</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span>
<span class="k">if</span> <span class="n">locale</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">fallback</span>
<span class="c1"># normalize to a SearXNG locale that can be passed to get_engine_locale</span>
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">:</span>
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
<span class="c1"># clean up locale_tag_list</span>
<span class="n">tag_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">tag</span> <span class="ow">in</span> <span class="n">locale_tag_list</span><span class="p">:</span>
<span class="k">if</span> <span class="n">tag</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">&#39;all&#39;</span><span class="p">,</span> <span class="s1">&#39;auto&#39;</span><span class="p">)</span> <span class="ow">or</span> <span class="n">tag</span> <span class="ow">in</span> <span class="n">ADDITIONAL_TRANSLATIONS</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">tag_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tag</span><span class="p">)</span>
<span class="c1"># emulate fetch_traits</span>
<span class="n">engine_locales</span> <span class="o">=</span> <span class="n">build_engine_locales</span><span class="p">(</span><span class="n">tag_list</span><span class="p">)</span>
<span class="k">return</span> <span class="n">get_engine_locale</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">,</span> <span class="n">engine_locales</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">fallback</span><span class="p">)</span></div>
<div class="viewcode-block" id="build_engine_locales">
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.build_engine_locales">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">build_engine_locales</span><span class="p">(</span><span class="n">tag_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;From a list of locale tags a dictionary is build that can be passed by</span>
<span class="sd"> argument ``engine_locales`` to :py:obj:`get_engine_locale`. This function</span>
<span class="sd"> is mainly used by :py:obj:`match_locale` and is similar to what the</span>
<span class="sd"> ``fetch_traits(..)`` function of engines do.</span>
<span class="sd"> If there are territory codes in the ``tag_list`` that have a *script code*</span>
<span class="sd"> additional keys are added to the returned dictionary.</span>
<span class="sd"> .. code:: python</span>
<span class="sd"> &gt;&gt;&gt; import locales</span>
<span class="sd"> &gt;&gt;&gt; engine_locales = locales.build_engine_locales([&#39;en&#39;, &#39;en-US&#39;, &#39;zh&#39;, &#39;zh-CN&#39;, &#39;zh-TW&#39;])</span>
<span class="sd"> &gt;&gt;&gt; engine_locales</span>
<span class="sd"> {</span>
<span class="sd"> &#39;en&#39;: &#39;en&#39;, &#39;en-US&#39;: &#39;en-US&#39;,</span>
<span class="sd"> &#39;zh&#39;: &#39;zh&#39;, &#39;zh-CN&#39;: &#39;zh-CN&#39;, &#39;zh_Hans&#39;: &#39;zh-CN&#39;,</span>
<span class="sd"> &#39;zh-TW&#39;: &#39;zh-TW&#39;, &#39;zh_Hant&#39;: &#39;zh-TW&#39;</span>
<span class="sd"> }</span>
<span class="sd"> &gt;&gt;&gt; get_engine_locale(&#39;zh-Hans&#39;, engine_locales)</span>
<span class="sd"> &#39;zh-CN&#39;</span>
<span class="sd"> This function is a good example to understand the language/region model</span>
<span class="sd"> of SearXNG:</span>
<span class="sd"> SearXNG only distinguishes between **search languages** and **search</span>
<span class="sd"> regions**, by adding the *script-tags*, languages with *script-tags* can</span>
<span class="sd"> be assigned to the **regions** that SearXNG supports.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">engine_locales</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">for</span> <span class="n">tag</span> <span class="ow">in</span> <span class="n">tag_list</span><span class="p">:</span>
<span class="n">locale</span> <span class="o">=</span> <span class="n">get_locale</span><span class="p">(</span><span class="n">tag</span><span class="p">)</span>
<span class="k">if</span> <span class="n">locale</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;build_engine_locales: skip locale tag </span><span class="si">%s</span><span class="s2"> / unknown by babel&quot;</span><span class="p">,</span> <span class="n">tag</span><span class="p">)</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">:</span>
<span class="n">engine_locales</span><span class="p">[</span><span class="n">region_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)]</span> <span class="o">=</span> <span class="n">tag</span>
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">script</span><span class="p">:</span>
<span class="n">engine_locales</span><span class="p">[</span><span class="n">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)]</span> <span class="o">=</span> <span class="n">tag</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">engine_locales</span><span class="p">[</span><span class="n">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)]</span> <span class="o">=</span> <span class="n">tag</span>
<span class="k">return</span> <span class="n">engine_locales</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../index.html">
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../index.html">Overview</a>
<ul>
<li><a href="../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,446 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.plugins._core &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.plugins._core</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.plugins._core</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="c1"># pylint: disable=too-few-public-methods,missing-module-docstring</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;PluginInfo&quot;</span><span class="p">,</span> <span class="s2">&quot;Plugin&quot;</span><span class="p">,</span> <span class="s2">&quot;PluginCfg&quot;</span><span class="p">,</span> <span class="s2">&quot;PluginStorage&quot;</span><span class="p">]</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">abc</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">importlib</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">inspect</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">collections.abc</span><span class="w"> </span><span class="kn">import</span> <span class="n">Generator</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">dataclasses</span><span class="w"> </span><span class="kn">import</span> <span class="n">dataclass</span><span class="p">,</span> <span class="n">field</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearchWithPlugins</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">Result</span><span class="p">,</span> <span class="n">EngineResults</span><span class="p">,</span> <span class="n">LegacyResult</span> <span class="c1"># pyright: ignore[reportPrivateLocalImportUsage]</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
<span class="n">log</span><span class="p">:</span> <span class="n">logging</span><span class="o">.</span><span class="n">Logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;searx.plugins&quot;</span><span class="p">)</span>
<div class="viewcode-block" id="PluginInfo">
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.PluginInfo">[docs]</a>
<span class="nd">@dataclass</span>
<span class="k">class</span><span class="w"> </span><span class="nc">PluginInfo</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Object that holds information about a *plugin*, these infos are shown to</span>
<span class="sd"> the user in the Preferences menu.</span>
<span class="sd"> To be able to translate the information into other languages, the text must</span>
<span class="sd"> be written in English and translated with :py:obj:`flask_babel.gettext`.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="nb">id</span><span class="p">:</span> <span class="nb">str</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The ID-selector in HTML/CSS `#&lt;id&gt;`.&quot;&quot;&quot;</span>
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Name of the *plugin*.&quot;&quot;&quot;</span>
<span class="n">description</span><span class="p">:</span> <span class="nb">str</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Short description of the *answerer*.&quot;&quot;&quot;</span>
<span class="n">preference_section</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;general&quot;</span><span class="p">,</span> <span class="s2">&quot;ui&quot;</span><span class="p">,</span> <span class="s2">&quot;privacy&quot;</span><span class="p">,</span> <span class="s2">&quot;query&quot;</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="s2">&quot;general&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Section (tab/group) in the preferences where this plugin is shown to the</span>
<span class="sd"> user.</span>
<span class="sd"> The value ``query`` is reserved for plugins that are activated via a</span>
<span class="sd"> *keyword* as part of a search query, see:</span>
<span class="sd"> - :py:obj:`PluginInfo.examples`</span>
<span class="sd"> - :py:obj:`Plugin.keywords`</span>
<span class="sd"> Those plugins are shown in the preferences in tab *Special Queries*.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">examples</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">list</span><span class="p">)</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;List of short examples of the usage / of query terms.&quot;&quot;&quot;</span>
<span class="n">keywords</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">list</span><span class="p">)</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;See :py:obj:`Plugin.keywords`&quot;&quot;&quot;</span></div>
<span class="n">ID_REGXP</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="s2">&quot;[a-z][a-z0-9].*&quot;</span><span class="p">)</span>
<div class="viewcode-block" id="Plugin">
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.Plugin">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">Plugin</span><span class="p">(</span><span class="n">abc</span><span class="o">.</span><span class="n">ABC</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Abstract base class of all Plugins.&quot;&quot;&quot;</span>
<span class="nb">id</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The ID (suffix) in the HTML form.&quot;&quot;&quot;</span>
<span class="n">active</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">ClassVar</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Plugin is enabled/disabled by default (:py:obj:`PluginCfg.active`).&quot;&quot;&quot;</span>
<span class="n">keywords</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Keywords in the search query that activate the plugin. The *keyword* is</span>
<span class="sd"> the first word in a search query. If a plugin should be executed regardless</span>
<span class="sd"> of the search query, the list of keywords should be empty (which is also the</span>
<span class="sd"> default in the base class for Plugins).&quot;&quot;&quot;</span>
<span class="n">log</span><span class="p">:</span> <span class="n">logging</span><span class="o">.</span><span class="n">Logger</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A logger object, is automatically initialized when calling the</span>
<span class="sd"> constructor (if not already set in the subclass).&quot;&quot;&quot;</span>
<span class="n">info</span><span class="p">:</span> <span class="n">PluginInfo</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Information about the *plugin*, see :py:obj:`PluginInfo`.&quot;&quot;&quot;</span>
<span class="n">fqn</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">&quot;PluginCfg&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">fqn</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">fqn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__mro__</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="vm">__module__</span>
<span class="c1"># names from the configuration</span>
<span class="k">for</span> <span class="n">n</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">plg_cfg</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">n</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
<span class="c1"># names that must be set by the plugin implementation</span>
<span class="k">for</span> <span class="n">attr</span> <span class="ow">in</span> <span class="p">[</span>
<span class="s2">&quot;id&quot;</span><span class="p">,</span>
<span class="p">]:</span>
<span class="k">if</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attr</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;plugin </span><span class="si">{</span><span class="bp">self</span><span class="si">}</span><span class="s2"> is missing attribute </span><span class="si">{</span><span class="n">attr</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">ID_REGXP</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;plugin ID </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2"> contains invalid character (use lowercase ASCII)&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;log&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span>
<span class="n">pkg_name</span> <span class="o">=</span> <span class="n">inspect</span><span class="o">.</span><span class="n">getmodule</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">)</span><span class="o">.</span><span class="n">__package__</span> <span class="c1"># pyright: ignore[reportOptionalMemberAccess]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">pkg_name</span><span class="si">}</span><span class="s2">.</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The hash value is used in :py:obj:`set`, for example, when an object</span>
<span class="sd"> is added to the set. The hash value is also used in other contexts,</span>
<span class="sd"> e.g. when checking for equality to identify identical plugins from</span>
<span class="sd"> different sources (name collisions).&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="nb">id</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;py:obj:`Plugin` objects are equal if the hash values of the two</span>
<span class="sd"> objects are equal.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">hash</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<div class="viewcode-block" id="Plugin.init">
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.Plugin.init">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">app</span><span class="p">:</span> <span class="s2">&quot;flask.Flask&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Initialization of the plugin, the return value decides whether this</span>
<span class="sd"> plugin is active or not. Initialization only takes place once, at the</span>
<span class="sd"> time the WEB application is set up. The base method always returns</span>
<span class="sd"> ``True``, the method can be overwritten in the inheritances,</span>
<span class="sd"> - ``True`` plugin is active</span>
<span class="sd"> - ``False`` plugin is inactive</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="kc">True</span></div>
<span class="c1"># pylint: disable=unused-argument</span>
<div class="viewcode-block" id="Plugin.pre_search">
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.Plugin.pre_search">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">pre_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">&quot;SearchWithPlugins&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Runs BEFORE the search request and returns a boolean:</span>
<span class="sd"> - ``True`` to continue the search</span>
<span class="sd"> - ``False`` to stop the search</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="kc">True</span></div>
<div class="viewcode-block" id="Plugin.on_result">
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.Plugin.on_result">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">on_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">&quot;SearchWithPlugins&quot;</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="s2">&quot;Result&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Runs for each result of each engine and returns a boolean:</span>
<span class="sd"> - ``True`` to keep the result</span>
<span class="sd"> - ``False`` to remove the result from the result list</span>
<span class="sd"> The ``result`` can be modified to the needs.</span>
<span class="sd"> .. hint::</span>
<span class="sd"> If :py:obj:`Result.url &lt;searx.result_types._base.Result.url&gt;` is modified,</span>
<span class="sd"> :py:obj:`Result.parsed_url &lt;searx.result_types._base.Result.parsed_url&gt;` must</span>
<span class="sd"> be changed accordingly:</span>
<span class="sd"> .. code:: python</span>
<span class="sd"> result[&quot;parsed_url&quot;] = urlparse(result[&quot;url&quot;])</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="kc">True</span></div>
<div class="viewcode-block" id="Plugin.post_search">
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.Plugin.post_search">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">post_search</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">&quot;SearchWithPlugins&quot;</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;None | list[Result | LegacyResult] | EngineResults&quot;</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Runs AFTER the search request. Can return a list of</span>
<span class="sd"> :py:obj:`Result &lt;searx.result_types._base.Result&gt;` objects to be added to the</span>
<span class="sd"> final result list.&quot;&quot;&quot;</span>
<span class="k">return</span></div>
</div>
<div class="viewcode-block" id="PluginCfg">
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.PluginCfg">[docs]</a>
<span class="nd">@dataclass</span>
<span class="k">class</span><span class="w"> </span><span class="nc">PluginCfg</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Settings of a plugin.</span>
<span class="sd"> .. code:: yaml</span>
<span class="sd"> mypackage.mymodule.MyPlugin:</span>
<span class="sd"> active: true</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">active</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Plugin is active by default and the user can *opt-out* in the preferences.&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="PluginStorage">
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.PluginStorage">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">PluginStorage</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A storage for managing the *plugins* of SearXNG.&quot;&quot;&quot;</span>
<span class="n">plugin_list</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="n">Plugin</span><span class="p">]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The list of :py:obj:`Plugins` in this storage.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Generator</span><span class="p">[</span><span class="n">Plugin</span><span class="p">]:</span>
<span class="k">yield from</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span><span class="p">)</span>
<span class="nd">@property</span>
<span class="k">def</span><span class="w"> </span><span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="n">PluginInfo</span><span class="p">]:</span>
<span class="k">return</span> <span class="p">[</span><span class="n">p</span><span class="o">.</span><span class="n">info</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span><span class="p">]</span>
<div class="viewcode-block" id="PluginStorage.load_settings">
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.PluginStorage.load_settings">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">load_settings</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Load plugins configured in SearXNG&#39;s settings :ref:`settings</span>
<span class="sd"> plugins`.&quot;&quot;&quot;</span>
<span class="k">for</span> <span class="n">fqn</span><span class="p">,</span> <span class="n">plg_settings</span> <span class="ow">in</span> <span class="n">cfg</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="bp">cls</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">mod_name</span><span class="p">,</span> <span class="n">cls_name</span> <span class="o">=</span> <span class="n">fqn</span><span class="o">.</span><span class="n">rsplit</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">mod</span> <span class="o">=</span> <span class="n">importlib</span><span class="o">.</span><span class="n">import_module</span><span class="p">(</span><span class="n">mod_name</span><span class="p">)</span>
<span class="bp">cls</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">mod</span><span class="p">,</span> <span class="n">cls_name</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-exception-caught</span>
<span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">exc</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">cls</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;plugin </span><span class="si">{</span><span class="n">fqn</span><span class="si">}</span><span class="s2"> is not implemented&quot;</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
<span class="n">plg</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">(</span><span class="n">PluginCfg</span><span class="p">(</span><span class="o">**</span><span class="n">plg_settings</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">register</span><span class="p">(</span><span class="n">plg</span><span class="p">)</span></div>
<div class="viewcode-block" id="PluginStorage.register">
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.PluginStorage.register">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">register</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plugin</span><span class="p">:</span> <span class="n">Plugin</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Register a :py:obj:`Plugin`. In case of name collision (if two</span>
<span class="sd"> plugins have same ID) a :py:obj:`KeyError` exception is raised.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">plugin</span> <span class="ow">in</span> <span class="p">[</span><span class="n">p</span><span class="o">.</span><span class="n">id</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span><span class="p">]:</span>
<span class="n">msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;name collision &#39;</span><span class="si">{</span><span class="n">plugin</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">&#39;&quot;</span>
<span class="n">plugin</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">critical</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">plugin</span><span class="p">)</span>
<span class="n">plugin</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;plugin has been loaded&quot;</span><span class="p">)</span></div>
<div class="viewcode-block" id="PluginStorage.init">
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.PluginStorage.init">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">app</span><span class="p">:</span> <span class="s2">&quot;flask.Flask&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Calls the method :py:obj:`Plugin.init` of each plugin in this</span>
<span class="sd"> storage. Depending on its return value, the plugin is removed from</span>
<span class="sd"> *this* storage or not.&quot;&quot;&quot;</span>
<span class="k">for</span> <span class="n">plg</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span><span class="o">.</span><span class="n">copy</span><span class="p">():</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">plg</span><span class="o">.</span><span class="n">init</span><span class="p">(</span><span class="n">app</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">plg</span><span class="p">)</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">pre_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">&quot;SearchWithPlugins&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="n">ret</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">for</span> <span class="n">plugin</span> <span class="ow">in</span> <span class="p">[</span><span class="n">p</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span> <span class="k">if</span> <span class="n">p</span><span class="o">.</span><span class="n">id</span> <span class="ow">in</span> <span class="n">search</span><span class="o">.</span><span class="n">user_plugins</span><span class="p">]:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">ret</span> <span class="o">=</span> <span class="nb">bool</span><span class="p">(</span><span class="n">plugin</span><span class="o">.</span><span class="n">pre_search</span><span class="p">(</span><span class="n">request</span><span class="o">=</span><span class="n">request</span><span class="p">,</span> <span class="n">search</span><span class="o">=</span><span class="n">search</span><span class="p">))</span>
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
<span class="n">plugin</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">&quot;Exception while calling pre_search&quot;</span><span class="p">)</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">ret</span><span class="p">:</span>
<span class="c1"># skip this search on the first False from a plugin</span>
<span class="k">break</span>
<span class="k">return</span> <span class="n">ret</span>
<span class="k">def</span><span class="w"> </span><span class="nf">on_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">&quot;SearchWithPlugins&quot;</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="s2">&quot;Result&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="n">ret</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">for</span> <span class="n">plugin</span> <span class="ow">in</span> <span class="p">[</span><span class="n">p</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span> <span class="k">if</span> <span class="n">p</span><span class="o">.</span><span class="n">id</span> <span class="ow">in</span> <span class="n">search</span><span class="o">.</span><span class="n">user_plugins</span><span class="p">]:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">ret</span> <span class="o">=</span> <span class="nb">bool</span><span class="p">(</span><span class="n">plugin</span><span class="o">.</span><span class="n">on_result</span><span class="p">(</span><span class="n">request</span><span class="o">=</span><span class="n">request</span><span class="p">,</span> <span class="n">search</span><span class="o">=</span><span class="n">search</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
<span class="n">plugin</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">&quot;Exception while calling on_result&quot;</span><span class="p">)</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">ret</span><span class="p">:</span>
<span class="c1"># ignore this result item on the first False from a plugin</span>
<span class="k">break</span>
<span class="k">return</span> <span class="n">ret</span>
<div class="viewcode-block" id="PluginStorage.post_search">
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.PluginStorage.post_search">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">post_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">&quot;SearchWithPlugins&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Extend :py:obj:`search.result_container</span>
<span class="sd"> &lt;searx.results.ResultContainer`&gt; with result items from plugins listed</span>
<span class="sd"> in :py:obj:`search.user_plugins &lt;SearchWithPlugins.user_plugins&gt;`.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">keyword</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">for</span> <span class="n">keyword</span> <span class="ow">in</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">():</span>
<span class="k">if</span> <span class="n">keyword</span><span class="p">:</span>
<span class="k">break</span>
<span class="k">for</span> <span class="n">plugin</span> <span class="ow">in</span> <span class="p">[</span><span class="n">p</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span> <span class="k">if</span> <span class="n">p</span><span class="o">.</span><span class="n">id</span> <span class="ow">in</span> <span class="n">search</span><span class="o">.</span><span class="n">user_plugins</span><span class="p">]:</span>
<span class="k">if</span> <span class="n">plugin</span><span class="o">.</span><span class="n">keywords</span><span class="p">:</span>
<span class="c1"># plugin with keywords: skip plugin if no keyword match</span>
<span class="k">if</span> <span class="n">keyword</span> <span class="ow">and</span> <span class="n">keyword</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">plugin</span><span class="o">.</span><span class="n">keywords</span><span class="p">:</span>
<span class="k">continue</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">plugin</span><span class="o">.</span><span class="n">post_search</span><span class="p">(</span><span class="n">request</span><span class="o">=</span><span class="n">request</span><span class="p">,</span> <span class="n">search</span><span class="o">=</span><span class="n">search</span><span class="p">)</span> <span class="ow">or</span> <span class="p">[]</span>
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
<span class="n">plugin</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">&quot;Exception while calling post_search&quot;</span><span class="p">)</span>
<span class="k">continue</span>
<span class="c1"># In case of *plugins* prefix ``plugin:`` is set, see searx.result_types.Result</span>
<span class="n">search</span><span class="o">.</span><span class="n">result_container</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;plugin: </span><span class="si">{</span><span class="n">plugin</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">results</span><span class="p">)</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,135 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.plugins.calculator &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.plugins.calculator</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.plugins.calculator</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="c1"># pylint: disable=missing-module-docstring</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span> <span class="c1"># pyright: ignore[reportUnknownVariableType]</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
<div class="viewcode-block" id="SXNGPlugin">
<a class="viewcode-back" href="../../../dev/plugins/calculator.html#searx.plugins.calculator.SXNGPlugin">[docs]</a>
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Parses and solves mathematical expressions.&quot;&quot;&quot;</span>
<span class="nb">id</span> <span class="o">=</span> <span class="s2">&quot;calculator&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">&quot;PluginCfg&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Calculator&quot;</span><span class="p">),</span>
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Parses and solves mathematical expressions.&quot;</span><span class="p">),</span>
<span class="n">preference_section</span><span class="o">=</span><span class="s2">&quot;query&quot;</span><span class="p">,</span>
<span class="p">)</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,177 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.plugins.hash_plugin &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.plugins.hash_plugin</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.plugins.hash_plugin</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="c1"># pylint: disable=missing-module-docstring, missing-class-docstring</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">hashlib</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="k">if</span> <span class="n">typing</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearchWithPlugins</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
<div class="viewcode-block" id="SXNGPlugin">
<a class="viewcode-back" href="../../../dev/plugins/hash_plugin.html#searx.plugins.hash_plugin.SXNGPlugin">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Plugin converts strings to different hash digests. The results are</span>
<span class="sd"> displayed in area for the &quot;answers&quot;.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="nb">id</span> <span class="o">=</span> <span class="s2">&quot;hash_plugin&quot;</span>
<span class="n">keywords</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;md5&quot;</span><span class="p">,</span> <span class="s2">&quot;sha1&quot;</span><span class="p">,</span> <span class="s2">&quot;sha224&quot;</span><span class="p">,</span> <span class="s2">&quot;sha256&quot;</span><span class="p">,</span> <span class="s2">&quot;sha384&quot;</span><span class="p">,</span> <span class="s2">&quot;sha512&quot;</span><span class="p">]</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">&quot;PluginCfg&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">parser_re</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;(</span><span class="si">{</span><span class="s1">&#39;|&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keywords</span><span class="p">)</span><span class="si">}</span><span class="s2">) (.*)&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">I</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Hash plugin&quot;</span><span class="p">),</span>
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span>
<span class="s2">&quot;Converts strings to different hash digests. Available functions: md5, sha1, sha224, sha256, sha384, sha512.&quot;</span> <span class="c1"># pylint:disable=line-too-long</span>
<span class="p">),</span>
<span class="n">examples</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;sha512 The quick brown fox jumps over the lazy dog&quot;</span><span class="p">],</span>
<span class="n">preference_section</span><span class="o">=</span><span class="s2">&quot;query&quot;</span><span class="p">,</span>
<span class="p">)</span>
<div class="viewcode-block" id="SXNGPlugin.post_search">
<a class="viewcode-back" href="../../../dev/plugins/hash_plugin.html#searx.plugins.hash_plugin.SXNGPlugin.post_search">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">post_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="s2">&quot;SXNG_Request&quot;</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">&quot;SearchWithPlugins&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns a result list only for the first page.&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="k">if</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">pageno</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">m</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parser_re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">m</span><span class="p">:</span>
<span class="c1"># wrong query</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">function</span><span class="p">,</span> <span class="n">string</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="n">groups</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">string</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span>
<span class="c1"># end if the string is empty</span>
<span class="k">return</span> <span class="n">results</span>
<span class="c1"># select hash function</span>
<span class="n">f</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">new</span><span class="p">(</span><span class="n">function</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
<span class="c1"># make digest from the given string</span>
<span class="n">f</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">string</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s2">&quot;utf-8&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">())</span>
<span class="n">answer</span> <span class="o">=</span> <span class="n">function</span> <span class="o">+</span> <span class="s2">&quot; &quot;</span> <span class="o">+</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;hash digest&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;: &quot;</span> <span class="o">+</span> <span class="n">f</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="n">answer</span><span class="p">))</span>
<span class="k">return</span> <span class="n">results</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,316 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.plugins.hostnames &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.plugins.hostnames</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.plugins.hostnames</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="c1"># pylint: disable=too-many-branches, unused-argument</span>
<span class="sd">&quot;&quot;&quot;During the initialization phase, the plugin checks whether a ``hostnames:``</span>
<span class="sd">configuration exists. If this is not the case, the plugin is not included in the</span>
<span class="sd">PluginStorage (it is not available for selection).</span>
<span class="sd">- ``hostnames.replace``: A **mapping** of regular expressions to hostnames to be</span>
<span class="sd"> replaced by other hostnames.</span>
<span class="sd"> .. code:: yaml</span>
<span class="sd"> hostnames:</span>
<span class="sd"> replace:</span>
<span class="sd"> &#39;(.*\\.)?youtube\\.com$&#39;: &#39;invidious.example.com&#39;</span>
<span class="sd"> &#39;(.*\\.)?youtu\\.be$&#39;: &#39;invidious.example.com&#39;</span>
<span class="sd"> ...</span>
<span class="sd">- ``hostnames.remove``: A **list** of regular expressions of the hostnames whose</span>
<span class="sd"> results should be taken from the results list.</span>
<span class="sd"> .. code:: yaml</span>
<span class="sd"> hostnames:</span>
<span class="sd"> remove:</span>
<span class="sd"> - &#39;(.*\\.)?facebook.com$&#39;</span>
<span class="sd"> - ...</span>
<span class="sd">- ``hostnames.high_priority``: A **list** of regular expressions for hostnames</span>
<span class="sd"> whose result should be given higher priority. The results from these hosts are</span>
<span class="sd"> arranged higher in the results list.</span>
<span class="sd"> .. code:: yaml</span>
<span class="sd"> hostnames:</span>
<span class="sd"> high_priority:</span>
<span class="sd"> - &#39;(.*\\.)?wikipedia.org$&#39;</span>
<span class="sd"> - ...</span>
<span class="sd">- ``hostnames.lower_priority``: A **list** of regular expressions for hostnames</span>
<span class="sd"> whose result should be given lower priority. The results from these hosts are</span>
<span class="sd"> arranged lower in the results list.</span>
<span class="sd"> .. code:: yaml</span>
<span class="sd"> hostnames:</span>
<span class="sd"> low_priority:</span>
<span class="sd"> - &#39;(.*\\.)?google(\\..*)?$&#39;</span>
<span class="sd"> - ...</span>
<span class="sd">If the URL matches the pattern of ``high_priority`` AND ``low_priority``, the</span>
<span class="sd">higher priority wins over the lower priority.</span>
<span class="sd">Alternatively, you can also specify a file name for the **mappings** or</span>
<span class="sd">**lists** to load these from an external file:</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> hostnames:</span>
<span class="sd"> replace: &#39;rewrite-hosts.yml&#39;</span>
<span class="sd"> remove:</span>
<span class="sd"> - &#39;(.*\\.)?facebook.com$&#39;</span>
<span class="sd"> - ...</span>
<span class="sd"> low_priority:</span>
<span class="sd"> - &#39;(.*\\.)?google(\\..*)?$&#39;</span>
<span class="sd"> - ...</span>
<span class="sd"> high_priority:</span>
<span class="sd"> - &#39;(.*\\.)?wikipedia.org$&#39;</span>
<span class="sd"> - ...</span>
<span class="sd">The ``rewrite-hosts.yml`` from the example above must be in the folder in which</span>
<span class="sd">the ``settings.yml`` file is already located (``/etc/searxng``). The file then</span>
<span class="sd">only contains the lists or the mapping tables without further information on the</span>
<span class="sd">namespaces. In the example above, this would be a mapping table that looks</span>
<span class="sd">something like this:</span>
<span class="sd">.. code:: yaml</span>
<span class="sd"> &#39;(.*\\.)?youtube\\.com$&#39;: &#39;invidious.example.com&#39;</span>
<span class="sd"> &#39;(.*\\.)?youtu\\.be$&#39;: &#39;invidious.example.com&#39;</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlunparse</span><span class="p">,</span> <span class="n">urlparse</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span> <span class="c1"># pyright: ignore[reportUnknownVariableType]</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">settings</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types._base</span><span class="w"> </span><span class="kn">import</span> <span class="n">MainResult</span><span class="p">,</span> <span class="n">LegacyResult</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.settings_loader</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_yaml_cfg</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">._core</span><span class="w"> </span><span class="kn">import</span> <span class="n">log</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearchWithPlugins</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">Result</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
<span class="n">REPLACE</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="n">re</span><span class="o">.</span><span class="n">Pattern</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">REMOVE</span><span class="p">:</span> <span class="nb">set</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="n">HIGH</span><span class="p">:</span> <span class="nb">set</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="n">LOW</span><span class="p">:</span> <span class="nb">set</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<div class="viewcode-block" id="SXNGPlugin">
<a class="viewcode-back" href="../../../dev/plugins/hostnames.html#searx.plugins.hostnames.SXNGPlugin">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Rewrite hostnames, remove results or prioritize them.&quot;&quot;&quot;</span>
<span class="nb">id</span> <span class="o">=</span> <span class="s2">&quot;hostnames&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">&quot;PluginCfg&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Hostnames plugin&quot;</span><span class="p">),</span>
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Rewrite hostnames and remove or prioritize results based on the hostname&quot;</span><span class="p">),</span>
<span class="n">preference_section</span><span class="o">=</span><span class="s2">&quot;general&quot;</span><span class="p">,</span>
<span class="p">)</span>
<div class="viewcode-block" id="SXNGPlugin.on_result">
<a class="viewcode-back" href="../../../dev/plugins/hostnames.html#searx.plugins.hostnames.SXNGPlugin.on_result">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">on_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="s2">&quot;SXNG_Request&quot;</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">&quot;SearchWithPlugins&quot;</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="s2">&quot;Result&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="k">for</span> <span class="n">pattern</span> <span class="ow">in</span> <span class="n">REMOVE</span><span class="p">:</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="ow">and</span> <span class="n">pattern</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span><span class="p">):</span>
<span class="c1"># if the link (parsed_url) of the result match, then remove the</span>
<span class="c1"># result from the result list, in any other case, the result</span>
<span class="c1"># remains in the list / see final &quot;return True&quot; below.</span>
<span class="c1"># log.debug(&quot;FIXME: remove [url/parsed_url] %s %s&quot;, pattern.pattern, result.url)</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="n">result</span><span class="o">.</span><span class="n">filter_urls</span><span class="p">(</span><span class="n">filter_url_field</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="p">(</span><span class="n">MainResult</span><span class="p">,</span> <span class="n">LegacyResult</span><span class="p">)):</span>
<span class="k">for</span> <span class="n">pattern</span> <span class="ow">in</span> <span class="n">LOW</span><span class="p">:</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="ow">and</span> <span class="n">pattern</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span><span class="p">):</span>
<span class="n">result</span><span class="o">.</span><span class="n">priority</span> <span class="o">=</span> <span class="s2">&quot;low&quot;</span>
<span class="k">for</span> <span class="n">pattern</span> <span class="ow">in</span> <span class="n">HIGH</span><span class="p">:</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="ow">and</span> <span class="n">pattern</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span><span class="p">):</span>
<span class="n">result</span><span class="o">.</span><span class="n">priority</span> <span class="o">=</span> <span class="s2">&quot;high&quot;</span>
<span class="k">return</span> <span class="kc">True</span></div>
<div class="viewcode-block" id="SXNGPlugin.init">
<a class="viewcode-back" href="../../../dev/plugins/hostnames.html#searx.plugins.hostnames.SXNGPlugin.init">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">app</span><span class="p">:</span> <span class="s2">&quot;flask.Flask&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
<span class="k">global</span> <span class="n">REPLACE</span><span class="p">,</span> <span class="n">REMOVE</span><span class="p">,</span> <span class="n">HIGH</span><span class="p">,</span> <span class="n">LOW</span> <span class="c1"># pylint: disable=global-statement</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">):</span>
<span class="c1"># Remove plugin, if there isn&#39;t a &quot;hostnames:&quot; setting</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="n">REPLACE</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_load_regular_expressions</span><span class="p">(</span><span class="s2">&quot;replace&quot;</span><span class="p">)</span> <span class="ow">or</span> <span class="p">{}</span> <span class="c1"># type: ignore</span>
<span class="n">REMOVE</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_load_regular_expressions</span><span class="p">(</span><span class="s2">&quot;remove&quot;</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">set</span><span class="p">()</span> <span class="c1"># type: ignore</span>
<span class="n">HIGH</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_load_regular_expressions</span><span class="p">(</span><span class="s2">&quot;high_priority&quot;</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">set</span><span class="p">()</span> <span class="c1"># type: ignore</span>
<span class="n">LOW</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_load_regular_expressions</span><span class="p">(</span><span class="s2">&quot;low_priority&quot;</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">set</span><span class="p">()</span> <span class="c1"># type: ignore</span>
<span class="k">return</span> <span class="kc">True</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">_load_regular_expressions</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">settings_key</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="n">re</span><span class="o">.</span><span class="n">Pattern</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="nb">set</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">setting_value</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">settings_key</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">setting_value</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="c1"># load external file with configuration</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">setting_value</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="n">setting_value</span> <span class="o">=</span> <span class="n">get_yaml_cfg</span><span class="p">(</span><span class="n">setting_value</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">setting_value</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
<span class="k">return</span> <span class="p">{</span><span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">r</span><span class="p">)</span> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">setting_value</span><span class="p">}</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">setting_value</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">return</span> <span class="p">{</span><span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">p</span><span class="p">):</span> <span class="n">r</span> <span class="k">for</span> <span class="p">(</span><span class="n">p</span><span class="p">,</span> <span class="n">r</span><span class="p">)</span> <span class="ow">in</span> <span class="n">setting_value</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
<span class="k">return</span> <span class="kc">None</span></div>
<div class="viewcode-block" id="filter_url_field">
<a class="viewcode-back" href="../../../dev/plugins/hostnames.html#searx.plugins.hostnames.filter_url_field">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">filter_url_field</span><span class="p">(</span><span class="n">result</span><span class="p">:</span> <span class="s2">&quot;Result|LegacyResult&quot;</span><span class="p">,</span> <span class="n">field_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">url_src</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span> <span class="o">|</span> <span class="nb">str</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns bool ``True`` to use URL unchanged (``False`` to ignore URL).</span>
<span class="sd"> If URL should be modified, the returned string is the new URL to use.&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">url_src</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;missing a URL in field </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">field_name</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="n">url_src_parsed</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="o">=</span><span class="n">url_src</span><span class="p">)</span>
<span class="k">for</span> <span class="n">pattern</span> <span class="ow">in</span> <span class="n">REMOVE</span><span class="p">:</span>
<span class="k">if</span> <span class="n">pattern</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">url_src_parsed</span><span class="o">.</span><span class="n">netloc</span><span class="p">):</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="k">for</span> <span class="n">pattern</span><span class="p">,</span> <span class="n">replacement</span> <span class="ow">in</span> <span class="n">REPLACE</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="n">pattern</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">url_src_parsed</span><span class="o">.</span><span class="n">netloc</span><span class="p">):</span>
<span class="n">new_url</span> <span class="o">=</span> <span class="n">url_src_parsed</span><span class="o">.</span><span class="n">_replace</span><span class="p">(</span><span class="n">netloc</span><span class="o">=</span><span class="n">pattern</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="n">replacement</span><span class="p">,</span> <span class="n">url_src_parsed</span><span class="o">.</span><span class="n">netloc</span><span class="p">))</span>
<span class="n">new_url</span> <span class="o">=</span> <span class="n">urlunparse</span><span class="p">(</span><span class="n">new_url</span><span class="p">)</span>
<span class="k">return</span> <span class="n">new_url</span>
<span class="k">return</span> <span class="kc">True</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,135 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.plugins.infinite_scroll &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.plugins.infinite_scroll</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.plugins.infinite_scroll</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="c1"># pylint: disable=missing-module-docstring</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span> <span class="c1"># pyright: ignore[reportUnknownVariableType]</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
<div class="viewcode-block" id="SXNGPlugin">
<a class="viewcode-back" href="../../../dev/plugins/infinite_scroll.html#searx.plugins.infinite_scroll.SXNGPlugin">[docs]</a>
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Automatically loads the next page when scrolling to bottom of the current page.&quot;&quot;&quot;</span>
<span class="nb">id</span> <span class="o">=</span> <span class="s2">&quot;infiniteScroll&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">&quot;PluginCfg&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Infinite scroll&quot;</span><span class="p">),</span>
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Automatically loads the next page when scrolling to bottom of the current page&quot;</span><span class="p">),</span>
<span class="n">preference_section</span><span class="o">=</span><span class="s2">&quot;ui&quot;</span><span class="p">,</span>
<span class="p">)</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,169 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.plugins.self_info &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.plugins.self_info</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.plugins.self_info</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="c1"># pylint: disable=missing-module-docstring, missing-class-docstring</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="n">ip_address</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
<span class="k">if</span> <span class="n">typing</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearchWithPlugins</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
<div class="viewcode-block" id="SXNGPlugin">
<a class="viewcode-back" href="../../../dev/plugins/self_info.html#searx.plugins.self_info.SXNGPlugin">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Simple plugin that displays information about user&#39;s request, including</span>
<span class="sd"> the IP or HTTP User-Agent. The information is displayed in area for the</span>
<span class="sd"> &quot;answers&quot;.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="nb">id</span> <span class="o">=</span> <span class="s2">&quot;self_info&quot;</span>
<span class="n">keywords</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;ip&quot;</span><span class="p">,</span> <span class="s2">&quot;user-agent&quot;</span><span class="p">]</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">&quot;PluginCfg&quot;</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">ip_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;^ip&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">IGNORECASE</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">ua_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;^user-agent&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">IGNORECASE</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Self Information&quot;</span><span class="p">),</span>
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Displays your IP if the query is &quot;ip&quot; and your user agent if the query is &quot;user-agent&quot;.&quot;&quot;&quot;</span>
<span class="p">),</span>
<span class="n">preference_section</span><span class="o">=</span><span class="s2">&quot;query&quot;</span><span class="p">,</span>
<span class="p">)</span>
<div class="viewcode-block" id="SXNGPlugin.post_search">
<a class="viewcode-back" href="../../../dev/plugins/self_info.html#searx.plugins.self_info.SXNGPlugin.post_search">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">post_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="s2">&quot;SXNG_Request&quot;</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">&quot;SearchWithPlugins&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Returns a result list only for the first page.&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="k">if</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">pageno</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ip_regex</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span><span class="p">)</span> <span class="ow">and</span> <span class="n">request</span><span class="o">.</span><span class="n">remote_addr</span><span class="p">:</span>
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
<span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Your IP is: &quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">remote_addr</span><span class="p">)</span><span class="o">.</span><span class="n">compressed</span><span class="p">)</span>
<span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ua_regex</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span><span class="p">):</span>
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Your user-agent is: &quot;</span><span class="p">)</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">user_agent</span><span class="p">)))</span>
<span class="k">return</span> <span class="n">results</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,179 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.plugins.time_zone &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.plugins.time_zone</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.plugins.time_zone</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="c1"># pylint: disable=missing-module-docstring</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">datetime</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.weather</span><span class="w"> </span><span class="kn">import</span> <span class="n">DateTime</span><span class="p">,</span> <span class="n">GeoLocation</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearchWithPlugins</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
<div class="viewcode-block" id="SXNGPlugin">
<a class="viewcode-back" href="../../../dev/plugins/time_zone.html#searx.plugins.time_zone.SXNGPlugin">[docs]</a>
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Plugin to display the current time at different timezones (usually the</span>
<span class="sd"> query city).&quot;&quot;&quot;</span>
<span class="nb">id</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;time_zone&quot;</span>
<span class="n">keywords</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;time&quot;</span><span class="p">,</span> <span class="s2">&quot;timezone&quot;</span><span class="p">,</span> <span class="s2">&quot;now&quot;</span><span class="p">,</span> <span class="s2">&quot;clock&quot;</span><span class="p">,</span> <span class="s2">&quot;timezones&quot;</span><span class="p">]</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">&quot;PluginCfg&quot;</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Timezones plugin&quot;</span><span class="p">),</span>
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Display the current time on different time zones.&quot;</span><span class="p">),</span>
<span class="n">preference_section</span><span class="o">=</span><span class="s2">&quot;query&quot;</span><span class="p">,</span>
<span class="n">examples</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;time Berlin&quot;</span><span class="p">,</span> <span class="s2">&quot;clock Los Angeles&quot;</span><span class="p">],</span>
<span class="p">)</span>
<div class="viewcode-block" id="SXNGPlugin.post_search">
<a class="viewcode-back" href="../../../dev/plugins/time_zone.html#searx.plugins.time_zone.SXNGPlugin.post_search">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">post_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="s2">&quot;SXNG_Request&quot;</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">&quot;SearchWithPlugins&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The plugin uses the :py:obj:`searx.weather.GeoLocation` class, which</span>
<span class="sd"> is already implemented in the context of weather forecasts, to determine</span>
<span class="sd"> the time zone. The :py:obj:`searx.weather.DateTime` class is used for</span>
<span class="sd"> the localized display of date and time.&quot;&quot;&quot;</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="k">if</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">pageno</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="c1"># remove keywords from the query</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span>
<span class="n">query_parts</span> <span class="o">=</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">part</span><span class="p">:</span> <span class="n">part</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">keywords</span><span class="p">,</span> <span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">))</span>
<span class="n">search_term</span> <span class="o">=</span> <span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">query_parts</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">search_term</span><span class="p">:</span>
<span class="n">date_time</span> <span class="o">=</span> <span class="n">DateTime</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">())</span>
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="n">date_time</span><span class="o">.</span><span class="n">l10n</span><span class="p">()))</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">geo</span> <span class="o">=</span> <span class="n">GeoLocation</span><span class="o">.</span><span class="n">by_query</span><span class="p">(</span><span class="n">search_term</span><span class="o">=</span><span class="n">search_term</span><span class="p">)</span>
<span class="k">if</span> <span class="n">geo</span><span class="p">:</span>
<span class="n">date_time</span> <span class="o">=</span> <span class="n">DateTime</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">tz</span><span class="o">=</span><span class="n">geo</span><span class="o">.</span><span class="n">zoneinfo</span><span class="p">))</span>
<span class="n">tz_name</span> <span class="o">=</span> <span class="n">geo</span><span class="o">.</span><span class="n">timezone</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">,</span> <span class="s1">&#39; &#39;</span><span class="p">)</span>
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
<span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span>
<span class="n">answer</span><span class="o">=</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">tz_name</span><span class="si">}</span><span class="s2">:&quot;</span> <span class="sa">f</span><span class="s2">&quot; </span><span class="si">{</span><span class="n">date_time</span><span class="o">.</span><span class="n">l10n</span><span class="p">()</span><span class="si">}</span><span class="s2"> (</span><span class="si">{</span><span class="n">date_time</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">&#39;%Z&#39;</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span><span class="p">)</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">results</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,188 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.plugins.tor_check &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.plugins.tor_check</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.plugins.tor_check</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;A plugin to check if the ip address of the request is a Tor exit-node if the</span>
<span class="sd">user searches for ``tor-check``. It fetches the tor exit node list from</span>
<span class="sd">:py:obj:`url_exit_list` and parses all the IPs into a list, then checks if the</span>
<span class="sd">user&#39;s IP address is in it.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="n">ip_address</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">httpx</span><span class="w"> </span><span class="kn">import</span> <span class="n">HTTPError</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="k">if</span> <span class="n">typing</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearchWithPlugins</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
<span class="c1"># Regex for exit node addresses in the list.</span>
<span class="n">reg</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;(?&lt;=ExitAddress )\S+&quot;</span><span class="p">)</span>
<span class="n">url_exit_list</span> <span class="o">=</span> <span class="s2">&quot;https://check.torproject.org/exit-addresses&quot;</span>
<span class="sd">&quot;&quot;&quot;URL to load Tor exit list from.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="SXNGPlugin">
<a class="viewcode-back" href="../../../dev/plugins/tor_check.html#searx.plugins.tor_check.SXNGPlugin">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Rewrite hostnames, remove results or prioritize them.&quot;&quot;&quot;</span>
<span class="nb">id</span> <span class="o">=</span> <span class="s2">&quot;tor_check&quot;</span>
<span class="n">keywords</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;tor-check&quot;</span><span class="p">,</span> <span class="s2">&quot;tor_check&quot;</span><span class="p">,</span> <span class="s2">&quot;torcheck&quot;</span><span class="p">,</span> <span class="s2">&quot;tor&quot;</span><span class="p">,</span> <span class="s2">&quot;tor check&quot;</span><span class="p">]</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">&quot;PluginCfg&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Tor check plugin&quot;</span><span class="p">),</span>
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span>
<span class="s2">&quot;This plugin checks if the address of the request is a Tor exit-node, and&quot;</span>
<span class="s2">&quot; informs the user if it is; like check.torproject.org, but from SearXNG.&quot;</span>
<span class="p">),</span>
<span class="n">preference_section</span><span class="o">=</span><span class="s2">&quot;query&quot;</span><span class="p">,</span>
<span class="p">)</span>
<div class="viewcode-block" id="SXNGPlugin.post_search">
<a class="viewcode-back" href="../../../dev/plugins/tor_check.html#searx.plugins.tor_check.SXNGPlugin.post_search">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">post_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="s2">&quot;SXNG_Request&quot;</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">&quot;SearchWithPlugins&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="k">if</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">pageno</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">if</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">keywords</span><span class="p">:</span>
<span class="c1"># Request the list of tor exit nodes.</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url_exit_list</span><span class="p">)</span>
<span class="n">node_list</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">reg</span><span class="p">,</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="c1"># type: ignore</span>
<span class="k">except</span> <span class="n">HTTPError</span><span class="p">:</span>
<span class="c1"># No answer, return error</span>
<span class="n">msg</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Could not download the list of Tor exit-nodes from&quot;</span><span class="p">)</span>
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">msg</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">url_exit_list</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">))</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">real_ip</span> <span class="o">=</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">address</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">remote_addr</span><span class="p">))</span><span class="o">.</span><span class="n">compressed</span>
<span class="k">if</span> <span class="n">real_ip</span> <span class="ow">in</span> <span class="n">node_list</span><span class="p">:</span>
<span class="n">msg</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;You are using Tor and it looks like you have the external IP address&quot;</span><span class="p">)</span>
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">msg</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">real_ip</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">msg</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;You are not using Tor and you have the external IP address&quot;</span><span class="p">)</span>
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">msg</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">real_ip</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">))</span>
<span class="k">return</span> <span class="n">results</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,260 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.plugins.unit_converter &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.plugins.unit_converter</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.plugins.unit_converter</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;A plugin for converting measured values from one unit to another unit (a</span>
<span class="sd">unit converter).</span>
<span class="sd">The plugin looks up the symbols (given in the query term) in a list of</span>
<span class="sd">converters, each converter is one item in the list (compare</span>
<span class="sd">:py:obj:`ADDITIONAL_UNITS`). If the symbols are ambiguous, the matching units</span>
<span class="sd">of measurement are evaluated. The weighting in the evaluation results from the</span>
<span class="sd">sorting of the :py:obj:`list of unit converters&lt;symbol_to_si&gt;`.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.numbers</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span><span class="p">,</span> <span class="n">get_locale</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.wikidata_units</span><span class="w"> </span><span class="kn">import</span> <span class="n">symbol_to_si</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
<span class="k">if</span> <span class="n">typing</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearchWithPlugins</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
<span class="n">CONVERT_KEYWORDS</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;in&quot;</span><span class="p">,</span> <span class="s2">&quot;to&quot;</span><span class="p">,</span> <span class="s2">&quot;as&quot;</span><span class="p">]</span>
<div class="viewcode-block" id="SXNGPlugin">
<a class="viewcode-back" href="../../../dev/plugins/unit_converter.html#searx.plugins.unit_converter.SXNGPlugin">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Convert between units. The result is displayed in area for the</span>
<span class="sd"> &quot;answers&quot;.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="nb">id</span> <span class="o">=</span> <span class="s2">&quot;unit_converter&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">&quot;PluginCfg&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Unit converter plugin&quot;</span><span class="p">),</span>
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;Convert between units&quot;</span><span class="p">),</span>
<span class="n">preference_section</span><span class="o">=</span><span class="s2">&quot;general&quot;</span><span class="p">,</span>
<span class="p">)</span>
<div class="viewcode-block" id="SXNGPlugin.post_search">
<a class="viewcode-back" href="../../../dev/plugins/unit_converter.html#searx.plugins.unit_converter.SXNGPlugin.post_search">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">post_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="s2">&quot;SXNG_Request&quot;</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">&quot;SearchWithPlugins&quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EngineResults</span><span class="p">:</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
<span class="c1"># only convert between units on the first page</span>
<span class="k">if</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">pageno</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span>
<span class="n">query_parts</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">query_parts</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">3</span><span class="p">:</span>
<span class="k">return</span> <span class="n">results</span>
<span class="k">for</span> <span class="n">query_part</span> <span class="ow">in</span> <span class="n">query_parts</span><span class="p">:</span>
<span class="k">for</span> <span class="n">keyword</span> <span class="ow">in</span> <span class="n">CONVERT_KEYWORDS</span><span class="p">:</span>
<span class="k">if</span> <span class="n">query_part</span> <span class="o">==</span> <span class="n">keyword</span><span class="p">:</span>
<span class="n">from_query</span><span class="p">,</span> <span class="n">to_query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">keyword</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">target_val</span> <span class="o">=</span> <span class="n">_parse_text_and_convert</span><span class="p">(</span><span class="n">from_query</span><span class="o">.</span><span class="n">strip</span><span class="p">(),</span> <span class="n">to_query</span><span class="o">.</span><span class="n">strip</span><span class="p">())</span>
<span class="k">if</span> <span class="n">target_val</span><span class="p">:</span>
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="n">target_val</span><span class="p">))</span>
<span class="k">return</span> <span class="n">results</span></div>
</div>
<span class="c1"># inspired from https://stackoverflow.com/a/42475086</span>
<span class="n">RE_MEASURE</span> <span class="o">=</span> <span class="sa">r</span><span class="s1">&#39;&#39;&#39;</span>
<span class="s1">(?P&lt;sign&gt;[-+]?) # +/- or nothing for positive</span>
<span class="s1">(\s*) # separator: white space or nothing</span>
<span class="s1">(?P&lt;number&gt;[\d\.,]*) # number: 1,000.00 (en) or 1.000,00 (de)</span>
<span class="s1">(?P&lt;E&gt;[eE][-+]?\d+)? # scientific notation: e(+/-)2 (*10^2)</span>
<span class="s1">(\s*) # separator: white space or nothing</span>
<span class="s1">(?P&lt;unit&gt;\S+) # unit of measure</span>
<span class="s1">&#39;&#39;&#39;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_text_and_convert</span><span class="p">(</span><span class="n">from_query</span><span class="p">,</span> <span class="n">to_query</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="c1"># pylint: disable=too-many-branches, too-many-locals</span>
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">from_query</span> <span class="ow">and</span> <span class="n">to_query</span><span class="p">):</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">measured</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">RE_MEASURE</span><span class="p">,</span> <span class="n">from_query</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">VERBOSE</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">measured</span> <span class="ow">and</span> <span class="n">measured</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="s1">&#39;number&#39;</span><span class="p">),</span> <span class="n">measured</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="s1">&#39;unit&#39;</span><span class="p">)):</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="c1"># Symbols are not unique, if there are several hits for the from-unit, then</span>
<span class="c1"># the correct one must be determined by comparing it with the to-unit</span>
<span class="c1"># https://github.com/searxng/searxng/pull/3378#issuecomment-2080974863</span>
<span class="c1"># first: collecting possible units</span>
<span class="n">source_list</span><span class="p">,</span> <span class="n">target_list</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">symbol</span><span class="p">,</span> <span class="n">si_name</span><span class="p">,</span> <span class="n">from_si</span><span class="p">,</span> <span class="n">to_si</span><span class="p">,</span> <span class="n">orig_symbol</span> <span class="ow">in</span> <span class="n">symbol_to_si</span><span class="p">():</span>
<span class="k">if</span> <span class="n">symbol</span> <span class="o">==</span> <span class="n">measured</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="s1">&#39;unit&#39;</span><span class="p">):</span>
<span class="n">source_list</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">si_name</span><span class="p">,</span> <span class="n">to_si</span><span class="p">))</span>
<span class="k">if</span> <span class="n">symbol</span> <span class="o">==</span> <span class="n">to_query</span><span class="p">:</span>
<span class="n">target_list</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">si_name</span><span class="p">,</span> <span class="n">from_si</span><span class="p">,</span> <span class="n">orig_symbol</span><span class="p">))</span>
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">source_list</span> <span class="ow">and</span> <span class="n">target_list</span><span class="p">):</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">source_to_si</span> <span class="o">=</span> <span class="n">target_from_si</span> <span class="o">=</span> <span class="n">target_symbol</span> <span class="o">=</span> <span class="kc">None</span>
<span class="c1"># second: find the right unit by comparing list of from-units with list of to-units</span>
<span class="k">for</span> <span class="n">source</span> <span class="ow">in</span> <span class="n">source_list</span><span class="p">:</span>
<span class="k">for</span> <span class="n">target</span> <span class="ow">in</span> <span class="n">target_list</span><span class="p">:</span>
<span class="k">if</span> <span class="n">source</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="n">target</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span> <span class="c1"># compare si_name</span>
<span class="n">source_to_si</span> <span class="o">=</span> <span class="n">source</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="n">target_from_si</span> <span class="o">=</span> <span class="n">target</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="n">target_symbol</span> <span class="o">=</span> <span class="n">target</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span>
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">source_to_si</span> <span class="ow">and</span> <span class="n">target_from_si</span><span class="p">):</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="n">_locale</span> <span class="o">=</span> <span class="n">get_locale</span><span class="p">()</span> <span class="ow">or</span> <span class="s1">&#39;en_US&#39;</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">measured</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="s1">&#39;sign&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="n">measured</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="s1">&#39;number&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="p">(</span><span class="n">measured</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="s1">&#39;E&#39;</span><span class="p">)</span> <span class="ow">or</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">numbers</span><span class="o">.</span><span class="n">parse_decimal</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">_locale</span><span class="p">)</span>
<span class="c1"># convert value to SI unit</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">source_to_si</span><span class="p">,</span> <span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="nb">int</span><span class="p">)):</span>
<span class="n">value</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">value</span><span class="p">)</span> <span class="o">*</span> <span class="n">source_to_si</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">source_to_si</span><span class="p">(</span><span class="nb">float</span><span class="p">(</span><span class="n">value</span><span class="p">))</span>
<span class="c1"># convert value from SI unit to target unit</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">target_from_si</span><span class="p">,</span> <span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="nb">int</span><span class="p">)):</span>
<span class="n">value</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">value</span><span class="p">)</span> <span class="o">*</span> <span class="n">target_from_si</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">target_from_si</span><span class="p">(</span><span class="nb">float</span><span class="p">(</span><span class="n">value</span><span class="p">))</span>
<span class="k">if</span> <span class="n">measured</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="s1">&#39;E&#39;</span><span class="p">):</span>
<span class="c1"># when incoming notation is scientific, outgoing notation is scientific</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">numbers</span><span class="o">.</span><span class="n">format_scientific</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">_locale</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">numbers</span><span class="o">.</span><span class="n">format_decimal</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">_locale</span><span class="p">,</span> <span class="nb">format</span><span class="o">=</span><span class="s1">&#39;#,##0.##########;-#&#39;</span><span class="p">)</span>
<span class="k">return</span> <span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">result</span><span class="si">}</span><span class="s1"> </span><span class="si">{</span><span class="n">target_symbol</span><span class="si">}</span><span class="s1">&#39;</span>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,194 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.result_types &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.result_types</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.result_types</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Typification of the result items generated by the *engines*, *answerers* and</span>
<span class="sd">*plugins*.</span>
<span class="sd">.. note::</span>
<span class="sd"> We are at the beginning of typing the results. Further typing will follow,</span>
<span class="sd"> but this is a very large task that we will only be able to implement</span>
<span class="sd"> gradually. For more, please read :ref:`result types`.</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-few-public-methods</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span>
<span class="s2">&quot;Result&quot;</span><span class="p">,</span>
<span class="s2">&quot;MainResult&quot;</span><span class="p">,</span>
<span class="s2">&quot;KeyValue&quot;</span><span class="p">,</span>
<span class="s2">&quot;EngineResults&quot;</span><span class="p">,</span>
<span class="s2">&quot;AnswerSet&quot;</span><span class="p">,</span>
<span class="s2">&quot;Answer&quot;</span><span class="p">,</span>
<span class="s2">&quot;Translations&quot;</span><span class="p">,</span>
<span class="s2">&quot;WeatherAnswer&quot;</span><span class="p">,</span>
<span class="s2">&quot;Code&quot;</span><span class="p">,</span>
<span class="s2">&quot;Paper&quot;</span><span class="p">,</span>
<span class="s2">&quot;File&quot;</span><span class="p">,</span>
<span class="p">]</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">abc</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">._base</span><span class="w"> </span><span class="kn">import</span> <span class="n">Result</span><span class="p">,</span> <span class="n">MainResult</span><span class="p">,</span> <span class="n">LegacyResult</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.answer</span><span class="w"> </span><span class="kn">import</span> <span class="n">AnswerSet</span><span class="p">,</span> <span class="n">Answer</span><span class="p">,</span> <span class="n">Translations</span><span class="p">,</span> <span class="n">WeatherAnswer</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.keyvalue</span><span class="w"> </span><span class="kn">import</span> <span class="n">KeyValue</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.code</span><span class="w"> </span><span class="kn">import</span> <span class="n">Code</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.paper</span><span class="w"> </span><span class="kn">import</span> <span class="n">Paper</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">.file</span><span class="w"> </span><span class="kn">import</span> <span class="n">File</span>
<div class="viewcode-block" id="ResultList">
<a class="viewcode-back" href="../../dev/engines/index.html#searx.result_types.ResultList">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">ResultList</span><span class="p">(</span><span class="nb">list</span><span class="p">[</span><span class="n">Result</span> <span class="o">|</span> <span class="n">LegacyResult</span><span class="p">],</span> <span class="n">abc</span><span class="o">.</span><span class="n">ABC</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Base class of all result lists (abstract).&quot;&quot;&quot;</span>
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">types</span><span class="p">:</span> <span class="c1"># pylint: disable=invalid-name</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The collection of result types (which have already been</span>
<span class="sd"> implemented).&quot;&quot;&quot;</span>
<span class="n">Answer</span> <span class="o">=</span> <span class="n">Answer</span>
<span class="n">KeyValue</span> <span class="o">=</span> <span class="n">KeyValue</span>
<span class="n">Code</span> <span class="o">=</span> <span class="n">Code</span>
<span class="n">Paper</span> <span class="o">=</span> <span class="n">Paper</span>
<span class="n">File</span> <span class="o">=</span> <span class="n">File</span>
<span class="n">MainResult</span> <span class="o">=</span> <span class="n">MainResult</span>
<span class="n">Result</span> <span class="o">=</span> <span class="n">Result</span>
<span class="n">Translations</span> <span class="o">=</span> <span class="n">Translations</span>
<span class="n">WeatherAnswer</span> <span class="o">=</span> <span class="n">WeatherAnswer</span>
<span class="c1"># for backward compatibility</span>
<span class="n">LegacyResult</span> <span class="o">=</span> <span class="n">LegacyResult</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="c1"># pylint: disable=useless-parent-delegation</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">add</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">Result</span> <span class="o">|</span> <span class="n">LegacyResult</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Add a :py:`Result` item to the result list.&quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">result</span><span class="p">)</span></div>
<div class="viewcode-block" id="EngineResults">
<a class="viewcode-back" href="../../dev/engines/index.html#searx.result_types.EngineResults">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">EngineResults</span><span class="p">(</span><span class="n">ResultList</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Result list that should be used by engine developers. For convenience,</span>
<span class="sd"> engine developers don&#39;t need to import types / see :py:obj:`ResultList.types`.</span>
<span class="sd"> .. code:: python</span>
<span class="sd"> from searx.result_types import EngineResults</span>
<span class="sd"> ...</span>
<span class="sd"> def response(resp) -&gt; EngineResults:</span>
<span class="sd"> res = EngineResults()</span>
<span class="sd"> ...</span>
<span class="sd"> res.add( res.types.Answer(answer=&quot;lorem ipsum ..&quot;, url=&quot;https://example.org&quot;) )</span>
<span class="sd"> ...</span>
<span class="sd"> return res</span>
<span class="sd"> &quot;&quot;&quot;</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../index.html">
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../index.html">Overview</a>
<ul>
<li><a href="../index.html">Module code</a>
</ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,704 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.result_types._base &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../result_types.html" accesskey="U">searx.result_types</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.result_types._base</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.result_types._base</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="c1"># pylint: disable=too-few-public-methods, missing-module-docstring</span>
<span class="sd">&quot;&quot;&quot;Basic types for the typification of results.</span>
<span class="sd">- :py:obj:`Result` base class</span>
<span class="sd">- :py:obj:`LegacyResult` for internal use only</span>
<span class="sd">----</span>
<span class="sd">.. autoclass:: Result</span>
<span class="sd"> :members:</span>
<span class="sd">.. _LegacyResult:</span>
<span class="sd">.. autoclass:: LegacyResult</span>
<span class="sd"> :members:</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;Result&quot;</span><span class="p">]</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">urllib.parse</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">warnings</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">datetime</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">collections.abc</span><span class="w"> </span><span class="kn">import</span> <span class="n">Callable</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">msgspec</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span> <span class="k">as</span> <span class="n">log</span>
<span class="n">WHITESPACE_REGEX</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="s1">&#39;( |</span><span class="se">\t</span><span class="s1">|</span><span class="se">\n</span><span class="s1">)+&#39;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">M</span> <span class="o">|</span> <span class="n">re</span><span class="o">.</span><span class="n">U</span><span class="p">)</span>
<span class="n">UNKNOWN</span> <span class="o">=</span> <span class="nb">object</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_normalize_url_fields</span><span class="p">(</span><span class="n">result</span><span class="p">:</span> <span class="s2">&quot;Result | LegacyResult&quot;</span><span class="p">):</span>
<span class="c1"># As soon we need LegacyResult not any longer, we can move this function to</span>
<span class="c1"># method Result.normalize_result_fields</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">url</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;result: invalid URL: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">))</span>
<span class="n">result</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">url</span><span class="p">)</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="p">:</span>
<span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">_replace</span><span class="p">(</span>
<span class="c1"># if the result has no scheme, use http as default</span>
<span class="n">scheme</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">scheme</span> <span class="ow">or</span> <span class="s2">&quot;http&quot;</span><span class="p">,</span>
<span class="n">path</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">result</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">geturl</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">LegacyResult</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;infobox&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span>
<span class="c1"># As soon we have InfoboxResult, we can move this function to method</span>
<span class="c1"># InfoboxResult.normalize_result_fields</span>
<span class="n">infobox_urls</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;urls&quot;</span><span class="p">,</span> <span class="p">[])</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">infobox_urls</span><span class="p">:</span>
<span class="n">_url</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;url&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">_url</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">_url</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">_url</span><span class="p">)</span>
<span class="n">item</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">_url</span><span class="o">.</span><span class="n">_replace</span><span class="p">(</span>
<span class="n">scheme</span><span class="o">=</span><span class="n">_url</span><span class="o">.</span><span class="n">scheme</span> <span class="ow">or</span> <span class="s2">&quot;http&quot;</span><span class="p">,</span>
<span class="c1"># netloc=_url.netloc.replace(&quot;www.&quot;, &quot;&quot;),</span>
<span class="n">path</span><span class="o">=</span><span class="n">_url</span><span class="o">.</span><span class="n">path</span><span class="p">,</span>
<span class="p">)</span><span class="o">.</span><span class="n">geturl</span><span class="p">()</span>
<span class="n">infobox_id</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;id&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="n">infobox_id</span><span class="p">:</span>
<span class="n">_url</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">infobox_id</span><span class="p">)</span>
<span class="n">result</span><span class="o">.</span><span class="n">id</span> <span class="o">=</span> <span class="n">_url</span><span class="o">.</span><span class="n">_replace</span><span class="p">(</span>
<span class="n">scheme</span><span class="o">=</span><span class="n">_url</span><span class="o">.</span><span class="n">scheme</span> <span class="ow">or</span> <span class="s2">&quot;http&quot;</span><span class="p">,</span>
<span class="c1"># netloc=_url.netloc.replace(&quot;www.&quot;, &quot;&quot;),</span>
<span class="n">path</span><span class="o">=</span><span class="n">_url</span><span class="o">.</span><span class="n">path</span><span class="p">,</span>
<span class="p">)</span><span class="o">.</span><span class="n">geturl</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_normalize_text_fields</span><span class="p">(</span><span class="n">result</span><span class="p">:</span> <span class="s2">&quot;MainResult | LegacyResult&quot;</span><span class="p">):</span>
<span class="c1"># As soon we need LegacyResult not any longer, we can move this function to</span>
<span class="c1"># method MainResult.normalize_result_fields</span>
<span class="c1"># Actually, a type check should not be necessary if the engine is</span>
<span class="c1"># implemented correctly. Historically, however, we have always had a type</span>
<span class="c1"># check here.</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">title</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">title</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;result: invalid type of field &#39;title&#39;: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">))</span>
<span class="n">result</span><span class="o">.</span><span class="n">title</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">content</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">content</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;result: invalid type of field &#39;content&#39;: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">))</span>
<span class="n">result</span><span class="o">.</span><span class="n">content</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
<span class="c1"># normalize title and content</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">title</span><span class="p">:</span>
<span class="n">result</span><span class="o">.</span><span class="n">title</span> <span class="o">=</span> <span class="n">WHITESPACE_REGEX</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="n">result</span><span class="o">.</span><span class="n">title</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">content</span><span class="p">:</span>
<span class="n">result</span><span class="o">.</span><span class="n">content</span> <span class="o">=</span> <span class="n">WHITESPACE_REGEX</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="n">result</span><span class="o">.</span><span class="n">content</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">content</span> <span class="o">==</span> <span class="n">result</span><span class="o">.</span><span class="n">title</span><span class="p">:</span>
<span class="c1"># avoid duplicate content between the content and title fields</span>
<span class="n">result</span><span class="o">.</span><span class="n">content</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_filter_urls</span><span class="p">(</span>
<span class="n">result</span><span class="p">:</span> <span class="s2">&quot;Result | LegacyResult&quot;</span><span class="p">,</span> <span class="n">filter_func</span><span class="p">:</span> <span class="s2">&quot;Callable[[Result | LegacyResult, str, str], str | bool]&quot;</span>
<span class="p">):</span>
<span class="c1"># pylint: disable=too-many-branches, too-many-statements</span>
<span class="c1"># As soon we need LegacyResult not any longer, we can move this function to</span>
<span class="c1"># method Result.</span>
<span class="n">url_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">,</span> <span class="s2">&quot;iframe_src&quot;</span><span class="p">,</span> <span class="s2">&quot;audio_src&quot;</span><span class="p">,</span> <span class="s2">&quot;img_src&quot;</span><span class="p">,</span> <span class="s2">&quot;thumbnail_src&quot;</span><span class="p">,</span> <span class="s2">&quot;thumbnail&quot;</span><span class="p">]</span>
<span class="n">url_src</span><span class="p">:</span> <span class="nb">str</span>
<span class="k">for</span> <span class="n">field_name</span> <span class="ow">in</span> <span class="n">url_fields</span><span class="p">:</span>
<span class="n">url_src</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">url_src</span><span class="p">:</span>
<span class="k">continue</span>
<span class="n">new_url</span> <span class="o">=</span> <span class="n">filter_func</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="n">url_src</span><span class="p">)</span>
<span class="c1"># log.debug(&quot;filter_urls: filter_func(result, %s) &#39;%s&#39; -&gt; &#39;%s&#39;&quot;, field_name, field_value, new_url)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">new_url</span><span class="p">,</span> <span class="nb">bool</span><span class="p">):</span>
<span class="k">if</span> <span class="n">new_url</span><span class="p">:</span>
<span class="c1"># log.debug(&quot;filter_urls: unchanged field %s URL %s&quot;, field_name, field_value)</span>
<span class="k">continue</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;filter_urls: drop field </span><span class="si">%s</span><span class="s2"> URL </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="n">url_src</span><span class="p">)</span>
<span class="n">new_url</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;filter_urls: modify field </span><span class="si">%s</span><span class="s2"> URL </span><span class="si">%s</span><span class="s2"> -&gt; </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="n">url_src</span><span class="p">,</span> <span class="n">new_url</span><span class="p">)</span>
<span class="nb">setattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="n">new_url</span><span class="p">)</span>
<span class="k">if</span> <span class="n">field_name</span> <span class="o">==</span> <span class="s2">&quot;url&quot;</span><span class="p">:</span>
<span class="c1"># sync parsed_url with new_url</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">new_url</span><span class="p">:</span>
<span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">new_url</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">new_url</span><span class="p">)</span>
<span class="c1"># &quot;urls&quot;: are from infobox</span>
<span class="c1">#</span>
<span class="c1"># As soon we have InfoboxResult, we can move this function to method</span>
<span class="c1"># InfoboxResult.normalize_result_fields</span>
<span class="n">infobox_urls</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;urls&quot;</span><span class="p">,</span> <span class="p">[])</span>
<span class="k">if</span> <span class="n">infobox_urls</span><span class="p">:</span>
<span class="c1"># log.debug(&quot;filter_urls: infobox_urls .. %s&quot;, infobox_urls)</span>
<span class="n">new_infobox_urls</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">infobox_urls</span><span class="p">:</span>
<span class="n">url_src</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;url&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">url_src</span><span class="p">:</span>
<span class="n">new_infobox_urls</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">continue</span>
<span class="n">new_url</span> <span class="o">=</span> <span class="n">filter_func</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;infobox_urls&quot;</span><span class="p">,</span> <span class="n">url_src</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">new_url</span><span class="p">,</span> <span class="nb">bool</span><span class="p">):</span>
<span class="k">if</span> <span class="n">new_url</span><span class="p">:</span>
<span class="n">new_infobox_urls</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="c1"># log.debug(&quot;filter_urls: leave URL in field &#39;urls&#39; (&#39;infobox_urls&#39;) unchanged -&gt; %s&quot;, _url)</span>
<span class="k">continue</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;filter_urls: remove URL from field &#39;urls&#39; (&#39;infobox_urls&#39;) URL </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">url_src</span><span class="p">)</span>
<span class="n">new_url</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">new_url</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;filter_urls: modify URL from field &#39;urls&#39; (&#39;infobox_urls&#39;) URL </span><span class="si">%s</span><span class="s2"> -&gt; </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">url_src</span><span class="p">,</span> <span class="n">new_url</span><span class="p">)</span>
<span class="n">item</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_url</span>
<span class="n">new_infobox_urls</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="nb">setattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;urls&quot;</span><span class="p">,</span> <span class="n">new_infobox_urls</span><span class="p">)</span>
<span class="c1"># &quot;attributes&quot;: are from infobox</span>
<span class="c1">#</span>
<span class="c1"># The infobox has additional subsections for attributes, urls and relatedTopics:</span>
<span class="n">infobox_attributes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;attributes&quot;</span><span class="p">,</span> <span class="p">[])</span>
<span class="k">if</span> <span class="n">infobox_attributes</span><span class="p">:</span>
<span class="c1"># log.debug(&quot;filter_urls: infobox_attributes .. %s&quot;, infobox_attributes)</span>
<span class="n">new_infobox_attributes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]]]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">infobox_attributes</span><span class="p">:</span>
<span class="n">image</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;image&quot;</span><span class="p">,</span> <span class="p">{})</span>
<span class="n">url_src</span> <span class="o">=</span> <span class="n">image</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;src&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">url_src</span><span class="p">:</span>
<span class="n">new_infobox_attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">continue</span>
<span class="n">new_url</span> <span class="o">=</span> <span class="n">filter_func</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;infobox_attributes&quot;</span><span class="p">,</span> <span class="n">url_src</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">new_url</span><span class="p">,</span> <span class="nb">bool</span><span class="p">):</span>
<span class="k">if</span> <span class="n">new_url</span><span class="p">:</span>
<span class="n">new_infobox_attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="c1"># log.debug(&quot;filter_urls: leave URL in field &#39;image.src&#39; unchanged -&gt; %s&quot;, url_src)</span>
<span class="k">continue</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;filter_urls: drop field &#39;image.src&#39; (&#39;infobox_attributes&#39;) URL </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">url_src</span><span class="p">)</span>
<span class="n">new_url</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">new_url</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
<span class="s2">&quot;filter_urls: modify &#39;image.src&#39; (&#39;infobox_attributes&#39;) URL </span><span class="si">%s</span><span class="s2"> -&gt; </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span>
<span class="n">url_src</span><span class="p">,</span>
<span class="n">new_url</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">item</span><span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">][</span><span class="s2">&quot;src&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_url</span>
<span class="n">new_infobox_attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="nb">setattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">&quot;attributes&quot;</span><span class="p">,</span> <span class="n">new_infobox_attributes</span><span class="p">)</span>
<span class="n">result</span><span class="o">.</span><span class="n">normalize_result_fields</span><span class="p">()</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_normalize_date_fields</span><span class="p">(</span><span class="n">result</span><span class="p">:</span> <span class="s2">&quot;MainResult | LegacyResult&quot;</span><span class="p">):</span>
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">publishedDate</span><span class="p">:</span> <span class="c1"># do not try to get a date from an empty string or a None type</span>
<span class="k">try</span><span class="p">:</span> <span class="c1"># test if publishedDate &gt;= 1900 (datetime module bug)</span>
<span class="n">result</span><span class="o">.</span><span class="n">pubdate</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">publishedDate</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1"> %H:%M:%S%z&#39;</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="n">result</span><span class="o">.</span><span class="n">publishedDate</span> <span class="o">=</span> <span class="kc">None</span>
<div class="viewcode-block" id="Result">
<a class="viewcode-back" href="../../../dev/result_types/base_result.html#searx.result_types._base.Result">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">Result</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Base class of all result types :ref:`result types`.&quot;&quot;&quot;</span>
<span class="n">url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A link related to this *result*&quot;&quot;&quot;</span>
<span class="n">engine</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Name of the engine *this* result comes from. In case of *plugins* a</span>
<span class="sd"> prefix ``plugin:`` is set, in case of *answerer* prefix ``answerer:`` is</span>
<span class="sd"> set.</span>
<span class="sd"> The field is optional and is initialized from the context if necessary.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">parsed_url</span><span class="p">:</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">ParseResult</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;:py:obj:`urllib.parse.ParseResult` of :py:obj:`Result.url`.</span>
<span class="sd"> The field is optional and is initialized from the context if necessary.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<div class="viewcode-block" id="Result.normalize_result_fields">
<a class="viewcode-back" href="../../../dev/result_types/base_result.html#searx.result_types._base.Result.normalize_result_fields">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">normalize_result_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Normalize fields ``url`` and ``parse_sql``.</span>
<span class="sd"> - If field ``url`` is set and field ``parse_url`` is unset, init</span>
<span class="sd"> ``parse_url`` from field ``url``. The ``url`` field is initialized</span>
<span class="sd"> with the resulting value in ``parse_url``, if ``url`` and</span>
<span class="sd"> ``parse_url`` are not equal.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">_normalize_url_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span></div>
<span class="k">def</span><span class="w"> </span><span class="nf">__post_init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">pass</span>
<div class="viewcode-block" id="Result.filter_urls">
<a class="viewcode-back" href="../../../dev/result_types/base_result.html#searx.result_types._base.Result.filter_urls">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">filter_urls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filter_func</span><span class="p">:</span> <span class="s2">&quot;Callable[[Result | LegacyResult, str, str], str | bool]&quot;</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A filter function is passed in the ``filter_func`` argument to</span>
<span class="sd"> filter and/or modify the URLs.</span>
<span class="sd"> The filter function receives the :py:obj:`result object &lt;Result&gt;` as</span>
<span class="sd"> the first argument and the field name (``str``) in the second argument.</span>
<span class="sd"> In the third argument the URL string value is passed to the filter function.</span>
<span class="sd"> The filter function is applied to all fields that contain a URL,</span>
<span class="sd"> in addition to the familiar ``url`` field, these include fields such as::</span>
<span class="sd"> [&quot;url&quot;, &quot;iframe_src&quot;, &quot;audio_src&quot;, &quot;img_src&quot;, &quot;thumbnail_src&quot;, &quot;thumbnail&quot;]</span>
<span class="sd"> and the ``urls`` list of items of the infobox.</span>
<span class="sd"> For each field, the filter function is called and returns a bool or a</span>
<span class="sd"> string value:</span>
<span class="sd"> - ``True``: leave URL in field unchanged</span>
<span class="sd"> - ``False``: remove URL field from result (or remove entire result)</span>
<span class="sd"> - ``str``: modified URL to be used instead</span>
<span class="sd"> See :ref:`filter urls example`.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">_filter_urls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filter_func</span><span class="o">=</span><span class="n">filter_func</span><span class="p">)</span></div>
<span class="k">def</span><span class="w"> </span><span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Generates a hash value that uniquely identifies the content of *this*</span>
<span class="sd"> result. The method can be adapted in the inheritance to compare results</span>
<span class="sd"> from different sources.</span>
<span class="sd"> If two result objects are not identical but have the same content, their</span>
<span class="sd"> hash values should also be identical.</span>
<span class="sd"> The hash value is used in contexts, e.g. when checking for equality to</span>
<span class="sd"> identify identical results from different sources (engines).</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="nb">id</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="nb">object</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;py:obj:`Result` objects are equal if the hash values of the two</span>
<span class="sd"> objects are equal. If needed, its recommended to overwrite</span>
<span class="sd"> &quot;py:obj:`Result.__hash__`.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">hash</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="c1"># for legacy code where a result is treated as a Python dict</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__setitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
<span class="k">if</span> <span class="n">field_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">__struct_fields__</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">field_name</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_name</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">iter</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__struct_fields__</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">as_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">{</span><span class="n">f</span><span class="p">:</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">f</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">__struct_fields__</span><span class="p">}</span>
<div class="viewcode-block" id="Result.defaults_from">
<a class="viewcode-back" href="../../../dev/result_types/base_result.html#searx.result_types._base.Result.defaults_from">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">defaults_from</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="s2">&quot;Result&quot;</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Fields not set in *self* will be updated from the field values of the</span>
<span class="sd"> *other*.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">for</span> <span class="n">field_name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">__struct_fields__</span><span class="p">:</span>
<span class="n">self_val</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
<span class="n">other_val</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
<span class="k">if</span> <span class="n">self_val</span><span class="p">:</span>
<span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="n">other_val</span><span class="p">)</span></div>
</div>
<div class="viewcode-block" id="MainResult">
<a class="viewcode-back" href="../../../dev/result_types/main/mainresult.html#searx.result_types._base.MainResult">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">MainResult</span><span class="p">(</span><span class="n">Result</span><span class="p">):</span> <span class="c1"># pylint: disable=missing-class-docstring</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Base class of all result types displayed in :ref:`area main results`.&quot;&quot;&quot;</span>
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;default.html&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Name of the template used to render the result.</span>
<span class="sd"> By default :origin:`result_templates/default.html</span>
<span class="sd"> &lt;searx/templates/simple/result_templates/default.html&gt;` is used.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">title</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Link title of the result item.&quot;&quot;&quot;</span>
<span class="n">content</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Extract or description of the result item&quot;&quot;&quot;</span>
<span class="n">img_src</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;URL of a image that is displayed in the result item.&quot;&quot;&quot;</span>
<span class="n">iframe_src</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;URL of an embedded ``&lt;iframe&gt;`` / the frame is collapsible.&quot;&quot;&quot;</span>
<span class="n">audio_src</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;URL of an embedded ``&lt;audio controls&gt;``.&quot;&quot;&quot;</span>
<span class="n">thumbnail</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;URL of a thumbnail that is displayed in the result item.&quot;&quot;&quot;</span>
<span class="n">publishedDate</span><span class="p">:</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The date on which the object was published.&quot;&quot;&quot;</span>
<span class="n">pubdate</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;String representation of :py:obj:`MainResult.publishedDate`</span>
<span class="sd"> Deprecated: it is still partially used in the templates, but will one day be</span>
<span class="sd"> completely eliminated.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">length</span><span class="p">:</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timedelta</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Playing duration in seconds.&quot;&quot;&quot;</span>
<span class="n">views</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;View count in humanized number format.&quot;&quot;&quot;</span>
<span class="n">author</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Author of the title.&quot;&quot;&quot;</span>
<span class="n">metadata</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Miscellaneous metadata.&quot;&quot;&quot;</span>
<span class="n">PriorityType</span> <span class="o">=</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="s2">&quot;high&quot;</span><span class="p">,</span> <span class="s2">&quot;low&quot;</span><span class="p">]</span> <span class="c1"># pyright: ignore[reportUnannotatedClassAttribute]</span>
<span class="n">priority</span><span class="p">:</span> <span class="s2">&quot;MainResult.PriorityType&quot;</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The priority can be set via :ref:`hostnames plugin`, for example.&quot;&quot;&quot;</span>
<span class="n">engines</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;In a merged results list, the names of the engines that found this result</span>
<span class="sd"> are listed in this field.&quot;&quot;&quot;</span>
<span class="c1"># open_group and close_group should not manged in the Result</span>
<span class="c1"># class (we should drop it from here!)</span>
<span class="n">open_group</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">close_group</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">positions</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">score</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">0</span>
<span class="n">category</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Ordinary url-results are equal if their values for</span>
<span class="sd"> :py:obj:`Result.template`, :py:obj:`Result.parsed_url` (without scheme)</span>
<span class="sd"> and :py:obj:`MainResult.img_src` are equal.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">parsed_url</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;missing a value in field &#39;parsed_url&#39;: </span><span class="si">{</span><span class="bp">self</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parsed_url</span>
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">template</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="o">+</span> <span class="sa">f</span><span class="s2">&quot;|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">netloc</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">path</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">params</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">query</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">fragment</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="o">+</span> <span class="sa">f</span><span class="s2">&quot;|</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">img_src</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="p">)</span>
<div class="viewcode-block" id="MainResult.normalize_result_fields">
<a class="viewcode-back" href="../../../dev/result_types/main/mainresult.html#searx.result_types._base.MainResult.normalize_result_fields">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">normalize_result_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">normalize_result_fields</span><span class="p">()</span>
<span class="n">_normalize_text_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="n">_normalize_date_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">engine</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">engines</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">engine</span><span class="p">)</span></div>
</div>
<div class="viewcode-block" id="LegacyResult">
<a class="viewcode-back" href="../../../dev/result_types/base_result.html#searx.result_types._base.LegacyResult">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">LegacyResult</span><span class="p">(</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A wrapper around a legacy result item. The SearXNG core uses this class</span>
<span class="sd"> for untyped dictionaries / to be downward compatible.</span>
<span class="sd"> This class is needed until we have implemented an :py:obj:`Result` class for</span>
<span class="sd"> each result type and the old usages in the codebase have been fully</span>
<span class="sd"> migrated.</span>
<span class="sd"> There is only one place where this class is used, in the</span>
<span class="sd"> :py:obj:`searx.results.ResultContainer`.</span>
<span class="sd"> .. attention::</span>
<span class="sd"> Do not use this class in your own implementations!</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">UNSET</span><span class="p">:</span> <span class="nb">object</span> <span class="o">=</span> <span class="nb">object</span><span class="p">()</span>
<span class="c1"># emulate field types from type class Result</span>
<span class="n">url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span>
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span>
<span class="n">engine</span><span class="p">:</span> <span class="nb">str</span>
<span class="n">parsed_url</span><span class="p">:</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">ParseResult</span> <span class="o">|</span> <span class="kc">None</span>
<span class="c1"># emulate field types from type class MainResult</span>
<span class="n">title</span><span class="p">:</span> <span class="nb">str</span>
<span class="n">content</span><span class="p">:</span> <span class="nb">str</span>
<span class="n">img_src</span><span class="p">:</span> <span class="nb">str</span>
<span class="n">thumbnail</span><span class="p">:</span> <span class="nb">str</span>
<span class="n">priority</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="s2">&quot;high&quot;</span><span class="p">,</span> <span class="s2">&quot;low&quot;</span><span class="p">]</span>
<span class="n">engines</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
<span class="n">positions</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span>
<span class="n">score</span><span class="p">:</span> <span class="nb">float</span>
<span class="n">category</span><span class="p">:</span> <span class="nb">str</span>
<span class="n">publishedDate</span><span class="p">:</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span> <span class="o">|</span> <span class="kc">None</span>
<span class="n">pubdate</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="c1"># infobox result</span>
<span class="n">urls</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span>
<span class="n">attributes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span>
<span class="k">def</span><span class="w"> </span><span class="nf">as_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="c1"># emulate field types from type class Result</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;url&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;template&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;template&quot;</span><span class="p">,</span> <span class="s2">&quot;default.html&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;engine&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;engine&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;parsed_url&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;parsed_url&quot;</span><span class="p">)</span>
<span class="c1"># emulate field types from type class MainResult</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;title&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;title&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;content&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;content&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;img_src&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;img_src&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;thumbnail&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;thumbnail&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;priority&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;priority&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;engines&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;engines&quot;</span><span class="p">,</span> <span class="nb">set</span><span class="p">())</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;positions&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;positions&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;score&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;score&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;category&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;category&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;publishedDate&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;publishedDate&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;infobox&quot;</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">:</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;urls&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;urls&quot;</span><span class="p">,</span> <span class="p">[])</span>
<span class="bp">self</span><span class="p">[</span><span class="s2">&quot;attributes&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;attributes&quot;</span><span class="p">,</span> <span class="p">[])</span>
<span class="c1"># Legacy types that have already been ported to a type ..</span>
<span class="k">if</span> <span class="s2">&quot;answer&quot;</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">:</span>
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;engine </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">engine</span><span class="si">}</span><span class="s2"> is using deprecated `dict` for answers&quot;</span>
<span class="sa">f</span><span class="s2">&quot; / use a class from searx.result_types.answer&quot;</span><span class="p">,</span>
<span class="ne">DeprecationWarning</span><span class="p">,</span>
<span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">template</span> <span class="o">=</span> <span class="s2">&quot;answer/legacy.html&quot;</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">template</span> <span class="o">==</span> <span class="s2">&quot;keyvalue.html&quot;</span><span class="p">:</span>
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;engine </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">engine</span><span class="si">}</span><span class="s2"> is using deprecated `dict` for key/value results&quot;</span>
<span class="sa">f</span><span class="s2">&quot; / use a class from searx.result_types&quot;</span><span class="p">,</span>
<span class="ne">DeprecationWarning</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="n">UNSET</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
<span class="k">if</span> <span class="n">default</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">UNSET</span> <span class="ow">and</span> <span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;LegacyResult object has no field named: </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">val</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">):</span>
<span class="bp">self</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">val</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span> <span class="c1"># pyright: ignore[reportIncompatibleVariableOverride]</span>
<span class="k">if</span> <span class="s2">&quot;answer&quot;</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">:</span>
<span class="c1"># deprecated ..</span>
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="p">[</span><span class="s2">&quot;answer&quot;</span><span class="p">])</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">template</span> <span class="o">==</span> <span class="s2">&quot;images.html&quot;</span><span class="p">:</span>
<span class="c1"># image results are equal if their values for template, the url and</span>
<span class="c1"># the img_src are equal.</span>
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">template</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">img_src</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">any</span><span class="p">(</span><span class="bp">cls</span> <span class="ow">in</span> <span class="bp">self</span> <span class="k">for</span> <span class="bp">cls</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;suggestion&quot;</span><span class="p">,</span> <span class="s2">&quot;correction&quot;</span><span class="p">,</span> <span class="s2">&quot;infobox&quot;</span><span class="p">,</span> <span class="s2">&quot;number_of_results&quot;</span><span class="p">,</span> <span class="s2">&quot;engine_data&quot;</span><span class="p">]):</span>
<span class="c1"># Ordinary url-results are equal if their values for template,</span>
<span class="c1"># parsed_url (without schema) and img_src` are equal.</span>
<span class="c1"># Code copied from with MainResult.__hash__:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">parsed_url</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;missing a value in field &#39;parsed_url&#39;: </span><span class="si">{</span><span class="bp">self</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parsed_url</span>
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">template</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="o">+</span> <span class="sa">f</span><span class="s2">&quot;|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">netloc</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">path</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">params</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">query</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">fragment</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="o">+</span> <span class="sa">f</span><span class="s2">&quot;|</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">img_src</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="p">)</span>
<span class="k">return</span> <span class="nb">id</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="nb">object</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">hash</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
<span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;LegacyResult: </span><span class="si">{</span><span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__repr__</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">normalize_result_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">_normalize_date_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="n">_normalize_url_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="n">_normalize_text_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">engine</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">engines</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">engine</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">defaults_from</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="s2">&quot;LegacyResult&quot;</span><span class="p">):</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">other</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">k</span><span class="p">):</span>
<span class="bp">self</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">v</span>
<div class="viewcode-block" id="LegacyResult.filter_urls">
<a class="viewcode-back" href="../../../dev/result_types/base_result.html#searx.result_types._base.LegacyResult.filter_urls">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">filter_urls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filter_func</span><span class="p">:</span> <span class="s2">&quot;Callable[[Result | LegacyResult, str, str], str | bool]&quot;</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;See :py:obj:`Result.filter_urls`&quot;&quot;&quot;</span>
<span class="n">_filter_urls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filter_func</span><span class="o">=</span><span class="n">filter_func</span><span class="p">)</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../result_types.html">searx.result_types</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,366 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.result_types.answer &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../result_types.html" accesskey="U">searx.result_types</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.result_types.answer</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.result_types.answer</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">Typification of the *answer* results. Results of this type are rendered in</span>
<span class="sd">the :origin:`answers.html &lt;searx/templates/simple/elements/answers.html&gt;`</span>
<span class="sd">template.</span>
<span class="sd">----</span>
<span class="sd">.. autoclass:: BaseAnswer</span>
<span class="sd"> :members:</span>
<span class="sd"> :show-inheritance:</span>
<span class="sd">.. autoclass:: Answer</span>
<span class="sd"> :members:</span>
<span class="sd"> :show-inheritance:</span>
<span class="sd">.. autoclass:: Translations</span>
<span class="sd"> :members:</span>
<span class="sd"> :show-inheritance:</span>
<span class="sd">.. autoclass:: WeatherAnswer</span>
<span class="sd"> :members:</span>
<span class="sd"> :show-inheritance:</span>
<span class="sd">.. autoclass:: AnswerSet</span>
<span class="sd"> :members:</span>
<span class="sd"> :show-inheritance:</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-few-public-methods</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;AnswerSet&quot;</span><span class="p">,</span> <span class="s2">&quot;Answer&quot;</span><span class="p">,</span> <span class="s2">&quot;Translations&quot;</span><span class="p">,</span> <span class="s2">&quot;WeatherAnswer&quot;</span><span class="p">]</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">msgspec</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">weather</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">._base</span><span class="w"> </span><span class="kn">import</span> <span class="n">Result</span>
<div class="viewcode-block" id="BaseAnswer">
<a class="viewcode-back" href="../../../dev/result_types/answer.html#searx.result_types.answer.BaseAnswer">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">BaseAnswer</span><span class="p">(</span><span class="n">Result</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Base class of all answer types. It is not intended to build instances of</span>
<span class="sd"> this class (aka *abstract*).&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="AnswerSet">
<a class="viewcode-back" href="../../../dev/result_types/answer.html#searx.result_types.answer.AnswerSet">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">AnswerSet</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Aggregator for :py:obj:`BaseAnswer` items in a result container.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__bool__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">add</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">answer</span><span class="p">:</span> <span class="n">BaseAnswer</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">a_hash</span> <span class="o">=</span> <span class="nb">hash</span><span class="p">(</span><span class="n">answer</span><span class="p">)</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">hash</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="o">==</span> <span class="n">a_hash</span><span class="p">:</span>
<span class="k">return</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">answer</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Sort items in this set and iterate over the items.&quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">answer</span><span class="p">:</span> <span class="n">answer</span><span class="o">.</span><span class="n">template</span><span class="p">)</span>
<span class="k">yield from</span> <span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__contains__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">answer</span><span class="p">:</span> <span class="n">BaseAnswer</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="n">a_hash</span> <span class="o">=</span> <span class="nb">hash</span><span class="p">(</span><span class="n">answer</span><span class="p">)</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">hash</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="o">==</span> <span class="n">a_hash</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="k">return</span> <span class="kc">False</span></div>
<div class="viewcode-block" id="Answer">
<a class="viewcode-back" href="../../../dev/result_types/answer.html#searx.result_types.answer.Answer">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">Answer</span><span class="p">(</span><span class="n">BaseAnswer</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Simple answer type where the *answer* is a simple string with an optional</span>
<span class="sd"> :py:obj:`url field &lt;Result.url&gt;` field to link a resource (article, map, ..)</span>
<span class="sd"> related to the answer.&quot;&quot;&quot;</span>
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;answer/legacy.html&quot;</span>
<span class="n">answer</span><span class="p">:</span> <span class="nb">str</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Text of the answer.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The hash value of field *answer* is the hash value of the</span>
<span class="sd"> :py:obj:`Answer` object. :py:obj:`Answer &lt;Result.__eq__&gt;` objects are</span>
<span class="sd"> equal, when the hash values of both objects are equal.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">answer</span><span class="p">)</span></div>
<div class="viewcode-block" id="Translations">
<a class="viewcode-back" href="../../../dev/result_types/answer.html#searx.result_types.answer.Translations">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">Translations</span><span class="p">(</span><span class="n">BaseAnswer</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Answer type with a list of translations.</span>
<span class="sd"> The items in the list of :py:obj:`Translations.translations` are of type</span>
<span class="sd"> :py:obj:`Translations.Item`:</span>
<span class="sd"> .. code:: python</span>
<span class="sd"> def response(resp):</span>
<span class="sd"> results = []</span>
<span class="sd"> ...</span>
<span class="sd"> foo_1 = Translations.Item(</span>
<span class="sd"> text=&quot;foobar&quot;,</span>
<span class="sd"> synonyms=[&quot;bar&quot;, &quot;foo&quot;],</span>
<span class="sd"> examples=[&quot;foo and bar are placeholders&quot;],</span>
<span class="sd"> )</span>
<span class="sd"> foo_url=&quot;https://www.deepl.com/de/translator#en/de/foo&quot;</span>
<span class="sd"> ...</span>
<span class="sd"> Translations(results=results, translations=[foo], url=foo_url)</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;answer/translations.html&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The template in :origin:`answer/translations.html</span>
<span class="sd"> &lt;searx/templates/simple/answer/translations.html&gt;`&quot;&quot;&quot;</span>
<span class="n">translations</span><span class="p">:</span> <span class="s2">&quot;list[Translations.Item]&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;List of translations.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">__post_init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">translations</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Translation does not have an item in the list translations&quot;</span><span class="p">)</span>
<div class="viewcode-block" id="Translations.Item">
<a class="viewcode-back" href="../../../dev/result_types/answer.html#searx.result_types.answer.Translations.Item">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">Item</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A single element of the translations / a translation. A translation</span>
<span class="sd"> consists of at least a mandatory ``text`` property (the translation) ,</span>
<span class="sd"> optional properties such as *definitions*, *synonyms* and *examples* are</span>
<span class="sd"> possible.&quot;&quot;&quot;</span>
<span class="n">text</span><span class="p">:</span> <span class="nb">str</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Translated text.&quot;&quot;&quot;</span>
<span class="n">transliteration</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Transliteration_ of the requested translation.</span>
<span class="sd"> .. _Transliteration: https://en.wikipedia.org/wiki/Transliteration</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">examples</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;List of examples for the requested translation.&quot;&quot;&quot;</span>
<span class="n">definitions</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;List of definitions for the requested translation.&quot;&quot;&quot;</span>
<span class="n">synonyms</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;List of synonyms for the requested translation.&quot;&quot;&quot;</span></div>
</div>
<div class="viewcode-block" id="WeatherAnswer">
<a class="viewcode-back" href="../../../dev/result_types/answer.html#searx.result_types.answer.WeatherAnswer">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">WeatherAnswer</span><span class="p">(</span><span class="n">BaseAnswer</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Answer type for weather data.&quot;&quot;&quot;</span>
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;answer/weather.html&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The template is located at :origin:`answer/weather.html</span>
<span class="sd"> &lt;searx/templates/simple/answer/weather.html&gt;`&quot;&quot;&quot;</span>
<span class="n">current</span><span class="p">:</span> <span class="s2">&quot;WeatherAnswer.Item&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Current weather at ``location``.&quot;&quot;&quot;</span>
<span class="n">forecasts</span><span class="p">:</span> <span class="s2">&quot;list[WeatherAnswer.Item]&quot;</span> <span class="o">=</span> <span class="p">[]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Weather forecasts for ``location``.&quot;&quot;&quot;</span>
<span class="n">service</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Weather service from which this information was provided.&quot;&quot;&quot;</span>
<div class="viewcode-block" id="WeatherAnswer.Item">
<a class="viewcode-back" href="../../../dev/result_types/answer.html#searx.result_types.answer.WeatherAnswer.Item">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">Item</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Weather parameters valid for a specific point in time.&quot;&quot;&quot;</span>
<span class="n">location</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">GeoLocation</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The geo-location the weather data is from (e.g. `Berlin, Germany`).&quot;&quot;&quot;</span>
<span class="n">temperature</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">Temperature</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Air temperature at 2m above the ground.&quot;&quot;&quot;</span>
<span class="n">condition</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">WeatherConditionType</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Standardized designations that summarize the weather situation</span>
<span class="sd"> (e.g. ``light sleet showers and thunder``).&quot;&quot;&quot;</span>
<span class="c1"># optional fields</span>
<span class="n">datetime</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">DateTime</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Time of the forecast - not needed for the current weather.&quot;&quot;&quot;</span>
<span class="n">summary</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;One-liner about the weather forecast / current weather conditions.</span>
<span class="sd"> If unset, a summary is build up from temperature and current weather</span>
<span class="sd"> conditions.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">feels_like</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">Temperature</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Apparent temperature, the temperature equivalent perceived by</span>
<span class="sd"> humans, caused by the combined effects of air temperature, relative</span>
<span class="sd"> humidity and wind speed. The measure is most commonly applied to the</span>
<span class="sd"> perceived outdoor temperature.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">pressure</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">Pressure</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Air pressure at sea level (e.g. 1030 hPa) &quot;&quot;&quot;</span>
<span class="n">humidity</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">RelativeHumidity</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Amount of relative humidity in the air at 2m above the ground. The</span>
<span class="sd"> unit is ``%``, e.g. 60%)</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">wind_from</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">Compass</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The directon which moves towards / direction the wind is coming from.&quot;&quot;&quot;</span>
<span class="n">wind_speed</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">WindSpeed</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Speed of wind / wind speed at 10m above the ground (10 min average).&quot;&quot;&quot;</span>
<span class="n">cloud_cover</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Amount of sky covered by clouds / total cloud cover for all heights</span>
<span class="sd"> (cloudiness, unit: %)&quot;&quot;&quot;</span>
<span class="c1"># attributes: dict[str, str | int] = {}</span>
<span class="c1"># &quot;&quot;&quot;Key-Value dict of additional typeless weather attributes.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">__post_init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">summary</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">summary</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{location}</span><span class="s2">: </span><span class="si">{temperature}</span><span class="s2">, </span><span class="si">{condition}</span><span class="s2">&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span>
<span class="n">temperature</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">temperature</span><span class="p">,</span>
<span class="n">condition</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">condition</span><span class="o">.</span><span class="n">capitalize</span><span class="p">()),</span>
<span class="p">)</span>
<span class="nd">@property</span>
<span class="k">def</span><span class="w"> </span><span class="nf">url</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Determines a `data URL`_ with a symbol for the weather</span>
<span class="sd"> conditions. If no symbol can be assigned, ``None`` is returned.</span>
<span class="sd"> .. _data URL:</span>
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/URI/Reference/Schemes/data</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">weather</span><span class="o">.</span><span class="n">symbol_url</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">condition</span><span class="p">)</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../result_types.html">searx.result_types</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,294 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.result_types.code &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../result_types.html" accesskey="U">searx.result_types</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.result_types.code</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.result_types.code</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Typification of the *code* results. Results of this type are rendered in</span>
<span class="sd">the :origin:`code.html &lt;searx/templates/simple/result_templates/code.html&gt;`</span>
<span class="sd">template. For highlighting the code passages, Pygments_ is used.</span>
<span class="sd">.. _Pygments: https://pygments.org</span>
<span class="sd">----</span>
<span class="sd">.. autoclass:: Code</span>
<span class="sd"> :members:</span>
<span class="sd"> :show-inheritance:</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-few-public-methods, disable=invalid-name</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;Code&quot;</span><span class="p">]</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pygments</span><span class="w"> </span><span class="kn">import</span> <span class="n">highlight</span> <span class="c1"># pyright: ignore[reportUnknownVariableType]</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pygments.lexers._mapping</span><span class="w"> </span><span class="kn">import</span> <span class="n">LEXERS</span> <span class="c1"># pyright: ignore[reportMissingTypeStubs]</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pygments.lexers</span><span class="w"> </span><span class="kn">import</span> <span class="n">guess_lexer</span><span class="p">,</span> <span class="n">get_lexer_by_name</span><span class="p">,</span> <span class="n">guess_lexer_for_filename</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pygments.util</span><span class="w"> </span><span class="kn">import</span> <span class="n">ClassNotFound</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">pygments.formatters</span><span class="w"> </span><span class="kn">import</span> <span class="n">HtmlFormatter</span> <span class="c1"># pylint: disable=no-name-in-module</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">._base</span><span class="w"> </span><span class="kn">import</span> <span class="n">MainResult</span>
<span class="n">_pygments_languages</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">def</span><span class="w"> </span><span class="nf">is_valid_language</span><span class="p">(</span><span class="n">code_language</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Checks if the specified ``code_language`` is known in Pygments.&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">_pygments_languages</span><span class="p">:</span>
<span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">LEXERS</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
<span class="c1"># l[2] is the tuple with the alias names</span>
<span class="k">for</span> <span class="n">alias_name</span> <span class="ow">in</span> <span class="n">l</span><span class="p">[</span><span class="mi">2</span><span class="p">]:</span>
<span class="n">_pygments_languages</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">alias_name</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
<span class="k">return</span> <span class="n">code_language</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">in</span> <span class="n">_pygments_languages</span>
<div class="viewcode-block" id="Code">
<a class="viewcode-back" href="../../../dev/result_types/main/code.html#searx.result_types.code.Code">[docs]</a>
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">Code</span><span class="p">(</span><span class="n">MainResult</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Result type suitable for displaying code passages.&quot;&quot;&quot;</span>
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;code.html&quot;</span>
<span class="n">repository</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A link related to a repository related to the *result*.&quot;&quot;&quot;</span>
<span class="n">codelines</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A list of two digit tuples where the first item is the line number and</span>
<span class="sd"> the second item is the code line.&quot;&quot;&quot;</span>
<span class="n">hl_lines</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;A list of line numbers to highlight.&quot;&quot;&quot;</span>
<span class="n">code_language</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&lt;guess&gt;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Pygment&#39;s short name of the lexer, e.g. ``text`` for the</span>
<span class="sd"> :py:obj:`pygments.lexers.special.TextLexer`. For a list of available</span>
<span class="sd"> languages consult: `Pygments languages`_. If the language is not in this</span>
<span class="sd"> list, a :py:obj:`ValueError` is raised.</span>
<span class="sd"> The default is ``&lt;guess&gt;`` which has a special meaning;</span>
<span class="sd"> - If :py:obj:`Code.filename` is set, Pygment&#39;s factory method</span>
<span class="sd"> :py:obj:`pygments.lexers.guess_lexer_for_filename` is used to determine</span>
<span class="sd"> the language of the ``codelines``.</span>
<span class="sd"> - else Pygment&#39;s :py:obj:`pygments.lexers.guess_lexer` factory is used.</span>
<span class="sd"> In case the language can&#39;t be detected, the fallback is ``text``.</span>
<span class="sd"> .. _Pygments languages: https://pygments.org/languages/</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Optional file name, can help to ``&lt;guess&gt;`` the language of the code (in</span>
<span class="sd"> case of ambiguous short code examples). If :py:obj:`Code.title` is not set,</span>
<span class="sd"> its default is the filename.&quot;&quot;&quot;</span>
<span class="n">strip_new_lines</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Strip leading and trailing newlines for each returned fragment (default:</span>
<span class="sd"> ``True``). Single file might return multiple code fragments.&quot;&quot;&quot;</span>
<span class="n">strip_whitespace</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Strip all leading and trailing whitespace for each returned fragment</span>
<span class="sd"> (default: ``False``). Single file might return multiple code fragments.</span>
<span class="sd"> Enabling this might break code indentation.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">__post_init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">__post_init__</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">title</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">title</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">code_language</span> <span class="o">!=</span> <span class="s2">&quot;&lt;guess&gt;&quot;</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">is_valid_language</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">code_language</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;unknown code_language: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">code_language</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The hash value is build up from URL and code lines. :py:obj:`Code</span>
<span class="sd"> &lt;Result.__eq__&gt;` objects are equal, when the hash values of both objects</span>
<span class="sd"> are equal.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">codelines</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">def</span><span class="w"> </span><span class="nf">get_lexer</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">code_language</span> <span class="o">!=</span> <span class="s2">&quot;&lt;guess&gt;&quot;</span><span class="p">:</span>
<span class="k">return</span> <span class="n">get_lexer_by_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">code_language</span><span class="p">)</span>
<span class="n">src_code</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">l</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">codelines</span><span class="p">])</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">return</span> <span class="n">guess_lexer_for_filename</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">,</span> <span class="n">src_code</span><span class="p">)</span>
<span class="k">except</span> <span class="n">ClassNotFound</span><span class="p">:</span>
<span class="k">pass</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">return</span> <span class="n">guess_lexer</span><span class="p">(</span><span class="n">src_code</span><span class="p">)</span>
<span class="k">except</span> <span class="n">ClassNotFound</span><span class="p">:</span>
<span class="k">pass</span>
<span class="k">return</span> <span class="n">get_lexer_by_name</span><span class="p">(</span><span class="s2">&quot;text&quot;</span><span class="p">)</span>
<div class="viewcode-block" id="Code.HTML">
<a class="viewcode-back" href="../../../dev/result_types/main/code.html#searx.result_types.code.Code.HTML">[docs]</a>
<span class="k">def</span><span class="w"> </span><span class="nf">HTML</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">options</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span> <span class="c1"># pyright: ignore[reportUnknownParameterType, reportMissingParameterType]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Rendered HTML, additional options are accepted, for more details have</span>
<span class="sd"> a look at HtmlFormatter_.</span>
<span class="sd"> .. _HtmlFormatter: https://pygments.org/docs/formatters/#HtmlFormatter</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">lexer</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_lexer</span><span class="p">()</span>
<span class="n">line_no</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span> <span class="c1"># current line number</span>
<span class="n">code_block_start</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span> <span class="c1"># line where the current code block starts</span>
<span class="n">code_block_end</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span> <span class="c1"># line where the current code ends</span>
<span class="n">code_block</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span> <span class="c1"># lines of the current code block</span>
<span class="n">html_code_blocks</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span> <span class="c1"># HTML representation of all code blocks</span>
<span class="k">def</span><span class="w"> </span><span class="nf">_render</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="c1"># pyright: ignore[reportUnknownParameterType, reportMissingParameterType]</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">default</span> <span class="ow">in</span> <span class="p">[</span>
<span class="p">(</span><span class="s2">&quot;linenos&quot;</span><span class="p">,</span> <span class="s2">&quot;inline&quot;</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;linenostart&quot;</span><span class="p">,</span> <span class="n">code_block_start</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;cssclass&quot;</span><span class="p">,</span> <span class="s2">&quot;code-highlight&quot;</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;hl_lines&quot;</span><span class="p">,</span> <span class="p">[</span><span class="n">hl</span> <span class="o">-</span> <span class="n">code_block_start</span> <span class="o">+</span> <span class="mi">1</span> <span class="k">for</span> <span class="n">hl</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">hl_lines</span><span class="p">]),</span>
<span class="p">]:</span>
<span class="n">kwargs</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">default</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownMemberType]</span>
<span class="c1"># Wrap the code inside &lt;pre&gt; blocks using &lt;code&gt;, as recommended by</span>
<span class="c1"># the HTML5 specification (default is False). Do we need this?</span>
<span class="n">kwargs</span><span class="p">[</span><span class="s2">&quot;wrapcode&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;wrapcode&quot;</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
<span class="n">html_code_blocks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="n">highlight</span><span class="p">(</span>
<span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">code_block</span><span class="p">),</span>
<span class="n">lexer</span><span class="p">,</span>
<span class="n">HtmlFormatter</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">),</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
<span class="p">)</span>
<span class="p">)</span>
<span class="k">for</span> <span class="n">line_no</span><span class="p">,</span> <span class="n">code_line</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">codelines</span><span class="p">:</span>
<span class="k">if</span> <span class="n">code_block_end</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="c1"># initial start condition</span>
<span class="n">code_block_start</span> <span class="o">=</span> <span class="n">line_no</span>
<span class="k">if</span> <span class="n">code_block_end</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">code_block_end</span> <span class="o">+</span> <span class="mi">1</span> <span class="o">!=</span> <span class="n">line_no</span><span class="p">:</span>
<span class="c1"># new code block is detected, render current code block</span>
<span class="n">_render</span><span class="p">(</span><span class="o">**</span><span class="n">options</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
<span class="c1"># reset conditions for next code block, which first line is the</span>
<span class="c1"># current code line</span>
<span class="n">code_block</span> <span class="o">=</span> <span class="p">[</span><span class="n">code_line</span><span class="p">]</span>
<span class="n">code_block_start</span> <span class="o">=</span> <span class="n">line_no</span>
<span class="n">code_block_end</span> <span class="o">=</span> <span class="n">line_no</span>
<span class="k">continue</span>
<span class="c1"># add line to the current code block and update last line n</span>
<span class="n">code_block</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">code_line</span><span class="p">)</span>
<span class="n">code_block_end</span> <span class="o">=</span> <span class="n">line_no</span>
<span class="c1"># highlight (last) code block</span>
<span class="n">_render</span><span class="p">(</span><span class="o">**</span><span class="n">options</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
<span class="k">return</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">html_code_blocks</span><span class="p">)</span></div>
</div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../result_types.html">searx.result_types</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,205 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.result_types.file &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../result_types.html" accesskey="U">searx.result_types</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.result_types.file</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.result_types.file</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">Typification of the *file* results. Results of this type are rendered in</span>
<span class="sd">the :origin:`file.html &lt;searx/templates/simple/result_templates/file.html&gt;`</span>
<span class="sd">template.</span>
<span class="sd">----</span>
<span class="sd">.. autoclass:: File</span>
<span class="sd"> :members:</span>
<span class="sd"> :show-inheritance:</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-few-public-methods</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;File&quot;</span><span class="p">]</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">mimetypes</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">._base</span><span class="w"> </span><span class="kn">import</span> <span class="n">MainResult</span>
<div class="viewcode-block" id="File">
<a class="viewcode-back" href="../../../dev/result_types/main/file.html#searx.result_types.file.File">[docs]</a>
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">File</span><span class="p">(</span><span class="n">MainResult</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Class for results of type *file*&quot;&quot;&quot;</span>
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;file.html&quot;</span>
<span class="n">filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Name of the file.&quot;&quot;&quot;</span>
<span class="n">size</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Size of bytes in human readable notation (``MB`` for 1024 * 1024 Bytes</span>
<span class="sd"> file size.)&quot;&quot;&quot;</span>
<span class="n">time</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Indication of a time, such as the date of the last modification or the</span>
<span class="sd"> date of creation. This is a simple string, the *date* of which can be freely</span>
<span class="sd"> chosen according to the context.&quot;&quot;&quot;</span>
<span class="n">mimetype</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Mimetype/Subtype of the file. For ``audio`` and ``video``, a URL can be</span>
<span class="sd"> passed in the :py:obj:`File.embedded` field to embed the referenced media in</span>
<span class="sd"> the result. If no value is specified, the MIME type is determined from</span>
<span class="sd"> ``self.filename`` or, alternatively, from ``self.embedded`` (if either of</span>
<span class="sd"> the two values is set).&quot;&quot;&quot;</span>
<span class="n">abstract</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Abstract of the file.&quot;&quot;&quot;</span>
<span class="n">author</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Author of the file.&quot;&quot;&quot;</span>
<span class="n">embedded</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;URL of an embedded media type (audio or video) / is collapsible.&quot;&quot;&quot;</span>
<span class="n">mtype</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Used for displaying :py:obj:`File.embedded`. Its value is automatically</span>
<span class="sd"> populated from the base type of :py:obj:`File.mimetype`, and can be</span>
<span class="sd"> explicitly set to enforce e.g. ``audio`` or ``video`` when mimetype is</span>
<span class="sd"> something like &quot;application/ogg&quot; but its know the content is for example a</span>
<span class="sd"> video.&quot;&quot;&quot;</span>
<span class="n">subtype</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Used for displaying :py:obj:`File.embedded`. Its value is automatically</span>
<span class="sd"> populated from the subtype type of :py:obj:`File.mimetype`, and can be</span>
<span class="sd"> explicitly set to enforce a subtype for the :py:obj:`File.embedded`</span>
<span class="sd"> element.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">__post_init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">__post_init__</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">mtype</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">subtype</span><span class="p">:</span>
<span class="n">fn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedded</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">mimetype</span> <span class="ow">and</span> <span class="n">fn</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">mimetype</span> <span class="o">=</span> <span class="n">mimetypes</span><span class="o">.</span><span class="n">guess_type</span><span class="p">(</span><span class="n">fn</span><span class="p">,</span> <span class="n">strict</span><span class="o">=</span><span class="kc">False</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="ow">or</span> <span class="s2">&quot;&quot;</span>
<span class="n">mtype</span><span class="p">,</span> <span class="n">subtype</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mimetype</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="p">[</span><span class="s2">&quot;&quot;</span><span class="p">])[:</span><span class="mi">2</span><span class="p">]</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">mtype</span><span class="p">:</span>
<span class="c1"># I don&#39;t know why, but the ogg video stream is not displayed,</span>
<span class="c1"># may https://github.com/videojs/video.js can help?</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedded</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;.ogv&quot;</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">mtype</span> <span class="o">=</span> <span class="s2">&quot;video&quot;</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedded</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;.oga&quot;</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">mtype</span> <span class="o">=</span> <span class="s2">&quot;audio&quot;</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">mtype</span> <span class="o">=</span> <span class="n">mtype</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">subtype</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">subtype</span> <span class="o">=</span> <span class="n">subtype</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../result_types.html">searx.result_types</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,159 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.result_types.keyvalue &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../result_types.html" accesskey="U">searx.result_types</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.result_types.keyvalue</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.result_types.keyvalue</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">Typification of the *keyvalue* results. Results of this type are rendered in</span>
<span class="sd">the :origin:`keyvalue.html &lt;searx/templates/simple/result_templates/keyvalue.html&gt;`</span>
<span class="sd">template.</span>
<span class="sd">----</span>
<span class="sd">.. autoclass:: KeyValue</span>
<span class="sd"> :members:</span>
<span class="sd"> :show-inheritance:</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-few-public-methods</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;KeyValue&quot;</span><span class="p">]</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">OrderedDict</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">._base</span><span class="w"> </span><span class="kn">import</span> <span class="n">MainResult</span>
<div class="viewcode-block" id="KeyValue">
<a class="viewcode-back" href="../../../dev/result_types/main/keyvalue.html#searx.result_types.keyvalue.KeyValue">[docs]</a>
<span class="k">class</span><span class="w"> </span><span class="nc">KeyValue</span><span class="p">(</span><span class="n">MainResult</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Simple table view which maps *key* names (first col) to *values*</span>
<span class="sd"> (second col).&quot;&quot;&quot;</span>
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;keyvalue.html&quot;</span>
<span class="n">kvmap</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">|</span> <span class="n">OrderedDict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Dictionary with keys and values. To sort keys, use :py:obj:`OrderedDict`.&quot;&quot;&quot;</span>
<span class="n">caption</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Optional caption for this result.&quot;&quot;&quot;</span>
<span class="n">key_title</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Optional title for the *key column*.&quot;&quot;&quot;</span>
<span class="n">value_title</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Optional title for the *value column*.&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;The KeyValues objects are checked for object identity, even if all</span>
<span class="sd"> fields of two results have the same values, they are different from each</span>
<span class="sd"> other.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="nb">id</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../result_types.html">searx.result_types</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

View File

@@ -0,0 +1,205 @@
<!DOCTYPE html>
<html lang="en" data-content_root="../../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>searx.result_types.paper &#8212; SearXNG Documentation (2026.1.16+97814c62a)</title>
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
</head><body>
<div class="related" role="navigation" aria-label="Related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="../../../genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="../../../py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> &#187;</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li>
<li class="nav-item nav-item-2"><a href="../result_types.html" accesskey="U">searx.result_types</a> &#187;</li>
<li class="nav-item nav-item-this"><a href="">searx.result_types.paper</a></li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for searx.result_types.paper</h1><div class="highlight"><pre>
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
<span class="sd">&quot;&quot;&quot;Typification of the *paper* results.</span>
<span class="sd">.. _BibTeX field types: https://en.wikipedia.org/wiki/BibTeX#Field_types</span>
<span class="sd">.. _BibTeX format: https://www.bibtex.com/g/bibtex-format/</span>
<span class="sd">Results of this type are rendered in the :origin:`paper.html</span>
<span class="sd">&lt;searx/templates/simple/result_templates/paper.html&gt;` template.</span>
<span class="sd">Related topics:</span>
<span class="sd">- `BibTeX field types`_</span>
<span class="sd">- `BibTeX format`_</span>
<span class="sd">----</span>
<span class="sd">.. autoclass:: Paper</span>
<span class="sd"> :members:</span>
<span class="sd"> :show-inheritance:</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="c1"># pylint: disable=too-few-public-methods, disable=invalid-name</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;Paper&quot;</span><span class="p">]</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.weather</span><span class="w"> </span><span class="kn">import</span> <span class="n">DateTime</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">._base</span><span class="w"> </span><span class="kn">import</span> <span class="n">MainResult</span>
<div class="viewcode-block" id="Paper">
<a class="viewcode-back" href="../../../dev/result_types/main/paper.html#searx.result_types.paper.Paper">[docs]</a>
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
<span class="k">class</span><span class="w"> </span><span class="nc">Paper</span><span class="p">(</span><span class="n">MainResult</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Result type suitable for displaying scientific papers and other</span>
<span class="sd"> documents.&quot;&quot;&quot;</span>
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;paper.html&quot;</span>
<span class="n">date_of_publication</span><span class="p">:</span> <span class="n">DateTime</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Date the document was published.&quot;&quot;&quot;</span>
<span class="n">content</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;An abstract or excerpt from the document.&quot;&quot;&quot;</span>
<span class="n">comments</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Free text display in italic below the content.&quot;&quot;&quot;</span>
<span class="n">tags</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Free tag list.&quot;&quot;&quot;</span>
<span class="nb">type</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Short description of medium type, e.g. *book*, *pdf* or *html* ...&quot;&quot;&quot;</span>
<span class="n">authors</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="nb">set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;List of authors of the work (authors with a &quot;s&quot; suffix, the &quot;author&quot; is</span>
<span class="sd"> in the :py:obj:`MainResult.author`).&quot;&quot;&quot;</span>
<span class="n">editor</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Editor of the book/paper.&quot;&quot;&quot;</span>
<span class="n">publisher</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Name of the publisher.&quot;&quot;&quot;</span>
<span class="n">journal</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Name of the journal or magazine the article was published in.&quot;&quot;&quot;</span>
<span class="n">volume</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">int</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Volume number.&quot;&quot;&quot;</span>
<span class="n">pages</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Page range where the article is.&quot;&quot;&quot;</span>
<span class="n">number</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Number of the report or the issue number for a journal article.&quot;&quot;&quot;</span>
<span class="n">doi</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;DOI number (like ``10.1038/d41586-018-07848-2``).&quot;&quot;&quot;</span>
<span class="n">issn</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;List of ISSN numbers like ``1476-4687``&quot;&quot;&quot;</span>
<span class="n">isbn</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;List of ISBN numbers like ``9780201896831``&quot;&quot;&quot;</span>
<span class="n">pdf_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;URL to the full article, the PDF version&quot;&quot;&quot;</span>
<span class="n">html_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;URL to full article, HTML version&quot;&quot;&quot;</span>
<span class="k">def</span><span class="w"> </span><span class="nf">__post_init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">__post_init__</span><span class="p">()</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">date_of_publication</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">publishedDate</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">date_of_publication</span> <span class="o">=</span> <span class="n">DateTime</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">publishedDate</span><span class="p">)</span></div>
</pre></div>
<div class="clearer"></div>
</div>
</div>
</div>
<span id="sidebar-top"></span>
<div class="sphinxsidebar" role="navigation" aria-label="Main">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
</a></p>
<h3><a href="../../../index.html">Table of Contents</a></h3>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
</ul>
<h3>Project Links</h3>
<ul>
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
<li><a href="https://searx.space">Public instances</a>
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
</ul><h3>Navigation</h3>
<ul>
<li><a href="../../../index.html">Overview</a>
<ul>
<li><a href="../../index.html">Module code</a>
<ul>
<li><a href="../result_types.html">searx.result_types</a>
</ul>
</li></ul>
</li>
</ul>
</li>
</ul>
<search id="searchbox" style="display: none" role="search">
<h3 id="searchlabel">Quick search</h3>
<div class="searchformwrapper">
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
<input type="submit" value="Go" />
</form>
</div>
</search>
<script>document.getElementById('searchbox').style.display = "block"</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="footer" role="contentinfo">
&#169; Copyright SearXNG team.
</div>
</body>
</html>

Some files were not shown because too many files have changed in this diff Show More