mirror of
https://github.com/searxng/searxng.git
synced 2026-07-31 12:21:24 +00:00
Compare commits
6 Commits
b8085d27ac
...
b93cc2f9f8
| Author | SHA1 | Date | |
|---|---|---|---|
|
b93cc2f9f8 | ||
|
501327ddf1 | ||
|
932fb22c80 | ||
|
|
e5d2685d8d | ||
|
|
fb5fd8c819 | ||
|
|
9ac9c8c4f5 |
@@ -10,7 +10,7 @@
|
||||
|
||||
.code-highlight {
|
||||
|
||||
pre { line-height: 100%; }
|
||||
pre { line-height: 125%; }
|
||||
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
|
||||
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
|
||||
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
|
||||
@@ -89,89 +89,89 @@
|
||||
.code-highlight-dark(){
|
||||
.code-highlight {
|
||||
|
||||
pre { line-height: 100%; }
|
||||
td.linenos .normal { color: #3c4354; background-color: transparent; padding-left: 5px; padding-right: 5px; }
|
||||
span.linenos { color: #3c4354; background-color: transparent; padding-left: 5px; padding-right: 5px; }
|
||||
td.linenos .special { color: #3c4354; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
|
||||
span.linenos.special { color: #3c4354; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
|
||||
.hll { background-color: #6e7681 }
|
||||
.c { color: #7E8AA1 } /* Comment */
|
||||
.err { color: #F88F7F } /* Error */
|
||||
.esc { color: #D4D2C8 } /* Escape */
|
||||
.g { color: #D4D2C8 } /* Generic */
|
||||
.k { color: #FFAD66 } /* Keyword */
|
||||
.l { color: #D5FF80 } /* Literal */
|
||||
.n { color: #D4D2C8 } /* Name */
|
||||
.o { color: #FFAD66 } /* Operator */
|
||||
.x { color: #D4D2C8 } /* Other */
|
||||
.p { color: #D4D2C8 } /* Punctuation */
|
||||
.ch { color: #F88F7F; font-style: italic } /* Comment.Hashbang */
|
||||
.cm { color: #7E8AA1 } /* Comment.Multiline */
|
||||
.cp { color: #FFAD66; font-weight: bold } /* Comment.Preproc */
|
||||
.cpf { color: #7E8AA1 } /* Comment.PreprocFile */
|
||||
.c1 { color: #7E8AA1 } /* Comment.Single */
|
||||
.cs { color: #7E8AA1; font-style: italic } /* Comment.Special */
|
||||
.gd { color: #F88F7F; background-color: #3D1E20 } /* Generic.Deleted */
|
||||
.ge { color: #D4D2C8; font-style: italic } /* Generic.Emph */
|
||||
.ges { color: #D4D2C8 } /* Generic.EmphStrong */
|
||||
.gr { color: #F88F7F } /* Generic.Error */
|
||||
.gh { color: #D4D2C8 } /* Generic.Heading */
|
||||
.gi { color: #6AD4AF; background-color: #19362C } /* Generic.Inserted */
|
||||
.go { color: #7E8AA1 } /* Generic.Output */
|
||||
.gp { color: #D4D2C8 } /* Generic.Prompt */
|
||||
.gs { color: #D4D2C8; font-weight: bold } /* Generic.Strong */
|
||||
.gu { color: #D4D2C8 } /* Generic.Subheading */
|
||||
.gt { color: #F88F7F } /* Generic.Traceback */
|
||||
.kc { color: #FFAD66 } /* Keyword.Constant */
|
||||
.kd { color: #FFAD66 } /* Keyword.Declaration */
|
||||
.kn { color: #FFAD66 } /* Keyword.Namespace */
|
||||
.kp { color: #FFAD66 } /* Keyword.Pseudo */
|
||||
.kr { color: #FFAD66 } /* Keyword.Reserved */
|
||||
.kt { color: #73D0FF } /* Keyword.Type */
|
||||
.ld { color: #D5FF80 } /* Literal.Date */
|
||||
.m { color: #DFBFFF } /* Literal.Number */
|
||||
.s { color: #D5FF80 } /* Literal.String */
|
||||
.na { color: #FFD173 } /* Name.Attribute */
|
||||
.nb { color: #FFD173 } /* Name.Builtin */
|
||||
.nc { color: #73D0FF } /* Name.Class */
|
||||
.no { color: #FFD173 } /* Name.Constant */
|
||||
.nd { color: #7E8AA1; font-weight: bold; font-style: italic } /* Name.Decorator */
|
||||
.ni { color: #95E6CB } /* Name.Entity */
|
||||
.ne { color: #73D0FF } /* Name.Exception */
|
||||
.nf { color: #FFD173 } /* Name.Function */
|
||||
.nl { color: #D4D2C8 } /* Name.Label */
|
||||
.nn { color: #D4D2C8 } /* Name.Namespace */
|
||||
.nx { color: #D4D2C8 } /* Name.Other */
|
||||
.py { color: #FFD173 } /* Name.Property */
|
||||
.nt { color: #5CCFE6 } /* Name.Tag */
|
||||
.nv { color: #D4D2C8 } /* Name.Variable */
|
||||
.ow { color: #FFAD66 } /* Operator.Word */
|
||||
.pm { color: #D4D2C8 } /* Punctuation.Marker */
|
||||
.w { color: #D4D2C8 } /* Text.Whitespace */
|
||||
.mb { color: #DFBFFF } /* Literal.Number.Bin */
|
||||
.mf { color: #DFBFFF } /* Literal.Number.Float */
|
||||
.mh { color: #DFBFFF } /* Literal.Number.Hex */
|
||||
.mi { color: #DFBFFF } /* Literal.Number.Integer */
|
||||
.mo { color: #DFBFFF } /* Literal.Number.Oct */
|
||||
.sa { color: #F29E74 } /* Literal.String.Affix */
|
||||
.sb { color: #D5FF80 } /* Literal.String.Backtick */
|
||||
.sc { color: #D5FF80 } /* Literal.String.Char */
|
||||
.dl { color: #D5FF80 } /* Literal.String.Delimiter */
|
||||
.sd { color: #7E8AA1 } /* Literal.String.Doc */
|
||||
.s2 { color: #D5FF80 } /* Literal.String.Double */
|
||||
.se { color: #95E6CB } /* Literal.String.Escape */
|
||||
.sh { color: #D5FF80 } /* Literal.String.Heredoc */
|
||||
.si { color: #95E6CB } /* Literal.String.Interpol */
|
||||
.sx { color: #95E6CB } /* Literal.String.Other */
|
||||
.sr { color: #95E6CB } /* Literal.String.Regex */
|
||||
.s1 { color: #D5FF80 } /* Literal.String.Single */
|
||||
.ss { color: #DFBFFF } /* Literal.String.Symbol */
|
||||
.bp { color: #5CCFE6 } /* Name.Builtin.Pseudo */
|
||||
.fm { color: #FFD173 } /* Name.Function.Magic */
|
||||
.vc { color: #D4D2C8 } /* Name.Variable.Class */
|
||||
.vg { color: #D4D2C8 } /* Name.Variable.Global */
|
||||
.vi { color: #D4D2C8 } /* Name.Variable.Instance */
|
||||
.vm { color: #D4D2C8 } /* Name.Variable.Magic */
|
||||
.il { color: #DFBFFF } /* Literal.Number.Integer.Long */
|
||||
pre { line-height: 125%; }
|
||||
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
|
||||
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
|
||||
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
|
||||
span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
|
||||
.hll { background-color: #49483e }
|
||||
.c { color: #959077 } /* Comment */
|
||||
.err { color: #ED007E; background-color: #1E0010 } /* Error */
|
||||
.esc { color: #F8F8F2 } /* Escape */
|
||||
.g { color: #F8F8F2 } /* Generic */
|
||||
.k { color: #66D9EF } /* Keyword */
|
||||
.l { color: #AE81FF } /* Literal */
|
||||
.n { color: #F8F8F2 } /* Name */
|
||||
.o { color: #FF4689 } /* Operator */
|
||||
.x { color: #F8F8F2 } /* Other */
|
||||
.p { color: #F8F8F2 } /* Punctuation */
|
||||
.ch { color: #959077 } /* Comment.Hashbang */
|
||||
.cm { color: #959077 } /* Comment.Multiline */
|
||||
.cp { color: #959077 } /* Comment.Preproc */
|
||||
.cpf { color: #959077 } /* Comment.PreprocFile */
|
||||
.c1 { color: #959077 } /* Comment.Single */
|
||||
.cs { color: #959077 } /* Comment.Special */
|
||||
.gd { color: #FF4689 } /* Generic.Deleted */
|
||||
.ge { color: #F8F8F2; font-style: italic } /* Generic.Emph */
|
||||
.ges { color: #F8F8F2; font-weight: bold; font-style: italic } /* Generic.EmphStrong */
|
||||
.gr { color: #F8F8F2 } /* Generic.Error */
|
||||
.gh { color: #F8F8F2 } /* Generic.Heading */
|
||||
.gi { color: #A6E22E } /* Generic.Inserted */
|
||||
.go { color: #66D9EF } /* Generic.Output */
|
||||
.gp { color: #FF4689; font-weight: bold } /* Generic.Prompt */
|
||||
.gs { color: #F8F8F2; font-weight: bold } /* Generic.Strong */
|
||||
.gu { color: #959077 } /* Generic.Subheading */
|
||||
.gt { color: #F8F8F2 } /* Generic.Traceback */
|
||||
.kc { color: #66D9EF } /* Keyword.Constant */
|
||||
.kd { color: #66D9EF } /* Keyword.Declaration */
|
||||
.kn { color: #FF4689 } /* Keyword.Namespace */
|
||||
.kp { color: #66D9EF } /* Keyword.Pseudo */
|
||||
.kr { color: #66D9EF } /* Keyword.Reserved */
|
||||
.kt { color: #66D9EF } /* Keyword.Type */
|
||||
.ld { color: #E6DB74 } /* Literal.Date */
|
||||
.m { color: #AE81FF } /* Literal.Number */
|
||||
.s { color: #E6DB74 } /* Literal.String */
|
||||
.na { color: #A6E22E } /* Name.Attribute */
|
||||
.nb { color: #F8F8F2 } /* Name.Builtin */
|
||||
.nc { color: #A6E22E } /* Name.Class */
|
||||
.no { color: #66D9EF } /* Name.Constant */
|
||||
.nd { color: #A6E22E } /* Name.Decorator */
|
||||
.ni { color: #F8F8F2 } /* Name.Entity */
|
||||
.ne { color: #A6E22E } /* Name.Exception */
|
||||
.nf { color: #A6E22E } /* Name.Function */
|
||||
.nl { color: #F8F8F2 } /* Name.Label */
|
||||
.nn { color: #F8F8F2 } /* Name.Namespace */
|
||||
.nx { color: #A6E22E } /* Name.Other */
|
||||
.py { color: #F8F8F2 } /* Name.Property */
|
||||
.nt { color: #FF4689 } /* Name.Tag */
|
||||
.nv { color: #F8F8F2 } /* Name.Variable */
|
||||
.ow { color: #FF4689 } /* Operator.Word */
|
||||
.pm { color: #F8F8F2 } /* Punctuation.Marker */
|
||||
.w { color: #F8F8F2 } /* Text.Whitespace */
|
||||
.mb { color: #AE81FF } /* Literal.Number.Bin */
|
||||
.mf { color: #AE81FF } /* Literal.Number.Float */
|
||||
.mh { color: #AE81FF } /* Literal.Number.Hex */
|
||||
.mi { color: #AE81FF } /* Literal.Number.Integer */
|
||||
.mo { color: #AE81FF } /* Literal.Number.Oct */
|
||||
.sa { color: #E6DB74 } /* Literal.String.Affix */
|
||||
.sb { color: #E6DB74 } /* Literal.String.Backtick */
|
||||
.sc { color: #E6DB74 } /* Literal.String.Char */
|
||||
.dl { color: #E6DB74 } /* Literal.String.Delimiter */
|
||||
.sd { color: #E6DB74 } /* Literal.String.Doc */
|
||||
.s2 { color: #E6DB74 } /* Literal.String.Double */
|
||||
.se { color: #AE81FF } /* Literal.String.Escape */
|
||||
.sh { color: #E6DB74 } /* Literal.String.Heredoc */
|
||||
.si { color: #E6DB74 } /* Literal.String.Interpol */
|
||||
.sx { color: #E6DB74 } /* Literal.String.Other */
|
||||
.sr { color: #E6DB74 } /* Literal.String.Regex */
|
||||
.s1 { color: #E6DB74 } /* Literal.String.Single */
|
||||
.ss { color: #E6DB74 } /* Literal.String.Symbol */
|
||||
.bp { color: #F8F8F2 } /* Name.Builtin.Pseudo */
|
||||
.fm { color: #A6E22E } /* Name.Function.Magic */
|
||||
.vc { color: #F8F8F2 } /* Name.Variable.Class */
|
||||
.vg { color: #F8F8F2 } /* Name.Variable.Global */
|
||||
.vi { color: #F8F8F2 } /* Name.Variable.Instance */
|
||||
.vm { color: #F8F8F2 } /* Name.Variable.Magic */
|
||||
.il { color: #AE81FF } /* Literal.Number.Integer.Long */
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,6 +30,12 @@
|
||||
span.linenos {
|
||||
color: var(--color-line-number);
|
||||
}
|
||||
|
||||
.err {
|
||||
// The code view in the result list should not act as a code-checker.
|
||||
border: none;
|
||||
color: inherit;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -150,6 +150,7 @@ intersphinx_mapping = {
|
||||
"linuxdoc" : ("https://return42.github.io/linuxdoc/", None),
|
||||
"sphinx" : ("https://www.sphinx-doc.org/en/master/", None),
|
||||
"valkey": ('https://valkey-py.readthedocs.io/en/stable/', None),
|
||||
"pygments": ("https://pygments.org/", None),
|
||||
}
|
||||
|
||||
issues_github_path = "searxng/searxng"
|
||||
|
||||
8
docs/dev/engines/online/repology.rst
Normal file
8
docs/dev/engines/online/repology.rst
Normal file
@@ -0,0 +1,8 @@
|
||||
.. _repology engine:
|
||||
|
||||
========
|
||||
Repology
|
||||
========
|
||||
|
||||
.. automodule:: searx.engines.repology
|
||||
:members:
|
||||
7
docs/dev/result_types/main/code.rst
Normal file
7
docs/dev/result_types/main/code.rst
Normal file
@@ -0,0 +1,7 @@
|
||||
.. _result_types.code:
|
||||
|
||||
============
|
||||
Code Results
|
||||
============
|
||||
|
||||
.. automodule:: searx.result_types.code
|
||||
@@ -15,6 +15,7 @@ following types have been implemented so far ..
|
||||
|
||||
main/mainresult
|
||||
main/keyvalue
|
||||
main/code
|
||||
|
||||
The :ref:`LegacyResult <LegacyResult>` is used internally for the results that
|
||||
have not yet been typed. The templates can be used as orientation until the
|
||||
@@ -27,6 +28,5 @@ final typing is complete.
|
||||
- :ref:`template map`
|
||||
- :ref:`template paper`
|
||||
- :ref:`template packages`
|
||||
- :ref:`template code`
|
||||
- :ref:`template files`
|
||||
- :ref:`template products`
|
||||
|
||||
@@ -469,33 +469,6 @@ links : :py:class:`dict`
|
||||
Additional links in the form of ``{'link_name': 'http://example.com'}``
|
||||
|
||||
|
||||
.. _template code:
|
||||
|
||||
``code.html``
|
||||
-------------
|
||||
|
||||
Displays result fields from:
|
||||
|
||||
- :ref:`macro result_header` and
|
||||
- :ref:`macro result_sub_header`
|
||||
|
||||
Additional fields used in the :origin:`code.html
|
||||
<searx/templates/simple/result_templates/code.html>`:
|
||||
|
||||
content : :py:class:`str`
|
||||
Description of the code fragment.
|
||||
|
||||
codelines : ``[line1, line2, ...]``
|
||||
Lines of the code fragment.
|
||||
|
||||
code_language : :py:class:`str`
|
||||
Name of the code language, the value is passed to
|
||||
:py:obj:`pygments.lexers.get_lexer_by_name`.
|
||||
|
||||
repository : :py:class:`str`
|
||||
URL of the repository of the code fragment.
|
||||
|
||||
|
||||
.. _template files:
|
||||
|
||||
``files.html``
|
||||
|
||||
@@ -54,7 +54,9 @@ Implementations
|
||||
|
||||
"""
|
||||
|
||||
import base64
|
||||
import typing
|
||||
import secrets
|
||||
|
||||
from urllib.parse import urlencode
|
||||
from datetime import datetime
|
||||
@@ -140,6 +142,10 @@ def request(query, params):
|
||||
query_params.update(category_config[chinaso_category]['params'])
|
||||
|
||||
params["url"] = f"{base_url}{category_config[chinaso_category]['endpoint']}?{urlencode(query_params)}"
|
||||
cookie = {
|
||||
"uid": base64.b64encode(secrets.token_bytes(16)).decode("utf-8"),
|
||||
}
|
||||
params["cookies"] = cookie
|
||||
|
||||
return params
|
||||
|
||||
|
||||
@@ -68,10 +68,8 @@ code blocks in a single file might be returned from the API).
|
||||
from __future__ import annotations
|
||||
|
||||
import typing as t
|
||||
from urllib.parse import urlencode, urlparse
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from pygments.lexers import guess_lexer_for_filename
|
||||
from pygments.util import ClassNotFound
|
||||
from searx.result_types import EngineResults
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.network import raise_for_httperror
|
||||
@@ -162,26 +160,10 @@ def request(query: str, params: dict[str, t.Any]) -> None:
|
||||
params['raise_for_httperror'] = False
|
||||
|
||||
|
||||
def get_code_language_name(filename: str, code_snippet: str) -> str | None:
|
||||
"""Returns a code language name by pulling information from the filename if
|
||||
possible otherwise by scanning the passed code snippet. In case there is any
|
||||
parsing error just default to no syntax highlighting."""
|
||||
try:
|
||||
lexer = guess_lexer_for_filename(filename, _text=code_snippet)
|
||||
if lexer is None:
|
||||
return None
|
||||
code_name_aliases = lexer.aliases
|
||||
if len(code_name_aliases) == 0:
|
||||
return None
|
||||
return code_name_aliases[0]
|
||||
except ClassNotFound:
|
||||
return None
|
||||
|
||||
|
||||
def extract_code(code_matches: list[dict[str, t.Any]]) -> tuple[list[str], set[int]]:
|
||||
"""
|
||||
Iterate over multiple possible matches, for each extract a code fragment.
|
||||
GitHub additionally sends context for _word_ highlights; pygments supports
|
||||
Github additionally sends context for _word_ highlights; pygments supports
|
||||
highlighting lines, as such we calculate which lines to highlight while
|
||||
traversing the text.
|
||||
"""
|
||||
@@ -231,18 +213,18 @@ def extract_code(code_matches: list[dict[str, t.Any]]) -> tuple[list[str], set[i
|
||||
|
||||
|
||||
def response(resp: SXNG_Response) -> EngineResults:
|
||||
results = EngineResults()
|
||||
res = EngineResults()
|
||||
|
||||
if resp.status_code == 422:
|
||||
# on a invalid search term the status code 422 "Unprocessable Content"
|
||||
# is returned / e.g. search term is "user: foo" instead "user:foo"
|
||||
return results
|
||||
return res
|
||||
# raise for other errors
|
||||
raise_for_httperror(resp)
|
||||
|
||||
for item in resp.json().get('items', []):
|
||||
repo = item['repository']
|
||||
text_matches = item['text_matches']
|
||||
repo: dict[str, str] = item['repository'] # pyright: ignore[reportAny]
|
||||
text_matches: list[dict[str, str]] = item['text_matches'] # pyright: ignore[reportAny]
|
||||
# ensure picking only the code contents in the blob
|
||||
code_matches = [
|
||||
match for match in text_matches if match["object_type"] == "FileContent" and match["property"] == "content"
|
||||
@@ -251,22 +233,18 @@ def response(resp: SXNG_Response) -> EngineResults:
|
||||
if not ghc_highlight_matching_lines:
|
||||
highlighted_lines_index: set[int] = set()
|
||||
|
||||
code_snippet = "\n".join(lines)
|
||||
res.add(
|
||||
res.types.Code(
|
||||
url=item["html_url"], # pyright: ignore[reportAny]
|
||||
title=f"{repo['full_name']} · {item['name']}",
|
||||
filename=f"{item['path']}",
|
||||
content=repo['description'],
|
||||
repository=repo['html_url'],
|
||||
codelines=[(i + 1, line) for (i, line) in enumerate(lines)],
|
||||
hl_lines=highlighted_lines_index,
|
||||
strip_whitespace=ghc_strip_whitespace,
|
||||
strip_new_lines=ghc_strip_new_lines,
|
||||
)
|
||||
)
|
||||
|
||||
kwargs: dict[str, t.Any] = {
|
||||
'template': 'code.html',
|
||||
'url': item['html_url'],
|
||||
'title': f"{repo['full_name']} · {item['path']}",
|
||||
'content': repo['description'],
|
||||
'repository': repo['html_url'],
|
||||
'codelines': [(i + 1, line) for (i, line) in enumerate(lines)],
|
||||
'hl_lines': highlighted_lines_index,
|
||||
'code_language': get_code_language_name(filename=item['name'], code_snippet=code_snippet),
|
||||
# important to set for highlighing
|
||||
'strip_whitespace': ghc_strip_whitespace,
|
||||
'strip_new_lines': ghc_strip_new_lines,
|
||||
'parsed_url': urlparse(item['html_url']),
|
||||
}
|
||||
results.add(results.types.LegacyResult(**kwargs))
|
||||
|
||||
return results
|
||||
return res
|
||||
|
||||
95
searx/engines/repology.py
Normal file
95
searx/engines/repology.py
Normal file
@@ -0,0 +1,95 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Repology_ monitors a huge number of package repositories and other sources
|
||||
comparing packages versions across them and gathering other information.
|
||||
|
||||
Repology_ shows you in which repositories a given project is packaged, which
|
||||
version is the latest and which needs updating, who maintains the package, and
|
||||
other related information.
|
||||
|
||||
.. _Repology: https://repology.org/docs/about
|
||||
|
||||
Configuration
|
||||
=============
|
||||
|
||||
The engine is inactive by default, meaning it is not available in the service.
|
||||
If you want to offer the engine, the ``inactive`` flag must be set to ``false``.
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: repology
|
||||
inactive: false
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
|
||||
from urllib.parse import urlencode
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
|
||||
about = {
|
||||
'website': 'https://repology.org',
|
||||
'wikidata_id': 'Q107409859',
|
||||
'use_official_api': True,
|
||||
'official_api_documentation': 'https://repology.org/api/v1',
|
||||
'require_api_key': False,
|
||||
'results': 'JSON',
|
||||
}
|
||||
categories: list[str] = ['packages', 'it']
|
||||
base_url: str = 'https://repology.org'
|
||||
|
||||
|
||||
def request(query: str, params: dict[str, t.Any]) -> None:
|
||||
args = {
|
||||
'search': query,
|
||||
}
|
||||
params['url'] = f"{base_url}/api/v1/projects/?{urlencode(args)}"
|
||||
|
||||
|
||||
def _get_most_common(items: list[str | None]) -> str | None:
|
||||
counts: dict[str | None, int] = {}
|
||||
for item in items:
|
||||
if item:
|
||||
counts[item] = counts.get(item, 0) + 1
|
||||
|
||||
if len(counts) == 0:
|
||||
return None
|
||||
return max(counts, key=counts.get)
|
||||
|
||||
|
||||
def _flatten(xss):
|
||||
return [x for xs in xss for x in xs]
|
||||
|
||||
|
||||
def response(resp: 'SXNG_Response') -> EngineResults:
|
||||
res = EngineResults()
|
||||
|
||||
resp_json = resp.json()
|
||||
for pkgname, repositories in resp_json.items():
|
||||
|
||||
# either there's a package with status "newest" or we assume that the
|
||||
# most commonly used version is the latest released (non-alpha) version
|
||||
latest_version = None
|
||||
for repo in repositories:
|
||||
if repo.get("status") == "newest":
|
||||
latest_version = repo["version"]
|
||||
break
|
||||
else:
|
||||
latest_version = _get_most_common([repo.get("version") for repo in repositories])
|
||||
|
||||
res.add(
|
||||
res.types.LegacyResult(
|
||||
template='packages.html',
|
||||
url=f"{base_url}/project/{pkgname}/versions",
|
||||
title=pkgname,
|
||||
content=_get_most_common([pkg.get("summary") for pkg in repositories]),
|
||||
package_name=_get_most_common([pkg.get("visiblename") for pkg in repositories]),
|
||||
version=latest_version,
|
||||
license_name=_get_most_common(_flatten([pkg.get("licenses", []) for pkg in repositories])),
|
||||
tags=list({pkg.get("repo") for pkg in repositories}), # ensure that tags are unique
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
@@ -1,79 +1,62 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Searchcode (IT)
|
||||
"""Searchcode (IT)"""
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import typing as t
|
||||
|
||||
from json import loads
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from searx.result_types import EngineResults
|
||||
from searx.extended_types import SXNG_Response
|
||||
|
||||
# about
|
||||
about = {
|
||||
"website": 'https://searchcode.com/',
|
||||
"website": "https://searchcode.com/",
|
||||
"wikidata_id": None,
|
||||
"official_api_documentation": 'https://searchcode.com/api/',
|
||||
"official_api_documentation": "https://searchcode.com/api/",
|
||||
"use_official_api": True,
|
||||
"require_api_key": False,
|
||||
"results": 'JSON',
|
||||
"results": "JSON",
|
||||
}
|
||||
|
||||
# engine dependent config
|
||||
categories = ['it']
|
||||
search_api = 'https://searchcode.com/api/codesearch_I/?'
|
||||
|
||||
# special code-endings which are not recognised by the file ending
|
||||
code_endings = {'cs': 'c#', 'h': 'c', 'hpp': 'cpp', 'cxx': 'cpp'}
|
||||
categories = ["it"]
|
||||
search_api = "https://searchcode.com/api/codesearch_I/?"
|
||||
|
||||
# paging is broken in searchcode.com's API .. not sure it will ever been fixed
|
||||
# paging = True
|
||||
|
||||
|
||||
def request(query, params):
|
||||
args = urlencode(
|
||||
{
|
||||
'q': query,
|
||||
# paging is broken in searchcode.com's API
|
||||
# 'p': params['pageno'] - 1,
|
||||
# 'per_page': 10,
|
||||
}
|
||||
)
|
||||
params['url'] = search_api + args
|
||||
logger.debug("query_url --> %s", params['url'])
|
||||
return params
|
||||
def request(query: str, params: dict[str, t.Any]) -> None:
|
||||
args = {
|
||||
"q": query,
|
||||
# paging is broken in searchcode.com's API
|
||||
# "p": params["pageno"] - 1,
|
||||
# "per_page": 10,
|
||||
}
|
||||
|
||||
params["url"] = search_api + urlencode(args)
|
||||
logger.debug("query_url --> %s", params["url"])
|
||||
|
||||
|
||||
def response(resp):
|
||||
results = []
|
||||
|
||||
search_results = loads(resp.text)
|
||||
def response(resp: SXNG_Response) -> EngineResults:
|
||||
res = EngineResults()
|
||||
|
||||
# parse results
|
||||
for result in search_results.get('results', []):
|
||||
href = result['url']
|
||||
title = "" + result['name'] + " - " + result['filename']
|
||||
repo = result['repo']
|
||||
|
||||
for result in resp.json().get("results", []):
|
||||
lines = {}
|
||||
for line, code in result['lines'].items():
|
||||
for line, code in result["lines"].items():
|
||||
lines[int(line)] = code
|
||||
|
||||
code_language = code_endings.get(
|
||||
result['filename'].split('.')[-1].lower(), result['filename'].split('.')[-1].lower()
|
||||
res.add(
|
||||
res.types.Code(
|
||||
url=result["url"],
|
||||
title=f'{result["name"]} - {result["filename"]}',
|
||||
repository=result["repo"],
|
||||
filename=result["filename"],
|
||||
codelines=sorted(lines.items()),
|
||||
strip_whitespace=True,
|
||||
)
|
||||
)
|
||||
|
||||
# append result
|
||||
results.append(
|
||||
{
|
||||
'url': href,
|
||||
'title': title,
|
||||
'content': '',
|
||||
'repository': repo,
|
||||
'codelines': sorted(lines.items()),
|
||||
'code_language': code_language,
|
||||
'template': 'code.html',
|
||||
'strip_whitespace': True,
|
||||
'strip_new_lines': True,
|
||||
}
|
||||
)
|
||||
|
||||
# return results
|
||||
return results
|
||||
return res
|
||||
|
||||
@@ -13,25 +13,38 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["Result", "MainResult", "KeyValue", "EngineResults", "AnswerSet", "Answer", "Translations", "WeatherAnswer"]
|
||||
__all__ = [
|
||||
"Result",
|
||||
"MainResult",
|
||||
"KeyValue",
|
||||
"EngineResults",
|
||||
"AnswerSet",
|
||||
"Answer",
|
||||
"Translations",
|
||||
"WeatherAnswer",
|
||||
"Code",
|
||||
]
|
||||
|
||||
import typing as t
|
||||
import abc
|
||||
|
||||
from searx import enginelib
|
||||
|
||||
from ._base import Result, MainResult, LegacyResult
|
||||
from .answer import AnswerSet, Answer, Translations, WeatherAnswer
|
||||
from .keyvalue import KeyValue
|
||||
from .code import Code
|
||||
|
||||
|
||||
class ResultList(list, abc.ABC):
|
||||
class ResultList(list, abc.ABC): # pyright: ignore[reportMissingTypeArgument]
|
||||
"""Base class of all result lists (abstract)."""
|
||||
|
||||
@t.final
|
||||
class types: # pylint: disable=invalid-name
|
||||
"""The collection of result types (which have already been implemented)."""
|
||||
"""The collection of result types (which have already been
|
||||
implemented)."""
|
||||
|
||||
Answer = Answer
|
||||
KeyValue = KeyValue
|
||||
Code = Code
|
||||
MainResult = MainResult
|
||||
Result = Result
|
||||
Translations = Translations
|
||||
@@ -42,11 +55,11 @@ class ResultList(list, abc.ABC):
|
||||
|
||||
def __init__(self):
|
||||
# pylint: disable=useless-parent-delegation
|
||||
super().__init__()
|
||||
super().__init__() # pyright: ignore[reportUnknownMemberType]
|
||||
|
||||
def add(self, result: Result | LegacyResult):
|
||||
"""Add a :py:`Result` item to the result list."""
|
||||
self.append(result)
|
||||
self.append(result) # pyright: ignore[reportUnknownMemberType]
|
||||
|
||||
|
||||
class EngineResults(ResultList):
|
||||
|
||||
185
searx/result_types/code.py
Normal file
185
searx/result_types/code.py
Normal file
@@ -0,0 +1,185 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Typification of the *code* results. Results of this type are rendered in
|
||||
the :origin:`code.html <searx/templates/simple/result_templates/code.html>`
|
||||
template. For highlighting the code passages, Pygments is used.
|
||||
|
||||
.. _Pygments: https://pygments.org
|
||||
|
||||
----
|
||||
|
||||
.. autoclass:: Code
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
"""
|
||||
# pylint: disable=too-few-public-methods, disable=invalid-name
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["Code"]
|
||||
|
||||
import typing as t
|
||||
|
||||
from pygments import highlight # pyright: ignore[reportUnknownVariableType]
|
||||
from pygments.lexers._mapping import LEXERS # pyright: ignore[reportMissingTypeStubs]
|
||||
from pygments.lexers import guess_lexer, get_lexer_by_name, guess_lexer_for_filename
|
||||
from pygments.util import ClassNotFound
|
||||
from pygments.formatters import HtmlFormatter # pylint: disable=no-name-in-module
|
||||
|
||||
from ._base import MainResult
|
||||
|
||||
|
||||
_pygments_languages: list[str] = []
|
||||
|
||||
|
||||
def is_valid_language(code_language: str) -> bool:
|
||||
"""Checks if the specified ``code_language`` is known in Pygments."""
|
||||
if not _pygments_languages:
|
||||
for l in LEXERS.values():
|
||||
# l[2] is the tuple with the alias names
|
||||
for alias_name in l[2]:
|
||||
_pygments_languages.append(alias_name.lower())
|
||||
return code_language.lower() in _pygments_languages
|
||||
|
||||
|
||||
@t.final
|
||||
class Code(MainResult, kw_only=True):
|
||||
"""Simple table view which maps *key* names (first col) to *values*
|
||||
(second col)."""
|
||||
|
||||
template: str = "code.html"
|
||||
|
||||
repository: str | None = None
|
||||
"""A link related to a repository related to the *result*"""
|
||||
|
||||
codelines: list[tuple[int, str]] = []
|
||||
"""A list of two digit tuples where the first item is the line number and
|
||||
the second item is the code line."""
|
||||
|
||||
hl_lines: set[int] = set()
|
||||
"""A list of line numbers to highlight"""
|
||||
|
||||
code_language: str = "<guess>"
|
||||
"""Pygment's short name of the lexer, e.g. ``text`` for the
|
||||
:py:obj:`pygments.lexers.special.TextLexer`. For a list of available
|
||||
languages consult: `Pygments languages`_. If the language is not in this
|
||||
list, a :py:obj:`ValueError` is raised.
|
||||
|
||||
The default is ``<guess>`` which has a special meaning;
|
||||
|
||||
- If :py:obj:`Code.filename` is set, Pygment's factory method
|
||||
:py:obj:`pygments.lexers.guess_lexer_for_filename` is used to determine
|
||||
the language of the ``codelines``.
|
||||
|
||||
- else Pygment's :py:obj:`pygments.lexers.guess_lexer` factory is used.
|
||||
|
||||
In case the language can't be detected, the fallback is ``text``.
|
||||
|
||||
.. _Pygments languages: https://pygments.org/languages/
|
||||
"""
|
||||
|
||||
filename: str | None = None
|
||||
"""Optional file name, can help to ``<guess>`` the language of the code (in
|
||||
case of ambiguous short code examples). If :py:obj:`Code.title` is not set,
|
||||
its default is the filename."""
|
||||
|
||||
strip_new_lines: bool = True
|
||||
"""Strip leading and trailing newlines for each returned fragment.
|
||||
Single file might return multiple code fragments.
|
||||
"""
|
||||
|
||||
strip_whitespace: bool = False
|
||||
"""Strip all leading and trailing whitespace for each returned fragment.
|
||||
Single file might return multiple code fragments. Enabling this might break
|
||||
code indentation.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
|
||||
if not self.title and self.filename:
|
||||
self.title = self.filename
|
||||
|
||||
if self.code_language != "<guess>" and not is_valid_language(self.code_language):
|
||||
raise ValueError(f"unknown code_language: {self.code_language}")
|
||||
|
||||
def __hash__(self):
|
||||
"""The hash value is build up from URL and code lines. :py:obj:`Code
|
||||
<Result.__eq__>` objects are equal, when the hash values of both objects
|
||||
are equal.
|
||||
"""
|
||||
return hash(f"{self.url} {self.codelines}")
|
||||
|
||||
def get_lexer(self):
|
||||
if self.code_language != "<guess>":
|
||||
return get_lexer_by_name(self.code_language)
|
||||
|
||||
src_code = "\n".join([l[1] for l in self.codelines])
|
||||
if self.filename:
|
||||
try:
|
||||
return guess_lexer_for_filename(self.filename, src_code)
|
||||
except ClassNotFound:
|
||||
pass
|
||||
try:
|
||||
return guess_lexer(src_code)
|
||||
except ClassNotFound:
|
||||
pass
|
||||
return get_lexer_by_name("text")
|
||||
|
||||
def HTML(self, **options) -> str: # pyright: ignore[reportUnknownParameterType, reportMissingParameterType]
|
||||
"""Rendered HTML, additional options are accepted, for more details have
|
||||
a look at HtmlFormatter_.
|
||||
|
||||
.. _HtmlFormatter: https://pygments.org/docs/formatters/#HtmlFormatter
|
||||
"""
|
||||
lexer = self.get_lexer()
|
||||
|
||||
line_no: int = 0 # current line number
|
||||
code_block_start: int = 0 # line where the current code block starts
|
||||
code_block_end: int | None = None # line where the current code ends
|
||||
code_block: list[str] = [] # lines of the current code block
|
||||
html_code_blocks: list[str] = [] # HTML representation of all code blocks
|
||||
|
||||
def _render(**kwargs): # pyright: ignore[reportUnknownParameterType, reportMissingParameterType]
|
||||
for k, default in [
|
||||
("linenos", "inline"),
|
||||
("linenostart", code_block_start),
|
||||
("cssclass", "code-highlight"),
|
||||
("hl_lines", [hl - code_block_start + 1 for hl in self.hl_lines]),
|
||||
]:
|
||||
kwargs[k] = kwargs.get(k, default) # pyright: ignore[reportUnknownMemberType]
|
||||
|
||||
# Wrap the code inside <pre> blocks using <code>, as recommended by
|
||||
# the HTML5 specification (default is False). Do we need this?
|
||||
kwargs["wrapcode"] = kwargs.get("wrapcode", True)
|
||||
|
||||
html_code_blocks.append(
|
||||
highlight(
|
||||
"\n".join(code_block),
|
||||
lexer,
|
||||
HtmlFormatter(**kwargs), # pyright: ignore[reportUnknownArgumentType]
|
||||
)
|
||||
)
|
||||
|
||||
for line_no, code_line in self.codelines:
|
||||
if code_block_end is None:
|
||||
# initial start condition
|
||||
code_block_start = line_no
|
||||
|
||||
if code_block_end is not None and code_block_end + 1 != line_no:
|
||||
# new code block is detected, render current code block
|
||||
_render(**options) # pyright: ignore[reportUnknownArgumentType]
|
||||
# reset conditions for next code block, which first line is the
|
||||
# current code line
|
||||
code_block = [code_line]
|
||||
code_block_start = line_no
|
||||
code_block_end = line_no
|
||||
continue
|
||||
|
||||
# add line to the current code block and update last line n
|
||||
code_block.append(code_line)
|
||||
code_block_end = line_no
|
||||
|
||||
# highlight (last) code block
|
||||
_render(**options) # pyright: ignore[reportUnknownArgumentType]
|
||||
return "\n".join(html_code_blocks)
|
||||
@@ -1955,6 +1955,7 @@ engines:
|
||||
engine: searchcode_code
|
||||
shortcut: scc
|
||||
disabled: true
|
||||
inactive: true
|
||||
|
||||
# - name: searx
|
||||
# engine: searx_engine
|
||||
@@ -2524,6 +2525,12 @@ engines:
|
||||
categories: videos
|
||||
disabled: true
|
||||
|
||||
- name: repology
|
||||
engine: repology
|
||||
shortcut: rep
|
||||
disabled: true
|
||||
inactive: true
|
||||
|
||||
- name: livespace
|
||||
engine: livespace
|
||||
shortcut: ls
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -10,22 +10,28 @@
|
||||
{%- endif -%}
|
||||
{%- if result.repository -%}
|
||||
<p class="content">{{- '' -}}
|
||||
{{ _('repo') }}: {{- ' ' -}}
|
||||
{{ _('Repository') }}: {{- ' ' -}}
|
||||
<a href="{{ result.repository|safe }}"{{- ' ' -}}
|
||||
{% if results_on_new_tab %}
|
||||
target="_blank" {{- ' ' -}}
|
||||
rel="noopener noreferrer"
|
||||
{%- else -%}
|
||||
rel="noreferrer"
|
||||
{%- endif -%}
|
||||
>
|
||||
{{- result.repository -}}
|
||||
{% if results_on_new_tab %}
|
||||
target="_blank" {{- ' ' -}}
|
||||
rel="noopener noreferrer"
|
||||
{%- else -%}
|
||||
rel="noreferrer"
|
||||
{%- endif -%}
|
||||
>
|
||||
{{- result.repository -}}
|
||||
</a>{{- '' -}}
|
||||
</p>
|
||||
{%- endif -%}
|
||||
|
||||
{%- if result.filename %}
|
||||
<p class="content">
|
||||
{{ _('Filename') }}: {{ result.filename|safe }}
|
||||
</p>
|
||||
{% endif -%}
|
||||
|
||||
<div dir="ltr" class="codelines">
|
||||
{{- result.codelines|code_highlighter(result.code_language, result.hl_lines, result.strip_whitespace, result.strip_new_lines)|safe -}}
|
||||
{{- result.HTML()|safe -}}
|
||||
</div>
|
||||
|
||||
{{- result_sub_footer(result) -}}
|
||||
|
||||
@@ -47,9 +47,6 @@ END_DARK_THEME = """
|
||||
|
||||
|
||||
class Formatter(HtmlFormatter): # pylint: disable=missing-class-docstring
|
||||
@property
|
||||
def _pre_style(self):
|
||||
return 'line-height: 100%;'
|
||||
|
||||
def get_style_lines(self, arg=None):
|
||||
style_lines = []
|
||||
@@ -73,4 +70,4 @@ def generat_css(light_style, dark_style) -> str:
|
||||
if __name__ == '__main__':
|
||||
print("update: %s" % LESS_FILE)
|
||||
with LESS_FILE.open('w', encoding='utf8') as f:
|
||||
f.write(generat_css('default', 'lightbulb'))
|
||||
f.write(generat_css('default', 'monokai'))
|
||||
|
||||
@@ -142,29 +142,26 @@ class GithubCodeTests(SearxTestCase):
|
||||
results = self.ghc.response(response)
|
||||
expected_results = EngineResults()
|
||||
expected_results.add(
|
||||
expected_results.types.LegacyResult(
|
||||
**{
|
||||
'url': "https://github.com/folke/dot/blob/3140f4f5720c3cc6b5034c624eb7706f8533a82c/TODO.md",
|
||||
'title': "folke/dot · TODO.md",
|
||||
'content': "☕️ My Dot Files",
|
||||
'repository': "https://github.com/folke/dot",
|
||||
'codelines': [
|
||||
(1, "- [x] windows picker"),
|
||||
(2, "- [x] toggle cwd / root (LazyVim)"),
|
||||
(3, "- [x] dynamic workspace symbol"),
|
||||
(4, "- [x] smart stops working after custom"),
|
||||
(5, "- [x] edit in empty buffer"),
|
||||
(6, "- [x] support toggling line nr for preview"),
|
||||
],
|
||||
'hl_lines': {2, 5, 6},
|
||||
'code_language': "markdown",
|
||||
'template': 'code.html',
|
||||
'strip_whitespace': False,
|
||||
'strip_new_lines': True,
|
||||
'parsed_url': urlparse(
|
||||
"https://github.com/folke/dot/blob/3140f4f5720c3cc6b5034c624eb7706f8533a82c/TODO.md"
|
||||
),
|
||||
}
|
||||
expected_results.types.Code(
|
||||
url="https://github.com/folke/dot/blob/3140f4f5720c3cc6b5034c624eb7706f8533a82c/TODO.md",
|
||||
title="folke/dot · TODO.md",
|
||||
content="☕️ My Dot Files",
|
||||
repository="https://github.com/folke/dot",
|
||||
codelines=[
|
||||
(1, "- [x] windows picker"),
|
||||
(2, "- [x] toggle cwd / root (LazyVim)"),
|
||||
(3, "- [x] dynamic workspace symbol"),
|
||||
(4, "- [x] smart stops working after custom"),
|
||||
(5, "- [x] edit in empty buffer"),
|
||||
(6, "- [x] support toggling line nr for preview"),
|
||||
],
|
||||
hl_lines={2, 5, 6},
|
||||
code_language="markdown",
|
||||
strip_whitespace=False,
|
||||
strip_new_lines=True,
|
||||
parsed_url=urlparse(
|
||||
"https://github.com/folke/dot/blob/3140f4f5720c3cc6b5034c624eb7706f8533a82c/TODO.md"
|
||||
),
|
||||
)
|
||||
)
|
||||
self.assertEqual(results, expected_results)
|
||||
|
||||
Reference in New Issue
Block a user