mirror of https://github.com/searxng/searxng.git
Compare commits
8 Commits
24e4ccee8f
...
0acc9a4f73
Author | SHA1 | Date |
---|---|---|
Bnyro | 0acc9a4f73 | |
Bnyro | 8744dd3c71 | |
Markus Heiser | 7927baf545 | |
Bnyro | ca81860c0b | |
Bnyro | e4961142e3 | |
Bnyro | f31a3a2053 | |
Markus Heiser | 0253c10b52 | |
Bnyro | f20a7632f1 |
|
@ -31,7 +31,7 @@ jobs:
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
|
@ -61,7 +61,7 @@ jobs:
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
|
@ -93,7 +93,7 @@ jobs:
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
|
@ -137,7 +137,7 @@ jobs:
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
|
@ -181,7 +181,7 @@ jobs:
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
|
|
|
@ -22,7 +22,7 @@ jobs:
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
|
|
|
@ -13,6 +13,7 @@
|
||||||
donation_url: false
|
donation_url: false
|
||||||
contact_url: false
|
contact_url: false
|
||||||
enable_metrics: true
|
enable_metrics: true
|
||||||
|
open_metrics: ''
|
||||||
|
|
||||||
``debug`` : ``$SEARXNG_DEBUG``
|
``debug`` : ``$SEARXNG_DEBUG``
|
||||||
Allow a more detailed log if you run SearXNG directly. Display *detailed* error
|
Allow a more detailed log if you run SearXNG directly. Display *detailed* error
|
||||||
|
@ -32,3 +33,10 @@
|
||||||
``enable_metrics``:
|
``enable_metrics``:
|
||||||
Enabled by default. Record various anonymous metrics available at ``/stats``,
|
Enabled by default. Record various anonymous metrics available at ``/stats``,
|
||||||
``/stats/errors`` and ``/preferences``.
|
``/stats/errors`` and ``/preferences``.
|
||||||
|
|
||||||
|
``open_metrics``:
|
||||||
|
Disabled by default. Set to a secret password to expose an
|
||||||
|
`OpenMetrics API <https://github.com/prometheus/OpenMetrics>`_ at ``/metrics``,
|
||||||
|
e.g. for usage with Prometheus. The ``/metrics`` endpoint is using HTTP Basic Auth,
|
||||||
|
where the password is the value of ``open_metrics`` set above. The username used for
|
||||||
|
Basic Auth can be randomly chosen as only the password is being validated.
|
||||||
|
|
|
@ -93,7 +93,7 @@ Online Currency
|
||||||
|
|
||||||
- :py:obj:`processors.online_currency <searx.search.processors.online_currency>`
|
- :py:obj:`processors.online_currency <searx.search.processors.online_currency>`
|
||||||
|
|
||||||
*no engine of this type is documented yet / comming soon*
|
*no engine of this type is documented yet / coming soon*
|
||||||
|
|
||||||
.. _online dictionary:
|
.. _online dictionary:
|
||||||
|
|
||||||
|
@ -104,4 +104,4 @@ Online Dictionary
|
||||||
|
|
||||||
- :py:obj:`processors.online_dictionary <searx.search.processors.online_dictionary>`
|
- :py:obj:`processors.online_dictionary <searx.search.processors.online_dictionary>`
|
||||||
|
|
||||||
*no engine of this type is documented yet / comming soon*
|
*no engine of this type is documented yet / coming soon*
|
||||||
|
|
|
@ -0,0 +1,13 @@
|
||||||
|
.. _adobe stock engine:
|
||||||
|
|
||||||
|
===========
|
||||||
|
Adobe Stock
|
||||||
|
===========
|
||||||
|
|
||||||
|
.. contents:: Contents
|
||||||
|
:depth: 2
|
||||||
|
:local:
|
||||||
|
:backlinks: entry
|
||||||
|
|
||||||
|
.. automodule:: searx.engines.adobe_stock
|
||||||
|
:members:
|
|
@ -19,3 +19,4 @@ tomli==2.0.2; python_version < '3.11'
|
||||||
msgspec==0.18.6
|
msgspec==0.18.6
|
||||||
eval_type_backport; python_version < '3.9'
|
eval_type_backport; python_version < '3.9'
|
||||||
typer-slim==0.13.1
|
typer-slim==0.13.1
|
||||||
|
isodate==0.7.2
|
||||||
|
|
|
@ -22,7 +22,7 @@ log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class FALSE:
|
class FALSE:
|
||||||
"""Class of ``False`` singelton"""
|
"""Class of ``False`` singleton"""
|
||||||
|
|
||||||
# pylint: disable=multiple-statements
|
# pylint: disable=multiple-statements
|
||||||
def __init__(self, msg):
|
def __init__(self, msg):
|
||||||
|
@ -81,7 +81,7 @@ class Config:
|
||||||
return cfg
|
return cfg
|
||||||
|
|
||||||
def __init__(self, cfg_schema: typing.Dict, deprecated: typing.Dict[str, str]):
|
def __init__(self, cfg_schema: typing.Dict, deprecated: typing.Dict[str, str]):
|
||||||
"""Construtor of class Config.
|
"""Constructor of class Config.
|
||||||
|
|
||||||
:param cfg_schema: Schema of the configuration
|
:param cfg_schema: Schema of the configuration
|
||||||
:param deprecated: dictionary that maps deprecated configuration names to a messages
|
:param deprecated: dictionary that maps deprecated configuration names to a messages
|
||||||
|
@ -159,7 +159,7 @@ class Config:
|
||||||
return pathlib.Path(str(val))
|
return pathlib.Path(str(val))
|
||||||
|
|
||||||
def pyobj(self, name, default=UNSET):
|
def pyobj(self, name, default=UNSET):
|
||||||
"""Get python object refered by full qualiffied name (FQN) in the config
|
"""Get python object referred by full qualiffied name (FQN) in the config
|
||||||
string."""
|
string."""
|
||||||
|
|
||||||
fqn = self.get(name, default)
|
fqn = self.get(name, default)
|
||||||
|
|
|
@ -55,10 +55,10 @@ from ._helpers import (
|
||||||
)
|
)
|
||||||
|
|
||||||
TOKEN_LIVE_TIME = 600
|
TOKEN_LIVE_TIME = 600
|
||||||
"""Livetime (sec) of limiter's CSS token."""
|
"""Lifetime (sec) of limiter's CSS token."""
|
||||||
|
|
||||||
PING_LIVE_TIME = 3600
|
PING_LIVE_TIME = 3600
|
||||||
"""Livetime (sec) of the ping-key from a client (request)"""
|
"""Lifetime (sec) of the ping-key from a client (request)"""
|
||||||
|
|
||||||
PING_KEY = 'SearXNG_limiter.ping'
|
PING_KEY = 'SearXNG_limiter.ping'
|
||||||
"""Prefix of all ping-keys generated by :py:obj:`get_ping_key`"""
|
"""Prefix of all ping-keys generated by :py:obj:`get_ping_key`"""
|
||||||
|
|
|
@ -0,0 +1,229 @@
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
"""`Adobe Stock`_ is a service that gives access to millions of royalty-free
|
||||||
|
assets. Assets types include photos, vectors, illustrations, templates, 3D
|
||||||
|
assets, videos, motion graphics templates and audio tracks.
|
||||||
|
|
||||||
|
.. Adobe Stock: https://stock.adobe.com/
|
||||||
|
|
||||||
|
Configuration
|
||||||
|
=============
|
||||||
|
|
||||||
|
The engine has the following mandatory setting:
|
||||||
|
|
||||||
|
- SearXNG's :ref:`engine categories`
|
||||||
|
- Adobe-Stock's :py:obj:`adobe_order`
|
||||||
|
- Adobe-Stock's :py:obj:`adobe_content_types`
|
||||||
|
|
||||||
|
.. code:: yaml
|
||||||
|
|
||||||
|
- name: adobe stock
|
||||||
|
engine: adobe_stock
|
||||||
|
shortcut: asi
|
||||||
|
categories: [images]
|
||||||
|
adobe_order: relevance
|
||||||
|
adobe_content_types: ["photo", "illustration", "zip_vector", "template", "3d", "image"]
|
||||||
|
|
||||||
|
- name: adobe stock video
|
||||||
|
engine: adobe_stock
|
||||||
|
network: adobe stock
|
||||||
|
shortcut: asi
|
||||||
|
categories: [videos]
|
||||||
|
adobe_order: relevance
|
||||||
|
adobe_content_types: ["video"]
|
||||||
|
|
||||||
|
Implementation
|
||||||
|
==============
|
||||||
|
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from urllib.parse import urlencode
|
||||||
|
|
||||||
|
import isodate
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger: logging.Logger
|
||||||
|
|
||||||
|
about = {
|
||||||
|
"website": "https://stock.adobe.com/",
|
||||||
|
"wikidata_id": "Q5977430",
|
||||||
|
"official_api_documentation": None,
|
||||||
|
"use_official_api": False,
|
||||||
|
"require_api_key": False,
|
||||||
|
"results": "JSON",
|
||||||
|
}
|
||||||
|
|
||||||
|
categories = []
|
||||||
|
paging = True
|
||||||
|
send_accept_language_header = True
|
||||||
|
results_per_page = 10
|
||||||
|
|
||||||
|
base_url = "https://stock.adobe.com"
|
||||||
|
|
||||||
|
adobe_order: str = ""
|
||||||
|
"""Sort order, can be one of:
|
||||||
|
|
||||||
|
- ``relevance`` or
|
||||||
|
- ``featured`` or
|
||||||
|
- ``creation`` (most recent) or
|
||||||
|
- ``nb_downloads`` (number of downloads)
|
||||||
|
"""
|
||||||
|
|
||||||
|
ADOBE_VALID_TYPES = ["photo", "illustration", "zip_vector", "video", "template", "3d", "audio", "image"]
|
||||||
|
adobe_content_types: list = []
|
||||||
|
"""A list of of content types. The following content types are offered:
|
||||||
|
|
||||||
|
- Images: ``image``
|
||||||
|
- Videos: ``video``
|
||||||
|
- Templates: ``template``
|
||||||
|
- 3D: ``3d``
|
||||||
|
- Audio ``audio``
|
||||||
|
|
||||||
|
Additional subcategories:
|
||||||
|
|
||||||
|
- Photos: ``photo``
|
||||||
|
- Illustrations: ``illustration``
|
||||||
|
- Vectors: ``zip_vector`` (Vectors),
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Do we need support for "free_collection" and "include_stock_enterprise"?
|
||||||
|
|
||||||
|
|
||||||
|
def init(_):
|
||||||
|
if not categories:
|
||||||
|
raise ValueError("adobe_stock engine: categories is unset")
|
||||||
|
|
||||||
|
# adobe_order
|
||||||
|
if not adobe_order:
|
||||||
|
raise ValueError("adobe_stock engine: adobe_order is unset")
|
||||||
|
if adobe_order not in ["relevance", "featured", "creation", "nb_downloads"]:
|
||||||
|
raise ValueError(f"unsupported adobe_order: {adobe_order}")
|
||||||
|
|
||||||
|
# adobe_content_types
|
||||||
|
if not adobe_content_types:
|
||||||
|
raise ValueError("adobe_stock engine: adobe_content_types is unset")
|
||||||
|
|
||||||
|
if isinstance(adobe_content_types, list):
|
||||||
|
for t in adobe_content_types:
|
||||||
|
if t not in ADOBE_VALID_TYPES:
|
||||||
|
raise ValueError("adobe_stock engine: adobe_content_types: '%s' is invalid" % t)
|
||||||
|
else:
|
||||||
|
raise ValueError(
|
||||||
|
"adobe_stock engine: adobe_content_types must be a list of strings not %s" % type(adobe_content_types)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def request(query, params):
|
||||||
|
|
||||||
|
args = {
|
||||||
|
"k": query,
|
||||||
|
"limit": results_per_page,
|
||||||
|
"order": adobe_order,
|
||||||
|
"search_page": params["pageno"],
|
||||||
|
"search_type": "pagination",
|
||||||
|
}
|
||||||
|
|
||||||
|
for content_type in ADOBE_VALID_TYPES:
|
||||||
|
args[f"filters[content_type:{content_type}]"] = 1 if content_type in adobe_content_types else 0
|
||||||
|
|
||||||
|
params["url"] = f"{base_url}/de/Ajax/Search?{urlencode(args)}"
|
||||||
|
|
||||||
|
# headers required to bypass bot-detection
|
||||||
|
if params["searxng_locale"] == "all":
|
||||||
|
params["headers"]["Accept-Language"] = "en-US,en;q=0.5"
|
||||||
|
|
||||||
|
return params
|
||||||
|
|
||||||
|
|
||||||
|
def parse_image_item(item):
|
||||||
|
return {
|
||||||
|
"template": "images.html",
|
||||||
|
"url": item["content_url"],
|
||||||
|
"title": item["title"],
|
||||||
|
"content": item["asset_type"],
|
||||||
|
"img_src": item["content_thumb_extra_large_url"],
|
||||||
|
"thumbnail_src": item["thumbnail_url"],
|
||||||
|
"resolution": f"{item['content_original_width']}x{item['content_original_height']}",
|
||||||
|
"img_format": item["format"],
|
||||||
|
"author": item["author"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_video_item(item):
|
||||||
|
|
||||||
|
# in video items, the title is more or less a "content description", we try
|
||||||
|
# to reduce the lenght of the title ..
|
||||||
|
|
||||||
|
title = item["title"]
|
||||||
|
content = ""
|
||||||
|
if "." in title.strip()[:-1]:
|
||||||
|
content = title
|
||||||
|
title = title.split(".", 1)[0]
|
||||||
|
elif "," in title:
|
||||||
|
content = title
|
||||||
|
title = title.split(",", 1)[0]
|
||||||
|
elif len(title) > 50:
|
||||||
|
content = title
|
||||||
|
title = ""
|
||||||
|
for w in content.split(" "):
|
||||||
|
title += f" {w}"
|
||||||
|
if len(title) > 50:
|
||||||
|
title = title.strip() + "\u2026"
|
||||||
|
break
|
||||||
|
|
||||||
|
return {
|
||||||
|
"template": "videos.html",
|
||||||
|
"url": item["content_url"],
|
||||||
|
"title": title,
|
||||||
|
"content": content,
|
||||||
|
# https://en.wikipedia.org/wiki/ISO_8601#Durations
|
||||||
|
"length": isodate.parse_duration(item["time_duration"]),
|
||||||
|
"publishedDate": datetime.strptime(item["creation_date"], "%Y-%m-%d"),
|
||||||
|
"thumbnail": item["thumbnail_url"],
|
||||||
|
"iframe_src": item["video_small_preview_url"],
|
||||||
|
"metadata": item["asset_type"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_audio_item(item):
|
||||||
|
audio_data = item["audio_data"]
|
||||||
|
content = audio_data.get("description") or ""
|
||||||
|
if audio_data.get("album"):
|
||||||
|
content = audio_data["album"] + " - " + content
|
||||||
|
|
||||||
|
return {
|
||||||
|
"url": item["content_url"],
|
||||||
|
"title": item["title"],
|
||||||
|
"content": content,
|
||||||
|
# "thumbnail": base_url + item["thumbnail_url"],
|
||||||
|
"iframe_src": audio_data["preview"]["url"],
|
||||||
|
"publishedDate": datetime.fromisoformat(audio_data["release_date"]) if audio_data["release_date"] else None,
|
||||||
|
"length": timedelta(seconds=round(audio_data["duration"] / 1000)) if audio_data["duration"] else None,
|
||||||
|
"author": item.get("artist_name"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def response(resp):
|
||||||
|
results = []
|
||||||
|
|
||||||
|
json_resp = resp.json()
|
||||||
|
|
||||||
|
if isinstance(json_resp["items"], list):
|
||||||
|
return None
|
||||||
|
for item in json_resp["items"].values():
|
||||||
|
if item["asset_type"].lower() in ["image", "premium-image", "illustration", "vector"]:
|
||||||
|
result = parse_image_item(item)
|
||||||
|
elif item["asset_type"].lower() == "video":
|
||||||
|
result = parse_video_item(item)
|
||||||
|
elif item["asset_type"].lower() == "audio":
|
||||||
|
result = parse_audio_item(item)
|
||||||
|
else:
|
||||||
|
logger.error("no handle for %s --> %s", item["asset_type"], item)
|
||||||
|
continue
|
||||||
|
results.append(result)
|
||||||
|
|
||||||
|
return results
|
|
@ -31,7 +31,7 @@ paging = True
|
||||||
number_of_results = 10
|
number_of_results = 10
|
||||||
|
|
||||||
# shortcuts for advanced search
|
# shortcuts for advanced search
|
||||||
shorcut_dict = {
|
shortcut_dict = {
|
||||||
# user-friendly keywords
|
# user-friendly keywords
|
||||||
'format:': 'dcformat:',
|
'format:': 'dcformat:',
|
||||||
'author:': 'dccreator:',
|
'author:': 'dccreator:',
|
||||||
|
@ -55,7 +55,7 @@ shorcut_dict = {
|
||||||
|
|
||||||
def request(query, params):
|
def request(query, params):
|
||||||
# replace shortcuts with API advanced search keywords
|
# replace shortcuts with API advanced search keywords
|
||||||
for key, val in shorcut_dict.items():
|
for key, val in shortcut_dict.items():
|
||||||
query = re.sub(key, val, query)
|
query = re.sub(key, val, query)
|
||||||
|
|
||||||
# basic search
|
# basic search
|
||||||
|
|
|
@ -10,7 +10,7 @@ On the `preference page`_ Bing offers a lot of languages an regions (see section
|
||||||
LANGUAGE and COUNTRY/REGION). The Language is the language of the UI, we need
|
LANGUAGE and COUNTRY/REGION). The Language is the language of the UI, we need
|
||||||
in SearXNG to get the translations of data such as *"published last week"*.
|
in SearXNG to get the translations of data such as *"published last week"*.
|
||||||
|
|
||||||
There is a description of the offical search-APIs_, unfortunately this is not
|
There is a description of the official search-APIs_, unfortunately this is not
|
||||||
the API we can use or that bing itself would use. You can look up some things
|
the API we can use or that bing itself would use. You can look up some things
|
||||||
in the API to get a better picture of bing, but the value specifications like
|
in the API to get a better picture of bing, but the value specifications like
|
||||||
the market codes are usually outdated or at least no longer used by bing itself.
|
the market codes are usually outdated or at least no longer used by bing itself.
|
||||||
|
@ -91,7 +91,7 @@ def request(query, params):
|
||||||
page = params.get('pageno', 1)
|
page = params.get('pageno', 1)
|
||||||
query_params = {
|
query_params = {
|
||||||
'q': query,
|
'q': query,
|
||||||
# if arg 'pq' is missed, somtimes on page 4 we get results from page 1,
|
# if arg 'pq' is missed, sometimes on page 4 we get results from page 1,
|
||||||
# don't ask why it is only sometimes / its M$ and they have never been
|
# don't ask why it is only sometimes / its M$ and they have never been
|
||||||
# deterministic ;)
|
# deterministic ;)
|
||||||
'pq': query,
|
'pq': query,
|
||||||
|
@ -177,7 +177,7 @@ def response(resp):
|
||||||
logger.debug('result error :\n%s', e)
|
logger.debug('result error :\n%s', e)
|
||||||
|
|
||||||
if result_len and _page_offset(resp.search_params.get("pageno", 0)) > result_len:
|
if result_len and _page_offset(resp.search_params.get("pageno", 0)) > result_len:
|
||||||
# Avoid reading more results than avalaible.
|
# Avoid reading more results than available.
|
||||||
# For example, if there is 100 results from some search and we try to get results from 120 to 130,
|
# For example, if there is 100 results from some search and we try to get results from 120 to 130,
|
||||||
# Bing will send back the results from 0 to 10 and no error.
|
# Bing will send back the results from 0 to 10 and no error.
|
||||||
# If we compare results count with the first parameter of the request we can avoid this "invalid" results.
|
# If we compare results count with the first parameter of the request we can avoid this "invalid" results.
|
||||||
|
|
|
@ -6,7 +6,7 @@ DuckDuckGo Lite
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
import re
|
import re
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode, quote_plus
|
||||||
import json
|
import json
|
||||||
import babel
|
import babel
|
||||||
import lxml.html
|
import lxml.html
|
||||||
|
@ -18,12 +18,12 @@ from searx import (
|
||||||
)
|
)
|
||||||
from searx.utils import (
|
from searx.utils import (
|
||||||
eval_xpath,
|
eval_xpath,
|
||||||
|
extr,
|
||||||
extract_text,
|
extract_text,
|
||||||
)
|
)
|
||||||
from searx.network import get # see https://github.com/searxng/searxng/issues/762
|
from searx.network import get # see https://github.com/searxng/searxng/issues/762
|
||||||
from searx import redisdb
|
from searx import redisdb
|
||||||
from searx.enginelib.traits import EngineTraits
|
from searx.enginelib.traits import EngineTraits
|
||||||
from searx.utils import extr
|
|
||||||
from searx.exceptions import SearxEngineCaptchaException
|
from searx.exceptions import SearxEngineCaptchaException
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
@ -42,7 +42,7 @@ about = {
|
||||||
}
|
}
|
||||||
|
|
||||||
send_accept_language_header = True
|
send_accept_language_header = True
|
||||||
"""DuckDuckGo-Lite tries to guess user's prefered language from the HTTP
|
"""DuckDuckGo-Lite tries to guess user's preferred language from the HTTP
|
||||||
``Accept-Language``. Optional the user can select a region filter (but not a
|
``Accept-Language``. Optional the user can select a region filter (but not a
|
||||||
language).
|
language).
|
||||||
"""
|
"""
|
||||||
|
@ -60,25 +60,25 @@ form_data = {'v': 'l', 'api': 'd.js', 'o': 'json'}
|
||||||
__CACHE = []
|
__CACHE = []
|
||||||
|
|
||||||
|
|
||||||
def _cache_key(data: dict):
|
def _cache_key(query, region):
|
||||||
return 'SearXNG_ddg_web_vqd' + redislib.secret_hash(f"{data['q']}//{data['kl']}")
|
return 'SearXNG_ddg_web_vqd' + redislib.secret_hash(f"{query}//{region}")
|
||||||
|
|
||||||
|
|
||||||
def cache_vqd(data: dict, value):
|
def cache_vqd(query, region, value):
|
||||||
"""Caches a ``vqd`` value from a query."""
|
"""Caches a ``vqd`` value from a query."""
|
||||||
c = redisdb.client()
|
c = redisdb.client()
|
||||||
if c:
|
if c:
|
||||||
logger.debug("cache vqd value: %s", value)
|
logger.debug("cache vqd value: %s", value)
|
||||||
c.set(_cache_key(data), value, ex=600)
|
c.set(_cache_key(query, region), value, ex=600)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
logger.debug("MEM cache vqd value: %s", value)
|
logger.debug("MEM cache vqd value: %s", value)
|
||||||
if len(__CACHE) > 100: # cache vqd from last 100 queries
|
if len(__CACHE) > 100: # cache vqd from last 100 queries
|
||||||
__CACHE.pop(0)
|
__CACHE.pop(0)
|
||||||
__CACHE.append((_cache_key(data), value))
|
__CACHE.append((_cache_key(query, region), value))
|
||||||
|
|
||||||
|
|
||||||
def get_vqd(data):
|
def get_vqd(query, region):
|
||||||
"""Returns the ``vqd`` that fits to the *query* (``data`` from HTTP POST).
|
"""Returns the ``vqd`` that fits to the *query* (``data`` from HTTP POST).
|
||||||
|
|
||||||
DDG's bot detection is sensitive to the ``vqd`` value. For some search terms
|
DDG's bot detection is sensitive to the ``vqd`` value. For some search terms
|
||||||
|
@ -108,7 +108,7 @@ def get_vqd(data):
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
key = _cache_key(data)
|
key = _cache_key(query, region)
|
||||||
value = None
|
value = None
|
||||||
c = redisdb.client()
|
c = redisdb.client()
|
||||||
if c:
|
if c:
|
||||||
|
@ -126,6 +126,23 @@ def get_vqd(data):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def extract_vqd(query):
|
||||||
|
"""
|
||||||
|
Scrape the vqd value matching the query from DuckDuckGo Web.
|
||||||
|
That function is currently not required for general results,
|
||||||
|
only for extra results such as images, videos, ...
|
||||||
|
|
||||||
|
Also see ``get_vqd(query, region)`` above.
|
||||||
|
"""
|
||||||
|
resp = get(f'https://duckduckgo.com/?q={quote_plus(query)}')
|
||||||
|
if resp.status_code != 200:
|
||||||
|
return None
|
||||||
|
|
||||||
|
vqd = extr(resp.text, 'vqd="', '"')
|
||||||
|
|
||||||
|
return vqd
|
||||||
|
|
||||||
|
|
||||||
def get_ddg_lang(eng_traits: EngineTraits, sxng_locale, default='en_US'):
|
def get_ddg_lang(eng_traits: EngineTraits, sxng_locale, default='en_US'):
|
||||||
"""Get DuckDuckGo's language identifier from SearXNG's locale.
|
"""Get DuckDuckGo's language identifier from SearXNG's locale.
|
||||||
|
|
||||||
|
@ -313,7 +330,7 @@ def request(query, params):
|
||||||
# from here on no more params['data'] shuld be set, since this dict is
|
# from here on no more params['data'] shuld be set, since this dict is
|
||||||
# needed to get a vqd value from the cache ..
|
# needed to get a vqd value from the cache ..
|
||||||
|
|
||||||
vqd = get_vqd(params['data'])
|
vqd = get_vqd(query, eng_region)
|
||||||
|
|
||||||
# Certain conditions must be met in order to call up one of the
|
# Certain conditions must be met in order to call up one of the
|
||||||
# following pages ...
|
# following pages ...
|
||||||
|
@ -362,7 +379,7 @@ def response(resp):
|
||||||
form = form[0]
|
form = form[0]
|
||||||
form_vqd = eval_xpath(form, '//input[@name="vqd"]/@value')[0]
|
form_vqd = eval_xpath(form, '//input[@name="vqd"]/@value')[0]
|
||||||
|
|
||||||
cache_vqd(resp.search_params["data"], form_vqd)
|
cache_vqd(resp.search_params['data']['q'], resp.search_params['data']['kl'], form_vqd)
|
||||||
|
|
||||||
# just select "web-result" and ignore results of class "result--ad result--ad--small"
|
# just select "web-result" and ignore results of class "result--ad result--ad--small"
|
||||||
for div_result in eval_xpath(doc, '//div[@id="links"]/div[contains(@class, "web-result")]'):
|
for div_result in eval_xpath(doc, '//div[@id="links"]/div[contains(@class, "web-result")]'):
|
||||||
|
@ -402,7 +419,7 @@ def fetch_traits(engine_traits: EngineTraits):
|
||||||
"""Fetch languages & regions from DuckDuckGo.
|
"""Fetch languages & regions from DuckDuckGo.
|
||||||
|
|
||||||
SearXNG's ``all`` locale maps DuckDuckGo's "Alle regions" (``wt-wt``).
|
SearXNG's ``all`` locale maps DuckDuckGo's "Alle regions" (``wt-wt``).
|
||||||
DuckDuckGo's language "Browsers prefered language" (``wt_WT``) makes no
|
DuckDuckGo's language "Browsers preferred language" (``wt_WT``) makes no
|
||||||
sense in a SearXNG request since SearXNG's ``all`` will not add a
|
sense in a SearXNG request since SearXNG's ``all`` will not add a
|
||||||
``Accept-Language`` HTTP header. The value in ``engine_traits.all_locale``
|
``Accept-Language`` HTTP header. The value in ``engine_traits.all_locale``
|
||||||
is ``wt-wt`` (the region).
|
is ``wt-wt`` (the region).
|
||||||
|
|
|
@ -11,6 +11,8 @@ from searx.utils import get_embeded_stream_url
|
||||||
|
|
||||||
from searx.engines.duckduckgo import fetch_traits # pylint: disable=unused-import
|
from searx.engines.duckduckgo import fetch_traits # pylint: disable=unused-import
|
||||||
from searx.engines.duckduckgo import (
|
from searx.engines.duckduckgo import (
|
||||||
|
cache_vqd,
|
||||||
|
extract_vqd,
|
||||||
get_ddg_lang,
|
get_ddg_lang,
|
||||||
get_vqd,
|
get_vqd,
|
||||||
)
|
)
|
||||||
|
@ -48,15 +50,20 @@ search_path_map = {'images': 'i', 'videos': 'v', 'news': 'news'}
|
||||||
|
|
||||||
|
|
||||||
def request(query, params):
|
def request(query, params):
|
||||||
|
eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
|
||||||
|
|
||||||
# request needs a vqd argument
|
# request needs a vqd argument
|
||||||
vqd = get_vqd(query)
|
vqd = get_vqd(query, eng_region)
|
||||||
|
if not vqd:
|
||||||
|
vqd = extract_vqd(query)
|
||||||
|
if vqd:
|
||||||
|
cache_vqd(query, eng_region, vqd)
|
||||||
|
|
||||||
if not vqd:
|
if not vqd:
|
||||||
# some search terms do not have results and therefore no vqd value
|
# some search terms do not have results and therefore no vqd value
|
||||||
params['url'] = None
|
params['url'] = None
|
||||||
return params
|
return params
|
||||||
|
|
||||||
eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
|
|
||||||
eng_lang = get_ddg_lang(traits, params['searxng_locale'])
|
eng_lang = get_ddg_lang(traits, params['searxng_locale'])
|
||||||
|
|
||||||
args = {
|
args = {
|
||||||
|
@ -86,6 +93,12 @@ def request(query, params):
|
||||||
|
|
||||||
params['url'] = f'https://duckduckgo.com/{search_path_map[ddg_category]}.js?{urlencode(args)}'
|
params['url'] = f'https://duckduckgo.com/{search_path_map[ddg_category]}.js?{urlencode(args)}'
|
||||||
|
|
||||||
|
# sending these two headers prevents rate limiting for the query
|
||||||
|
params['headers'] = {
|
||||||
|
'Referer': 'https://duckduckgo.com/',
|
||||||
|
'X-Requested-With': 'XMLHttpRequest',
|
||||||
|
}
|
||||||
|
|
||||||
return params
|
return params
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -142,7 +142,7 @@ search_url = base_url + '/sp/search'
|
||||||
|
|
||||||
# specific xpath variables
|
# specific xpath variables
|
||||||
# ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
|
# ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
|
||||||
# not ads: div[@class="result"] are the direct childs of div[@id="results"]
|
# not ads: div[@class="result"] are the direct children of div[@id="results"]
|
||||||
search_form_xpath = '//form[@id="search"]'
|
search_form_xpath = '//form[@id="search"]'
|
||||||
"""XPath of Startpage's origin search form
|
"""XPath of Startpage's origin search form
|
||||||
|
|
||||||
|
|
|
@ -208,7 +208,7 @@ def response(resp):
|
||||||
'title': backlink['image_name'],
|
'title': backlink['image_name'],
|
||||||
'img_src': backlink['url'],
|
'img_src': backlink['url'],
|
||||||
'format': tineye_match['image_format'],
|
'format': tineye_match['image_format'],
|
||||||
'widht': tineye_match['width'],
|
'width': tineye_match['width'],
|
||||||
'height': tineye_match['height'],
|
'height': tineye_match['height'],
|
||||||
'publishedDate': backlink['crawl_date'],
|
'publishedDate': backlink['crawl_date'],
|
||||||
}
|
}
|
||||||
|
|
|
@ -32,7 +32,7 @@ void_arch = 'x86_64'
|
||||||
"""Default architecture to search for. For valid values see :py:obj:`ARCH_RE`"""
|
"""Default architecture to search for. For valid values see :py:obj:`ARCH_RE`"""
|
||||||
|
|
||||||
ARCH_RE = re.compile('aarch64-musl|armv6l-musl|armv7l-musl|x86_64-musl|aarch64|armv6l|armv7l|i686|x86_64')
|
ARCH_RE = re.compile('aarch64-musl|armv6l-musl|armv7l-musl|x86_64-musl|aarch64|armv6l|armv7l|i686|x86_64')
|
||||||
"""Regular expresion that match a architecture in the query string."""
|
"""Regular expression that match a architecture in the query string."""
|
||||||
|
|
||||||
|
|
||||||
def request(query, params):
|
def request(query, params):
|
||||||
|
|
|
@ -12,7 +12,7 @@ ipv6_prefix = 48
|
||||||
|
|
||||||
[botdetection.ip_limit]
|
[botdetection.ip_limit]
|
||||||
|
|
||||||
# To get unlimited access in a local network, by default link-lokal addresses
|
# To get unlimited access in a local network, by default link-local addresses
|
||||||
# (networks) are not monitored by the ip_limit
|
# (networks) are not monitored by the ip_limit
|
||||||
filter_link_local = false
|
filter_link_local = false
|
||||||
|
|
||||||
|
|
|
@ -120,7 +120,7 @@ _TR_LOCALES: list[str] = []
|
||||||
|
|
||||||
|
|
||||||
def get_translation_locales() -> list[str]:
|
def get_translation_locales() -> list[str]:
|
||||||
"""Returns the list of transaltion locales (*underscore*). The list is
|
"""Returns the list of translation locales (*underscore*). The list is
|
||||||
generated from the translation folders in :origin:`searx/translations`"""
|
generated from the translation folders in :origin:`searx/translations`"""
|
||||||
|
|
||||||
global _TR_LOCALES # pylint:disable=global-statement
|
global _TR_LOCALES # pylint:disable=global-statement
|
||||||
|
|
|
@ -8,6 +8,7 @@ from timeit import default_timer
|
||||||
from operator import itemgetter
|
from operator import itemgetter
|
||||||
|
|
||||||
from searx.engines import engines
|
from searx.engines import engines
|
||||||
|
from searx.openmetrics import OpenMetricsFamily
|
||||||
from .models import HistogramStorage, CounterStorage, VoidHistogram, VoidCounterStorage
|
from .models import HistogramStorage, CounterStorage, VoidHistogram, VoidCounterStorage
|
||||||
from .error_recorder import count_error, count_exception, errors_per_engines
|
from .error_recorder import count_error, count_exception, errors_per_engines
|
||||||
|
|
||||||
|
@ -149,7 +150,9 @@ def get_reliabilities(engline_name_list, checker_results):
|
||||||
checker_result = checker_results.get(engine_name, {})
|
checker_result = checker_results.get(engine_name, {})
|
||||||
checker_success = checker_result.get('success', True)
|
checker_success = checker_result.get('success', True)
|
||||||
errors = engine_errors.get(engine_name) or []
|
errors = engine_errors.get(engine_name) or []
|
||||||
if counter('engine', engine_name, 'search', 'count', 'sent') == 0:
|
sent_count = counter('engine', engine_name, 'search', 'count', 'sent')
|
||||||
|
|
||||||
|
if sent_count == 0:
|
||||||
# no request
|
# no request
|
||||||
reliability = None
|
reliability = None
|
||||||
elif checker_success and not errors:
|
elif checker_success and not errors:
|
||||||
|
@ -164,8 +167,9 @@ def get_reliabilities(engline_name_list, checker_results):
|
||||||
|
|
||||||
reliabilities[engine_name] = {
|
reliabilities[engine_name] = {
|
||||||
'reliability': reliability,
|
'reliability': reliability,
|
||||||
|
'sent_count': sent_count,
|
||||||
'errors': errors,
|
'errors': errors,
|
||||||
'checker': checker_results.get(engine_name, {}).get('errors', {}),
|
'checker': checker_result.get('errors', {}),
|
||||||
}
|
}
|
||||||
return reliabilities
|
return reliabilities
|
||||||
|
|
||||||
|
@ -245,3 +249,53 @@ def get_engines_stats(engine_name_list):
|
||||||
'max_time': math.ceil(max_time_total or 0),
|
'max_time': math.ceil(max_time_total or 0),
|
||||||
'max_result_count': math.ceil(max_result_count or 0),
|
'max_result_count': math.ceil(max_result_count or 0),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def openmetrics(engine_stats, engine_reliabilities):
|
||||||
|
metrics = [
|
||||||
|
OpenMetricsFamily(
|
||||||
|
key="searxng_engines_response_time_total_seconds",
|
||||||
|
type_hint="gauge",
|
||||||
|
help_hint="The average total response time of the engine",
|
||||||
|
data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
|
||||||
|
data=[engine['total'] for engine in engine_stats['time']],
|
||||||
|
),
|
||||||
|
OpenMetricsFamily(
|
||||||
|
key="searxng_engines_response_time_processing_seconds",
|
||||||
|
type_hint="gauge",
|
||||||
|
help_hint="The average processing response time of the engine",
|
||||||
|
data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
|
||||||
|
data=[engine['processing'] for engine in engine_stats['time']],
|
||||||
|
),
|
||||||
|
OpenMetricsFamily(
|
||||||
|
key="searxng_engines_response_time_http_seconds",
|
||||||
|
type_hint="gauge",
|
||||||
|
help_hint="The average HTTP response time of the engine",
|
||||||
|
data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
|
||||||
|
data=[engine['http'] for engine in engine_stats['time']],
|
||||||
|
),
|
||||||
|
OpenMetricsFamily(
|
||||||
|
key="searxng_engines_result_count_total",
|
||||||
|
type_hint="counter",
|
||||||
|
help_hint="The total amount of results returned by the engine",
|
||||||
|
data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
|
||||||
|
data=[engine['result_count'] for engine in engine_stats['time']],
|
||||||
|
),
|
||||||
|
OpenMetricsFamily(
|
||||||
|
key="searxng_engines_request_count_total",
|
||||||
|
type_hint="counter",
|
||||||
|
help_hint="The total amount of user requests made to this engine",
|
||||||
|
data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
|
||||||
|
data=[engine_reliabilities.get(engine['name'], {}).get('sent_count', 0) for engine in engine_stats['time']],
|
||||||
|
),
|
||||||
|
OpenMetricsFamily(
|
||||||
|
key="searxng_engines_reliability_total",
|
||||||
|
type_hint="counter",
|
||||||
|
help_hint="The overall reliability of the engine",
|
||||||
|
data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
|
||||||
|
data=[
|
||||||
|
engine_reliabilities.get(engine['name'], {}).get('reliability', 0) for engine in engine_stats['time']
|
||||||
|
],
|
||||||
|
),
|
||||||
|
]
|
||||||
|
return "".join([str(metric) for metric in metrics])
|
||||||
|
|
|
@ -0,0 +1,35 @@
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
"""Module providing support for displaying data in OpenMetrics format"""
|
||||||
|
|
||||||
|
|
||||||
|
class OpenMetricsFamily: # pylint: disable=too-few-public-methods
|
||||||
|
"""A family of metrics.
|
||||||
|
The key parameter is the metric name that should be used (snake case).
|
||||||
|
The type_hint parameter must be one of 'counter', 'gauge', 'histogram', 'summary'.
|
||||||
|
The help_hint parameter is a short string explaining the metric.
|
||||||
|
The data_info parameter is a dictionary of descriptionary parameters for the data point (e.g. request method/path).
|
||||||
|
The data parameter is a flat list of the actual data in shape of a primive type.
|
||||||
|
|
||||||
|
See https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md for more information.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, key: str, type_hint: str, help_hint: str, data_info: list, data: list):
|
||||||
|
self.key = key
|
||||||
|
self.type_hint = type_hint
|
||||||
|
self.help_hint = help_hint
|
||||||
|
self.data_info = data_info
|
||||||
|
self.data = data
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
text_representation = f"""# HELP {self.key} {self.help_hint}
|
||||||
|
# TYPE {self.key} {self.type_hint}
|
||||||
|
"""
|
||||||
|
|
||||||
|
for i, data_info_dict in enumerate(self.data_info):
|
||||||
|
if not data_info_dict and data_info_dict != 0:
|
||||||
|
continue
|
||||||
|
|
||||||
|
info_representation = ','.join([f"{key}=\"{value}\"" for (key, value) in data_info_dict.items()])
|
||||||
|
text_representation += f"{self.key}{{{info_representation}}} {self.data[i]}\n"
|
||||||
|
|
||||||
|
return text_representation
|
|
@ -234,7 +234,7 @@ def _parse_text_and_convert(search, from_query, to_query):
|
||||||
value = target_from_si(float(value))
|
value = target_from_si(float(value))
|
||||||
|
|
||||||
if measured.group('E'):
|
if measured.group('E'):
|
||||||
# when incomming notation is scientific, outgoing notation is scientific
|
# when incoming notation is scientific, outgoing notation is scientific
|
||||||
result = babel.numbers.format_scientific(value, locale=_locale)
|
result = babel.numbers.format_scientific(value, locale=_locale)
|
||||||
else:
|
else:
|
||||||
result = babel.numbers.format_decimal(value, locale=_locale, format='#,##0.##########;-#')
|
result = babel.numbers.format_decimal(value, locale=_locale, format='#,##0.##########;-#')
|
||||||
|
|
|
@ -325,7 +325,7 @@ class ClientPref:
|
||||||
# hint: searx.webapp.get_client_settings should be moved into this class
|
# hint: searx.webapp.get_client_settings should be moved into this class
|
||||||
|
|
||||||
locale: babel.Locale
|
locale: babel.Locale
|
||||||
"""Locale prefered by the client."""
|
"""Locale preferred by the client."""
|
||||||
|
|
||||||
def __init__(self, locale: Optional[babel.Locale] = None):
|
def __init__(self, locale: Optional[babel.Locale] = None):
|
||||||
self.locale = locale
|
self.locale = locale
|
||||||
|
|
|
@ -12,6 +12,10 @@ general:
|
||||||
contact_url: false
|
contact_url: false
|
||||||
# record stats
|
# record stats
|
||||||
enable_metrics: true
|
enable_metrics: true
|
||||||
|
# expose stats in open metrics format at /metrics
|
||||||
|
# leave empty to disable (no password set)
|
||||||
|
# open_metrics: <password>
|
||||||
|
open_metrics: ''
|
||||||
|
|
||||||
brand:
|
brand:
|
||||||
new_issue_url: https://github.com/searxng/searxng/issues/new
|
new_issue_url: https://github.com/searxng/searxng/issues/new
|
||||||
|
@ -325,6 +329,36 @@ engines:
|
||||||
shortcut: 9g
|
shortcut: 9g
|
||||||
disabled: true
|
disabled: true
|
||||||
|
|
||||||
|
- name: adobe stock
|
||||||
|
engine: adobe_stock
|
||||||
|
shortcut: asi
|
||||||
|
categories: ["images"]
|
||||||
|
# https://docs.searxng.org/dev/engines/online/adobe_stock.html
|
||||||
|
adobe_order: relevance
|
||||||
|
adobe_content_types: ["photo", "illustration", "zip_vector", "template", "3d", "image"]
|
||||||
|
timeout: 6
|
||||||
|
disabled: true
|
||||||
|
|
||||||
|
- name: adobe stock video
|
||||||
|
engine: adobe_stock
|
||||||
|
shortcut: asv
|
||||||
|
network: adobe stock
|
||||||
|
categories: ["videos"]
|
||||||
|
adobe_order: relevance
|
||||||
|
adobe_content_types: ["video"]
|
||||||
|
timeout: 6
|
||||||
|
disabled: true
|
||||||
|
|
||||||
|
- name: adobe stock audio
|
||||||
|
engine: adobe_stock
|
||||||
|
shortcut: asa
|
||||||
|
network: adobe stock
|
||||||
|
categories: ["music"]
|
||||||
|
adobe_order: relevance
|
||||||
|
adobe_content_types: ["audio"]
|
||||||
|
timeout: 6
|
||||||
|
disabled: true
|
||||||
|
|
||||||
- name: alpine linux packages
|
- name: alpine linux packages
|
||||||
engine: alpinelinux
|
engine: alpinelinux
|
||||||
disabled: true
|
disabled: true
|
||||||
|
|
|
@ -143,6 +143,7 @@ SCHEMA = {
|
||||||
'contact_url': SettingsValue((None, False, str), None),
|
'contact_url': SettingsValue((None, False, str), None),
|
||||||
'donation_url': SettingsValue((bool, str), "https://docs.searxng.org/donate.html"),
|
'donation_url': SettingsValue((bool, str), "https://docs.searxng.org/donate.html"),
|
||||||
'enable_metrics': SettingsValue(bool, True),
|
'enable_metrics': SettingsValue(bool, True),
|
||||||
|
'open_metrics': SettingsValue(str, ''),
|
||||||
},
|
},
|
||||||
'brand': {
|
'brand': {
|
||||||
'issue_url': SettingsValue(str, 'https://github.com/searxng/searxng/issues'),
|
'issue_url': SettingsValue(str, 'https://github.com/searxng/searxng/issues'),
|
||||||
|
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
@ -1 +1 @@
|
||||||
{"version":3,"file":"searxng.head.min.js","sources":["../src/js/head/00_init.js"],"sourcesContent":["/* SPDX-License-Identifier: AGPL-3.0-or-later */\n(function (w, d) {\n 'use strict';\n\n // add data- properties\n var script = d.currentScript || (function () {\n var scripts = d.getElementsByTagName('script');\n return scripts[scripts.length - 1];\n })();\n\n w.searxng = {\n settings: JSON.parse(atob(script.getAttribute('client_settings')))\n };\n\n // update the css\n var hmtlElement = d.getElementsByTagName(\"html\")[0];\n hmtlElement.classList.remove('no-js');\n hmtlElement.classList.add('js');\n\n})(window, document);\n"],"names":["w","d","script","currentScript","scripts","getElementsByTagName","length","searxng","settings","JSON","parse","atob","getAttribute","hmtlElement","classList","remove","add","window","document"],"mappings":"CACA,SAAWA,EAAGC,GACZ,aAGA,IAAIC,EAASD,EAAEE,eAAkB,WAC/B,IAAIC,EAAUH,EAAEI,qBAAqB,QAAQ,EAC7C,OAAOD,EAAQA,EAAQE,OAAS,EACjC,EAAE,EAEHN,EAAEO,QAAU,CACVC,SAAUC,KAAKC,MAAMC,KAAKT,EAAOU,aAAa,iBAAiB,CAAC,CAAC,CACnE,EAGA,IAAIC,EAAcZ,EAAEI,qBAAqB,MAAM,EAAE,GACjDQ,EAAYC,UAAUC,OAAO,OAAO,EACpCF,EAAYC,UAAUE,IAAI,IAAI,CAE/B,GAAEC,OAAQC,QAAQ"}
|
{"version":3,"file":"searxng.head.min.js","sources":["../src/js/head/00_init.js"],"sourcesContent":["/* SPDX-License-Identifier: AGPL-3.0-or-later */\n(function (w, d) {\n 'use strict';\n\n // add data- properties\n var script = d.currentScript || (function () {\n var scripts = d.getElementsByTagName('script');\n return scripts[scripts.length - 1];\n })();\n\n w.searxng = {\n settings: JSON.parse(atob(script.getAttribute('client_settings')))\n };\n\n // update the css\n var htmlElement = d.getElementsByTagName(\"html\")[0];\n htmlElement.classList.remove('no-js');\n htmlElement.classList.add('js');\n\n})(window, document);\n"],"names":["w","d","script","currentScript","scripts","getElementsByTagName","length","searxng","settings","JSON","parse","atob","getAttribute","htmlElement","classList","remove","add","window","document"],"mappings":"CACA,SAAWA,EAAGC,GACZ,aAGA,IAAIC,EAASD,EAAEE,eAAkB,WAC/B,IAAIC,EAAUH,EAAEI,qBAAqB,QAAQ,EAC7C,OAAOD,EAAQA,EAAQE,OAAS,EACjC,EAAE,EAEHN,EAAEO,QAAU,CACVC,SAAUC,KAAKC,MAAMC,KAAKT,EAAOU,aAAa,iBAAiB,CAAC,CAAC,CACnE,EAGA,IAAIC,EAAcZ,EAAEI,qBAAqB,MAAM,EAAE,GACjDQ,EAAYC,UAAUC,OAAO,OAAO,EACpCF,EAAYC,UAAUE,IAAI,IAAI,CAE/B,GAAEC,OAAQC,QAAQ"}
|
File diff suppressed because one or more lines are too long
|
@ -13,8 +13,8 @@
|
||||||
};
|
};
|
||||||
|
|
||||||
// update the css
|
// update the css
|
||||||
var hmtlElement = d.getElementsByTagName("html")[0];
|
var htmlElement = d.getElementsByTagName("html")[0];
|
||||||
hmtlElement.classList.remove('no-js');
|
htmlElement.classList.remove('no-js');
|
||||||
hmtlElement.classList.add('js');
|
htmlElement.classList.add('js');
|
||||||
|
|
||||||
})(window, document);
|
})(window, document);
|
||||||
|
|
|
@ -8,7 +8,7 @@ window.searxng = (function (w, d) {
|
||||||
|
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
// not invented here tookit with bugs fixed elsewhere
|
// not invented here toolkit with bugs fixed elsewhere
|
||||||
// purposes : be just good enough and as small as possible
|
// purposes : be just good enough and as small as possible
|
||||||
|
|
||||||
// from https://plainjs.com/javascript/events/live-binding-event-handlers-14/
|
// from https://plainjs.com/javascript/events/live-binding-event-handlers-14/
|
||||||
|
|
|
@ -441,7 +441,7 @@ searxng.ready(function () {
|
||||||
var body = document.getElementsByTagName('body')[0];
|
var body = document.getElementsByTagName('body')[0];
|
||||||
body.appendChild(helpPanel);
|
body.appendChild(helpPanel);
|
||||||
} else {
|
} else {
|
||||||
// togggle hidden
|
// toggle hidden
|
||||||
helpPanel.classList.toggle('invisible');
|
helpPanel.classList.toggle('invisible');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
|
@ -67,7 +67,7 @@
|
||||||
} */
|
} */
|
||||||
});
|
});
|
||||||
|
|
||||||
// this event occour only once per element
|
// this event occur only once per element
|
||||||
event.preventDefault();
|
event.preventDefault();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -13,8 +13,8 @@ iframe[src^="https://www.mixcloud.com"] {
|
||||||
// issue with an image URL that is blocked since it is an a Cross-Origin
|
// issue with an image URL that is blocked since it is an a Cross-Origin
|
||||||
// request. The alternative text (<img alt='Mixcloud Logo'> then cause an
|
// request. The alternative text (<img alt='Mixcloud Logo'> then cause an
|
||||||
// scrollbar in the inner of the iframe we can't avoid. Another quirk comes
|
// scrollbar in the inner of the iframe we can't avoid. Another quirk comes
|
||||||
// when pressing the play button, somtimes the shown player has an height of
|
// when pressing the play button, sometimes the shown player has an height of
|
||||||
// 200px, somtimes 250px.
|
// 200px, sometimes 250px.
|
||||||
height: 250px;
|
height: 250px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -4,31 +4,31 @@ see the CSS rules for #results in style.less ( grid-template-columns and gap).
|
||||||
|
|
||||||
In this file, the --center-page-width values comes from the Oscar theme (Bootstrap 3).
|
In this file, the --center-page-width values comes from the Oscar theme (Bootstrap 3).
|
||||||
|
|
||||||
All rules starts with ".center-aligment-yes #main_results" to be enabled only
|
All rules starts with ".center-alignment-yes #main_results" to be enabled only
|
||||||
on the /search URL and when the "center alignment" preference is enabled.
|
on the /search URL and when the "center alignment" preference is enabled.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
@media screen and (min-width: @phone) {
|
@media screen and (min-width: @phone) {
|
||||||
.center-aligment-yes #main_results {
|
.center-alignment-yes #main_results {
|
||||||
--center-page-width: 48rem;
|
--center-page-width: 48rem;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@media screen and (min-width: 62rem) {
|
@media screen and (min-width: 62rem) {
|
||||||
.center-aligment-yes #main_results {
|
.center-alignment-yes #main_results {
|
||||||
--center-page-width: 60rem;
|
--center-page-width: 60rem;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@media screen and (min-width: @tablet) {
|
@media screen and (min-width: @tablet) {
|
||||||
.center-aligment-yes #main_results {
|
.center-alignment-yes #main_results {
|
||||||
--center-page-width: 73rem;
|
--center-page-width: 73rem;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@media screen and (min-width: @phone) and (max-width: @tablet) {
|
@media screen and (min-width: @phone) and (max-width: @tablet) {
|
||||||
// any change must be reset in @media screen and (min-width: @tablet) { ... }
|
// any change must be reset in @media screen and (min-width: @tablet) { ... }
|
||||||
.center-aligment-yes #main_results {
|
.center-alignment-yes #main_results {
|
||||||
#results {
|
#results {
|
||||||
grid-template-columns: 60% calc(40% - @results-gap);
|
grid-template-columns: 60% calc(40% - @results-gap);
|
||||||
margin-left: 0;
|
margin-left: 0;
|
||||||
|
@ -50,7 +50,7 @@ on the /search URL and when the "center alignment" preference is enabled.
|
||||||
}
|
}
|
||||||
|
|
||||||
@media screen and (min-width: @tablet) {
|
@media screen and (min-width: @tablet) {
|
||||||
.center-aligment-yes #main_results {
|
.center-alignment-yes #main_results {
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
|
|
|
@ -297,7 +297,7 @@ article[data-vim-selected].category-social {
|
||||||
padding-top: 0.6rem;
|
padding-top: 0.6rem;
|
||||||
.ltr-padding-right(1rem);
|
.ltr-padding-right(1rem);
|
||||||
width: 7rem;
|
width: 7rem;
|
||||||
height: unset; // remove heigth value that was needed for lazy loading
|
height: unset; // remove height value that was needed for lazy loading
|
||||||
}
|
}
|
||||||
|
|
||||||
.break {
|
.break {
|
||||||
|
@ -399,7 +399,7 @@ article[data-vim-selected].category-social {
|
||||||
padding-top: 0.6rem;
|
padding-top: 0.6rem;
|
||||||
.ltr-padding-right(1rem);
|
.ltr-padding-right(1rem);
|
||||||
width: 20rem;
|
width: 20rem;
|
||||||
height: unset; // remove heigth value that was needed for lazy loading
|
height: unset; // remove height value that was needed for lazy loading
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -639,12 +639,12 @@ summary.title {
|
||||||
padding-top: 1em;
|
padding-top: 1em;
|
||||||
}
|
}
|
||||||
|
|
||||||
.sidebar-collapsable {
|
.sidebar-collapsible {
|
||||||
border-top: 1px solid var(--color-sidebar-border);
|
border-top: 1px solid var(--color-sidebar-border);
|
||||||
padding-bottom: 0.5em;
|
padding-bottom: 0.5em;
|
||||||
}
|
}
|
||||||
|
|
||||||
#sidebar-end-collapsable {
|
#sidebar-end-collapsible {
|
||||||
border-bottom: 1px solid var(--color-sidebar-border);
|
border-bottom: 1px solid var(--color-sidebar-border);
|
||||||
width: 100%;
|
width: 100%;
|
||||||
}
|
}
|
||||||
|
@ -1007,10 +1007,10 @@ summary.title {
|
||||||
}
|
}
|
||||||
|
|
||||||
@media screen and (min-width: @phone) and (max-width: @tablet) {
|
@media screen and (min-width: @phone) and (max-width: @tablet) {
|
||||||
// when .center-aligment-yes, see style-center.less
|
// when .center-alignment-yes, see style-center.less
|
||||||
// the media query includes "min-width: @phone"
|
// the media query includes "min-width: @phone"
|
||||||
// because the phone layout includes the tablet layout unconditionally.
|
// because the phone layout includes the tablet layout unconditionally.
|
||||||
.center-aligment-no {
|
.center-alignment-no {
|
||||||
.results-tablet();
|
.results-tablet();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html class="no-js theme-{{ preferences.get_value('simple_style') or 'auto' }} center-aligment-{{ preferences.get_value('center_alignment') and 'yes' or 'no' }}" lang="{{ locale_rfc5646 }}" {% if rtl %} dir="rtl"{% endif %}>
|
<html class="no-js theme-{{ preferences.get_value('simple_style') or 'auto' }} center-alignment-{{ preferences.get_value('center_alignment') and 'yes' or 'no' }}" lang="{{ locale_rfc5646 }}" {% if rtl %} dir="rtl"{% endif %}>
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="description" content="SearXNG — a privacy-respecting, open metasearch engine">
|
<meta name="description" content="SearXNG — a privacy-respecting, open metasearch engine">
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
<div id="apis" role="complementary" aria-labelledby="apis-title">
|
<div id="apis" role="complementary" aria-labelledby="apis-title">
|
||||||
<details class="sidebar-collapsable">
|
<details class="sidebar-collapsible">
|
||||||
<summary class="title" id="apis-title">{{ _('Download results') }}</summary>
|
<summary class="title" id="apis-title">{{ _('Download results') }}</summary>
|
||||||
<div class="wrapper">
|
<div class="wrapper">
|
||||||
{%- for output_type in search_formats -%}
|
{%- for output_type in search_formats -%}
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
<div id="engines_msg">
|
<div id="engines_msg">
|
||||||
{% if (not results and not answers) or not max_response_time %}
|
{% if (not results and not answers) or not max_response_time %}
|
||||||
<details class="sidebar-collapsable" open>
|
<details class="sidebar-collapsible" open>
|
||||||
<summary class="title" id="engines_msg-title">{{ _('Messages from the search engines') }}</summary>
|
<summary class="title" id="engines_msg-title">{{ _('Messages from the search engines') }}</summary>
|
||||||
{% else %}
|
{% else %}
|
||||||
<details class="sidebar-collapsable">
|
<details class="sidebar-collapsible">
|
||||||
<summary class="title" id="engines_msg-title">{{ _('Response time') }}: {{ max_response_time | round(1) }} {{ _('seconds') }}</summary>
|
<summary class="title" id="engines_msg-title">{{ _('Response time') }}: {{ max_response_time | round(1) }} {{ _('seconds') }}</summary>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<table class="engine-stats" id="engines_msg-table">
|
<table class="engine-stats" id="engines_msg-table">
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
<div id="search_url" role="complementary" aria-labelledby="search_url-title">
|
<div id="search_url" role="complementary" aria-labelledby="search_url-title">
|
||||||
<details class="sidebar-collapsable">
|
<details class="sidebar-collapsible">
|
||||||
<summary class="title" id="search_url-title">{{ _('Search URL') }}</summary>
|
<summary class="title" id="search_url-title">{{ _('Search URL') }}</summary>
|
||||||
<button id="copy_url" type="submit" data-copied-text="{{ _('Copied') }}">{{ _('Copy') }}</button>
|
<button id="copy_url" type="submit" data-copied-text="{{ _('Copied') }}">{{ _('Copy') }}</button>
|
||||||
<div class="selectable_url">
|
<div class="selectable_url">
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
<div id="suggestions" role="complementary" aria-labelledby="suggestions-title">
|
<div id="suggestions" role="complementary" aria-labelledby="suggestions-title">
|
||||||
<details class="sidebar-collapsable">
|
<details class="sidebar-collapsible">
|
||||||
<summary class="title" id="suggestions-title">{{ _('Suggestions') }}</summary>
|
<summary class="title" id="suggestions-title">{{ _('Suggestions') }}</summary>
|
||||||
<div class="wrapper">
|
<div class="wrapper">
|
||||||
{%- for suggestion in suggestions -%}
|
{%- for suggestion in suggestions -%}
|
||||||
|
|
|
@ -61,7 +61,7 @@ or manually by executing the searx/webapp.py file? -->
|
||||||
{%- endfor -%}
|
{%- endfor -%}
|
||||||
</textarea>
|
</textarea>
|
||||||
<input type="checkbox" id="step1">
|
<input type="checkbox" id="step1">
|
||||||
<label for="step1">{{ _('Start submiting a new issue on GitHub') }}</label>
|
<label for="step1">{{ _('Start submitting a new issue on GitHub') }}</label>
|
||||||
<div class="step1 step_content">
|
<div class="step1 step_content">
|
||||||
<p><a href="{{ get_setting('brand.issue_url') }}?q=is%3Aissue+Bug:%20{{ engine_name }} {{ technical_report }}" target="_blank" rel="noreferrer noreferrer">{{ _('Please check for existing bugs about this engine on GitHub') }}</a></p>
|
<p><a href="{{ get_setting('brand.issue_url') }}?q=is%3Aissue+Bug:%20{{ engine_name }} {{ technical_report }}" target="_blank" rel="noreferrer noreferrer">{{ _('Please check for existing bugs about this engine on GitHub') }}</a></p>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -44,7 +44,7 @@
|
||||||
|
|
||||||
{%- if infoboxes -%}
|
{%- if infoboxes -%}
|
||||||
<div id="infoboxes">
|
<div id="infoboxes">
|
||||||
<details open class="sidebar-collapsable">
|
<details open class="sidebar-collapsible">
|
||||||
<summary class="title">{{ _('Info') }}</summary>
|
<summary class="title">{{ _('Info') }}</summary>
|
||||||
{%- for infobox in infoboxes -%}
|
{%- for infobox in infoboxes -%}
|
||||||
{%- include 'simple/elements/infobox.html' -%}
|
{%- include 'simple/elements/infobox.html' -%}
|
||||||
|
@ -67,7 +67,7 @@
|
||||||
{%- include 'simple/elements/apis.html' -%}
|
{%- include 'simple/elements/apis.html' -%}
|
||||||
{%- endif -%}
|
{%- endif -%}
|
||||||
|
|
||||||
<div id="sidebar-end-collapsable"></div>
|
<div id="sidebar-end-collapsible"></div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{% if corrections %}
|
{% if corrections %}
|
||||||
|
|
|
@ -677,7 +677,7 @@ msgid "proxied"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: searx/templates/simple/new_issue.html:64
|
#: searx/templates/simple/new_issue.html:64
|
||||||
msgid "Start submiting a new issue on GitHub"
|
msgid "Start submitting a new issue on GitHub"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: searx/templates/simple/new_issue.html:66
|
#: searx/templates/simple/new_issue.html:66
|
||||||
|
|
|
@ -48,7 +48,7 @@ _XPATH_CACHE: Dict[str, XPath] = {}
|
||||||
_LANG_TO_LC_CACHE: Dict[str, Dict[str, str]] = {}
|
_LANG_TO_LC_CACHE: Dict[str, Dict[str, str]] = {}
|
||||||
|
|
||||||
_FASTTEXT_MODEL: Optional["fasttext.FastText._FastText"] = None # type: ignore
|
_FASTTEXT_MODEL: Optional["fasttext.FastText._FastText"] = None # type: ignore
|
||||||
"""fasttext model to predict laguage of a search term"""
|
"""fasttext model to predict language of a search term"""
|
||||||
|
|
||||||
SEARCH_LANGUAGE_CODES = frozenset([searxng_locale[0].split('-')[0] for searxng_locale in sxng_locales])
|
SEARCH_LANGUAGE_CODES = frozenset([searxng_locale[0].split('-')[0] for searxng_locale in sxng_locales])
|
||||||
"""Languages supported by most searxng engines (:py:obj:`searx.sxng_locales.sxng_locales`)."""
|
"""Languages supported by most searxng engines (:py:obj:`searx.sxng_locales.sxng_locales`)."""
|
||||||
|
|
|
@ -225,7 +225,7 @@ def get_search_query_from_webapp(
|
||||||
"""Assemble data from preferences and request.form (from the HTML form) needed
|
"""Assemble data from preferences and request.form (from the HTML form) needed
|
||||||
in a search query.
|
in a search query.
|
||||||
|
|
||||||
The returned tuple consits of:
|
The returned tuple consists of:
|
||||||
|
|
||||||
1. instance of :py:obj:`searx.search.SearchQuery`
|
1. instance of :py:obj:`searx.search.SearchQuery`
|
||||||
2. instance of :py:obj:`searx.query.RawTextQuery`
|
2. instance of :py:obj:`searx.query.RawTextQuery`
|
||||||
|
|
|
@ -87,10 +87,7 @@ from searx.webadapter import (
|
||||||
get_selected_categories,
|
get_selected_categories,
|
||||||
parse_lang,
|
parse_lang,
|
||||||
)
|
)
|
||||||
from searx.utils import (
|
from searx.utils import gen_useragent, dict_subset
|
||||||
gen_useragent,
|
|
||||||
dict_subset,
|
|
||||||
)
|
|
||||||
from searx.version import VERSION_STRING, GIT_URL, GIT_BRANCH
|
from searx.version import VERSION_STRING, GIT_URL, GIT_BRANCH
|
||||||
from searx.query import RawTextQuery
|
from searx.query import RawTextQuery
|
||||||
from searx.plugins import Plugin, plugins, initialize as plugin_initialize
|
from searx.plugins import Plugin, plugins, initialize as plugin_initialize
|
||||||
|
@ -104,13 +101,7 @@ from searx.answerers import (
|
||||||
answerers,
|
answerers,
|
||||||
ask,
|
ask,
|
||||||
)
|
)
|
||||||
from searx.metrics import (
|
from searx.metrics import get_engines_stats, get_engine_errors, get_reliabilities, histogram, counter, openmetrics
|
||||||
get_engines_stats,
|
|
||||||
get_engine_errors,
|
|
||||||
get_reliabilities,
|
|
||||||
histogram,
|
|
||||||
counter,
|
|
||||||
)
|
|
||||||
from searx.flaskfix import patch_application
|
from searx.flaskfix import patch_application
|
||||||
|
|
||||||
from searx.locales import (
|
from searx.locales import (
|
||||||
|
@ -1218,6 +1209,30 @@ def stats_checker():
|
||||||
return jsonify(result)
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
|
@app.route('/metrics')
|
||||||
|
def stats_open_metrics():
|
||||||
|
password = settings['general'].get("open_metrics")
|
||||||
|
|
||||||
|
if not (settings['general'].get("enable_metrics") and password):
|
||||||
|
return Response('open metrics is disabled', status=404, mimetype='text/plain')
|
||||||
|
|
||||||
|
if not request.authorization or request.authorization.password != password:
|
||||||
|
return Response('access forbidden', status=401, mimetype='text/plain')
|
||||||
|
|
||||||
|
filtered_engines = dict(filter(lambda kv: request.preferences.validate_token(kv[1]), engines.items()))
|
||||||
|
|
||||||
|
checker_results = checker_get_result()
|
||||||
|
checker_results = (
|
||||||
|
checker_results['engines'] if checker_results['status'] == 'ok' and 'engines' in checker_results else {}
|
||||||
|
)
|
||||||
|
|
||||||
|
engine_stats = get_engines_stats(filtered_engines)
|
||||||
|
engine_reliabilities = get_reliabilities(filtered_engines, checker_results)
|
||||||
|
metrics_text = openmetrics(engine_stats, engine_reliabilities)
|
||||||
|
|
||||||
|
return Response(metrics_text, mimetype='text/plain')
|
||||||
|
|
||||||
|
|
||||||
@app.route('/robots.txt', methods=['GET'])
|
@app.route('/robots.txt', methods=['GET'])
|
||||||
def robots():
|
def robots():
|
||||||
return Response(
|
return Response(
|
||||||
|
|
|
@ -72,7 +72,7 @@ def _instance_infosetset_ctx(base_url):
|
||||||
# from searx.network import network
|
# from searx.network import network
|
||||||
# network.done()
|
# network.done()
|
||||||
|
|
||||||
# waiting some seconds before ending the comand line was the only solution I
|
# waiting some seconds before ending the command line was the only solution I
|
||||||
# found ..
|
# found ..
|
||||||
|
|
||||||
time.sleep(3)
|
time.sleep(3)
|
||||||
|
|
|
@ -85,7 +85,7 @@ def main():
|
||||||
|
|
||||||
|
|
||||||
def fetch_traits_map():
|
def fetch_traits_map():
|
||||||
"""Fetchs supported languages for each engine and writes json file with those."""
|
"""Fetches supported languages for each engine and writes json file with those."""
|
||||||
network.set_timeout_for_thread(10.0)
|
network.set_timeout_for_thread(10.0)
|
||||||
|
|
||||||
def log(msg):
|
def log(msg):
|
||||||
|
|
|
@ -25,7 +25,7 @@ from searx.locales import (
|
||||||
)
|
)
|
||||||
|
|
||||||
LOCALE_DATA_FILE = Path(searx_dir) / 'data' / 'locales.json'
|
LOCALE_DATA_FILE = Path(searx_dir) / 'data' / 'locales.json'
|
||||||
TRANSLATOINS_FOLDER = Path(searx_dir) / 'translations'
|
TRANSLATIONS_FOLDER = Path(searx_dir) / 'translations'
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|
|
@ -65,7 +65,7 @@ def main():
|
||||||
test_layer.setUp()
|
test_layer.setUp()
|
||||||
run_robot_tests([getattr(test_webapp, x) for x in dir(test_webapp) if x.startswith('test_')])
|
run_robot_tests([getattr(test_webapp, x) for x in dir(test_webapp) if x.startswith('test_')])
|
||||||
except Exception: # pylint: disable=broad-except
|
except Exception: # pylint: disable=broad-except
|
||||||
print('Error occured: {0}'.format(traceback.format_exc()))
|
print('Error occurred: {0}'.format(traceback.format_exc()))
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
finally:
|
finally:
|
||||||
test_layer.tearDown()
|
test_layer.tearDown()
|
||||||
|
|
10
utils/lib.sh
10
utils/lib.sh
|
@ -772,7 +772,7 @@ docs.clean() {
|
||||||
|
|
||||||
docs.prebuild() {
|
docs.prebuild() {
|
||||||
# Dummy function to run some actions before sphinx-doc build gets started.
|
# Dummy function to run some actions before sphinx-doc build gets started.
|
||||||
# This finction needs to be overwritten by the application script.
|
# This function needs to be overwritten by the application script.
|
||||||
true
|
true
|
||||||
dump_return $?
|
dump_return $?
|
||||||
}
|
}
|
||||||
|
@ -1065,7 +1065,7 @@ nginx_remove_app() {
|
||||||
# usage: nginx_remove_app <myapp.conf>
|
# usage: nginx_remove_app <myapp.conf>
|
||||||
|
|
||||||
info_msg "remove nginx app: $1"
|
info_msg "remove nginx app: $1"
|
||||||
nginx_dissable_app "$1"
|
nginx_disable_app "$1"
|
||||||
rm -f "${NGINX_APPS_AVAILABLE}/$1"
|
rm -f "${NGINX_APPS_AVAILABLE}/$1"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1082,7 +1082,7 @@ nginx_enable_app() {
|
||||||
nginx_reload
|
nginx_reload
|
||||||
}
|
}
|
||||||
|
|
||||||
nginx_dissable_app() {
|
nginx_disable_app() {
|
||||||
|
|
||||||
# usage: nginx_disable_app <myapp.conf>
|
# usage: nginx_disable_app <myapp.conf>
|
||||||
|
|
||||||
|
@ -1192,7 +1192,7 @@ apache_remove_site() {
|
||||||
# usage: apache_remove_site <mysite.conf>
|
# usage: apache_remove_site <mysite.conf>
|
||||||
|
|
||||||
info_msg "remove apache site: $1"
|
info_msg "remove apache site: $1"
|
||||||
apache_dissable_site "$1"
|
apache_disable_site "$1"
|
||||||
rm -f "${APACHE_SITES_AVAILABLE}/$1"
|
rm -f "${APACHE_SITES_AVAILABLE}/$1"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1222,7 +1222,7 @@ apache_enable_site() {
|
||||||
apache_reload
|
apache_reload
|
||||||
}
|
}
|
||||||
|
|
||||||
apache_dissable_site() {
|
apache_disable_site() {
|
||||||
|
|
||||||
# usage: apache_disable_site <mysite.conf>
|
# usage: apache_disable_site <mysite.conf>
|
||||||
|
|
||||||
|
|
|
@ -56,7 +56,7 @@ EOF
|
||||||
|
|
||||||
go.ver_info(){
|
go.ver_info(){
|
||||||
|
|
||||||
# print informations about a golang distribution. To print filename
|
# print information about a golang distribution. To print filename
|
||||||
# sha256 and size of the archive that fits to your OS and host:
|
# sha256 and size of the archive that fits to your OS and host:
|
||||||
#
|
#
|
||||||
# go.ver_info "${GO_VERSION}" archive "$(go.os)" "$(go.arch)" filename sha256 size
|
# go.ver_info "${GO_VERSION}" archive "$(go.os)" "$(go.arch)" filename sha256 size
|
||||||
|
|
|
@ -36,7 +36,7 @@ nvm.is_installed() {
|
||||||
}
|
}
|
||||||
|
|
||||||
if [[ -z "${NVM_DIR}" ]]; then
|
if [[ -z "${NVM_DIR}" ]]; then
|
||||||
# nvm is not pre-intalled in $HOME. Prepare for using nvm from <repo-root>
|
# nvm is not pre-installed in $HOME. Prepare for using nvm from <repo-root>
|
||||||
NVM_DIR="$(git rev-parse --show-toplevel)/${NVM_LOCAL_FOLDER}"
|
NVM_DIR="$(git rev-parse --show-toplevel)/${NVM_LOCAL_FOLDER}"
|
||||||
fi
|
fi
|
||||||
export NVM_DIR
|
export NVM_DIR
|
||||||
|
@ -93,7 +93,7 @@ nvm.help() {
|
||||||
nvm.: use nvm (without dot) to execute nvm commands directly
|
nvm.: use nvm (without dot) to execute nvm commands directly
|
||||||
install : install NVM locally at $(git rev-parse --show-toplevel)/${NVM_LOCAL_FOLDER}
|
install : install NVM locally at $(git rev-parse --show-toplevel)/${NVM_LOCAL_FOLDER}
|
||||||
clean : remove NVM installation
|
clean : remove NVM installation
|
||||||
status : prompt some status informations about nvm & node
|
status : prompt some status information about nvm & node
|
||||||
nodejs : install Node.js latest LTS
|
nodejs : install Node.js latest LTS
|
||||||
cmd ... : run command ... in NVM environment
|
cmd ... : run command ... in NVM environment
|
||||||
bash : start bash interpreter with NVM environment sourced
|
bash : start bash interpreter with NVM environment sourced
|
||||||
|
@ -108,7 +108,7 @@ nvm.install() {
|
||||||
pushd "${NVM_DIR}" &> /dev/null
|
pushd "${NVM_DIR}" &> /dev/null
|
||||||
git fetch --all | prefix_stdout " ${_Yellow}||${_creset} "
|
git fetch --all | prefix_stdout " ${_Yellow}||${_creset} "
|
||||||
else
|
else
|
||||||
# delete any leftovers from previos installations
|
# delete any leftovers from previous installations
|
||||||
if nvm.is_local; then
|
if nvm.is_local; then
|
||||||
rm -rf "${NVM_DIR}"
|
rm -rf "${NVM_DIR}"
|
||||||
fi
|
fi
|
||||||
|
|
|
@ -96,7 +96,7 @@ static.build.commit() {
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# drop existing commit from previos build
|
# drop existing commit from previous build
|
||||||
static.build.drop &>/dev/null
|
static.build.drop &>/dev/null
|
||||||
|
|
||||||
( set -e
|
( set -e
|
||||||
|
|
10
utils/lxc.sh
10
utils/lxc.sh
|
@ -159,7 +159,7 @@ main() {
|
||||||
;;
|
;;
|
||||||
copy)
|
copy)
|
||||||
case $2 in
|
case $2 in
|
||||||
''|images) lxc_copy_images_localy;;
|
''|images) lxc_copy_images_locally;;
|
||||||
*) usage "$_usage"; exit 42;;
|
*) usage "$_usage"; exit 42;;
|
||||||
esac
|
esac
|
||||||
;;
|
;;
|
||||||
|
@ -167,7 +167,7 @@ main() {
|
||||||
sudo_or_exit
|
sudo_or_exit
|
||||||
case $2 in
|
case $2 in
|
||||||
''|--|containers) remove_containers ;;
|
''|--|containers) remove_containers ;;
|
||||||
images) lxc_delete_images_localy ;;
|
images) lxc_delete_images_locally ;;
|
||||||
${LXC_HOST_PREFIX}-*)
|
${LXC_HOST_PREFIX}-*)
|
||||||
! lxc_exists "$2" && warn_msg "container not yet exists: $2" && exit 0
|
! lxc_exists "$2" && warn_msg "container not yet exists: $2" && exit 0
|
||||||
if ask_yn "Do you really want to delete container $2"; then
|
if ask_yn "Do you really want to delete container $2"; then
|
||||||
|
@ -291,7 +291,7 @@ build_all_containers() {
|
||||||
rst_title "Build all LXC containers of suite"
|
rst_title "Build all LXC containers of suite"
|
||||||
echo
|
echo
|
||||||
usage_containers
|
usage_containers
|
||||||
lxc_copy_images_localy
|
lxc_copy_images_locally
|
||||||
lxc_init_all_containers
|
lxc_init_all_containers
|
||||||
lxc_config_all_containers
|
lxc_config_all_containers
|
||||||
lxc_boilerplate_all_containers
|
lxc_boilerplate_all_containers
|
||||||
|
@ -361,7 +361,7 @@ remove_containers() {
|
||||||
# images
|
# images
|
||||||
# ------
|
# ------
|
||||||
|
|
||||||
lxc_copy_images_localy() {
|
lxc_copy_images_locally() {
|
||||||
rst_title "copy images" section
|
rst_title "copy images" section
|
||||||
for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
|
for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
|
||||||
lxc_image_copy "${LXC_SUITE[i]}" "${LXC_SUITE[i+1]}"
|
lxc_image_copy "${LXC_SUITE[i]}" "${LXC_SUITE[i+1]}"
|
||||||
|
@ -369,7 +369,7 @@ lxc_copy_images_localy() {
|
||||||
# lxc image list local: && wait_key
|
# lxc image list local: && wait_key
|
||||||
}
|
}
|
||||||
|
|
||||||
lxc_delete_images_localy() {
|
lxc_delete_images_locally() {
|
||||||
rst_title "Delete LXC images"
|
rst_title "Delete LXC images"
|
||||||
rst_para "local existing images"
|
rst_para "local existing images"
|
||||||
echo
|
echo
|
||||||
|
|
|
@ -33,7 +33,7 @@ ui:
|
||||||
|
|
||||||
enabled_plugins:
|
enabled_plugins:
|
||||||
- 'Hash plugin'
|
- 'Hash plugin'
|
||||||
- 'Self Informations'
|
- 'Self Information'
|
||||||
- 'Tracker URL remover'
|
- 'Tracker URL remover'
|
||||||
- 'Ahmia blacklist'
|
- 'Ahmia blacklist'
|
||||||
# - 'Hostnames plugin' # see 'hostnames' configuration below
|
# - 'Hostnames plugin' # see 'hostnames' configuration below
|
||||||
|
|
Loading…
Reference in New Issue