mirror of https://github.com/searxng/searxng.git
Compare commits
6 Commits
1b40f118a6
...
890c2c1b59
Author | SHA1 | Date |
---|---|---|
Allen | 890c2c1b59 | |
dependabot[bot] | cd384a8a60 | |
Markus Heiser | c4055e449f | |
Markus Heiser | 2fdbf2622b | |
Markus Heiser | 21dd524a12 | |
Allen | 0476de443e |
|
@ -1,5 +1,5 @@
|
||||||
name: "Checker"
|
name: "Checker"
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
schedule:
|
schedule:
|
||||||
- cron: "0 4 * * 5"
|
- cron: "0 4 * * 5"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
name: "Update searx.data"
|
name: "Update searx.data"
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
schedule:
|
schedule:
|
||||||
- cron: "59 23 28 * *"
|
- cron: "59 23 28 * *"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
name: Integration
|
name: Integration
|
||||||
|
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
push:
|
push:
|
||||||
branches: ["master"]
|
branches: ["master"]
|
||||||
pull_request:
|
pull_request:
|
||||||
|
@ -16,62 +16,62 @@ jobs:
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
os: [ubuntu-20.04]
|
os: [ubuntu-20.04]
|
||||||
python-version: ["3.9", "3.10", "3.11", "3.12",]
|
python-version: ["3.9", "3.10", "3.11", "3.12"]
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
- name: Install Ubuntu packages
|
- name: Install Ubuntu packages
|
||||||
run: |
|
run: |
|
||||||
sudo ./utils/searxng.sh install packages
|
sudo ./utils/searxng.sh install packages
|
||||||
sudo apt install firefox
|
sudo apt install firefox
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python-version }}
|
python-version: ${{ matrix.python-version }}
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v3
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
./.nvm
|
./.nvm
|
||||||
./node_modules
|
./node_modules
|
||||||
key: python-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements*.txt', 'setup.py') }}
|
key: python-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements*.txt', 'setup.py') }}
|
||||||
- name: Install Python dependencies
|
- name: Install Python dependencies
|
||||||
if: steps.cache-python.outputs.cache-hit != 'true'
|
if: steps.cache-python.outputs.cache-hit != 'true'
|
||||||
run: |
|
run: |
|
||||||
make V=1 install
|
make V=1 install
|
||||||
make V=1 gecko.driver
|
make V=1 gecko.driver
|
||||||
- name: Run tests
|
- name: Run tests
|
||||||
run: make V=1 ci.test
|
run: make V=1 ci.test
|
||||||
|
|
||||||
themes:
|
themes:
|
||||||
name: Themes
|
name: Themes
|
||||||
runs-on: ubuntu-20.04
|
runs-on: ubuntu-20.04
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
- name: Install Ubuntu packages
|
- name: Install Ubuntu packages
|
||||||
run: sudo ./utils/searxng.sh install buildhost
|
run: sudo ./utils/searxng.sh install buildhost
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: '3.12'
|
python-version: '3.12'
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v3
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
./.nvm
|
./.nvm
|
||||||
./node_modules
|
./node_modules
|
||||||
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
||||||
- name: Install node dependencies
|
- name: Install node dependencies
|
||||||
run: make V=1 node.env
|
run: make V=1 node.env
|
||||||
- name: Build themes
|
- name: Build themes
|
||||||
run: make V=1 themes.all
|
run: make V=1 themes.all
|
||||||
|
|
||||||
documentation:
|
documentation:
|
||||||
name: Documentation
|
name: Documentation
|
||||||
|
@ -79,40 +79,40 @@ jobs:
|
||||||
permissions:
|
permissions:
|
||||||
contents: write # for JamesIves/github-pages-deploy-action to push changes in repo
|
contents: write # for JamesIves/github-pages-deploy-action to push changes in repo
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: '0'
|
fetch-depth: '0'
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
- name: Install Ubuntu packages
|
- name: Install Ubuntu packages
|
||||||
run: sudo ./utils/searxng.sh install buildhost
|
run: sudo ./utils/searxng.sh install buildhost
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: '3.12'
|
python-version: '3.12'
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v3
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
./.nvm
|
./.nvm
|
||||||
./node_modules
|
./node_modules
|
||||||
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
||||||
- name: Build documentation
|
- name: Build documentation
|
||||||
run: |
|
run: |
|
||||||
make V=1 docs.clean docs.html
|
make V=1 docs.clean docs.html
|
||||||
- name: Deploy
|
- name: Deploy
|
||||||
if: github.ref == 'refs/heads/master'
|
if: github.ref == 'refs/heads/master'
|
||||||
uses: JamesIves/github-pages-deploy-action@3.7.1
|
uses: JamesIves/github-pages-deploy-action@3.7.1
|
||||||
with:
|
with:
|
||||||
GITHUB_TOKEN: ${{ github.token }}
|
GITHUB_TOKEN: ${{ github.token }}
|
||||||
BRANCH: gh-pages
|
BRANCH: gh-pages
|
||||||
FOLDER: dist/docs
|
FOLDER: dist/docs
|
||||||
CLEAN: true # Automatically remove deleted files from the deploy branch
|
CLEAN: true # Automatically remove deleted files from the deploy branch
|
||||||
SINGLE_COMMIT: True
|
SINGLE_COMMIT: true
|
||||||
COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}'
|
COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}'
|
||||||
|
|
||||||
babel:
|
babel:
|
||||||
name: Update translations branch
|
name: Update translations branch
|
||||||
|
@ -125,37 +125,37 @@ jobs:
|
||||||
permissions:
|
permissions:
|
||||||
contents: write # for make V=1 weblate.push.translations
|
contents: write # for make V=1 weblate.push.translations
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: '0'
|
fetch-depth: '0'
|
||||||
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: '3.12'
|
python-version: '3.12'
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v3
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
./.nvm
|
./.nvm
|
||||||
./node_modules
|
./node_modules
|
||||||
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
||||||
- name: weblate & git setup
|
- name: weblate & git setup
|
||||||
env:
|
env:
|
||||||
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
|
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
|
||||||
run: |
|
run: |
|
||||||
mkdir -p ~/.config
|
mkdir -p ~/.config
|
||||||
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
|
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
|
||||||
git config --global user.email "searxng-bot@users.noreply.github.com"
|
git config --global user.email "searxng-bot@users.noreply.github.com"
|
||||||
git config --global user.name "searxng-bot"
|
git config --global user.name "searxng-bot"
|
||||||
- name: Update transations
|
- name: Update transations
|
||||||
id: update
|
id: update
|
||||||
run: |
|
run: |
|
||||||
make V=1 weblate.push.translations
|
make V=1 weblate.push.translations
|
||||||
|
|
||||||
dockers:
|
dockers:
|
||||||
name: Docker
|
name: Docker
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
name: "Security checks"
|
name: "Security checks"
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
schedule:
|
schedule:
|
||||||
- cron: "42 05 * * *"
|
- cron: "42 05 * * *"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
name: "Update translations"
|
name: "Update translations"
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
schedule:
|
schedule:
|
||||||
- cron: "05 07 * * 5"
|
- cron: "05 07 * * 5"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
@ -10,50 +10,50 @@ jobs:
|
||||||
runs-on: ubuntu-20.04
|
runs-on: ubuntu-20.04
|
||||||
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
|
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: '0'
|
fetch-depth: '0'
|
||||||
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: '3.12'
|
python-version: '3.12'
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v3
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
./.nvm
|
./.nvm
|
||||||
./node_modules
|
./node_modules
|
||||||
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
||||||
- name: weblate & git setup
|
- name: weblate & git setup
|
||||||
env:
|
env:
|
||||||
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
|
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
|
||||||
run: |
|
run: |
|
||||||
mkdir -p ~/.config
|
mkdir -p ~/.config
|
||||||
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
|
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
|
||||||
git config --global user.email "searxng-bot@users.noreply.github.com"
|
git config --global user.email "searxng-bot@users.noreply.github.com"
|
||||||
git config --global user.name "searxng-bot"
|
git config --global user.name "searxng-bot"
|
||||||
- name: Merge and push transation updates
|
- name: Merge and push transation updates
|
||||||
run: |
|
run: |
|
||||||
make V=1 weblate.translations.commit
|
make V=1 weblate.translations.commit
|
||||||
- name: Create Pull Request
|
- name: Create Pull Request
|
||||||
id: cpr
|
id: cpr
|
||||||
uses: peter-evans/create-pull-request@v3
|
uses: peter-evans/create-pull-request@v3
|
||||||
with:
|
with:
|
||||||
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
||||||
commit-message: '[l10n] update translations from Weblate'
|
commit-message: '[l10n] update translations from Weblate'
|
||||||
committer: searxng-bot <searxng-bot@users.noreply.github.com>
|
committer: searxng-bot <searxng-bot@users.noreply.github.com>
|
||||||
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
|
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
|
||||||
signoff: false
|
signoff: false
|
||||||
branch: translations_update
|
branch: translations_update
|
||||||
delete-branch: true
|
delete-branch: true
|
||||||
draft: false
|
draft: false
|
||||||
title: '[l10n] update translations from Weblate'
|
title: '[l10n] update translations from Weblate'
|
||||||
body: |
|
body: |
|
||||||
update translations from Weblate
|
update translations from Weblate
|
||||||
labels: |
|
labels: |
|
||||||
translation
|
translation
|
||||||
|
|
2
manage
2
manage
|
@ -57,7 +57,7 @@ while IFS= read -r line; do
|
||||||
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
|
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
|
||||||
YAMLLINT_FILES+=("$line")
|
YAMLLINT_FILES+=("$line")
|
||||||
fi
|
fi
|
||||||
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')"
|
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')"
|
||||||
|
|
||||||
RST_FILES=(
|
RST_FILES=(
|
||||||
'README.rst'
|
'README.rst'
|
||||||
|
|
|
@ -4,7 +4,7 @@ cov-core==1.15.0
|
||||||
black==24.3.0
|
black==24.3.0
|
||||||
pylint==3.3.1
|
pylint==3.3.1
|
||||||
splinter==0.21.0
|
splinter==0.21.0
|
||||||
selenium==4.25.0
|
selenium==4.26.1
|
||||||
Pallets-Sphinx-Themes==2.3.0
|
Pallets-Sphinx-Themes==2.3.0
|
||||||
Sphinx==7.4.7
|
Sphinx==7.4.7
|
||||||
sphinx-issues==5.0.0
|
sphinx-issues==5.0.0
|
||||||
|
|
|
@ -12,7 +12,6 @@ from searx import logger
|
||||||
from searx.engines import engines
|
from searx.engines import engines
|
||||||
from searx.metrics import histogram_observe, counter_add, count_error
|
from searx.metrics import histogram_observe, counter_add, count_error
|
||||||
|
|
||||||
|
|
||||||
CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)
|
CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)
|
||||||
WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)
|
WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)
|
||||||
|
|
||||||
|
@ -133,7 +132,7 @@ def result_score(result, priority):
|
||||||
weight = 1.0
|
weight = 1.0
|
||||||
|
|
||||||
for result_engine in result['engines']:
|
for result_engine in result['engines']:
|
||||||
if hasattr(engines[result_engine], 'weight'):
|
if hasattr(engines.get(result_engine), 'weight'):
|
||||||
weight *= float(engines[result_engine].weight)
|
weight *= float(engines[result_engine].weight)
|
||||||
|
|
||||||
weight *= len(result['positions'])
|
weight *= len(result['positions'])
|
||||||
|
@ -332,10 +331,14 @@ class ResultContainer:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def __merge_duplicated_http_result(self, duplicated, result, position):
|
def __merge_duplicated_http_result(self, duplicated, result, position):
|
||||||
# using content with more text
|
# use content with more text
|
||||||
if result_content_len(result.get('content', '')) > result_content_len(duplicated.get('content', '')):
|
if result_content_len(result.get('content', '')) > result_content_len(duplicated.get('content', '')):
|
||||||
duplicated['content'] = result['content']
|
duplicated['content'] = result['content']
|
||||||
|
|
||||||
|
# use title with more text
|
||||||
|
if result_content_len(result.get('title', '')) > len(duplicated.get('title', '')):
|
||||||
|
duplicated['title'] = result['title']
|
||||||
|
|
||||||
# merge all result's parameters not found in duplicate
|
# merge all result's parameters not found in duplicate
|
||||||
for key in result.keys():
|
for key in result.keys():
|
||||||
if not duplicated.get(key):
|
if not duplicated.get(key):
|
||||||
|
@ -347,7 +350,7 @@ class ResultContainer:
|
||||||
# add engine to list of result-engines
|
# add engine to list of result-engines
|
||||||
duplicated['engines'].add(result['engine'])
|
duplicated['engines'].add(result['engine'])
|
||||||
|
|
||||||
# using https if possible
|
# use https if possible
|
||||||
if duplicated['parsed_url'].scheme != 'https' and result['parsed_url'].scheme == 'https':
|
if duplicated['parsed_url'].scheme != 'https' and result['parsed_url'].scheme == 'https':
|
||||||
duplicated['url'] = result['parsed_url'].geturl()
|
duplicated['url'] = result['parsed_url'].geturl()
|
||||||
duplicated['parsed_url'] = result['parsed_url']
|
duplicated['parsed_url'] = result['parsed_url']
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
|
|
||||||
from searx.search import SearchQuery, EngineRef
|
from searx.search import SearchQuery, EngineRef
|
||||||
from searx.search.processors import online
|
from searx.search.processors import online
|
||||||
from searx.engines import load_engines
|
import searx.search
|
||||||
from searx import engines
|
from searx import engines
|
||||||
|
|
||||||
from tests import SearxTestCase
|
from tests import SearxTestCase
|
||||||
|
@ -22,10 +22,10 @@ TEST_ENGINE = {
|
||||||
class TestOnlineProcessor(SearxTestCase): # pylint: disable=missing-class-docstring
|
class TestOnlineProcessor(SearxTestCase): # pylint: disable=missing-class-docstring
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
load_engines([TEST_ENGINE])
|
searx.search.initialize([TEST_ENGINE])
|
||||||
|
|
||||||
def tearDown(self):
|
def tearDown(self):
|
||||||
load_engines([])
|
searx.search.load_engines([])
|
||||||
|
|
||||||
def _get_params(self, online_processor, search_query, engine_category):
|
def _get_params(self, online_processor, search_query, engine_category):
|
||||||
params = online_processor.get_params(search_query, engine_category)
|
params = online_processor.get_params(search_query, engine_category)
|
||||||
|
|
|
@ -2,26 +2,11 @@
|
||||||
# pylint: disable=missing-module-docstring
|
# pylint: disable=missing-module-docstring
|
||||||
|
|
||||||
from unittest.mock import MagicMock, Mock
|
from unittest.mock import MagicMock, Mock
|
||||||
from searx.engines import load_engines, mariadb_server
|
from searx.engines import mariadb_server
|
||||||
from tests import SearxTestCase
|
from tests import SearxTestCase
|
||||||
|
|
||||||
|
|
||||||
class MariadbServerTests(SearxTestCase): # pylint: disable=missing-class-docstring
|
class MariadbServerTests(SearxTestCase): # pylint: disable=missing-class-docstring
|
||||||
def setUp(self):
|
|
||||||
load_engines(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
'name': 'mariadb server',
|
|
||||||
'engine': 'mariadb_server',
|
|
||||||
'shortcut': 'mdb',
|
|
||||||
'timeout': 9.0,
|
|
||||||
'disabled': True,
|
|
||||||
}
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
def tearDown(self):
|
|
||||||
load_engines([])
|
|
||||||
|
|
||||||
def test_init_no_query_str_raises(self):
|
def test_init_no_query_str_raises(self):
|
||||||
self.assertRaises(ValueError, lambda: mariadb_server.init({}))
|
self.assertRaises(ValueError, lambda: mariadb_server.init({}))
|
||||||
|
|
|
@ -1,28 +1,34 @@
|
||||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
# pylint: disable=missing-module-docstring
|
# pylint: disable=missing-module-docstring
|
||||||
|
|
||||||
|
import logging
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from unittest.mock import Mock
|
from unittest.mock import Mock
|
||||||
from requests import HTTPError
|
from requests import HTTPError
|
||||||
from parameterized import parameterized
|
from parameterized import parameterized
|
||||||
from searx.engines import load_engines, tineye
|
import searx.search
|
||||||
|
import searx.engines
|
||||||
from tests import SearxTestCase
|
from tests import SearxTestCase
|
||||||
|
|
||||||
|
|
||||||
class TinEyeTests(SearxTestCase): # pylint: disable=missing-class-docstring
|
class TinEyeTests(SearxTestCase): # pylint: disable=missing-class-docstring
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
load_engines([{'name': 'tineye', 'engine': 'tineye', 'shortcut': 'tin', 'timeout': 9.0, 'disabled': True}])
|
searx.search.initialize(
|
||||||
|
[{'name': 'tineye', 'engine': 'tineye', 'shortcut': 'tin', 'timeout': 9.0, 'disabled': True}]
|
||||||
|
)
|
||||||
|
|
||||||
|
self.tineye = searx.engines.engines['tineye']
|
||||||
|
self.tineye.logger.setLevel(logging.CRITICAL)
|
||||||
|
|
||||||
def tearDown(self):
|
def tearDown(self):
|
||||||
load_engines([])
|
searx.search.load_engines([])
|
||||||
|
|
||||||
def test_status_code_raises(self):
|
def test_status_code_raises(self):
|
||||||
response = Mock()
|
response = Mock()
|
||||||
response.status_code = 401
|
response.status_code = 401
|
||||||
response.raise_for_status.side_effect = HTTPError()
|
response.raise_for_status.side_effect = HTTPError()
|
||||||
self.assertRaises(HTTPError, lambda: tineye.response(response))
|
self.assertRaises(HTTPError, lambda: self.tineye.response(response))
|
||||||
|
|
||||||
@parameterized.expand([(400), (422)])
|
@parameterized.expand([(400), (422)])
|
||||||
def test_returns_empty_list(self, status_code):
|
def test_returns_empty_list(self, status_code):
|
||||||
|
@ -30,7 +36,7 @@ class TinEyeTests(SearxTestCase): # pylint: disable=missing-class-docstring
|
||||||
response.json.return_value = {}
|
response.json.return_value = {}
|
||||||
response.status_code = status_code
|
response.status_code = status_code
|
||||||
response.raise_for_status.side_effect = HTTPError()
|
response.raise_for_status.side_effect = HTTPError()
|
||||||
results = tineye.response(response)
|
results = self.tineye.response(response)
|
||||||
self.assertEqual(0, len(results))
|
self.assertEqual(0, len(results))
|
||||||
|
|
||||||
def test_logs_format_for_422(self):
|
def test_logs_format_for_422(self):
|
||||||
|
@ -39,9 +45,9 @@ class TinEyeTests(SearxTestCase): # pylint: disable=missing-class-docstring
|
||||||
response.status_code = 422
|
response.status_code = 422
|
||||||
response.raise_for_status.side_effect = HTTPError()
|
response.raise_for_status.side_effect = HTTPError()
|
||||||
|
|
||||||
with self.assertLogs(tineye.logger) as assert_logs_context:
|
with self.assertLogs(self.tineye.logger) as assert_logs_context:
|
||||||
tineye.response(response)
|
self.tineye.response(response)
|
||||||
self.assertIn(tineye.FORMAT_NOT_SUPPORTED, ','.join(assert_logs_context.output))
|
self.assertIn(self.tineye.FORMAT_NOT_SUPPORTED, ','.join(assert_logs_context.output))
|
||||||
|
|
||||||
def test_logs_signature_for_422(self):
|
def test_logs_signature_for_422(self):
|
||||||
response = Mock()
|
response = Mock()
|
||||||
|
@ -49,9 +55,9 @@ class TinEyeTests(SearxTestCase): # pylint: disable=missing-class-docstring
|
||||||
response.status_code = 422
|
response.status_code = 422
|
||||||
response.raise_for_status.side_effect = HTTPError()
|
response.raise_for_status.side_effect = HTTPError()
|
||||||
|
|
||||||
with self.assertLogs(tineye.logger) as assert_logs_context:
|
with self.assertLogs(self.tineye.logger) as assert_logs_context:
|
||||||
tineye.response(response)
|
self.tineye.response(response)
|
||||||
self.assertIn(tineye.NO_SIGNATURE_ERROR, ','.join(assert_logs_context.output))
|
self.assertIn(self.tineye.NO_SIGNATURE_ERROR, ','.join(assert_logs_context.output))
|
||||||
|
|
||||||
def test_logs_download_for_422(self):
|
def test_logs_download_for_422(self):
|
||||||
response = Mock()
|
response = Mock()
|
||||||
|
@ -59,9 +65,9 @@ class TinEyeTests(SearxTestCase): # pylint: disable=missing-class-docstring
|
||||||
response.status_code = 422
|
response.status_code = 422
|
||||||
response.raise_for_status.side_effect = HTTPError()
|
response.raise_for_status.side_effect = HTTPError()
|
||||||
|
|
||||||
with self.assertLogs(tineye.logger) as assert_logs_context:
|
with self.assertLogs(self.tineye.logger) as assert_logs_context:
|
||||||
tineye.response(response)
|
self.tineye.response(response)
|
||||||
self.assertIn(tineye.DOWNLOAD_ERROR, ','.join(assert_logs_context.output))
|
self.assertIn(self.tineye.DOWNLOAD_ERROR, ','.join(assert_logs_context.output))
|
||||||
|
|
||||||
def test_logs_description_for_400(self):
|
def test_logs_description_for_400(self):
|
||||||
description = 'There was a problem with that request. Error ID: ad5fc955-a934-43c1-8187-f9a61d301645'
|
description = 'There was a problem with that request. Error ID: ad5fc955-a934-43c1-8187-f9a61d301645'
|
||||||
|
@ -70,8 +76,8 @@ class TinEyeTests(SearxTestCase): # pylint: disable=missing-class-docstring
|
||||||
response.status_code = 400
|
response.status_code = 400
|
||||||
response.raise_for_status.side_effect = HTTPError()
|
response.raise_for_status.side_effect = HTTPError()
|
||||||
|
|
||||||
with self.assertLogs(tineye.logger) as assert_logs_context:
|
with self.assertLogs(self.tineye.logger) as assert_logs_context:
|
||||||
tineye.response(response)
|
self.tineye.response(response)
|
||||||
self.assertIn(description, ','.join(assert_logs_context.output))
|
self.assertIn(description, ','.join(assert_logs_context.output))
|
||||||
|
|
||||||
def test_crawl_date_parses(self):
|
def test_crawl_date_parses(self):
|
||||||
|
@ -90,5 +96,5 @@ class TinEyeTests(SearxTestCase): # pylint: disable=missing-class-docstring
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
response.status_code = 200
|
response.status_code = 200
|
||||||
results = tineye.response(response)
|
results = self.tineye.response(response)
|
||||||
self.assertEqual(date, results[0]['publishedDate'])
|
self.assertEqual(date, results[0]['publishedDate'])
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
# pylint: disable=missing-module-docstring
|
# pylint: disable=missing-module-docstring
|
||||||
|
|
||||||
from parameterized.parameterized import parameterized
|
from parameterized.parameterized import parameterized
|
||||||
from searx.engines import load_engines
|
import searx.search
|
||||||
from searx.query import RawTextQuery
|
from searx.query import RawTextQuery
|
||||||
from tests import SearxTestCase
|
from tests import SearxTestCase
|
||||||
|
|
||||||
|
@ -218,10 +218,10 @@ class TestBang(SearxTestCase): # pylint:disable=missing-class-docstring
|
||||||
THE_QUERY = 'the query'
|
THE_QUERY = 'the query'
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
load_engines(TEST_ENGINES)
|
searx.search.initialize(TEST_ENGINES)
|
||||||
|
|
||||||
def tearDown(self):
|
def tearDown(self):
|
||||||
load_engines([])
|
searx.search.load_engines([])
|
||||||
|
|
||||||
@parameterized.expand(SPECIFIC_BANGS)
|
@parameterized.expand(SPECIFIC_BANGS)
|
||||||
def test_bang(self, bang: str):
|
def test_bang(self, bang: str):
|
||||||
|
|
|
@ -2,9 +2,26 @@
|
||||||
# pylint: disable=missing-module-docstring
|
# pylint: disable=missing-module-docstring
|
||||||
|
|
||||||
from searx.results import ResultContainer
|
from searx.results import ResultContainer
|
||||||
|
import searx.search
|
||||||
from tests import SearxTestCase
|
from tests import SearxTestCase
|
||||||
|
|
||||||
|
|
||||||
|
def make_test_engine_dict(**kwargs) -> dict:
|
||||||
|
test_engine = {
|
||||||
|
# fmt: off
|
||||||
|
'name': None,
|
||||||
|
'engine': None,
|
||||||
|
'categories': 'general',
|
||||||
|
'shortcut': 'dummy',
|
||||||
|
'timeout': 3.0,
|
||||||
|
'tokens': [],
|
||||||
|
# fmt: on
|
||||||
|
}
|
||||||
|
|
||||||
|
test_engine.update(**kwargs)
|
||||||
|
return test_engine
|
||||||
|
|
||||||
|
|
||||||
def fake_result(url='https://aa.bb/cc?dd=ee#ff', title='aaa', content='bbb', engine='wikipedia', **kwargs):
|
def fake_result(url='https://aa.bb/cc?dd=ee#ff', title='aaa', content='bbb', engine='wikipedia', **kwargs):
|
||||||
result = {
|
result = {
|
||||||
# fmt: off
|
# fmt: off
|
||||||
|
@ -19,23 +36,40 @@ def fake_result(url='https://aa.bb/cc?dd=ee#ff', title='aaa', content='bbb', eng
|
||||||
|
|
||||||
|
|
||||||
class ResultContainerTestCase(SearxTestCase): # pylint: disable=missing-class-docstring
|
class ResultContainerTestCase(SearxTestCase): # pylint: disable=missing-class-docstring
|
||||||
|
|
||||||
|
def setUp(self) -> None:
|
||||||
|
stract_engine = make_test_engine_dict(name="stract", engine="stract", shortcut="stra")
|
||||||
|
duckduckgo_engine = make_test_engine_dict(name="duckduckgo", engine="duckduckgo", shortcut="ddg")
|
||||||
|
mojeek_engine = make_test_engine_dict(name="mojeek", engine="mojeek", shortcut="mjk")
|
||||||
|
searx.search.initialize([stract_engine, duckduckgo_engine, mojeek_engine])
|
||||||
|
self.container = ResultContainer()
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
searx.search.load_engines([])
|
||||||
|
|
||||||
def test_empty(self):
|
def test_empty(self):
|
||||||
c = ResultContainer()
|
self.assertEqual(self.container.get_ordered_results(), [])
|
||||||
self.assertEqual(c.get_ordered_results(), [])
|
|
||||||
|
|
||||||
def test_one_result(self):
|
def test_one_result(self):
|
||||||
c = ResultContainer()
|
self.container.extend('wikipedia', [fake_result()])
|
||||||
c.extend('wikipedia', [fake_result()])
|
|
||||||
self.assertEqual(c.results_length(), 1)
|
self.assertEqual(self.container.results_length(), 1)
|
||||||
|
|
||||||
def test_one_suggestion(self):
|
def test_one_suggestion(self):
|
||||||
c = ResultContainer()
|
self.container.extend('wikipedia', [fake_result(suggestion=True)])
|
||||||
c.extend('wikipedia', [fake_result(suggestion=True)])
|
|
||||||
self.assertEqual(len(c.suggestions), 1)
|
self.assertEqual(len(self.container.suggestions), 1)
|
||||||
self.assertEqual(c.results_length(), 0)
|
self.assertEqual(self.container.results_length(), 0)
|
||||||
|
|
||||||
def test_result_merge(self):
|
def test_result_merge(self):
|
||||||
c = ResultContainer()
|
self.container.extend('wikipedia', [fake_result()])
|
||||||
c.extend('wikipedia', [fake_result()])
|
self.container.extend('wikidata', [fake_result(), fake_result(url='https://example.com/')])
|
||||||
c.extend('wikidata', [fake_result(), fake_result(url='https://example.com/')])
|
|
||||||
self.assertEqual(c.results_length(), 2)
|
self.assertEqual(self.container.results_length(), 2)
|
||||||
|
|
||||||
|
def test_result_merge_by_title(self):
|
||||||
|
self.container.extend('stract', [fake_result(engine='stract', title='short title')])
|
||||||
|
self.container.extend('duckduckgo', [fake_result(engine='duckduckgo', title='normal title')])
|
||||||
|
self.container.extend('mojeek', [fake_result(engine='mojeek', title='this long long title')])
|
||||||
|
|
||||||
|
self.assertEqual(self.container.get_ordered_results()[0].get('title', ''), 'this long long title')
|
||||||
|
|
Loading…
Reference in New Issue