jandieman
/
searxng
mirror of https://github.com/searxng/searxng.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into fix-infinite-scroll

This commit is contained in:
Markus Heiser 2020-01-28 10:59:03 +00:00 committed by GitHub
parent 0e7b6c9a03 bda1895655
commit e64ff38217
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
157 changed files with 6509 additions and 10928 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -19,5 +19,8 @@ node_modules/
.tx/ .tx/
build/
dist/
local/ local/
gh-pages/
searx.egg-info/ searx.egg-info/

27
Makefile
View File

@ -1,19 +1,32 @@
# -*- coding: utf-8; mode: makefile-gmake -*- # -*- coding: utf-8; mode: makefile-gmake -*-
export GIT_URL=https://github.com/asciimoo/searx
export SEARX_URL=https://searx.me
export DOCS_URL=https://asciimoo.github.io/searx
PYOBJECTS = searx PYOBJECTS = searx
DOC = docs
PY_SETUP_EXTRAS ?= \[test\] PY_SETUP_EXTRAS ?= \[test\]
PYDIST=./dist/py
PYBUILD=./build/py
include utils/makefile.include include utils/makefile.include
include utils/makefile.python include utils/makefile.python
include utils/makefile.sphinx
all: clean install all: clean install
PHONY += help PHONY += help
help: help:
@echo ' test - run developer tests' @echo ' test - run developer tests'
@echo ' docs - build documentation'
@echo ' docs-live - autobuild HTML documentation while editing'
@echo ' run - run developer instance' @echo ' run - run developer instance'
@echo ' install - developer install (./local)' @echo ' install - developer install (./local)'
@echo ' uninstall - uninstall (./local)' @echo ' uninstall - uninstall (./local)'
@echo ' gh-pages - build docs & deploy on gh-pages branch'
@echo ' clean - drop builds and environments'
@echo '' @echo ''
@$(MAKE) -s -f utils/makefile.include make-help @$(MAKE) -s -f utils/makefile.include make-help
@echo '' @echo ''
@ -40,6 +53,20 @@ run: pyenvinstall
) & ) &
$(PY_ENV)/bin/python ./searx/webapp.py $(PY_ENV)/bin/python ./searx/webapp.py
# docs
# ----
PHONY += docs
docs: pyenvinstall sphinx-doc
$(call cmd,sphinx,html,docs,docs)
PHONY += docs-live
docs-live: pyenvinstall sphinx-live
$(call cmd,sphinx_autobuild,html,docs,docs)
$(GH_PAGES)::
@echo "doc available at --> $(DOCS_URL)"
# test # test
# ---- # ----

6
README.rst
View File

@ -23,13 +23,13 @@ Go to the `searx-docker <https://github.com/searx/searx-docker>`__ project.
Without Docker Without Docker
------ ------
For all the details, follow this `step by step installation <https://asciimoo.github.io/searx/dev/install/installation.html>`__. For all of the details, follow this `step by step installation <https://asciimoo.github.io/searx/dev/install/installation.html>`__.
Note: the documentation needs to be updated. Note: the documentation needs to be updated.
If you are in hurry If you are in a hurry
------ ------
- clone source: - clone the source:
``git clone https://github.com/asciimoo/searx.git && cd searx`` ``git clone https://github.com/asciimoo/searx.git && cd searx``
- install dependencies: ``./manage.sh update_packages`` - install dependencies: ``./manage.sh update_packages``
- edit your - edit your

130
docs/_themes/searx/static/searx.css vendored Normal file
View File

@ -0,0 +1,130 @@
@import url("pocoo.css");
a, a.reference, a.footnote-reference {
color: #004b6b;
border-color: #004b6b;
}
a:hover {
color: #6d4100;
border-color: #6d4100;
}
p.version-warning {
background-color: #004b6b;
}
div.sidebar {
background-color: whitesmoke;
border-color: lightsteelblue;
border-radius: 3pt;
}
p.sidebar-title, .sidebar p {
margin: 6pt;
}
.sidebar li,
.hlist li {
list-style-type: disclosure-closed;
}
/* admonitions
*/
div.admonition, div.topic {
background-color: #fafafa;
margin: 8px 0px;
padding: 1em;
border-radius: 3pt 0 0 3pt;
border-top: none;
border-right: none;
border-bottom: none;
border-left: 5pt solid #ccc;
}
p.admonition-title:after {
content: none;
}
.admonition.hint { border-color: #416dc0b0; }
.admonition.note { border-color: #6c856cb0; }
.admonition.tip { border-color: #85c5c2b0; }
.admonition.attention { border-color: #ecec97b0; }
.admonition.caution { border-color: #a6c677b0; }
.admonition.danger { border-color: #d46262b0; }
.admonition.important { border-color: #dfa3a3b0; }
.admonition.error { border-color: red; }
.admonition.warning { border-color: darkred; }
.admonition.admonition-generic-admonition-title {
border-color: #416dc0b0;
}
/* admonitions with (rendered) reST markup examples (:class: rst-example)
*
* .. admonition:: title of the example
* :class: rst-example
* ....
*/
div.rst-example {
background-color: inherit;
margin: 0;
border-top: none;
border-right: 1px solid #ccc;
border-bottom: none;
border-left: none;
border-radius: none;
padding: 0;
}
div.rst-example > p.admonition-title {
font-family: Sans Serif;
font-style: italic;
font-size: 0.8em;
display: block;
border-bottom: 1px solid #ccc;
padding: 0.5em 1em;
text-align: right;
}
/* code block in figures
*/
div.highlight pre {
text-align: left;
}
/* Table theme
*/
thead, tfoot {
background-color: #fff;
}
th:hover, td:hover {
background-color: #ffc;
}
thead th, tfoot th, tfoot td, tbody th {
background-color: #fffaef;
}
tbody tr:nth-child(odd) {
background-color: #fff;
}
tbody tr:nth-child(even) {
background-color: #fafafa;
}
caption {
font-family: Sans Serif;
padding: 0.5em;
margin: 0.5em 0 0.5em 0;
caption-side: top;
text-align: left;
}

6
docs/_themes/searx/theme.conf vendored Normal file
View File

@ -0,0 +1,6 @@
[theme]
inherit = pocoo
stylesheet = searx.css
[options]
touch_icon =

96
docs/admin/api.rst Normal file
View File

@ -0,0 +1,96 @@
.. _adminapi:
==================
Administration API
==================
Get configuration data
======================
.. code:: http
GET /config HTTP/1.1
Sample response
---------------
.. code:: json
{
"autocomplete": "",
"categories": [
"map",
"it",
"images",
],
"default_locale": "",
"default_theme": "oscar",
"engines": [
{
"categories": [
"map"
],
"enabled": true,
"name": "openstreetmap",
"shortcut": "osm"
},
{
"categories": [
"it"
],
"enabled": true,
"name": "arch linux wiki",
"shortcut": "al"
},
{
"categories": [
"images"
],
"enabled": true,
"name": "google images",
"shortcut": "goi"
},
{
"categories": [
"it"
],
"enabled": false,
"name": "bitbucket",
"shortcut": "bb"
},
],
"instance_name": "searx",
"locales": {
"de": "Deutsch (German)",
"en": "English",
"eo": "Esperanto (Esperanto)",
},
"plugins": [
{
"enabled": true,
"name": "HTTPS rewrite"
},
{
"enabled": false,
"name": "Vim-like hotkeys"
}
],
"safe_search": 0
}
Embed search bar
================
The search bar can be embedded into websites. Just paste the example into the
HTML of the site. URL of the searx instance and values are customizable.
.. code:: html
<form method="post" action="https://searx.me/">
<!-- search --> <input type="text" name="q" />
<!-- categories --> <input type="hidden" name="categories" value="general,social media" />
<!-- language --> <input type="hidden" name="lang" value="all" />
<!-- locale --> <input type="hidden" name="locale" value="en" />
<!-- date filter --> <input type="hidden" name="time_range" value="month" />
</form>

33
docs/admin/arch_public.dot Normal file
View File

@ -0,0 +1,33 @@
digraph G {
node [style=filled, shape=box, fillcolor="#ffffcc", fontname="Sans"];
edge [fontname="Sans"];
browser [label="Browser", shape=Mdiamond];
rp [label="Reverse Proxy", href="url to configure reverse proxy"];
filtron [label="Filtron", href="https://github.com/asciimoo/filtron"];
morty [label="Morty", href="https://github.com/asciimoo/morty"];
static [label="Static files", href="url to configure static files"];
uwsgi [label="uwsgi", href="url to configure uwsgi"]
searx1 [label="Searx #1"];
searx2 [label="Searx #2"];
searx3 [label="Searx #3"];
searx4 [label="Searx #4"];
browser -> rp [label="HTTPS"]
subgraph cluster_searx {
label = "Searx instance" fontname="Sans";
bgcolor="#fafafa";
{ rank=same; static rp };
rp -> morty [label="optional: images and HTML pages proxy"];
rp -> static [label="optional: reverse proxy serves directly static files"];
rp -> filtron [label="HTTP"];
filtron -> uwsgi [label="HTTP"];
uwsgi -> searx1;
uwsgi -> searx2;
uwsgi -> searx3;
uwsgi -> searx4;
}
}

24
docs/admin/architecture.rst Normal file
View File

@ -0,0 +1,24 @@
.. _architecture:
============
Architecture
============
.. sidebar:: Needs work!
This article needs some work / Searx is a collaborative effort. If you have
any contribution, feel welcome to send us your :pull:`PR <../pulls>`, see
:ref:`how to contribute`.
Herein you will find some hints and suggestions about typical architectures of
searx infrastructures.
We start with a contribution from :pull:`@dalf <1776#issuecomment-567917320>`.
It shows a *reference* setup for public searx instances.
.. _arch public:
.. kernel-figure:: arch_public.dot
:alt: arch_public.dot
Reference architecture of a public searx setup.

103
docs/admin/buildhosts.rst Normal file
View File

@ -0,0 +1,103 @@
.. _buildhosts:
==========
Buildhosts
==========
.. sidebar:: This article needs some work
If you have any contribution send us your :pull:`PR <../pulls>`, see
:ref:`how to contribute`.
To get best results from build, its recommend to install additional packages
on build hosts.
.. _docs build:
Build docs
==========
.. _Graphviz: https://graphviz.gitlab.io
.. _ImageMagick: https://www.imagemagick.org
.. _XeTeX: https://tug.org/xetex/
.. _dvisvgm: https://dvisvgm.de/
.. sidebar:: Sphinx build needs
- ImageMagick_
- Graphviz_
- XeTeX_
- dvisvgm_
Most of the sphinx requirements are installed from :origin:`setup.py` and the
docs can be build from scratch with ``make docs``. For better math and image
processing additional packages are needed. The XeTeX_ needed not only for PDF
creation, its also needed for :ref:`math` when HTML output is build.
To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered
as images (``sphinx.ext.imgmath`` extension). If your docs build (``make
docs``) shows warnings like this::
WARNING: dot(1) not found, for better output quality install \
graphviz from http://www.graphviz.org
..
WARNING: LaTeX command 'latex' cannot be run (needed for math \
display), check the imgmath_latex setting
you need to install additional packages on your build host, to get better HTML
output.
.. _system requirements:
.. tabs::
.. group-tab:: Ubuntu / debian
.. code-block:: sh
$ sudo apt install graphviz imagemagick texlive-xetex librsvg2-bin
.. group-tab:: Arch Linux
.. code-block:: sh
$ sudo pacman -S graphviz imagemagick texlive-bin extra/librsvg
.. group-tab:: Fedora / RHEL
.. code-block:: sh
$ sudo dnf install graphviz graphviz-gd texlive-xetex-bin librsvg2-tools
For PDF output you also need:
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
$ sudo apt texlive-latex-recommended texlive-extra-utils ttf-dejavu
.. group-tab:: Arch Linux
.. code:: sh
$ sudo pacman -S texlive-core texlive-latexextra ttf-dejavu
.. group-tab:: Fedora / RHEL
.. code:: sh
$ sudo dnf install \
texlive-collection-fontsrecommended texlive-collection-latex \
dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts
.. _system requirements END:
.. literalinclude:: ../conf.py
:language: python
:start-after: # sphinx.ext.imgmath setup
:end-before: # sphinx.ext.imgmath setup END

71
docs/admin/engines.rst Normal file
View File

@ -0,0 +1,71 @@
.. _engines generic:
=======
Engines
=======
.. sidebar:: Further reading ..
- :ref:`settings engine`
- :ref:`engine settings`
- :ref:`engine file`
============= =========== ==================== ============
:ref:`engine settings` :ref:`engine file`
------------------------- ---------------------------------
Name (cfg) Categories
------------------------- ---------------------------------
Engine .. Paging support **P**
------------------------- -------------------- ------------
Shortcut **S** Language support **L**
Timeout **TO** Time range support **TR**
Disabled **D** Offline **O**
------------- ----------- -------------------- ------------
Safe search **SS**
------------- ----------- ---------------------------------
Weigth **W**
------------- ----------- ---------------------------------
Disabled **D**
============= =========== =================================
Configuration defaults (at built time):
.. _configured engines:
.. jinja:: webapp
.. flat-table:: Engines configured at built time (defaults)
:header-rows: 1
:stub-columns: 2
* - Name (cfg)
- S
- Engine
- TO
- Categories
- P
- L
- SS
- D
- TR
- O
- W
- D
{% for name, mod in engines.items() %}
* - {{name}}
- !{{mod.shortcut}}
- {{mod.__name__}}
- {{mod.timeout}}
- {{", ".join(mod.categories)}}
- {{(mod.paging and "y") or ""}}
- {{(mod.language_support and "y") or ""}}
- {{(mod.safesearch and "y") or ""}}
- {{(mod.disabled and "y") or ""}}
- {{(mod.time_range_support and "y") or ""}}
- {{(mod.offline and "y") or ""}}
- {{mod.weight or 1 }}
- {{(mod.disabled and "y") or ""}}
{% endfor %}

148
docs/admin/filtron.rst Normal file
View File

@ -0,0 +1,148 @@
==========================
How to protect an instance
==========================
Searx depens on external search services. To avoid the abuse of these services
it is advised to limit the number of requests processed by searx.
An application firewall, ``filtron`` solves exactly this problem. Information
on how to install it can be found at the `project page of filtron
<https://github.com/asciimoo/filtron>`__.
Sample configuration of filtron
===============================
An example configuration can be find below. This configuration limits the access
of:
- scripts or applications (roboagent limit)
- webcrawlers (botlimit)
- IPs which send too many requests (IP limit)
- too many json, csv, etc. requests (rss/json limit)
- the same UserAgent of if too many requests (useragent limit)
.. code:: json
[{
"name":"search request",
"filters":[
"Param:q",
"Path=^(/|/search)$"
],
"interval":"<time-interval-in-sec (int)>",
"limit":"<max-request-number-in-interval (int)>",
"subrules":[
{
"name":"roboagent limit",
"interval":"<time-interval-in-sec (int)>",
"limit":"<max-request-number-in-interval (int)>",
"filters":[
"Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
],
"actions":[
{
"name":"block",
"params":{
"message":"Rate limit exceeded"
}
}
]
},
{
"name":"botlimit",
"limit":0,
"stop":true,
"filters":[
"Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
],
"actions":[
{
"name":"block",
"params":{
"message":"Rate limit exceeded"
}
}
]
},
{
"name":"IP limit",
"interval":"<time-interval-in-sec (int)>",
"limit":"<max-request-number-in-interval (int)>",
"stop":true,
"aggregations":[
"Header:X-Forwarded-For"
],
"actions":[
{
"name":"block",
"params":{
"message":"Rate limit exceeded"
}
}
]
},
{
"name":"rss/json limit",
"interval":"<time-interval-in-sec (int)>",
"limit":"<max-request-number-in-interval (int)>",
"stop":true,
"filters":[
"Param:format=(csv|json|rss)"
],
"actions":[
{
"name":"block",
"params":{
"message":"Rate limit exceeded"
}
}
]
},
{
"name":"useragent limit",
"interval":"<time-interval-in-sec (int)>",
"limit":"<max-request-number-in-interval (int)>",
"aggregations":[
"Header:User-Agent"
],
"actions":[
{
"name":"block",
"params":{
"message":"Rate limit exceeded"
}
}
]
}
]
}]
Route request through filtron
=============================
Filtron can be started using the following command:
.. code:: sh
$ filtron -rules rules.json
It listens on ``127.0.0.1:4004`` and forwards filtered requests to
``127.0.0.1:8888`` by default.
Use it along with ``nginx`` with the following example configuration.
.. code:: nginx
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_pass http://127.0.0.1:4004/;
}
Requests are coming from port 4004 going through filtron and then forwarded to
port 8888 where a searx is being run.

16
docs/admin/index.rst Normal file
View File

@ -0,0 +1,16 @@
===========================
Administrator documentation
===========================
.. toctree::
:maxdepth: 1
installation
settings
api
architecture
filtron
morty
engines
plugins
buildhosts

349
docs/admin/installation.rst Normal file
View File

@ -0,0 +1,349 @@
.. _installation:
============
Installation
============
.. contents::
:depth: 3
Basic installation
==================
Step by step installation for Debian/Ubuntu with virtualenv. For Ubuntu, be sure
to have enable universe repository.
Install packages:
.. code:: sh
$ sudo -H apt-get install \
git build-essential libxslt-dev \
python-dev python-virtualenv python-babel \
zlib1g-dev libffi-dev libssl-dev
Install searx:
.. code:: sh
cd /usr/local
sudo -H git clone https://github.com/asciimoo/searx.git
sudo -H useradd searx -d /usr/local/searx
sudo -H chown searx:searx -R /usr/local/searx
Install dependencies in a virtualenv:
.. code:: sh
cd /usr/local/searx
sudo -H -u searx -i
.. code:: sh
(searx)$ virtualenv searx-ve
(searx)$ . ./searx-ve/bin/activate
(searx)$ ./manage.sh update_packages
Configuration
==============
.. code:: sh
sed -i -e "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml
Edit searx/settings.yml if necessary.
Check
=====
Start searx:
.. code:: sh
python searx/webapp.py
Go to http://localhost:8888
If everything works fine, disable the debug option in settings.yml:
.. code:: sh
sed -i -e "s/debug : True/debug : False/g" searx/settings.yml
At this point searx is not demonized ; uwsgi allows this.
You can exit the virtualenv and the searx user bash (enter exit command
twice).
uwsgi
=====
Install packages:
.. code:: sh
sudo -H apt-get install \
uwsgi uwsgi-plugin-python
Create the configuration file ``/etc/uwsgi/apps-available/searx.ini`` with this
content:
.. code:: ini
[uwsgi]
# Who will run the code
uid = searx
gid = searx
# disable logging for privacy
disable-logging = true
# Number of workers (usually CPU count)
workers = 4
# The right granted on the created socket
chmod-socket = 666
# Plugin to use and interpretor config
single-interpreter = true
master = true
plugin = python
lazy-apps = true
enable-threads = true
# Module to import
module = searx.webapp
# Support running the module from a webserver subdirectory.
route-run = fixpathinfo:
# Virtualenv and python path
virtualenv = /usr/local/searx/searx-ve/
pythonpath = /usr/local/searx/
chdir = /usr/local/searx/searx/
Activate the uwsgi application and restart:
.. code:: sh
cd /etc/uwsgi/apps-enabled
ln -s ../apps-available/searx.ini
/etc/init.d/uwsgi restart
Web server
==========
with nginx
----------
If nginx is not installed (uwsgi will not work with the package
nginx-light):
.. code:: sh
sudo -H apt-get install nginx
Hosted at /
~~~~~~~~~~~
Create the configuration file ``/etc/nginx/sites-available/searx`` with this
content:
.. code:: nginx
server {
listen 80;
server_name searx.example.com;
root /usr/local/searx/searx;
location /static {
}
location / {
include uwsgi_params;
uwsgi_pass unix:/run/uwsgi/app/searx/socket;
}
}
Create a symlink to sites-enabled:
.. code:: sh
sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx
Restart service:
.. code:: sh
sudo -H service nginx restart
sudo -H service uwsgi restart
from subdirectory URL (/searx)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Add this configuration in the server config file
``/etc/nginx/sites-enabled/default``:
.. code:: nginx
location /searx/static {
alias /usr/local/searx/searx/static;
}
location /searx {
uwsgi_param SCRIPT_NAME /searx;
include uwsgi_params;
uwsgi_pass unix:/run/uwsgi/app/searx/socket;
}
**OR** using reverse proxy (Please, note that reverse proxy advised to be used
in case of single-user or low-traffic instances.)
.. code:: nginx
location /searx/static {
alias /usr/local/searx/searx/static;
}
location /searx {
proxy_pass http://127.0.0.1:8888;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_set_header X-Script-Name /searx;
proxy_buffering off;
}
Enable ``base_url`` in ``searx/settings.yml``
.. code:: yaml
base_url : http://your.domain.tld/searx/
Restart service:
.. code:: sh
sudo -H service nginx restart
sudo -H service uwsgi restart
disable logs
^^^^^^^^^^^^
for better privacy you can disable nginx logs about searx.
how to proceed: below ``uwsgi_pass`` in ``/etc/nginx/sites-available/default``
add:
.. code:: nginx
access_log /dev/null;
error_log /dev/null;
Restart service:
.. code:: sh
sudo -H service nginx restart
with apache
-----------
Add wsgi mod:
.. code:: sh
sudo -H apt-get install libapache2-mod-uwsgi
sudo -H a2enmod uwsgi
Add this configuration in the file ``/etc/apache2/apache2.conf``:
.. code:: apache
<Location />
Options FollowSymLinks Indexes
SetHandler uwsgi-handler
uWSGISocket /run/uwsgi/app/searx/socket
</Location>
Note that if your instance of searx is not at the root, you should change
``<Location />`` by the location of your instance, like ``<Location /searx>``.
Restart Apache:
.. code:: sh
sudo -H /etc/init.d/apache2 restart
disable logs
~~~~~~~~~~~~
For better privacy you can disable Apache logs.
.. warning::
You can only disable logs for the whole (virtual) server not for a specific
path.
Go back to ``/etc/apache2/apache2.conf`` and above ``<Location />`` add:
.. code:: apache
CustomLog /dev/null combined
Restart Apache:
.. code:: sh
sudo -H /etc/init.d/apache2 restart
How to update
=============
.. code:: sh
cd /usr/local/searx
sudo -H -u searx -i
.. code:: sh
(searx)$ . ./searx-ve/bin/activate
(searx)$ git stash
(searx)$ git pull origin master
(searx)$ git stash apply
(searx)$ ./manage.sh update_packages
.. code:: sh
sudo -H service uwsgi restart
Docker
======
Make sure you have installed Docker. For instance, you can deploy searx like this:
.. code:: sh
docker pull wonderfall/searx
docker run -d --name searx -p $PORT:8888 wonderfall/searx
Go to ``http://localhost:$PORT``.
See https://hub.docker.com/r/wonderfall/searx/ for more informations. It's also
possible to build searx from the embedded Dockerfile.
.. code:: sh
git clone https://github.com/asciimoo/searx.git
cd searx
docker build -t whatever/searx .
References
==========
* https://about.okhin.fr/posts/Searx/ with some additions
* How to: `Setup searx in a couple of hours with a free SSL certificate
<https://www.reddit.com/r/privacytoolsIO/comments/366kvn/how_to_setup_your_own_privacy_respecting_search/>`__

26
docs/admin/morty.rst Normal file
View File

@ -0,0 +1,26 @@
=========================
How to setup result proxy
=========================
.. _morty: https://github.com/asciimoo/morty
.. _morty's README: https://github.com/asciimoo/morty
By default searx can only act as an image proxy for result images, but it is
possible to proxify all the result URLs with an external service, morty_.
To use this feature, morty has to be installed and activated in searx's
``settings.yml``.
Add the following snippet to your ``settings.yml`` and restart searx:
.. code:: yaml
result_proxy:
url : http://127.0.0.1:3000/
key : your_morty_proxy_key
``url``
Is the address of the running morty service.
``key``
Is an optional argument, see `morty's README`_ for more information.

39
docs/admin/plugins.rst Normal file
View File

@ -0,0 +1,39 @@
.. _plugins generic:
===============
Plugins builtin
===============
.. sidebar:: Further reading ..
- :ref:`dev plugin`
Configuration defaults (at built time):
:DO: Default on
.. _configured plugins:
.. jinja:: webapp
.. flat-table:: Plugins configured at built time (defaults)
:header-rows: 1
:stub-columns: 1
:widths: 3 1 9
* - Name
- DO
- Description
JS & CSS dependencies
{% for plgin in plugins %}
* - {{plgin.name}}
- {{(plgin.default_on and "y") or ""}}
- {{plgin.description}}
{% for dep in (plgin.js_dependencies + plgin.css_dependencies) %}
| ``{{dep}}`` {% endfor %}
{% endfor %}

181
docs/admin/settings.rst Normal file
View File

@ -0,0 +1,181 @@
.. _settings.yml:
================
``settings.yml``
================
.. sidebar:: Further reading ..
- :ref:`search API`
This page describe the options possibilities of the settings.yml file.
.. _settings global:
Global Settings
===============
.. code:: yaml
server:
port : 8888
secret_key : "ultrasecretkey" # change this!
debug : False # debug mode, only for development
request_timeout : 2.0 # seconds
base_url : False # set custom base_url (or False)
themes_path : "" # custom ui themes path
default_theme : oscar # ui theme
useragent_suffix : "" # suffix of searx_useragent, could contain
# informations like admins email address
image_proxy : False # proxying image results through searx
default_locale : "" # default interface locale
# uncomment below section if you want to use a proxy
#outgoing_proxies :
# http : http://127.0.0.1:8080
# https: http://127.0.0.1:8080
# uncomment below section only if you have more than one network interface
# which can be the source of outgoing search requests
#source_ips:
# - 1.1.1.1
# - 1.1.1.2
locales:
en : English
de : Deutsch
he : Hebrew
hu : Magyar
fr : Français
es : Español
it : Italiano
nl : Nederlands
ja : 日本語 (Japanese)
tr : Türkçe
ru : Russian
ro : Romanian
``port`` :
Port number of the searx web application if you run it directly using ``python
searx/webapp.py``. Doesn't apply to searx running on Apache or Nginx.
``secret_key`` :
Used for cryptography purpose.
``debug`` :
Allow a more detailed log if you run searx directly. Display *detailed* error
messages in the browser too, so this must be deactivated in production.
``request_timeout`` :
Global timeout of the requests made to others engines in seconds. A bigger
timeout will allow to wait for answers from slow engines, but in consequence
will slow searx reactivity (the result page may take the time specified in the
timeout to load)
``base_url`` :
The base URL where searx is deployed. Used to create correct inbound links.
``themes_path`` :
Path to where the themes are located. If you didn't develop anything, leave it
blank.
``default_theme`` :
Name of the theme you want to use by default on you searx instance.
``useragent_suffix`` :
Suffix to the user-agent searx uses to send requests to others engines. If an
engine wish to block you, a contact info here may be useful to avoid that.
``image_proxy`` :
Allow your instance of searx of being able to proxy images. Uses memory space.
``default_locale`` :
Aearx interface language. If blank, the locale is detected by using the
browser language. If it doesn't work, or you are deploying a language
specific instance of searx, a locale can be defined using an ISO language
code, like ``fr``, ``en``, ``de``.
.. _requests proxies: http://docs.python-requests.org/en/latest/user/advanced/#proxies
.. _PR SOCKS support: https://github.com/kennethreitz/requests/pull/478
``outgoing_proxies`` :
Define a proxy you wish to use, see `requests proxies`_. SOCKS proxies are
not supported / see `PR SOCKS support`.
``source_ips`` :
If you use multiple nework interfaces, define from which IP the requests must
be made.
``locales`` :
Locales codes and their names. Available translations of searx interface.
.. _settings engine:
Engine settings
===============
.. sidebar:: Further reading ..
- :ref:`engines-dev`
.. code:: yaml
- name : bing
engine : bing
shortcut : bi
base_url : 'https://{language}.wikipedia.org/'
categories : general
timeout : 3.0
api_key : 'apikey'
disabled : True
language : en_US
``name`` :
Name that will be used accross searx to define this engine. In settings, on
the result page...
``engine`` :
Name of the python file used to handle requests and responses to and from this
search engine.
``shortcut`` :
Code used to execute bang requests (in this case using ``!bi`` or ``?bi``)
``base_url`` : optional
Part of the URL that should be stable accross every request. Can be useful to
use multiple sites using only one engine, or updating the site URL without
touching at the code.
``categories`` : optional
Define in which categories this engine will be active. Most of the time, it is
defined in the code of the engine, but in a few cases it is useful, like when
describing multiple search engine using the same code.
``timeout`` : optional
Timeout of the search with the current search engine. **Be careful, it will
modify the global timeout of searx.**
``api_key`` : optional
In a few cases, using an API needs the use of a secret key. How to obtain them
is described in the file.
``disabled`` : optional
To disable by default the engine, but not deleting it. It will allow the user
to manually activate it in the settings.
``language`` : optional
If you want to use another language for a specific engine, you can define it
by using the full ISO code of language and country, like ``fr_FR``, ``en_US``,
``de_DE``.
``weigth`` : default ``1``
Weighting of the results of this engine.
.. note::
A few more options are possible, but they are pretty specific to some
engines, and so won't be described here.

43
docs/blog/admin.rst Normal file
View File

@ -0,0 +1,43 @@
=============================================================
Searx admin interface
=============================================================
.. _searx-admin: https://github.com/kvch/searx-admin#searx-admin
.. _NLnet Foundation: https://nlnet.nl/
manage your instance from your browser
.. sidebar:: Installation
Installation guide can be found in the repository of searx-admin_.
One of the biggest advantages of searx is being extremely customizable. But at
first it can be daunting to newcomers. A barrier of taking advantage of this
feature is our ugly settings file which is sometimes hard to understand and
edit.
To make self-hosting searx more accessible a new tool is introduced, called
``searx-admin``. It is a web application which is capable of managing your
instance and manipulating its settings via a web UI. It aims to replace editing
of ``settings.yml`` for less experienced administrators or people who prefer
graphical admin interfaces.
.. figure:: searx-admin-engines.png
:alt: Screenshot of engine list
Configuration page of engines
Since ``searx-admin`` acts as a supervisor for searx, we have decided to
implement it as a standalone tool instead of part of searx. Another reason for
making it a standalone tool is that the codebase and dependencies of searx
should not grow because of a fully optional feature, which does not affect
existing instances.
Acknowledgements
================
This development was sponsored by `NLnet Foundation`_.
| Happy hacking.
| kvch // 2017.08.22 21:25

10
docs/blog/index.rst Normal file
View File

@ -0,0 +1,10 @@
====
Blog
====
.. toctree::
:maxdepth: 1
python3
admin
intro-offline

77
docs/blog/intro-offline.rst Normal file
View File

@ -0,0 +1,77 @@
===============================
Preparation for offline engines
===============================
Offline engines
===============
To extend the functionality of searx, offline engines are going to be
introduced. An offline engine is an engine which does not need Internet
connection to perform a search and does not use HTTP to communicate.
Offline engines can be configured as online engines, by adding those to the
`engines` list of :origin:`settings.yml <searx/settings.yml>`. Thus, searx
finds the engine file and imports it.
Example skeleton for the new engines:
.. code:: python
from subprocess import PIPE, Popen
categories = ['general']
offline = True
def init(settings):
pass
def search(query, params):
process = Popen(['ls', query], stdout=PIPE)
return_code = process.wait()
if return_code != 0:
raise RuntimeError('non-zero return code', return_code)
results = []
line = process.stdout.readline()
while line:
result = parse_line(line)
results.append(results)
line = process.stdout.readline()
return results
Development progress
====================
First, a proposal has been created as a Github issue. Then it was moved to the
wiki as a design document. You can read it here: :wiki:`Offline-engines`.
In this development step, searx core was prepared to accept and perform offline
searches. Offline search requests are scheduled together with regular offline
requests.
As offline searches can return arbitrary results depending on the engine, the
current result templates were insufficient to present such results. Thus, a new
template is introduced which is caplable of presenting arbitrary key value pairs
as a table. You can check out the pull request for more details see
:pull:`1700`.
Next steps
==========
Today, it is possible to create/run an offline engine. However, it is going to be publicly available for everyone who knows the searx instance. So the next step is to introduce token based access for engines. This way administrators are able to limit the access to private engines.
Acknowledgement
===============
This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .
.. _Search and Discovery Fund: https://nlnet.nl/discovery
.. _NLnet Foundation: https://nlnet.nl/
| Happy hacking.
| kvch // 2019.10.21 17:03

68
docs/blog/python3.rst Normal file
View File

@ -0,0 +1,68 @@
============================
Introducing Python 3 support
============================
.. _Python 2.7 clock: https://pythonclock.org/
.. sidebar:: Python 2.7 to 3 upgrade
This chapter exists of historical reasons. Python 2.7 release schedule ends
(`Python 2.7 clock`_) after 11 years Python 3 exists
As most operation systems are coming with Python3 installed by default. So it is
time for searx to support Python3. But don't worry support of Python2.7 won't be
dropped.
.. image:: searxpy3.png
:scale: 50 %
:alt: hurray
:align: center
How to run searx using Python 3
===============================
Please make sure that you run at least Python 3.5.
To run searx, first a Python3 virtualenv should be created. After entering the
virtualenv, dependencies must be installed. Then run searx with python3 instead
of the usual python command.
.. code:: sh
virtualenv -p python3 venv3
source venv3/bin/activate
pip3 install -r requirements.txt
python3 searx/webapp.py
If you want to run searx using Python2.7, you don't have to do anything
differently as before.
Fun facts
=========
- 115 files were changed when implementing the support for both Python versions.
- All of the dependencies was compatible except for the robotframework used for
browser tests. Thus, these tests were migrated to splinter. So from now on
both versions are being tested on Travis and can be tested locally.
If you found bugs
=================
Please open an issue on `GitHub`_. Make sure that you mention your Python
version in your issue, so we can investigate it properly.
.. _GitHub: https://github.com/asciimoo/searx/issues
Acknowledgment
==============
This development was sponsored by `NLnet Foundation`_.
.. _NLnet Foundation: https://nlnet.nl/
| Happy hacking.
| kvch // 2017.05.13 22:57

BIN
docs/blog/searx-admin-engines.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 50 KiB

BIN
docs/blog/searxpy3.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 30 KiB

114
docs/conf.py Normal file
View File

@ -0,0 +1,114 @@
# -*- coding: utf-8 -*-
import sys, os
from searx.version import VERSION_STRING
from pallets_sphinx_themes import ProjectLink
GIT_URL = os.environ.get("GIT_URL", "https://github.com/asciimoo/searx")
SEARX_URL = os.environ.get("SEARX_URL", "https://searx.me")
DOCS_URL = os.environ.get("DOCS_URL", "https://asciimoo.github.io/searx/")
# Project --------------------------------------------------------------
project = u'searx'
copyright = u'2015-2019, Adam Tauber, Noémi Ványi'
author = u'Adam Tauber'
release, version = VERSION_STRING, VERSION_STRING
highlight_language = 'none'
# General --------------------------------------------------------------
master_doc = "index"
source_suffix = '.rst'
numfig = True
from searx import webapp
jinja_contexts = {
'webapp': dict(**webapp.__dict__)
}
# usage:: lorem :patch:`f373169` ipsum
extlinks = {}
# upstream links
extlinks['wiki'] = ('https://github.com/asciimoo/searx/wiki/%s', ' ')
extlinks['pull'] = ('https://github.com/asciimoo/searx/pull/%s', 'PR ')
# links to custom brand
extlinks['origin'] = (GIT_URL + '/blob/master/%s', 'git://')
extlinks['patch'] = (GIT_URL + '/commit/%s', '#')
extlinks['search'] = (SEARX_URL + '/%s', '#')
extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ')
extlinks['pypi'] = ('https://pypi.org/project/%s', 'PyPi: ')
extlinks['man'] = ('https://manpages.debian.org/jump?q=%s', '')
#extlinks['role'] = (
# 'https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-%s', '')
extlinks['duref'] = (
'http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#%s', '')
extlinks['durole'] = (
'http://docutils.sourceforge.net/docs/ref/rst/roles.html#%s', '')
extlinks['dudir'] = (
'http://docutils.sourceforge.net/docs/ref/rst/directives.html#%s', '')
extlinks['ctan'] = (
'https://ctan.org/pkg/%s', 'CTAN: ')
extensions = [
'sphinx.ext.imgmath',
'sphinx.ext.extlinks',
'sphinx.ext.viewcode',
"sphinx.ext.autodoc",
"sphinx.ext.intersphinx",
"pallets_sphinx_themes",
"sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst
"sphinxcontrib.jinja", # https://github.com/tardyp/sphinx-jinja
'linuxdoc.rstFlatTable', # Implementation of the 'flat-table' reST-directive.
'linuxdoc.kfigure', # Sphinx extension which implements scalable image handling.
"sphinx_tabs.tabs", # https://github.com/djungelorm/sphinx-tabs
]
intersphinx_mapping = {
"python": ("https://docs.python.org/3/", None),
"flask": ("https://flask.palletsprojects.com/", None),
# "werkzeug": ("https://werkzeug.palletsprojects.com/", None),
"jinja": ("https://jinja.palletsprojects.com/", None),
"linuxdoc" : ("https://return42.github.io/linuxdoc/", None),
"sphinx" : ("https://www.sphinx-doc.org/en/master/", None),
}
issues_github_path = "asciimoo/searx"
# HTML -----------------------------------------------------------------
sys.path.append(os.path.abspath('_themes'))
html_theme_path = ['_themes']
html_theme = "searx"
# sphinx.ext.imgmath setup
html_math_renderer = 'imgmath'
imgmath_image_format = 'svg'
imgmath_font_size = 14
# sphinx.ext.imgmath setup END
html_theme_options = {"index_sidebar_logo": True}
html_context = {
"project_links": [
ProjectLink("Source", GIT_URL),
ProjectLink("Wiki", "https://github.com/asciimoo/searx/wiki"),
ProjectLink("Public instances", "https://asciimoo.github.io/searx/user/public_instances.html"),
ProjectLink("Twitter", "https://twitter.com/Searx_engine"),
]
}
html_sidebars = {
"**": ["project.html", "relations.html", "searchbox.html"],
}
singlehtml_sidebars = {"index": ["project.html", "localtoc.html"]}
html_static_path = ["static"]
html_logo = "static/img/searx_logo_small.png"
html_title = "Searx Documentation ({})".format("Searx-{}.tex".format(VERSION_STRING))
html_show_sourcelink = False
# LaTeX ----------------------------------------------------------------
latex_documents = [
(master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual")
]

180
docs/dev/contribution_guide.rst Normal file
View File

@ -0,0 +1,180 @@
.. _how to contribute:
=================
How to contribute
=================
Prime directives: Privacy, Hackability
======================================
Searx has two prime directives, **privacy-by-design and hackability** . The
hackability comes in three levels:
- support of search engines
- plugins to alter search behaviour
- hacking searx itself
Note the lack of "world domination" among the directives. Searx has no
intention of wide mass-adoption, rounded corners, etc. The prime directive
"privacy" deserves a separate chapter, as it's quite uncommon unfortunately.
Privacy-by-design
-----------------
Searx was born out of the need for a **privacy-respecting** search tool which
can be extended easily to maximize both, its search and its privacy protecting
capabilities.
A few widely used features work differently or turned off by default or not
implemented at all **as a consequence of privacy-by-design**.
If a feature reduces the privacy preserving aspects of searx, it should be
switched off by default or should not implemented at all. There are plenty of
search engines already providing such features. If a feature reduces the
protection of searx, users must be informed about the effect of choosing to
enable it. Features that protect privacy but differ from the expectations of
the user should also be explained.
Also, if you think that something works weird with searx, it's might be because
of the tool you use is designed in a way to interfere with the privacy respect.
Submitting a bugreport to the vendor of the tool that misbehaves might be a good
feedback to reconsider the disrespect to its customers (e.g. ``GET`` vs ``POST``
requests in various browsers).
Remember the other prime directive of searx is to be hackable, so if the above
privacy concerns do not fancy you, simply fork it.
*Happy hacking.*
Code
====
.. _PEP8: https://www.python.org/dev/peps/pep-0008/
.. _Conventional Commits: https://www.conventionalcommits.org/
.. _Git Commit Good Practice: https://wiki.openstack.org/wiki/GitCommitMessages
.. _Structural split of changes:
https://wiki.openstack.org/wiki/GitCommitMessages#Structural_split_of_changes
.. _gitmoji: https://gitmoji.carloscuesta.me/
.. _Semantic PR: https://github.com/zeke/semantic-pull-requests
.. sidebar:: Create good commits!
- `Structural split of changes`_
- `Conventional Commits`_
- `Git Commit Good Practice`_
- some like to use: gitmoji_
- not yet active: `Semantic PR`_
In order to submit a patch, please follow the steps below:
- Follow coding conventions.
- PEP8_ standards apply, except the convention of line length
- Maximum line length is 120 characters
- The cardinal rule for creating good commits is to ensure there is only one
*logical change* per commit / read `Structural split of changes`_
- Check if your code breaks existing tests. If so, update the tests or fix your
code.
- If your code can be unit-tested, add unit tests.
- Add yourself to the :origin:`AUTHORS.rst` file.
- Choose meaning full commit messages, read `Conventional Commits`_
.. code::
<type>[optional scope]: <description>
[optional body]
[optional footer(s)]
- Create a pull request.
For more help on getting started with searx development, see :ref:`devquickstart`.
Translation
===========
Translation currently takes place on :ref:`transifex <translation>`.
.. caution::
Please, do not update translation files in the repo.
.. _contrib docs:
Documentation
=============
.. _Sphinx: http://www.sphinx-doc.org
.. _reST: http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html
.. sidebar:: The reST sources
has been moved from ``gh-branch`` into ``master`` (:origin:`docs`).
The documentation is built using Sphinx_. So in order to be able to generate
the required files, you have to install it on your system. Much easier, use
our :ref:`makefile`.
Here is an example which makes a complete rebuild:
.. code:: sh
$ make docs-clean docs
...
The HTML pages are in dist/docs.
.. _make docs-live:
live build
----------
.. sidebar:: docs-clean
It is recommended to assert a complete rebuild before deploying (use
``docs-clean``).
Live build is like WYSIWYG. If you want to edit the documentation, its
recommended to use. The Makefile target ``docs-live`` builds the docs, opens
URL in your favorite browser and rebuilds every time a reST file has been
changed.
.. code:: sh
$ make docs-live
...
The HTML pages are in dist/docs.
... Serving on http://0.0.0.0:8080
... Start watching changes
.. _deploy on github.io:
deploy on github.io
-------------------
To deploy documentation at :docs:`github.io <.>` use Makefile target
:ref:`make gh-pages`, which will builds the documentation, clones searx into a sub
folder ``gh-pages``, cleans it, copies the doc build into and runs all the
needed git add, commit and push:
.. code:: sh
$ make docs-clean gh-pages
...
SPHINX docs --> file://<...>/dist/docs
The HTML pages are in dist/docs.
...
Cloning into 'gh-pages' ...
...
cd gh-pages; git checkout gh-pages >/dev/null
Switched to a new branch 'gh-pages'
...
doc available at --> https://asciimoo.github.io/searx

6
docs/dev/csv_table.txt Normal file
View File

@ -0,0 +1,6 @@
stub col row 1, column, "loremLorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy
eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam
voluptua."
stub col row 1, "At vero eos et accusam et justo duo dolores et ea rebum. Stet clita
kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.", column
stub col row 1, column, column

267
docs/dev/engine_overview.rst Normal file
View File

@ -0,0 +1,267 @@
.. _engines-dev:
===============
Engine overview
===============
.. _metasearch-engine: https://en.wikipedia.org/wiki/Metasearch_engine
searx is a metasearch-engine_, so it uses different search engines to provide
better results.
Because there is no general search API which could be used for every search
engine, an adapter has to be built between searx and the external search
engines. Adapters are stored under the folder :origin:`searx/engines`.
.. contents::
:depth: 3
:backlinks: entry
general engine configuration
============================
It is required to tell searx the type of results the engine provides. The
arguments can be set in the engine file or in the settings file
(normally ``settings.yml``). The arguments in the settings file override
the ones in the engine file.
It does not matter if an option is stored in the engine file or in the
settings. However, the standard way is the following:
.. _engine file:
engine file
-----------
======================= =========== ===========================================
argument type information
======================= =========== ===========================================
categories list pages, in which the engine is working
paging boolean support multible pages
language_support boolean support language choosing
time_range_support boolean support search time range
offline boolean engine runs offline
======================= =========== ===========================================
.. _engine settings:
settings.yml
------------
======================= =========== ===========================================
argument type information
======================= =========== ===========================================
name string name of search-engine
engine string name of searx-engine
(filename without ``.py``)
shortcut string shortcut of search-engine
timeout string specific timeout for search-engine
======================= =========== ===========================================
overrides
---------
A few of the options have default values in the engine, but are often
overwritten by the settings. If ``None`` is assigned to an option in the engine
file, it has to be redefined in the settings, otherwise searx will not start
with that engine.
The naming of overrides is arbitrary. But the recommended overrides are the
following:
======================= =========== ===========================================
argument type information
======================= =========== ===========================================
base_url string base-url, can be overwritten to use same
engine on other URL
number_of_results int maximum number of results per request
language string ISO code of language and country like en_US
api_key string api-key if required by engine
======================= =========== ===========================================
example code
------------
.. code:: python
# engine dependent config
categories = ['general']
paging = True
language_support = True
making a request
================
To perform a search an URL have to be specified. In addition to specifying an
URL, arguments can be passed to the query.
passed arguments
----------------
These arguments can be used to construct the search query. Furthermore,
parameters with default value can be redefined for special purposes.
====================== ============ ========================================================================
argument type default-value, information
====================== ============ ========================================================================
url string ``''``
method string ``'GET'``
headers set ``{}``
data set ``{}``
cookies set ``{}``
verify boolean ``True``
headers.User-Agent string a random User-Agent
category string current category, like ``'general'``
started datetime current date-time
pageno int current pagenumber
language string specific language code like ``'en_US'``, or ``'all'`` if unspecified
====================== ============ ========================================================================
parsed arguments
----------------
The function ``def request(query, params):`` always returns the ``params``
variable. Inside searx, the following paramters can be used to specify a search
request:
============ =========== =========================================================
argument type information
============ =========== =========================================================
url string requested url
method string HTTP request method
headers set HTTP header information
data set HTTP data information (parsed if ``method != 'GET'``)
cookies set HTTP cookies
verify boolean Performing SSL-Validity check
============ =========== =========================================================
example code
------------
.. code:: python
# search-url
base_url = 'https://example.com/'
search_string = 'search?{query}&page={page}'
# do search-request
def request(query, params):
search_path = search_string.format(
query=urlencode({'q': query}),
page=params['pageno'])
params['url'] = base_url + search_path
return params
returned results
================
Searx is able to return results of different media-types. Currently the
following media-types are supported:
- default_
- images_
- videos_
- torrent_
- map_
To set another media-type as default, the parameter ``template`` must be set to
the desired type.
default
-------
========================= =====================================================
result-parameter information
========================= =====================================================
url string, url of the result
title string, title of the result
content string, general result-text
publishedDate :py:class:`datetime.datetime`, time of publish
========================= =====================================================
images
------
To use this template, the parameter:
========================= =====================================================
result-parameter information
========================= =====================================================
template is set to ``images.html``
url string, url to the result site
title string, title of the result *(partly implemented)*
content *(partly implemented)*
publishedDate :py:class:`datetime.datetime`,
time of publish *(partly implemented)*
img\_src string, url to the result image
thumbnail\_src string, url to a small-preview image
========================= =====================================================
videos
------
========================= =====================================================
result-parameter information
========================= =====================================================
template is set to ``videos.html``
url string, url of the result
title string, title of the result
content *(not implemented yet)*
publishedDate :py:class:`datetime.datetime`, time of publish
thumbnail string, url to a small-preview image
========================= =====================================================
torrent
-------
.. _magnetlink: https://en.wikipedia.org/wiki/Magnet_URI_scheme
========================= =====================================================
result-parameter information
========================= =====================================================
template is set to ``torrent.html``
url string, url of the result
title string, title of the result
content string, general result-text
publishedDate :py:class:`datetime.datetime`,
time of publish *(not implemented yet)*
seed int, number of seeder
leech int, number of leecher
filesize int, size of file in bytes
files int, number of files
magnetlink string, magnetlink_ of the result
torrentfile string, torrentfile of the result
========================= =====================================================
map
---
========================= =====================================================
result-parameter information
========================= =====================================================
url string, url of the result
title string, title of the result
content string, general result-text
publishedDate :py:class:`datetime.datetime`, time of publish
latitude latitude of result (in decimal format)
longitude longitude of result (in decimal format)
boundingbox boundingbox of result (array of 4. values
``[lat-min, lat-max, lon-min, lon-max]``)
geojson geojson of result (http://geojson.org)
osm.type type of osm-object (if OSM-Result)
osm.id id of osm-object (if OSM-Result)
address.name name of object
address.road street name of object
address.house_number house number of object
address.locality city, place of object
address.postcode postcode of object
address.country country of object
========================= =====================================================

3
docs/dev/hello.dot Normal file
View File

@ -0,0 +1,3 @@
graph G {
Hello -- World
}

15
docs/dev/index.rst Normal file
View File

@ -0,0 +1,15 @@
=======================
Developer documentation
=======================
.. toctree::
:maxdepth: 1
quickstart
contribution_guide
engine_overview
search_api
plugins
translation
makefile
reST

221
docs/dev/makefile.rst Normal file
View File

@ -0,0 +1,221 @@
.. _makefile:
================
Makefile Targets
================
.. _gnu-make: https://www.gnu.org/software/make/manual/make.html#Introduction
.. sidebar:: build environment
Before looking deeper at the targets, first read about :ref:`makefile setup`
and :ref:`make pyenv`.
With the aim to simplify development cycles, started with :pull:`1756` a
``Makefile`` based boilerplate was added. If you are not familiar with
Makefiles, we recommend to read gnu-make_ introduction.
The usage is simple, just type ``make {target-name}`` to *build* a target.
Calling the ``help`` target gives a first overview::
$ make help
test - run developer tests
docs - build documentation
docs-live - autobuild HTML documentation while editing
run - run developer instance
install - developer install (./local)
uninstall - uninstall (./local)
gh-pages - build docs & deploy on gh-pages branch
clean - drop builds and environments
...
.. contents:: Contents
:depth: 2
:local:
:backlinks: entry
.. _makefile setup:
Setup
=====
.. _git stash: https://git-scm.com/docs/git-stash
The main setup is done in the :origin:`Makefile`::
export GIT_URL=https://github.com/asciimoo/searx
export SEARX_URL=https://searx.me
export DOCS_URL=https://asciimoo.github.io/searx
.. sidebar:: fork & upstream
Commit changes in your (local) branch, fork or whatever, but do not push them
upstream / `git stash`_ is your friend.
:GIT_URL: Changes this, to point to your searx fork.
:SEARX_URL: Changes this, to point to your searx instance.
:DOCS_URL: If you host your own (branded) documentation, change this URL.
.. _make pyenv:
Python environment
==================
.. sidebar:: activate environment
``source ./local/py3/bin/activate``
With Makefile we do no longer need to build up the virualenv manually (as
described in the :ref:`devquickstart` guide). Jump into your git working tree
and release a ``make pyenv``:
.. code:: sh
$ cd ~/searx-clone
$ make pyenv
PYENV usage: source ./local/py3/bin/activate
...
With target ``pyenv`` a development environment (aka virtualenv) was build up in
``./local/py3/``. To make a *developer install* of searx (:origin:`setup.py`)
into this environment, use make target ``install``:
.. code:: sh
$ make install
PYENV usage: source ./local/py3/bin/activate
PYENV using virtualenv from ./local/py3
PYENV install .
You have never to think about intermediate targets like ``pyenv`` or
``install``, the ``Makefile`` chains them as requisites. Just run your main
target.
.. sidebar:: drop environment
To get rid of the existing environment before re-build use :ref:`clean target
<make clean>` first.
If you think, something goes wrong with your ./local environment or you change
the :origin:`setup.py` file (or the requirements listed in
:origin:`requirements-dev.txt` and :origin:`requirements.txt`), you have to call
:ref:`make clean`.
.. _make run:
``make run``
============
To get up a running a developer instance simply call ``make run``. This enables
*debug* option in :origin:`searx/settings.yml`, starts a ``./searx/webapp.py``
instance, disables *debug* option again and opens the URL in your favorite WEB
browser (:man:`xdg-open`):
.. code:: sh
$ make run
PYENV usage: source ./local/py3/bin/activate
PYENV install .
./local/py3/bin/python ./searx/webapp.py
...
INFO:werkzeug: * Running on http://127.0.0.1:8888/ (Press CTRL+C to quit)
...
.. _make clean:
``make clean``
==============
Drop all intermediate files, all builds, but keep sources untouched. Includes
target ``pyclean`` which drops ./local environment. Before calling ``make
clean`` stop all processes using :ref:`make pyenv`.
.. code:: sh
$ make clean
CLEAN pyclean
CLEAN clean
.. _make docs:
``make docs docs-live docs-clean``
==================================
We describe the usage of the ``doc*`` targets in the :ref:`How to contribute /
Documentation <contrib docs>` section. If you want to edit the documentation
read our :ref:`make docs-live` section. If you are working in your own brand,
adjust your :ref:`Makefile setup <makefile setup>`.
.. _make gh-pages:
``make gh-pages``
=================
To deploy on github.io first adjust your :ref:`Makefile setup <makefile
setup>`. For any further read :ref:`deploy on github.io`.
.. _make test:
``make test``
=============
Runs a series of tests: ``test.pep8``, ``test.unit``, ``test.robot`` and does
additional :ref:`pylint checks <make pylint>`. You can run tests selective,
e.g.:
.. code:: sh
$ make test.pep8 test.unit
. ./local/py3/bin/activate; ./manage.sh pep8_check
[!] Running pep8 check
. ./local/py3/bin/activate; ./manage.sh unit_tests
[!] Running unit tests
.. _make pylint:
``make pylint``
===============
.. _Pylint: https://www.pylint.org/
Before commiting its recommend to do some (more) linting. Pylint_ is known as
one of the best source-code, bug and quality checker for the Python programming
language. Pylint_ is not yet a quality gate within our searx project (like
:ref:`test.pep8 <make test>` it is), but Pylint_ can help to improve code
quality anyway. The pylint profile we use at searx project is found in
project's root folder :origin:`.pylintrc`.
Code quality is a ongoing process. Don't try to fix all messages from Pylint,
run Pylint and check if your changed lines are bringing up new messages. If so,
fix it. By this, code quality gets incremental better and if there comes the
day, the linting is balanced out, we might decide to add Pylint as a quality
gate.
``make pybuild``
================
.. _PyPi: https://pypi.org/
.. _twine: https://twine.readthedocs.io/en/latest/
Build Python packages in ``./dist/py``.
.. code:: sh
$ make pybuild
...
BUILD pybuild
running sdist
running egg_info
...
$ ls ./dist/py/
searx-0.15.0-py3-none-any.whl searx-0.15.0.tar.gz
To upload packages to PyPi_, there is also a ``upload-pypi`` target. It needs
twine_ to be installed. Since you are not the owner of :pypi:`searx` you will
never need the latter.

54
docs/dev/plugins.rst Normal file
View File

@ -0,0 +1,54 @@
.. _dev plugin:
=======
Plugins
=======
.. sidebar:: Further reading ..
- :ref:`plugins generic`
Plugins can extend or replace functionality of various components of searx.
Example plugin
==============
.. code:: python
name = 'Example plugin'
description = 'This plugin extends the suggestions with the word "example"'
default_on = False # disabled by default
js_dependencies = tuple() # optional, list of static js files
css_dependencies = tuple() # optional, list of static css files
# attach callback to the post search hook
# request: flask request object
# ctx: the whole local context of the post search hook
def post_search(request, ctx):
ctx['search'].suggestions.add('example')
return True
Plugin entry points
===================
Entry points (hooks) define when a plugin runs. Right now only three hooks are
implemented. So feel free to implement a hook if it fits the behaviour of your
plugin.
Pre search hook
---------------
Runs BEFORE the search request. Function to implement: ``pre_search``
Post search hook
----------------
Runs AFTER the search request. Function to implement: ``post_search``
Result hook
-----------
Runs when a new result is added to the result list. Function to implement:
``on_result``

132
docs/dev/quickstart.rst Normal file
View File

@ -0,0 +1,132 @@
.. _devquickstart:
======================
Development Quickstart
======================
.. sidebar:: :ref:`makefile`
For additional developer purpose there are :ref:`makefile`.
This quickstart guide gets your environment set up with searx. Furthermore, it
gives a short introduction to the ``manage.sh`` script.
How to setup your development environment
=========================================
.. sidebar:: :ref:`make pyenv <make pyenv>`
Alternatively use the :ref:`make pyenv`.
First, clone the source code of searx to the desired folder. In this case the
source is cloned to ``~/myprojects/searx``. Then create and activate the
searx-ve virtualenv and install the required packages using ``manage.sh``.
.. code:: sh
cd ~/myprojects
git clone https://github.com/asciimoo/searx.git
cd searx
virtualenv searx-ve
. ./searx-ve/bin/activate
./manage.sh update_dev_packages
How to run tests
================
.. sidebar:: :ref:`make test.unit <make test>`
Alternatively use the ``test.pep8``, ``test.unit``, ``test.robot`` targets.
Tests can be run using the ``manage.sh`` script. Following tests and checks are
available:
- Unit tests
- Selenium tests
- PEP8 validation
- Unit test coverage check
For example unit tests are run with the command below:
.. code:: sh
./manage.sh unit_tests
For further test options, please consult the help of the ``manage.sh`` script or
read :ref:`make test`.
How to compile styles and javascript
====================================
.. _less: http://lesscss.org/
.. _NodeJS: https://nodejs.org
How to build styles
-------------------
Less_ is required to build the styles of searx. Less_ can be installed using
either NodeJS_ or Apt.
.. code:: sh
sudo -H apt-get install nodejs
sudo -H npm install -g less
OR
.. code:: sh
sudo -H apt-get install node-less
After satisfying the requirements styles can be build using ``manage.sh``
.. code:: sh
./manage.sh styles
How to build the source of the oscar theme
==========================================
.. _grunt: https://gruntjs.com/
Grunt_ must be installed in order to build the javascript sources. It depends on
NodeJS, so first Node has to be installed.
.. code:: sh
sudo -H apt-get install nodejs
sudo -H npm install -g grunt-cli
After installing grunt, the files can be built using the following command:
.. code:: sh
./manage.sh grunt_build
Tips for debugging/development
==============================
.. sidebar:: :ref:`make run`
Makefile target ``run`` already enables debug option for your developer
session / see :ref:`make run`.
Turn on debug logging
Whether you are working on a new engine or trying to eliminate a bug, it is
always a good idea to turn on debug logging. When debug logging is enabled a
stack trace appears, instead of the cryptic ``Internal Server Error``
message. It can be turned on by setting ``debug: False`` to ``debug: True`` in
:origin:`settings.yml <searx/settings.yml>`.
.. sidebar:: :ref:`make test`
Alternatively use the :ref:`make test` targets.
Run ``./manage.sh tests`` before creating a PR.
Failing build on Travis is common because of PEP8 checks. So a new commit
must be created containing these format fixes. This phase can be skipped if
``./manage.sh tests`` is run locally before creating a PR.

1428
docs/dev/reST.rst Normal file
View File

File diff suppressed because it is too large Load Diff

120
docs/dev/search_api.rst Normal file
View File

@ -0,0 +1,120 @@
.. _search API:
==========
Search API
==========
The search supports both ``GET`` and ``POST``.
Furthermore, two enpoints ``/`` and ``/search`` are available for querying.
``GET /``
``GET /search``
Parameters
==========
.. sidebar:: Further reading ..
- :ref:`engines-dev`
- :ref:`settings.yml`
- :ref:`engines generic`
``q`` : required
The search query. This string is passed to external search services. Thus,
searx supports syntax of each search service. For example, ``site:github.com
searx`` is a valid query for Google. However, if simply the query above is
passed to any search engine which does not filter its results based on this
syntax, you might not get the results you wanted.
See more at :ref:`search-syntax`
``categories`` : optional
Comma separated list, specifies the active search categories
``engines`` : optional
Comma separated list, specifies the active search engines.
``lang`` : default ``all``
Code of the language.
``pageno`` : default ``1``
Search page number.
``time_range`` : optional
[ ``day``, ``month``, ``year`` ]
Time range of search for engines which support it. See if an engine supports
time range search in the preferences page of an instance.
``format`` : optional
[ ``json``, ``csv``, ``rss`` ]
Output format of results.
``results_on_new_tab`` : default ``0``
[ ``0``, ``1`` ]
Open search results on new tab.
``image_proxy`` : default ``False``
[ ``True``, ``False`` ]
Proxy image results through searx.
``autocomplete`` : default *empty*
[ ``google``, ``dbpedia``, ``duckduckgo``, ``startpage``, ``wikipedia`` ]
Service which completes words as you type.
``safesearch`` : default ``None``
[ ``0``, ``1``, ``None`` ]
Filter search results of engines which support safe search. See if an engine
supports safe search in the preferences page of an instance.
``theme`` : default ``oscar``
[ ``oscar``, ``simple``, ``legacy``, ``pix-art``, ``courgette`` ]
Theme of instance.
Please note, available themes depend on an instance. It is possible that an
instance administrator deleted, created or renamed themes on his/her instance.
See the available options in the preferences page of the instance.
``oscar-style`` : default ``logicodev``
[ ``pointhi``, ``logicodev`` ]
Style of Oscar theme. It is only parsed if the theme of an instance is
``oscar``.
Please note, available styles depend on an instance. It is possible that an
instance administrator deleted, created or renamed styles on his/her
instance. See the available options in the preferences page of the instance.
``enabled_plugins`` : optional
List of enabled plugins.
:default: ``HTTPS_rewrite``, ``Self_Informations``,
``Search_on_category_select``, ``Tracker_URL_remover``
:values: [ ``DOAI_rewrite``, ``HTTPS_rewrite``, ``Infinite_scroll``,
``Vim-like_hotkeys``, ``Self_Informations``, ``Tracker_URL_remover``,
``Search_on_category_select`` ]
``disabled_plugins``: optional
List of disabled plugins.
:default: ``DOAI_rewrite``, ``Infinite_scroll``, ``Vim-like_hotkeys``
:values: ``DOAI_rewrite``, ``HTTPS_rewrite``, ``Infinite_scroll``,
``Vim-like_hotkeys``, ``Self_Informations``, ``Tracker_URL_remover``,
``Search_on_category_select``
``enabled_engines`` : optional : *all* :origin:`engines <searx/engines>`
List of enabled engines.
``disabled_engines`` : optional : *all* :origin:`engines <searx/engines>`
List of disabled engines.

10
docs/dev/svg_image.svg Normal file
View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- originate: https://commons.wikimedia.org/wiki/File:Variable_Resistor.svg -->
<svg xmlns="http://www.w3.org/2000/svg"
version="1.1" baseProfile="full"
width="70px" height="40px" viewBox="0 0 700 400">
<line x1="0" y1="200" x2="700" y2="200" stroke="black" stroke-width="20px"/>
<rect x="100" y="100" width="500" height="200" fill="white" stroke="black" stroke-width="20px"/>
<line x1="180" y1="370" x2="500" y2="50" stroke="black" stroke-width="15px"/>
<polygon points="585 0 525 25 585 50" transform="rotate(135 525 25)"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 580 B

71
docs/dev/translation.rst Normal file
View File

@ -0,0 +1,71 @@
.. _translation:
===========
Translation
===========
.. _searx@transifex: https://www.transifex.com/asciimoo/searx/
Translation currently takes place on `searx@transifex`_
Requirements
============
* Transifex account
* Installed CLI tool of Transifex
Init Transifex project
======================
After installing ``transifex`` using pip, run the following command to
initialize the project.
.. code:: sh
tx init # Transifex instance: https://www.transifex.com/asciimoo/searx/
After ``$HOME/.transifexrc`` is created, get a Transifex API key and insert it
into the configuration file.
Create a configuration file for ``tx`` named ``$HOME/.tx/config``.
.. code:: ini
[main]
host = https://www.transifex.com
[searx.messagespo]
file_filter = searx/translations/<lang>/LC_MESSAGES/messages.po
source_file = messages.pot
source_lang = en
type = PO
Then run ``tx set``:
.. code:: shell
tx set --auto-local -r searx.messagespo 'searx/translations/<lang>/LC_MESSAGES/messages.po' \
--source-lang en --type PO --source-file messages.pot --execute
Update translations
===================
To retrieve the latest translations, pull it from Transifex.
.. code:: sh
tx pull -a
Then check the new languages. If strings translated are not enough, delete those
folders, because those should not be compiled. Call the command below to compile
the ``.po`` files.
.. code:: shell
pybabel compile -d searx/translations
After the compilation is finished commit the ``.po`` and ``.mo`` files and
create a PR.

32
docs/index.rst Normal file
View File

@ -0,0 +1,32 @@
================
Welcome to searx
================
Search without being tracked.
.. sidebar:: Features
- Self hosted
- No user tracking
- No user profiling
- About 70 supported search engines
- Easy integration with any search engine
- Cookies are not used by default
- Secure, encrypted connections (HTTPS/SSL)
- Hosted by organizations, such as *La Quadrature du Net*, which promote
digital rights
Searx is a free internet metasearch engine which aggregates results from more
than 70 search services. Users are neither tracked nor profiled. Additionally,
searx can be used over Tor for online anonymity.
Get started with searx by using one of the :wiki:`Searx-instances`. If you
don't trust anyone, you can set up your own, see :ref:`installation`.
.. toctree::
:maxdepth: 2
user/index
admin/index
dev/index
blog/index

BIN
docs/static/img/searx_logo_small.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 6.3 KiB

10
docs/user/index.rst Normal file
View File

@ -0,0 +1,10 @@
==================
User documentation
==================
.. toctree::
:maxdepth: 1
public_instances
search_syntax
own-instance

77
docs/user/own-instance.rst Normal file
View File

@ -0,0 +1,77 @@
===========================
Why use a private instance?
===========================
"Is it worth to run my own instance?" is a common question among searx users.
Before answering this question, see what options a searx user has.
Public instances are open to everyone who has access to its URL. Usually, these
are operated by unknown parties (from the users' point of view). Private
instances can be used by a select group of people. It is for example a searx of
group of friends or a company which can be accessed through VPN. Also it can be
single user one which runs on the user's laptop.
To gain more insight on how these instances work let's dive into how searx
protects its users.
How does searx protect privacy?
===============================
Searx protects the privacy of its users in multiple ways regardless of the type
of the instance (private, public). Removal of private data from search requests
comes in three forms:
1. removal of private data from requests going to search services
2. not forwarding anything from a third party services through search services
(e.g. advertisement)
3. removal of private data from requests going to the result pages
Removing private data means not sending cookies to external search engines and
generating a random browser profile for every request. Thus, it does not matter
if a public or private instance handles the request, because it is anonymized in
both cases. IP addresses will be the IP of the instance. But searx can be
configured to use proxy or Tor. `Result proxy
<https://github.com/asciimoo/morty>`__ is supported, too.
Searx does not serve ads or tracking content unlike most search services. So
private data is not forwarded to third parties who might monetize it. Besides
protecting users from search services, both referring page and search query are
hidden from visited result pages.
What are the consequences of using public instances?
----------------------------------------------------
If someone uses a public instance, he/she has to trust the administrator of that
instance. This means that the user of the public instance does not know whether
his/her requests are logged, aggregated and sent or sold to a third party.
Also, public instances without proper protection are more vulnerable to abusing
the search service, In this case the external service in exchange returns
CAPTCHAs or bans the IP of the instance. Thus, search requests return less
results.
I see. What about private instances?
------------------------------------
If users run their own instances, everything is in their control: the source
code, logging settings and private data. Unknown instance administrators do not
have to be trusted.
Furthermore, as the default settings of their instance is editable, there is no
need to use cookies to tailor searx to their needs. So preferences will not be
reset to defaults when clearing browser cookies. As settings are stored on
their computer, it will not be accessible to others as long as their computer is
not compromised.
Conclusion
==========
Always use an instance which is operated by people you trust. The privacy
features of searx are available to users no matter what kind of instance they
use.
If someone is on the go or just wants to try searx for the first time public
instances are the best choices. Additionally, public instance are making a
world a better place, because those who cannot or do not want to run an
instance, have access to a privacy respecting search service.

318
docs/user/public_instances.rst Normal file
View File

@ -0,0 +1,318 @@
.. _public instances:
..
links has been ported from markdown to reST by::
regexpr: \[([^\]]*)\]\(([^)]*)\)
substitution: `\1 <\2>`__
======================
Public Searx instances
======================
.. _mailing list: mailto:searx-instances@autistici.org
.. _subscription page: https://www.autistici.org/mailman/listinfo/searx-instances
Useful information
==================
* Up-to-date health report available on https://stats.searx.xyz [1]_, for onion
(tor) services: https://stats.searx.xyz/tor.html
* Searx instances `mailing list`_ & `subscription page`_.
* Some of the Searx instances have a CAcert SSL certificate. You can install the
missing root cert `from here <http://www.cacert.org/index.php?id=3>`__.
* To add your own Searx instance to this page send us your PR. A GitHub account
is required to send PR or add an issue.
.. [1] Note that most of the instances with a A+ grade in CSP column in this
site are not fully functional - for example auto-completion may not work.
List of public Searx instances
==============================
Meta-searx instances
====================
These are websites that source from other searx instances. These are useful if
you can't decide which Searx instance to use:
.. flat-table:: Meta-searx instances
:header-rows: 1
:stub-columns: 0
:widths: 2 1 2 4 4
* - clearnet host
- onion host
- issuer
- source selection method
- extra privacy features
* - `Neocities <https://searx.neocities.org/>`__
- n/a
- Comodo (`Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.neocities.org>`__)
- Redirects users directly to a random selection of any known running
server after entering query. Requires
Javascript. `Changelog <https://searx.neocities.org/changelog.html>`__.
- Excludes servers with user tracking and analytics or are proxied through
Cloudflare.
* - `Searxes <https://searxes.danwin1210.me/>`__ @Danwin
- onion v3 `hidden service
<http://searxes.nmqnkngye4ct7bgss4bmv5ca3wpa55yugvxen5kz2bbq67lwy6ps54yd.onion/>`__
- Let's Encrypt (`Verification
<https://www.ssllabs.com/ssltest/analyze.html?d=searxes.danwin1210.me>`__)
- sources data from a randomly selected running server that satisfies
admin's quality standards which is used for post-processing
- filters out privacy-hostile websites (like CloudFlare) and either marks
them as such or folds them below the high ranking results.
Alive and running
=================
**BEFORE EDITING**: Please add your Searx instance by respecting the alphabetic order.
.. note::
Public instances listed here may yield less accurate results as they have
much higher traffic and consequently have a higher chance of being blocked by
search providers such as Google, Qwant, Bing, Startpage, etc. Hosting your
own instance or using an instance that isn't listed here may give you a more
consistent search experience.
* `ai.deafpray.wtf/searx <https://ai.deafpray.wtf/searx>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=ai.deafpray.wtf/searx>`__
* `bamboozle.it <https://bamboozle.it/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=bamboozle.it>`__
* `bee.jaekr.dev <https://bee.jaekr.dev>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=bee.jaekr.dev>`__
* `beezboo.com <https://beezboo.com/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=beezboo.com>`__
* `burtrum.org/searx <https://burtrum.org/searx>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=burtrum.org/searx>`__
* `darmarit.cloud/searx <https://darmarit.cloud/searx>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=darmarit.cloud/searx>`__
* `dc.ax <https://dc.ax>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=dc.ax>`__
* `dynabyte.ca <https://dynabyte.ca>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=dynabyte.ca>`__
* `goso.ga <https://goso.ga/search>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=goso.ga>`__
* `gruble.de <https://www.gruble.de/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=www.gruble.de>`__
* `haku.ahmia.fi <https://haku.ahmia.fi/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=haku.ahmia.fi&latest>`__
* `haku.lelux.fi <https://haku.lelux.fi/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=haku.lelux.fi>`__
* `huyo.me <https://huyo.me/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=huyo.me>`__
* `jsearch.pw <https://jsearch.pw>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=jsearch.pw>`__
* `le-dahut.com/searx <https://le-dahut.com/searx>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=le-dahut.com/searx>`__
* `mijisou.com <https://mijisou.com/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=mijisou.com>`__
* `null.media <https://null.media>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=null.media>`__
* `openworlds.info <https://openworlds.info/>`__ - Issuer: Let's Encrypt
* `perfectpixel.de/searx/ <https://www.perfectpixel.de/searx/>`__ - Issuer: LetsEncrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=www.perfectpixel.de>`__
* `ransack.i2p <http://ransack.i2p/>`__ - I2P eepsite, only accessible with `I2P <https://geti2p.net/>`__ (`base32 address <http://mqamk4cfykdvhw5kjez2gnvse56gmnqxn7vkvvbuor4k4j2lbbnq.b32.i2p>`__)
* `rapu.nz <https://rapu.nz/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=rapu.nz>`__
* `roflcopter.fr <https://wtf.roflcopter.fr/searx>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=wtf.roflcopter.fr>`__
* `roteserver.de/searx <https://roteserver.de/searx>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=roteserver.de>`__
* `s.cmd.gg <https://s.cmd.gg>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=s.cmd.gg>`__
* `search.activemail.de <https://search.activemail.de/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.activemail.de&latest>`__
* `search.anonymize.com <https://search.anonymize.com/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.anonymize.com>`__
* `search.azkware.net <https://search.azkware.net/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.azkware.net>`__
* `search.biboumail.fr <https://search.biboumail.fr/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.biboumail.fr>`__
* `search.blankenberg.eu <https://search.blankenberg.eu>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.blankenberg.eu>`__
* `search.d4networks.com <https://search.d4networks.com/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.d4networks.com>`__
* `search.datensturm.net <https://search.datensturm.net>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.datensturm.net>`__
* `search.disroot.org <https://search.disroot.org/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.disroot.org>`__
* `search.ethibox.fr <https://search.ethibox.fr>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.ethibox.fr>`__
* `search.fossdaily.xyz <https://search.fossdaily.xyz>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.fossdaily.xyz>`__
* `search.galaxy.cat <https://search.galaxy.cat>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.galaxy.cat>`__
* `search.gibberfish.org <https://search.gibberfish.org/>`__ (as `Hidden Service <http://o2jdk5mdsijm2b7l.onion/>`__ or `Proxied through Tor <https://search.gibberfish.org/tor/>`__) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.gibberfish.org>`__
* `search.koehn.com <https://search.koehn.com>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.koehn.com>`__
* `search.lgbtq.cool <https://search.lgbtq.cool/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.lgbtq.cool>`__
* `search.mdosch.de <https://search.mdosch.de/>`__ (as `Hidden Service <http://search.4bkxscubgtxwvhpe.onion/>`__) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.mdosch.de>`__
* `search.modalogi.com <https://search.modalogi.com/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.modalogi.com&latest>`__
* `search.moravit.com <https://search.moravit.com>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.moravit.com>`__
* `search.nebulacentre.net <https://search.nebulacentre.net>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.nebulacentre.net>`__
* `search.paulla.asso.fr <https://search.paulla.asso.fr/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.paulla.asso.fr>`__
* `search.pifferi.info <https://search.pifferi.info/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.pifferi.info&latest>`__
* `search.poal.co <https://search.poal.co/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.poal.co>`__
* `search.privacytools.io <https://search.privacytools.io/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.privacytools.io>`__ - Uses Matomo for user tracking and analytics
* `search.seds.nl <https://search.seds.nl/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.seds.nl&latest>`__
* `search.snopyta.org <https://search.snopyta.org/>`__ (as `Hidden Service <http://juy4e6eicawzdrz7.onion/>`__) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.snopyta.org>`__
* `search.spaeth.me <https://search.spaeth.me/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.spaeth.me&latest>`__
* `search.st8.at <https://search.st8.at/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.st8.at>`__
* `search.stinpriza.org <https://search.stinpriza.org>`__ (as `Hidden Service <http://z5vawdol25vrmorm4yydmohsd4u6rdoj2sylvoi3e3nqvxkvpqul7bqd.onion/>`__) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.stinpriza.org&hideResults=on>`__
* `search.sudo-i.net <https://search.sudo-i.net/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.sudo-i.net>`__
* `search.tolstoevsky.ml <https://search.tolstoevsky.ml>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.tolstoevsky.ml>`__
* `searchsin.com/searx <https://searchsin.com/searx>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searchsin.com/searx>`__
* `searx.anongoth.pl <https://searx.anongoth.pl>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.anongoth.pl&latest>`__
* `searx.be <https://searx.be>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.be>`__
* `searx.ca <https://searx.ca/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.ca>`__
* `searx.canox.net <https://searx.canox.net/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.canox.net>`__
* `searx.cybt.de <https://searx.cybt.de/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.cybt.de>`__
* `searx.de <https://www.searx.de/>`__ - Issuer: COMODO `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.de>`__
* `searx.decatec.de <https://searx.decatec.de>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.decatec.de>`__
* `searx.devol.it <https://searx.devol.it/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=sears.devol.it>`__
* `searx.dnswarden.com <https://searx.dnswarden.com>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.dnswarden.com>`__
* `searx.drakonix.net <https://searx.drakonix.net/>`__ - (down) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.drakonix.net>`__
* `searx.dresden.network <https://searx.dresden.network/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.dresden.network>`__
* `searx.elukerio.org <https://searx.elukerio.org/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.elukerio.org/>`__
* `searx.everdot.org <https://searx.everdot.org/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.everdot.org/>`__ - Crawls using YaCy
* `searx.foo.li <https://searx.foo.li>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.foo.li&hideResults=on>`__
* `searx.fossencdi.org <https://searx.fossencdi.org>`__ (as `Hidden Service <http://searx.cwuzdtzlubq5uual.onion/>`__) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.fossencdi.org>`__
* `searx.fr32k.de <https://searx.fr32k.de/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.fr32k.de>`__
* `searx.good.one.pl <https://searx.good.one.pl>`__ (as `Hidden Service <http://searxl7u2y6gvonm.onion/>`__) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.good.one.pl>`__
* `searx.gotrust.de <https://searx.gotrust.de/>`__ (as `Hidden Service <http://nxhhwbbxc4khvvlw.onion/>`__) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.gotrust.de>`__
* `searx.hardwired.link <https://searx.hardwired.link/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.hardwired.link>`__
* `searx.hlfh.space <https://searx.hlfh.space>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.hlfh.space>`__
* `searx.info <https://searx.info>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.info>`__
* `searx.itunix.eu <https://searx.itunix.eu/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.itunix.eu>`__
* `searx.john-at-me.net <https://searx.john-at-me.net/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.john-at-me.net>`__
* `searx.kvch.me <https://searx.kvch.me>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.kvch.me>`__
* `searx.laquadrature.net <https://searx.laquadrature.net>`__ (as `Hidden Service <http://searchb5a7tmimez.onion/>`__) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.laquadrature.net>`__
* `searx.lelux.fi <https://searx.lelux.fi/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=haku.lelux.fi>`__
* `searx.lhorn.de <https://searx.lhorn.de/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.lhorn.de&latest>`__
* `searx.li <https://searx.li/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.li>`__
* `searx.libmail.eu <https://searx.libmail.eu/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.libmail.eu/>`__
* `searx.linux.pizza <https://searx.linux.pizza>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.linux.pizza>`__
* `searx.lynnesbian.space <https://searx.lynnesbian.space/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.lynnesbian.space>`__
* `searx.mastodontech.de <https://searx.mastodontech.de/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.mastodontech.de>`__
* `searx.me <https://searx.me>`__ (as `Hidden Service <http://ulrn6sryqaifefld.onion/>`__) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.me>`__
* `searx.mxchange.org <https://searx.mxchange.org/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.mxchange.org>`__
* `searx.nakhan.net <https://searx.nakhan.net>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.nakhan.net>`__
* `searx.nixnet.xyz <https://searx.nixnet.xyz>`__ (as `Hidden Service <http://searx.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd.onion/>`__) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.nixnet.xyz>`__
* `searx.nnto.net <https://searx.nnto.net/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.nnto.net>`__
* `searx.openhoofd.nl <https://searx.openhoofd.nl/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=openhoofd.nl>`__
* `searx.openpandora.org <https://searx.openpandora.org/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.openpandora.org&latest>`__
* `searx.operationtulip.com <https://searx.operationtulip.com/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.operationtulip.com>`__
* `searx.orcadian.net <https://searx.orcadian.net/>`__ - Issuer: Comodo CA Limited `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.orcadian.net>`__
* `searx.ouahpit.info <https://searx.ouahpiti.info/>`__ - Issuer: Let's Encrypt
* `searx.pofilo.fr <https://searx.pofilo.fr/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.pofilo.fr>`__
* `searx.prvcy.eu <https://searx.prvcy.eu/>`__ (as `Hidden Service <http://hmfztxt3pfhevucl.onion/>`__) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.prvcy.eu>`__
* `searx.pwoss.org <https://searx.pwoss.org>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.pwoss.org>`__
* `searx.ro <https://searx.ro/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.ro>`__
* `searx.ru <https://searx.ru/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.ru>`__
* `searx.solusar.de <https://searx.solusar.de/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.solusar.de>`__
* `searx.targaryen.house <https://searx.targaryen.house/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.targaryen.house>`__
* `searx.tuxcloud.net <https://searx.tuxcloud.net>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.tuxcloud.net>`__
* `searx.tyil.nl <https://searx.tyil.nl>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.tyil.nl>`__
* `searx.wegeeks.win <https://searx.wegeeks.win>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.wegeeks.win>`__
* `searx.win <https://searx.win/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.win&latest>`__
* `searx.xyz <https://searx.xyz/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.xyz&latest>`__
* `searx.zareldyn.net <https://searx.zareldyn.net/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.zareldyn.net>`__
* `searx.zdechov.net <https://searx.zdechov.net>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.zdechov.net>`__
* `searxs.eu <https://www.searxs.eu>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=www.searxs.eu&hideResults=on>`__
* `seeks.hsbp.org <https://seeks.hsbp.org/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=seeks.hsbp.org>`__ - `PGP signed fingerprints of cert <https://seeks.hsbp.org/cert>`__
* `skyn3t.in/srx <https://skyn3t.in/srx/>`__ - Issuer: Let's Encrypt | onion `hidden service <http://skyn3tb3bas655mw.onion/srx/>`__
* `spot.ecloud.global <https://spot.ecloud.global/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=spot.ecloud.global>`__
* `srx.sx <https://srx.sx>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=srx.sx>`__
* `stemy.me/searx <https://stemy.me/searx>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=stemy.me>`__
* `suche.dasnetzundich.de <https://suche.dasnetzundich.de>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=suche.dasnetzundich.de>`__
* `suche.elaon.de <https://suche.elaon.de>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=suche.elaon.de>`__
* `suche.xyzco456vwisukfg.onion <http://suche.xyzco456vwisukfg.onion/>`__
* `suche.uferwerk.org <https://suche.uferwerk.org>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=suche.uferwerk.org>`__
* `timdor.noip.me/searx <https://timdor.noip.me/searx>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=timdor.noip.me/searx>`__
* `trovu.komun.org <https://trovu.komun.org>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=trovu.komun.org>`__
* `unmonito.red <https://unmonito.red/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=unmonito.red>`__
* `www.finden.tk <https://www.finden.tk/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=www.finden.tk>`__
* `zoek.anchel.nl <https://zoek.anchel.nl/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=zoek.anchel.nl>`__
Running in exclusive private walled-gardens
===========================================
These instances run in walled-gardens that exclude some segment of the general
public (e.g. Tor users and users sharing IPs with many other users). Caution:
privacy is also compromised on these sites due to exposure of cleartext traffic
to a third party other than the website operator.
* `intelme.com <https://intelme.com>`__ - Issuer: Cloudflare `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=intelme.com>`__
* `search404.io <https://www.search404.io/>`__ - Issuer: Cloudflare `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search404.io>`__
* `searx.com.au <https://searx.com.au/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.com.au>`__
* `searx.lavatech.top <https://searx.lavatech.top/>`__ - Issuer: Cloudflare `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.lavatech.top>`__
* `searchx.mobi <https://searchx.mobi/>`__ - Issuer: Cloudflare `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searchx.mobi>`__
* `searx.org <https://searx.org/>`__ - Issuer: Cloudflare `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.org>`__
* `searx.run <https://searx.run/>`__ - Issuer: Cloudflare `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.run>`__
* `searx.world <https://searx.world>`__ - Issuer: Cloudflare `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.world>`__ - Adds Amazon affiliate links
Running with an incorrect SSL certificate
=========================================
* `listi.me <https://listi.me/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=listi.me&latest>`__
* `s.matejc.com <https://s.matejc.com/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=s.matejc.com>`__
* `search.jollausers.de <https://search.jollausers.de>`__ - Incorrectly configured `SSL certificate <https://www.ssllabs.com/ssltest/analyze.html?d=search.jollausers.de>`__
* `search.paviro.de <https://search.paviro.de>`__ - Issuer: LetsEncrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.paviro.de>`__
* `searx.abenthung.it <https://searx.abenthung.it/>`__ - Issuer: Comodo CA Limited `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.abenthung.it>`__
* `searx.coding4schoki.org <https://searx.coding4schoki.org/>`__ - Incorrectly configured `SSL Certificate <https://www.ssllabs.com/ssltest/analyze.html?d=searx.coding4schoki.org>`__
* `searx.haxors.club <https://searx.haxors.club/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.haxors.club>`__
* `searx.nulltime.net <https://searx.nulltime.net/>`__ (as `Hidden Service <http://searx7gwtu5rh6wr.onion>`__) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.nulltime.net>`__
* `searx.ch <https://searx.ch/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.ch>`__ (cert clock problems)
Offline
=======
* `a.searx.space <https://a.searx.space>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=a.searx.space>`__ (unstable, under construction).
* `anyonething.de <https://anyonething.de>`__ - (was found to have become a pastebin on or before 2019-03-01) Issuer: Comodo CA Limited (Warning: uses Cloudflare) `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=anyonething.de>`__
* `h7jwxg5rakyfvikpi.onion <http://7jwxg5rakyfvikpi.onion/>`__ - available only as Tor Hidden Service (down on 2019-06-26)
* `hacktivis.me/searx <https://hacktivis.me/searx>`__ - (down) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=hacktivis.me/searx>`__
* `icebal.com <https://icebal.com>`__ - (down) Issuer: Let's Encrypt
* `netrangler.host <https://netrangler.host>`__ - (down) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=netrangler.host>`__
* `opengo.nl <https://www.opengo.nl>`__ - (down) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=www.opengo.nl>`__
* `p9e.de <https://p9e.de/>`__ - (down - timeout) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=p9e.de>`__
* `rubri.co <https://rubri.co>`__ - (down) Issuer: Let's Encrypt
* `s.bacafe.xyz <https://s.bacafe.xyz/>`__ - (down) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=s.bacafe.xyz&latest>`__
* `search.alecpap.com <https://search.alecpap.com/>`__ - (down) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.alecpap.com>`__
* `search.blackit.de <https://search.blackit.de/>`__ - (down) Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.blackit.de>`__
* `search.deblan.org <https://search.deblan.org/>`__ (down) - Issuer: COMODO via GANDI `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.deblan.org>`__
* `search.homecomputing.fr <https://search.homecomputing.fr/>`__ - (down) Issuer: CAcert `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.homecomputing.fr>`__
* `search.jpope.org <https://search.jpope.org>`__ - (down - timeout) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.jpope.org>`__
* `search.kakise.xyz <https://search.kakise.xyz/>`__ - down
* `search.kosebamse.com <https://search.kosebamse.com>`__ - Issuer: LetsEncrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.kosebamse.com>`__
* `search.kujiu.org <https://search.kujiu.org>`__ - (down) Issuer: Let's Encrypt
* `search.mailaender.coffee <https://search.mailaender.coffee/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.mailaender.coffee>`__
* `search.matrix.ac <https://search.matrix.ac>`__ - (down) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=matrix.ac>`__
* `search.mypsc.ca <https://search.mypsc.ca/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.mypsc.ca>`__
* `search.namedkitten.pw <https://search.namedkitten.pw>`__ - (SSL error) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.namedkitten.pw>`__
* `search.opentunisia.org <https://search.opentunisia.org>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.opentunisia.org>`__
* `search.r3d007.com <https://search.r3d007.com/>`__ - (down) Issuer: Let's Encrypt
* `search.static.lu <https://search.static.lu/>`__ - (down) Issuer: StartCom `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.static.lu>`__
* `search.teej.xyz <https://search.teej.xyz>`__ - (down) Issuer: LetsEncrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.teej.xyz>`__
* `search.wxzm.sx <https://search.wxzm.sx>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=search.wxzm.sx>`__
* `searx.4ray.co <https://searx.4ray.co/>`__ - (no longer an instance, redirects to main page) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.4ray.co>`__
* `searx.32bitflo.at <https://searx.32bitflo.at/>`__ - (down) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.32bitflo.at>`__
* `searx.ahh.si <https://searx.ahh.si/>`__ - (down) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.ahh.si>`__
* `searx.angristan.xyz <https://searx.angristan.xyz/>`__ - (down) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.angristan.xyz>`__
* `searx.antirep.net <https://searx.antirep.net/>`__ - (return a 502 HTTP error) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.antirep.net>`__
* `searx.aquilenet.fr <https://searx.aquilenet.fr/>`__ - (down - 429 HTTP error) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.aquilenet.fr>`__
* `searx.at <https://searx.at/>`__ - (return "request exception" at every search) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.at>`__
* `searx.cc <https://searx.cc/>`__ - (down on 2019-06-26) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.cc>`__
* `searx.dk <https://searx.dk/>`__ - (down - 429 HTTP error) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.dk>`__
* `searx.ehrmanns.ch <https://searx.ehrmanns.ch>`__ - (down) Issuer: Let's Encrypt
* `searx.glibre.net <https://searx.glibre.net>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.glibre.net>`__
* `searx.infini.fr <https://searx.infini.fr>`__ - (return a page stating that the website is not installed) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.infini.fr>`__
* `searx.jeanphilippemorvan.info <https://searx.jeanphilippemorvan.info/>`__ - (down) Issuer: StartCom `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.jeanphilippemorvan.info>`__
* `searx.lhorn.de <https://searx.lhorn.de/>`__ - (redirect the Searx's github repository page) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.lhorn.de&latest>`__ (only reachable from european countries)
* `searx.lvweb.host <https://searx.lvweb.host>`__ - (down) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.lvweb.host>`__
* `searx.mrtino.eu <https://searx.mrtino.eu>`__ - (down) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.mrtino.eu>`__
* `searx.netzspielplatz.de <https://searx.netzspielplatz.de/>`__ - (error page about GDPR even when browsing it from USA and Asia) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.netzspielplatz.de>`__
* `searx.new-admin.net <https://searx.new-admin.net>`__ - (down) Issuer: Let's Encrypt
* `searx.nogafa.org <https://searx.nogafa.org/>`__ - (broken CSS) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.nogafa.org>`__
* `searx.potato.hu <https://searx.potato.hu>`__ - (not a searx instance) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.potato.hu>`__
* `searx.rubbeldiekatz.info <https://searx.rubbeldiekatz.info/>`__ - (down) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.rubbeldiekatz.info/>`__
* `searx.s42.space <https://searx.s42.space>`__ - (down) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.s42.space>`__
* `searx.salcay.hu <https://searx.salcay.hu/>`__ - (down - blank page) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.salcay.hu>`__
* `searx.selea.se <https://searx.selea.se>`__ - (Leads to default Apache page) Issuer: RapidSSL (HSTS preloaded, DNSSEC) `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.selea.se>`__ | `HSTS Preload <https://hstspreload.org/?domain=searx.selea.se>`__
* `searx.steinscraft.net <https://searx.steinscraft.net/>`__ - (down) Issuer: Cloudflare
* `searx.techregion.de <https://searx.techregion.de/>`__ - (domain expired) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.techregion.de>`__
* `searx.tognella.com <https://searx.tognella.com/>`__ - (down) Issuer: Cloudflare
* `searx.xi.ht <https://searx.xi.ht/>`__ - (return a 502 HTTP error) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searx.xi.ht>`__
* `searxist.com <https://searxist.com/>`__ - (down) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=searxist.com>`__
* `so.sb <https://so.sb/>`__ - (down) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=so.sb>`__
* `srx.stdout.net <https://srx.stdout.net/>`__ - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=srx.stdout.net>`__
* `w6f7cgdm54cyvohcuhraaafhajctyj3ihenrovuxogoagrr5g43qmoid.onion <http://w6f7cgdm54cyvohcuhraaafhajctyj3ihenrovuxogoagrr5g43qmoid.onion/>`__ - Hidden Service
* `win8linux.nohost.me <https://win8linux.nohost.me/searx/>`__ - (down) Issuer: Let's Encrypt
* `wiznet.tech <https://wiznet.tech>`__ - (down) - Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=wiznet.tech>`__
* `www.mercurius.space <https://www.mercurius.space/>`__ - (down) Issuer: Let's Encrypt
* `www.ready.pm <https://www.ready.pm>`__ - Issuer: WoSign `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=www.ready.pm>`__
* `z.awsmppl.com <https://z.awsmppl.com>`__ - (down) Issuer: Let's Encrypt `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=z.awsmppl.com>`__
* `zlsdzh.tk <https://zlsdzh.tk>`__ - (down - 404 HTTP error) Issuer: TrustAsia Technologies, Inc. `Verification <https://www.ssllabs.com/ssltest/analyze.html?d=zlsdzh.tk>`__ *

42
docs/user/search_syntax.rst Normal file
View File

@ -0,0 +1,42 @@
.. _search-syntax:
=============
Search syntax
=============
Searx allows you to modify the default categories, engines and search language
via the search query.
Prefix ``!``
to set Category/engine
Prefix: ``:``
to set language
Prefix: ``?``
to add engines and categories to the currently selected categories
Abbrevations of the engines and languages are also accepted. Engine/category
modifiers are chainable and inclusive (e.g. with :search:`!it !ddg !wp qwer
<?q=%21it%20%21ddg%20%21wp%20qwer>` search in IT category **and** duckduckgo
**and** wikipedia for ``qwer``).
See the :search:`/preferences page <preferences>` for the list of engines,
categories and languages.
Examples
========
Search in wikipedia for ``qwer``:
- :search:`!wp qwer <?q=%21wp%20qwer>` or
- :search:`!wikipedia qwer :search:<?q=%21wikipedia%20qwer>`
Image search:
- :search:`!images Cthulhu <?q=%21images%20Cthulhu>`
Custom language in wikipedia:
- :search:`:hu !wp hackerspace <?q=%3Ahu%20%21wp%20hackerspace>`

10
manage.sh
View File

@ -18,12 +18,12 @@ ACTION="$1"
update_packages() { update_packages() {
pip install --upgrade pip pip install --upgrade pip
pip install --upgrade setuptools pip install --upgrade setuptools
pip install -r "$BASE_DIR/requirements.txt" pip install -Ur "$BASE_DIR/requirements.txt"
} }
update_dev_packages() { update_dev_packages() {
update_packages update_packages
pip install -r "$BASE_DIR/requirements-dev.txt" pip install -Ur "$BASE_DIR/requirements-dev.txt"
} }
install_geckodriver() { install_geckodriver() {
@ -70,6 +70,11 @@ locales() {
pybabel compile -d "$SEARX_DIR/translations" pybabel compile -d "$SEARX_DIR/translations"
} }
update_useragents() {
echo '[!] Updating user agent versions'
python utils/fetch_firefox_version.py
}
pep8_check() { pep8_check() {
echo '[!] Running pep8 check' echo '[!] Running pep8 check'
# ignored rules: # ignored rules:
@ -246,6 +251,7 @@ Commands
update_dev_packages - Check & update development and production dependency changes update_dev_packages - Check & update development and production dependency changes
install_geckodriver - Download & install geckodriver if not already installed (required for robot_tests) install_geckodriver - Download & install geckodriver if not already installed (required for robot_tests)
npm_packages - Download & install npm dependencies npm_packages - Download & install npm dependencies
update_useragents - Update useragents.json with the most recent versions of Firefox
Build Build
----- -----

6
requirements-dev.txt
View File

@ -1,3 +1,6 @@
pallets-sphinx-themes
Sphinx
sphinx-issues
mock==2.0.0 mock==2.0.0
nose2[coverage_plugin] nose2[coverage_plugin]
cov-core==1.15.0 cov-core==1.15.0
@ -8,3 +11,6 @@ transifex-client==0.12.2
unittest2==1.1.0 unittest2==1.1.0
zope.testrunner==4.5.1 zope.testrunner==4.5.1
selenium==3.141.0 selenium==3.141.0
linuxdoc @ git+http://github.com/return42/linuxdoc.git
sphinx-jinja
sphinx-tabs

15
searx/data/useragents.json
View File

@ -1,14 +1,15 @@
{ {
"ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}",
"versions": [ "versions": [
"61.0.1", "70.0.1",
"61.0", "70.0",
"60.0.2", "69.0.3",
"60.0.1", "69.0.2",
"60.0" "69.0.1",
"69.0"
], ],
"os": [ "os": [
"Windows NT 10; WOW64", "Windows NT 10; WOW64",
"X11; Linux x86_64" "X11; Linux x86_64"
] ],
"ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}"
} }

5
searx/engines/bing.py
View File

@ -89,8 +89,7 @@ def response(resp):
'content': content}) 'content': content})
try: try:
result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()')) result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]//text()'))
result_len_container = utils.to_string(result_len_container)
if "-" in result_len_container: if "-" in result_len_container:
# Remove the part "from-to" for paginated request ... # Remove the part "from-to" for paginated request ...
result_len_container = result_len_container[result_len_container.find("-") * 2 + 2:] result_len_container = result_len_container[result_len_container.find("-") * 2 + 2:]
@ -102,7 +101,7 @@ def response(resp):
logger.debug('result error :\n%s', e) logger.debug('result error :\n%s', e)
pass pass
if _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len: if result_len and _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len:
return [] return []
results.append({'number_of_results': result_len}) results.append({'number_of_results': result_len})

18
searx/engines/flickr_noapi.py
View File

@ -109,14 +109,22 @@ def response(resp):
else: else:
url = build_flickr_url(photo['ownerNsid'], photo['id']) url = build_flickr_url(photo['ownerNsid'], photo['id'])
results.append({'url': url, result = {
'title': title, 'url': url,
'img_src': img_src, 'img_src': img_src,
'thumbnail_src': thumbnail_src, 'thumbnail_src': thumbnail_src,
'content': content,
'author': author,
'source': source, 'source': source,
'img_format': img_format, 'img_format': img_format,
'template': 'images.html'}) 'template': 'images.html'
}
try:
result['author'] = author
result['title'] = title
result['content'] = content
except:
result['author'] = ''
result['title'] = ''
result['content'] = ''
results.append(result)
return results return results

39
searx/engines/gigablast.py
View File

@ -14,6 +14,7 @@ import random
from json import loads from json import loads
from time import time from time import time
from lxml.html import fromstring from lxml.html import fromstring
from searx.poolrequests import get
from searx.url_utils import urlencode from searx.url_utils import urlencode
from searx.utils import eval_xpath from searx.utils import eval_xpath
@ -31,13 +32,9 @@ search_string = 'search?{query}'\
'&c=main'\ '&c=main'\
'&s={offset}'\ '&s={offset}'\
'&format=json'\ '&format=json'\
'&qh=0'\ '&langcountry={lang}'\
'&qlang={lang}'\
'&ff={safesearch}'\ '&ff={safesearch}'\
'&rxiec={rxieu}'\ '&rand={rxikd}'
'&ulse={ulse}'\
'&rand={rxikd}'\
'&dbez={dbez}'
# specific xpath variables # specific xpath variables
results_xpath = '//response//result' results_xpath = '//response//result'
url_xpath = './/url' url_xpath = './/url'
@ -46,9 +43,26 @@ content_xpath = './/sum'
supported_languages_url = 'https://gigablast.com/search?&rxikd=1' supported_languages_url = 'https://gigablast.com/search?&rxikd=1'
extra_param = '' # gigablast requires a random extra parameter
# which can be extracted from the source code of the search page
def parse_extra_param(text):
global extra_param
param_lines = [x for x in text.splitlines() if x.startswith('var url=') or x.startswith('url=url+')]
extra_param = ''
for l in param_lines:
extra_param += l.split("'")[1]
extra_param = extra_param.split('&')[-1]
def init(engine_settings=None):
parse_extra_param(get('http://gigablast.com/search?c=main&qlangcountry=en-us&q=south&s=10').text)
# do search-request # do search-request
def request(query, params): def request(query, params):
print("EXTRAPARAM:", extra_param)
offset = (params['pageno'] - 1) * number_of_results offset = (params['pageno'] - 1) * number_of_results
if params['language'] == 'all': if params['language'] == 'all':
@ -67,14 +81,11 @@ def request(query, params):
search_path = search_string.format(query=urlencode({'q': query}), search_path = search_string.format(query=urlencode({'q': query}),
offset=offset, offset=offset,
number_of_results=number_of_results, number_of_results=number_of_results,
rxikd=int(time() * 1000),
rxieu=random.randint(1000000000, 9999999999),
ulse=random.randint(100000000, 999999999),
lang=language, lang=language,
safesearch=safesearch, rxikd=int(time() * 1000),
dbez=random.randint(100000000, 999999999)) safesearch=safesearch)
params['url'] = base_url + search_path params['url'] = base_url + search_path + '&' + extra_param
return params return params
@ -84,7 +95,11 @@ def response(resp):
results = [] results = []
# parse results # parse results
try:
response_json = loads(resp.text) response_json = loads(resp.text)
except:
parse_extra_param(resp.text)
raise Exception('extra param expired, please reload')
for result in response_json['results']: for result in response_json['results']:
# append result # append result

7
searx/engines/ina.py
View File

@ -32,7 +32,7 @@ base_url = 'https://www.ina.fr'
search_url = base_url + '/layout/set/ajax/recherche/result?autopromote=&hf={ps}&b={start}&type=Video&r=&{query}' search_url = base_url + '/layout/set/ajax/recherche/result?autopromote=&hf={ps}&b={start}&type=Video&r=&{query}'
# specific xpath variables # specific xpath variables
results_xpath = '//div[contains(@class,"search-results--list")]/div[@class="media"]' results_xpath = '//div[contains(@class,"search-results--list")]//div[@class="media-body"]'
url_xpath = './/a/@href' url_xpath = './/a/@href'
title_xpath = './/h3[@class="h3--title media-heading"]' title_xpath = './/h3[@class="h3--title media-heading"]'
thumbnail_xpath = './/img/@src' thumbnail_xpath = './/img/@src'
@ -65,8 +65,11 @@ def response(resp):
videoid = result.xpath(url_xpath)[0] videoid = result.xpath(url_xpath)[0]
url = base_url + videoid url = base_url + videoid
title = p.unescape(extract_text(result.xpath(title_xpath))) title = p.unescape(extract_text(result.xpath(title_xpath)))
try:
thumbnail = extract_text(result.xpath(thumbnail_xpath)[0]) thumbnail = extract_text(result.xpath(thumbnail_xpath)[0])
if thumbnail[0] == '/': except:
thumbnail = ''
if thumbnail and thumbnail[0] == '/':
thumbnail = base_url + thumbnail thumbnail = base_url + thumbnail
d = extract_text(result.xpath(publishedDate_xpath)[0]) d = extract_text(result.xpath(publishedDate_xpath)[0])
d = d.split('/') d = d.split('/')

2
searx/engines/microsoft_academic.py
View File

@ -45,6 +45,8 @@ def request(query, params):
def response(resp): def response(resp):
results = [] results = []
response_data = loads(resp.text) response_data = loads(resp.text)
if not response_data:
return results
for result in response_data['results']: for result in response_data['results']:
url = _get_url(result) url = _get_url(result)

2
searx/engines/openstreetmap.py
View File

@ -24,7 +24,7 @@ result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'
# do search-request # do search-request
def request(query, params): def request(query, params):
params['url'] = base_url + search_string.format(query=query) params['url'] = base_url + search_string.format(query=query.decode('utf-8'))
return params return params

1
searx/engines/qwant.py
View File

@ -50,6 +50,7 @@ def request(query, params):
language = match_language(params['language'], supported_languages, language_aliases) language = match_language(params['language'], supported_languages, language_aliases)
params['url'] += '&locale=' + language.replace('-', '_').lower() params['url'] += '&locale=' + language.replace('-', '_').lower()
params['headers']['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0'
return params return params

2
searx/engines/scanr_structures.py
View File

@ -29,7 +29,7 @@ def request(query, params):
params['url'] = search_url params['url'] = search_url
params['method'] = 'POST' params['method'] = 'POST'
params['headers']['Content-type'] = "application/json" params['headers']['Content-type'] = "application/json"
params['data'] = dumps({"query": query, params['data'] = dumps({"query": query.decode('utf-8'),
"searchField": "ALL", "searchField": "ALL",
"sortDirection": "ASC", "sortDirection": "ASC",
"sortOrder": "RELEVANCY", "sortOrder": "RELEVANCY",

14
searx/engines/spotify.py
View File

@ -12,10 +12,14 @@
from json import loads from json import loads
from searx.url_utils import urlencode from searx.url_utils import urlencode
import requests
import base64
# engine dependent config # engine dependent config
categories = ['music'] categories = ['music']
paging = True paging = True
api_client_id = None
api_client_secret = None
# search-url # search-url
url = 'https://api.spotify.com/' url = 'https://api.spotify.com/'
@ -31,6 +35,16 @@ def request(query, params):
params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset) params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)
r = requests.post(
'https://accounts.spotify.com/api/token',
data={'grant_type': 'client_credentials'},
headers={'Authorization': 'Basic ' + base64.b64encode(
"{}:{}".format(api_client_id, api_client_secret).encode('utf-8')
).decode('utf-8')}
)
j = loads(r.text)
params['headers'] = {'Authorization': 'Bearer {}'.format(j.get('access_token'))}
return params return params

9
searx/engines/wikipedia.py
View File

@ -21,7 +21,8 @@ search_url = base_url + u'w/api.php?'\
'action=query'\ 'action=query'\
'&format=json'\ '&format=json'\
'&{query}'\ '&{query}'\
'&prop=extracts|pageimages'\ '&prop=extracts|pageimages|pageprops'\
'&ppprop=disambiguation'\
'&exintro'\ '&exintro'\
'&explaintext'\ '&explaintext'\
'&pithumbsize=300'\ '&pithumbsize=300'\
@ -79,12 +80,15 @@ def response(resp):
# wikipedia article's unique id # wikipedia article's unique id
# first valid id is assumed to be the requested article # first valid id is assumed to be the requested article
if 'pages' not in search_result['query']:
return results
for article_id in search_result['query']['pages']: for article_id in search_result['query']['pages']:
page = search_result['query']['pages'][article_id] page = search_result['query']['pages'][article_id]
if int(article_id) > 0: if int(article_id) > 0:
break break
if int(article_id) < 0: if int(article_id) < 0 or 'disambiguation' in page.get('pageprops', {}):
return [] return []
title = page.get('title') title = page.get('title')
@ -96,6 +100,7 @@ def response(resp):
extract = page.get('extract') extract = page.get('extract')
summary = extract_first_paragraph(extract, title, image) summary = extract_first_paragraph(extract, title, image)
summary = summary.replace('() ', '')
# link to wikipedia article # link to wikipedia article
wikipedia_link = base_url.format(language=url_lang(resp.search_params['language'])) \ wikipedia_link = base_url.format(language=url_lang(resp.search_params['language'])) \

35
searx/settings.yml
View File

@ -79,9 +79,10 @@ engines:
categories : science categories : science
timeout : 4.0 timeout : 4.0
- name : base # tmp suspended: dh key too small
engine : base # - name : base
shortcut : bs # engine : base
# shortcut : bs
- name : wikipedia - name : wikipedia
engine : wikipedia engine : wikipedia
@ -408,7 +409,7 @@ engines:
- name : library genesis - name : library genesis
engine : xpath engine : xpath
search_url : http://libgen.io/search.php?req={query} search_url : https://libgen.is/search.php?req={query}
url_xpath : //a[contains(@href,"bookfi.net")]/@href url_xpath : //a[contains(@href,"bookfi.net")]/@href
title_xpath : //a[contains(@href,"book/")]/text()[1] title_xpath : //a[contains(@href,"book/")]/text()[1]
content_xpath : //td/a[1][contains(@href,"=author")]/text() content_xpath : //td/a[1][contains(@href,"=author")]/text()
@ -464,7 +465,7 @@ engines:
- name : openairedatasets - name : openairedatasets
engine : json_engine engine : json_engine
paging : True paging : True
search_url : http://api.openaire.eu/search/datasets?format=json&page={pageno}&size=10&title={query} search_url : https://api.openaire.eu/search/datasets?format=json&page={pageno}&size=10&title={query}
results_query : response/results/result results_query : response/results/result
url_query : metadata/oaf:entity/oaf:result/children/instance/webresource/url/$ url_query : metadata/oaf:entity/oaf:result/children/instance/webresource/url/$
title_query : metadata/oaf:entity/oaf:result/title/$ title_query : metadata/oaf:entity/oaf:result/title/$
@ -476,7 +477,7 @@ engines:
- name : openairepublications - name : openairepublications
engine : json_engine engine : json_engine
paging : True paging : True
search_url : http://api.openaire.eu/search/publications?format=json&page={pageno}&size=10&title={query} search_url : https://api.openaire.eu/search/publications?format=json&page={pageno}&size=10&title={query}
results_query : response/results/result results_query : response/results/result
url_query : metadata/oaf:entity/oaf:result/children/instance/webresource/url/$ url_query : metadata/oaf:entity/oaf:result/children/instance/webresource/url/$
title_query : metadata/oaf:entity/oaf:result/title/$ title_query : metadata/oaf:entity/oaf:result/title/$
@ -552,10 +553,11 @@ engines:
timeout : 10.0 timeout : 10.0
disabled : True disabled : True
- name : scanr structures # tmp suspended: bad certificate
shortcut: scs # - name : scanr structures
engine : scanr_structures # shortcut: scs
disabled : True # engine : scanr_structures
# disabled : True
- name : soundcloud - name : soundcloud
engine : soundcloud engine : soundcloud
@ -598,9 +600,12 @@ engines:
shortcut : se shortcut : se
categories : science categories : science
- name : spotify # Spotify needs API credentials
engine : spotify # - name : spotify
shortcut : stf # engine : spotify
# shortcut : stf
# api_client_id : *******
# api_client_secret : *******
- name : startpage - name : startpage
engine : startpage engine : startpage
@ -812,7 +817,7 @@ locales:
doi_resolvers : doi_resolvers :
oadoi.org : 'https://oadoi.org/' oadoi.org : 'https://oadoi.org/'
doi.org : 'https://doi.org/' doi.org : 'https://doi.org/'
doai.io : 'http://doai.io/' doai.io : 'https://doai.io/'
sci-hub.tw : 'http://sci-hub.tw/' sci-hub.tw : 'https://sci-hub.tw/'
default_doi_resolver : 'oadoi.org' default_doi_resolver : 'oadoi.org'

4
searx/settings_robot.yml
View File

@ -43,7 +43,7 @@ locales:
doi_resolvers : doi_resolvers :
oadoi.org : 'https://oadoi.org/' oadoi.org : 'https://oadoi.org/'
doi.org : 'https://doi.org/' doi.org : 'https://doi.org/'
doai.io : 'http://doai.io/' doai.io : 'https://doai.io/'
sci-hub.tw : 'http://sci-hub.tw/' sci-hub.tw : 'https://sci-hub.tw/'
default_doi_resolver : 'oadoi.org' default_doi_resolver : 'oadoi.org'

12
searx/static/plugins/js/vim_hotkeys.js
View File

@ -125,6 +125,14 @@ $(document).ready(function() {
} }
}); });
function nextResult(current, direction) {
var next = current[direction]();
while (!next.is('.result') && next.length !== 0) {
next = next[direction]();
}
return next
}
function highlightResult(which) { function highlightResult(which) {
return function() { return function() {
var current = $('.result[data-vim-selected]'); var current = $('.result[data-vim-selected]');
@ -157,13 +165,13 @@ $(document).ready(function() {
} }
break; break;
case 'down': case 'down':
next = current.next('.result'); next = nextResult(current, 'next');
if (next.length === 0) { if (next.length === 0) {
next = $('.result:first'); next = $('.result:first');
} }
break; break;
case 'up': case 'up':
next = current.prev('.result'); next = nextResult(current, 'prev');
if (next.length === 0) { if (next.length === 0) {
next = $('.result:last'); next = $('.result:last');
} }

2
searx/static/themes/oscar/less/logicodev/code.less
View File

@ -99,5 +99,3 @@ pre, code{
.highlight { .highlight {
font-weight: 700; font-weight: 700;
} }

1
searx/static/themes/oscar/less/logicodev/navbar.less
View File

@ -28,4 +28,3 @@
width: 80%; width: 80%;
} }
} }

2
searx/templates/oscar/infobox.html
View File

@ -6,7 +6,7 @@
<div class="panel-body"> <div class="panel-body">
{% if infobox.img_src %}<img class="img-responsive center-block infobox_part" src="{{ image_proxify(infobox.img_src) }}" alt="{{ infobox.infobox }}" />{% endif %} {% if infobox.img_src %}<img class="img-responsive center-block infobox_part" src="{{ image_proxify(infobox.img_src) }}" alt="{{ infobox.infobox }}" />{% endif %}
{% if infobox.content %}<bdi><p class="infobox_part">{{ infobox.content }}</p></bdi>{% endif %} {% if infobox.content %}<bdi><p class="infobox_part">{{ infobox.content | safe }}</p></bdi>{% endif %}
{% if infobox.attributes -%} {% if infobox.attributes -%}
<table class="table table-striped infobox_part"> <table class="table table-striped infobox_part">

10
searx/templates/oscar/languages.html
View File

@ -1,12 +1,8 @@
{% if preferences -%} <select class="language custom-select form-control" id="language" name="language" accesskey="l">
<select class="custom-select form-control" name='language'>
{%- else -%}
<select class="time_range custom-select form-control" id='language' name='language'>
{%- endif -%}
<option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option> <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
{%- for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) -%} {%- for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) -%}
<option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}> <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>
{{- lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id -}} {{- lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id -}}
</option> </option>
{%- endfor -%} {%- endfor -%}
</select> </select>

124
searx/templates/oscar/results.html
View File

@ -15,7 +15,68 @@
{% include 'oscar/search.html' %} {% include 'oscar/search.html' %}
<div class="row"> <div class="row">
<div class="col-sm-8" id="main_results"> <div class="col-sm-4 col-sm-push-8" id="sidebar_results">
{% if number_of_results != '0' -%}
<p><small>{{ _('Number of results') }}: {{ number_of_results }}</small></p>
{%- endif %}
{% if unresponsive_engines and results|length >= 1 -%}
<div class="alert alert-danger fade in" role="alert">
<p>{{ _('Engines cannot retrieve results') }}:</p>
{%- for engine_name, error_type in unresponsive_engines -%}
{{- engine_name }} ({{ error_type }}){% if not loop.last %}, {% endif %}{{- "" -}}
{%- endfor -%}
</div>
{%- endif %}
{% if infoboxes -%}
{% for infobox in infoboxes %}
{% include 'oscar/infobox.html' %}{{- "\n\n" -}}
{% endfor %}
{%- endif %}
{% if suggestions %}
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">{{ _('Suggestions') }}</h4>
</div>
<div class="panel-body">
{% for suggestion in suggestions %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" role="navigation" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} suggestion_item">
<input type="hidden" name="q" value="{{ suggestion.url }}">
<button type="submit" class="btn btn-default btn-xs">{{ suggestion.title }}</button>
</form>
{% endfor %}
</div>
</div>
{%- endif %}
<div class="panel panel-default">
<div class="panel-heading">{{- "" -}}
<h4 class="panel-title">{{ _('Links') }}</h4>{{- "" -}}
</div>
<div class="panel-body">
<form role="form">{{- "" -}}
<div class="form-group">{{- "" -}}
<label for="search_url">{{ _('Search URL') }}</label>{{- "" -}}
<input id="search_url" type="url" class="form-control select-all-on-click cursor-text" name="search_url" value="{{ search_url() }}" readonly>{{- "" -}}
</div>{{- "" -}}
</form>
<label>{{ _('Download results') }}</label>
<div class="clearfix"></div>
{% for output_type in ('csv', 'json', 'rss') %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} result_download">
{{- search_form_attrs(pageno) -}}
<input type="hidden" name="format" value="{{ output_type }}">{{- "" -}}
<button type="submit" class="btn btn-default">{{ output_type }}</button>{{- "" -}}
</form>
{% endfor %}
<div class="clearfix"></div>
</div>
</div>
</div><!-- /#sidebar_results -->
<div class="col-sm-8 col-sm-pull-4" id="main_results">
<h1 class="sr-only">{{ _('Search results') }}</h1> <h1 class="sr-only">{{ _('Search results') }}</h1>
{% if corrections -%} {% if corrections -%}
@ -91,66 +152,5 @@
{% endif %} {% endif %}
{% endif %} {% endif %}
</div><!-- /#main_results --> </div><!-- /#main_results -->
<div class="col-sm-4" id="sidebar_results">
{% if number_of_results != '0' -%}
<p><small>{{ _('Number of results') }}: {{ number_of_results }}</small></p>
{%- endif %}
{% if unresponsive_engines and results|length >= 1 -%}
<div class="alert alert-danger fade in" role="alert">
<p>{{ _('Engines cannot retrieve results') }}:</p>
{%- for engine_name, error_type in unresponsive_engines -%}AAAA
{{- engine_name }} ({{ error_type }}){% if not loop.last %}, {% endif %}{{- "" -}}
{%- endfor -%}
</div>
{%- endif %}
{% if infoboxes -%}
{% for infobox in infoboxes %}
{% include 'oscar/infobox.html' %}{{- "\n\n" -}}
{% endfor %}
{%- endif %}
{% if suggestions %}
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">{{ _('Suggestions') }}</h4>
</div>
<div class="panel-body">
{% for suggestion in suggestions %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" role="navigation" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} suggestion_item">
<input type="hidden" name="q" value="{{ suggestion.url }}">
<button type="submit" class="btn btn-default btn-xs">{{ suggestion.title }}</button>
</form>
{% endfor %}
</div>
</div>
{%- endif %}
<div class="panel panel-default">
<div class="panel-heading">{{- "" -}}
<h4 class="panel-title">{{ _('Links') }}</h4>{{- "" -}}
</div>
<div class="panel-body">
<form role="form">{{- "" -}}
<div class="form-group">{{- "" -}}
<label for="search_url">{{ _('Search URL') }}</label>{{- "" -}}
<input id="search_url" type="url" class="form-control select-all-on-click cursor-text" name="search_url" value="{{ search_url() }}" readonly>{{- "" -}}
</div>{{- "" -}}
</form>
<label>{{ _('Download results') }}</label>
<div class="clearfix"></div>
{% for output_type in ('csv', 'json', 'rss') %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} result_download">
{{- search_form_attrs(pageno) -}}
<input type="hidden" name="format" value="{{ output_type }}">{{- "" -}}
<button type="submit" class="btn btn-default">{{ output_type }}</button>{{- "" -}}
</form>
{% endfor %}
<div class="clearfix"></div>
</div>
</div>
</div><!-- /#sidebar_results -->
</div> </div>
{% endblock %} {% endblock %}

2
searx/templates/oscar/search.html
View File

@ -3,7 +3,7 @@
<div class="row"> <div class="row">
<div class="col-xs-12 col-md-8"> <div class="col-xs-12 col-md-8">
<div class="input-group search-margin"> <div class="input-group search-margin">
<input type="search" name="q" class="form-control" id="q" placeholder="{{ _('Search for...') }}" aria-label="{{ _('Search for...') }}" autocomplete="off" value="{{ q }}"> <input type="search" name="q" class="form-control" id="q" placeholder="{{ _('Search for...') }}" aria-label="{{ _('Search for...') }}" autocomplete="off" value="{{ q }}" accesskey="s">
<span class="input-group-btn"> <span class="input-group-btn">
<button type="submit" class="btn btn-default" aria-label="{{ _('Start search') }}"><span class="hide_if_nojs">{{ icon('search') }}</span><span class="hidden active_if_nojs">{{ _('Start search') }}</span></button> <button type="submit" class="btn btn-default" aria-label="{{ _('Start search') }}"><span class="hide_if_nojs">{{ icon('search') }}</span><span class="hidden active_if_nojs">{{ _('Start search') }}</span></button>
</span> </span>

2
searx/templates/oscar/search_full.html
View File

@ -6,7 +6,7 @@
{% else %} {% else %}
<div class="input-group col-md-8 col-md-offset-2"> <div class="input-group col-md-8 col-md-offset-2">
{% endif %} {% endif %}
<input type="search" name="q" class="form-control input-lg autofocus" id="q" placeholder="{{ _('Search for...') }}" aria-label="{{ _('Search for...') }}" autocomplete="off" value="{{ q }}"> <input type="search" name="q" class="form-control input-lg autofocus" id="q" placeholder="{{ _('Search for...') }}" aria-label="{{ _('Search for...') }}" autocomplete="off" value="{{ q }}" accesskey="s">
<span class="input-group-btn"> <span class="input-group-btn">
<button type="submit" class="btn btn-default input-lg" aria-label="{{ _('Start search') }}"><span class="hide_if_nojs">{{ icon('search') }}</span><span class="hidden active_if_nojs">{{ _('Start search') }}</span></button> <button type="submit" class="btn btn-default input-lg" aria-label="{{ _('Start search') }}"><span class="hide_if_nojs">{{ icon('search') }}</span><span class="hidden active_if_nojs">{{ _('Start search') }}</span></button>
</span> </span>

2
searx/templates/oscar/time-range.html
View File

@ -1,4 +1,4 @@
<select name="time_range" id="time-range" class="custom-select form-control">{{- "" -}} <select name="time_range" id="time-range" class="custom-select form-control" accesskey="t">{{- "" -}}
<option id="time-range-anytime" value="" {{ "selected" if time_range=="" or not time_range else ""}}> <option id="time-range-anytime" value="" {{ "selected" if time_range=="" or not time_range else ""}}>
{{- _('Anytime') -}} {{- _('Anytime') -}}
</option>{{- "" -}} </option>{{- "" -}}

4
searx/webapp.py
View File

@ -606,11 +606,11 @@ def index():
# HTML output format # HTML output format
# suggestions: use RawTextQuery to get the suggestion URLs with the same bang # suggestions: use RawTextQuery to get the suggestion URLs with the same bang
suggestion_urls = map(lambda suggestion: { suggestion_urls = list(map(lambda suggestion: {
'url': raw_text_query.changeSearchQuery(suggestion).getFullQuery(), 'url': raw_text_query.changeSearchQuery(suggestion).getFullQuery(),
'title': suggestion 'title': suggestion
}, },
result_container.suggestions) result_container.suggestions))
correction_urls = list(map(lambda correction: { correction_urls = list(map(lambda correction: {
'url': raw_text_query.changeSearchQuery(correction).getFullQuery(), 'url': raw_text_query.changeSearchQuery(correction).getFullQuery(),

0
tests/unit/engines/__init__.py
View File

37
tests/unit/engines/pubmed.py
View File

@ -1,37 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
from searx.engines import pubmed
from searx.testing import SearxTestCase
class TestPubmedEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
params = pubmed.request(query, dicto)
self.assertIn('url', params)
self.assertIn('eutils.ncbi.nlm.nih.gov/', params['url'])
self.assertIn('term', params['url'])
def test_response(self):
self.assertRaises(AttributeError, pubmed.response, None)
self.assertRaises(AttributeError, pubmed.response, [])
self.assertRaises(AttributeError, pubmed.response, '')
self.assertRaises(AttributeError, pubmed.response, '[]')
response = mock.Mock(text='<PubmedArticleSet></PubmedArticleSet>')
self.assertEqual(pubmed.response(response), [])
xml_mock = """<eSearchResult><Count>1</Count><RetMax>1</RetMax><RetStart>0</RetStart><IdList>
<Id>1</Id>
</IdList></eSearchResult>
"""
response = mock.Mock(text=xml_mock.encode('utf-8'))
results = pubmed.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['content'], 'No abstract is available for this publication.')

110
tests/unit/engines/seedpeer_fixture.html
View File

@ -1,110 +0,0 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
</head>
<body>
<div id="header">
<div id="whoIsYou">
<a href="/lang.php"><small>SeedPeer in your own language?</small></a>&nbsp;&nbsp;&nbsp;<a href="http://www.seedpeer.eu"><img src="/images/flags/uk.gif" width="16px" alt="Torrents EN" /></a> <a href="http://spanish.seedpeer.eu"><img src="/images/flags/es.gif" width="16px" alt="Torrents ES" /></a> <a href="http://german.seedpeer.eu"><img src="/images/flags/de.gif" width="16px" alt="Torrents DE" /></a> <a href="http://french.seedpeer.eu"><img src="/images/flags/fr.gif" width="16px" alt="Torrents FR" /></a> <a href="http://portuguese.seedpeer.eu"><img src="/images/flags/pt.gif" width="16px" alt="Torrents Portuguese" /></a> <a href="http://swedish.seedpeer.eu"><img src="/images/flags/se.gif" width="16px" alt="Torrents Sweden" /></a>
</div>
<script type="text/javascript">
whoIsYou();
</script>
<div id="search">
<form action="/search.php" method="get">
<input id="topsearchbar" name="search" value="narcos season 2" />
<input type="submit" class="searchbutton" value="Torrents" />
<input style="color:#000" type="submit" class="searchbutton" name="usenet" value="Usenet Binaries" />
</form>
<div id="suggestion"></div>
</div>
<div id="logo"><a href="/"><img src="/images/logo2.gif" alt="Seedpeer homepage" width="415" height="143" /></a></div>
<div id="subtext"><a href="/">Home</a> &gt; <a href="/search.html">Torrent search</a> &gt; Narcos season 2 | page 1</div>
</div>
<div id="nav">
<ul>
<!--
<li><font style="color:red;font-size:9px;font-weight:bold;">NEW</font><a title="Download TOP Games for FREE" rel="nofollow" href="http://www.bigrebelads.com/affiliate/index?ref=9301" target="_blank">FREE Games</a></li>
-->
<li style="border-left:none" id="categories"><a title="Browse Torrent Categories" href="/browse.html">Categories</a>
<ul>
<li><a title="Browse Anime Torrents" href="/browse.html#6">Anime</a></li>
<li><a title="Browse Game Torrents" href="/browse.html#4">Games</a></li>
<li><a title="Browse Movie Torrents" href="/browse.html#1">Movies</a></li>
<li><a title="Browse Music Torrents" href="/browse.html#3">Music</a></li>
<li><a title="Browse Software Torrents" href="/browse.html#5">Software</a></li>
<li><a title="Browse TV Torrents" href="/browse.html#2">TV Shows</a></li>
<li><a title="Browse Other Torrents" href="/browse.html#7">Others</a></li>
</ul>
</li>
<li><a title="Upload A Torrents" href="/upload.html">Upload torrent</a></li>
<li id="verified"><a title="Verified Torrents" href="/verified.html">Verified</a></li>
<li id="searchoptions"><a title="Search Torrents" href="/search.html">Torrent search</a></li>
<li id="newsgroups"><a style="color:#212b3e" title="News Groups" href="/usenet.html">Usenet Binaries</a></li>
<li id="about" style="border-right:none"><a rel="nofollow" href="/faq.html">About Us</a>
<ul>
<li><a title="SeedPeer Statistics" href="/stats.html">Statistics</a></li>
<li><a title="Contact Us" href="/contact.html">Contact</a></li>
<li><a title="Frequently Asked Questions" href="/faq.html">FAQ</a></li>
<li><a title="SeedPeer API" href="http://api.seedpeer.eu">Our API</a></li>
<li><a title="SeedPeer Blog" href="/blog">Blog</a></li>
</ul>
</li>
<!--<li><a href="/toolbar.php">Our Toolbar</a></li>-->
</ul>
<div class="clear"></div>
</div>
<div id="body"><div id="pageTop"></div>
<div id="headerbox"><h1>Verified <font class="colored">Narcos season 2</font> torrents</h1></div><table width="100%"><tr><th>
<span style="float:right">
<a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> |
<a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
</span>
<a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th><td class="tableAd" rowspan="6"><iframe src="http://creative.wwwpromoter.com/13689?d=300x250" width="300" height="250" style="border: none;" frameborder="0" scrolling="no"></iframe></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_1" href="" data-tad="431726" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Full Version</a></td><td class="right">20 hours</td><td class="right">681.3 MB</td><td class="right"><font color="green">28</font> </td><td class="right"><font color="navy">654</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_2" href="" data-tad="431727" data-url="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Trusted Source</a></td><td class="right">12 hours</td><td class="right">787.1 MB</td><td class="right"><font color="green">64</font> </td><td class="right"><font color="navy">220</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_3" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Full Narcos season 2 Download</strong></a> <small><a class="pblink" id="pblink_table_item_4" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Usenet</a></small></td><td class="right">24 hours</td><td class="right">775.5 MB</td><td class="right"><font color="green">60</font> </td><td class="right"><font color="navy">236</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_5" href="" data-tad="431730" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014 - DIRECT STREAMING</a> <small><a class="pblink" id="pblink_table_item_6" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">17 hours</td><td class="right">654.1 MB</td><td class="right"><font color="green">2</font> </td><td class="right"><font color="navy">391</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_7" href="" data-tad="431731" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014</a> <small><a class="pblink" id="pblink_table_item_8" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">20 hours</td><td class="right">754.5 MB</td><td class="right"><font color="green">21</font> </td><td class="right"><font color="navy">919</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr></table><br /><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=728x90' width='728' height='90' style='border: none;' frameborder='0' scrolling='no'></iframe><center><span style="float:right;margin:1em .2em 0 0"><a title="Download at the speed of your connection" href="/usenet.php?search=narcos+season+2"><img src="/images/dlf.gif" alt="Search Binaries" /></a></span><div style="margin-bottom:1em;margin-right:290px" id="headerbox"><h1><a href="/searchfeed/narcos+season+2.xml" target="_blank" title="SeedPeer RSS Torrent Search Feed fornarcos season 2"><img src="/images/feedIcon.png" border="0" /></a>&nbsp;2 <font class="colored">Narcos season 2</font> Torrents were found</h1></div><table width="100%"><tr><th>
<span style="float:right">
<a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> |
<a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
</span>
<a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th></tr><tr class=""><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong>%20 torrent&u=http://seedpeer.seedpeer.eu/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong> <small><a href="/browse.html#11686840"></a></small></a></td><td class="right">19 hours</td><td class="right">4.39 GB</td><td class="right"><font color="green">715</font> </td><td class="right"><font color="navy">183</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr><tr class="tdark"><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG%20 torrent&u=http://seedpeer.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG <small><a href="/browse.html#11685972"></a></small></a></td><td class="right">1 day</td><td class="right">2.48 GB</td><td class="right"><font color="green">861</font> </td><td class="right"><font color="navy">332</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr></table><div id="headerbox"><h1>Related searches for: <font class="colored">Narcos season 2</font></h1></div><div id="search_suggestions"><br />Other suggested searches: </div><br /><a href="http://torrentz2.eu/search?f=narcos-season-2">Search for "narcos-season-2" on Torrentz2.eu</a><br /><a href="http://torrent-finder.info/show.php?q=narcos-season-2">Search for "narcos-season-2" on Torrent-Finder</a><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe>&nbsp;<iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe>&nbsp;<iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe></center><div id="footer">
<table width="100%">
<tr>
<td width="30%">
<h2>Torrents Download</h2>
<a href="/">Torrent search</a><br />
<a href="/browse.html">Browse categories</a><br />
<a href="/verified.html">Verified Torrents</a><br />
<a href="/order-date.html">Today's torrents</a><br />
<a href="/yesterday.html">Yesterday's torrents</a><br />
<a href="/stats.html">Statistics</a><br />
<br />
<a href="/faq.html#copyright"><strong>Copyright & Removal</strong></a>
</td>
<td width="30%"><h2>Cool Stuff</h2>
<a href="/promotional.php">Promotional</a><br />
<a href="/contact.html">Advertising Information</a><br />
<strong><a href="/plugins.php" title="Add a search plugin to Firefox or Internet Explorer">Search Plugin <span style="color:red">*</span></a></strong><br />
<a href="http://www.utorrent.com">&micro;Torrent Client</a><br />
<a href="/blog">Seedpeer Blog</a><br />
</td>
<td width="30%"><h2>Links</h2>
<a href="http://www.sumotorrent.com" target="_blank"><strong>SumoTorrent</strong></a><br />
<a href="http://www.torrent-finder.info" target="_blank"><strong>Torrent Finder</strong></a><br />
<a href="http://www.torrentpond.com" target="_blank"><strong>TorrentPond</strong></a><br />
<a href="https://www.limetorrents.cc" target="_blank">LimeTorrents.cc</a><br />
<a href="http://www.torrents.to/" target="_blank">Torrents.to</a><br />
<a href="http://www.torrentfunk.com" target="_blank">TorrentFunk</a><br />
<a href="https://monova.org" target="_blank">Monova</a><br />
<a href="http://www.torrentroom.com" target="_blank">TorrentRoom</a><br />
<a href="http://www.katcr.co/" target="_blank">Kickass Torrents Community</a><br />
</td>
<td width="10%"><div id="bottomlogo"></div></td>
</tr>
</table>
<br />
<br />
</div>
</div>
</body>
</html>

78
tests/unit/engines/test_acgsou.py
View File

@ -1,78 +0,0 @@
# coding=utf-8
from collections import defaultdict
import mock
from searx.engines import acgsou
from searx.testing import SearxTestCase
class TestAcgsouEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dic = defaultdict(dict)
dic['pageno'] = 1
params = acgsou.request(query, dic)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('acgsou.com' in params['url'])
def test_response(self):
resp = mock.Mock(text='<html></html>')
self.assertEqual(acgsou.response(resp), [])
html = u"""
<html>
<table id="listTable" class="list_style table_fixed">
<thead class="tcat">
<tr>
<th axis="string" class="l1 tableHeaderOver">test</th>
<th axis="string" class="l2 tableHeaderOver">test</th>
<th axis="string" class="l3 tableHeaderOver">test</th>
<th axis="size" class="l4 tableHeaderOver">test</th>
<th axis="number" class="l5 tableHeaderOver">test</th>
<th axis="number" class="l6 tableHeaderOver">test</th>
<th axis="number" class="l7 tableHeaderOver">test</th>
<th axis="string" class="l8 tableHeaderOver">test</th>
</tr>
</thead>
<tbody class="tbody" id="data_list">
<tr class="alt1 ">
<td nowrap="nowrap">date</td>
<td><a href="category.html">testcategory テスト</a></td>
<td style="text-align:left;">
<a href="show-torrentid.html" target="_blank">torrentname テスト</a>
</td>
<td>1MB</td>
<td nowrap="nowrap">
<span class="bts_1">
29
</span>
</td>
<td nowrap="nowrap">
<span class="btl_1">
211
</span>
</td>
<td nowrap="nowrap">
<span class="btc_">
168
</span>
</td>
<td><a href="random.html">user</a></td>
</tr>
</tbody>
</table>
</html>
"""
resp = mock.Mock(text=html)
results = acgsou.response(resp)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
r = results[0]
self.assertEqual(r['url'], 'http://www.acgsou.com/show-torrentid.html')
self.assertEqual(r['content'], u'Category: "testcategory テスト".')
self.assertEqual(r['title'], u'torrentname テスト')
self.assertEqual(r['filesize'], 1048576)

111
tests/unit/engines/test_archlinux.py
View File

@ -1,111 +0,0 @@
from collections import defaultdict
import mock
from searx.engines import archlinux
from searx.testing import SearxTestCase
domains = {
'all': 'https://wiki.archlinux.org',
'de': 'https://wiki.archlinux.de',
'fr': 'https://wiki.archlinux.fr',
'ja': 'https://wiki.archlinuxjp.org',
'ro': 'http://wiki.archlinux.ro',
'tr': 'http://archtr.org/wiki'
}
class TestArchLinuxEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dic = defaultdict(dict)
dic['pageno'] = 1
dic['language'] = 'en-US'
params = archlinux.request(query, dic)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('wiki.archlinux.org' in params['url'])
for lang, name in archlinux.main_langs:
dic['language'] = lang
params = archlinux.request(query, dic)
self.assertTrue(name in params['url'])
for lang, domain in domains.items():
dic['language'] = lang
params = archlinux.request(query, dic)
self.assertTrue(domain in params['url'])
def test_response(self):
response = mock.Mock(text='<html></html>',
search_params={'language': 'en_US'})
self.assertEqual(archlinux.response(response), [])
html = """
<ul class="mw-search-results">
<li>
<div class="mw-search-result-heading">
<a href="/index.php/ATI" title="ATI">ATI</a>
</div>
<div class="searchresult">
Lorem ipsum dolor sit amet
</div>
<div class="mw-search-result-data">
30 KB (4,630 words) - 19:04, 17 March 2016</div>
</li>
<li>
<div class="mw-search-result-heading">
<a href="/index.php/Frequently_asked_questions" title="Frequently asked questions">
Frequently asked questions
</a>
</div>
<div class="searchresult">
CPUs with AMDs instruction set "AMD64"
</div>
<div class="mw-search-result-data">
17 KB (2,722 words) - 20:13, 21 March 2016
</div>
</li>
<li>
<div class="mw-search-result-heading">
<a href="/index.php/CPU_frequency_scaling" title="CPU frequency scaling">CPU frequency scaling</a>
</div>
<div class="searchresult">
ondemand for AMD and older Intel CPU
</div>
<div class="mw-search-result-data">
15 KB (2,319 words) - 23:46, 16 March 2016
</div>
</li>
</ul>
"""
expected = [
{
'title': 'ATI',
'url': 'https://wiki.archlinux.org/index.php/ATI'
},
{
'title': 'Frequently asked questions',
'url': 'https://wiki.archlinux.org/index.php/Frequently_asked_questions'
},
{
'title': 'CPU frequency scaling',
'url': 'https://wiki.archlinux.org/index.php/CPU_frequency_scaling'
}
]
response = mock.Mock(text=html)
response.search_params = {
'language': 'en_US'
}
results = archlinux.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), len(expected))
i = 0
for exp in expected:
res = results[i]
i += 1
for key, value in exp.items():
self.assertEqual(res[key], value)

58
tests/unit/engines/test_arxiv.py
View File

@ -1,58 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
from searx.engines import arxiv
from searx.testing import SearxTestCase
class TestBaseEngine(SearxTestCase):
def test_request(self):
query = 'test_query'.encode('utf-8')
dicto = defaultdict(dict)
dicto['pageno'] = 1
params = arxiv.request(query, dicto)
self.assertIn('url', params)
self.assertIn('export.arxiv.org/api/', params['url'])
def test_response(self):
self.assertRaises(AttributeError, arxiv.response, None)
self.assertRaises(AttributeError, arxiv.response, [])
self.assertRaises(AttributeError, arxiv.response, '')
self.assertRaises(AttributeError, arxiv.response, '[]')
response = mock.Mock(content=b'''<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom"></feed>''')
self.assertEqual(arxiv.response(response), [])
xml_mock = b'''<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title type="html">ArXiv Query: search_query=all:test_query&amp;id_list=&amp;start=0&amp;max_results=1</title>
<id>http://arxiv.org/api/1</id>
<updated>2000-01-21T00:00:00-01:00</updated>
<opensearch:totalResults xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/">1</opensearch:totalResults>
<opensearch:startIndex xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/">0</opensearch:startIndex>
<opensearch:itemsPerPage xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/">1</opensearch:itemsPerPage>
<entry>
<id>http://arxiv.org/1</id>
<updated>2000-01-01T00:00:01Z</updated>
<published>2000-01-01T00:00:01Z</published>
<title>Mathematical proof.</title>
<summary>Mathematical formula.</summary>
<author>
<name>A. B.</name>
</author>
<link href="http://arxiv.org/1" rel="alternate" type="text/html"/>
<link title="pdf" href="http://arxiv.org/1" rel="related" type="application/pdf"/>
<category term="math.QA" scheme="http://arxiv.org/schemas/atom"/>
<category term="1" scheme="http://arxiv.org/schemas/atom"/>
</entry>
</feed>
'''
response = mock.Mock(content=xml_mock)
results = arxiv.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Mathematical proof.')
self.assertEqual(results[0]['content'], 'Mathematical formula.')

91
tests/unit/engines/test_base.py
View File

@ -1,91 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
from searx.engines import base
from searx.testing import SearxTestCase
class TestBaseEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
params = base.request(query, dicto)
self.assertIn('url', params)
self.assertIn('base-search.net', params['url'])
def test_response(self):
self.assertRaises(AttributeError, base.response, None)
self.assertRaises(AttributeError, base.response, [])
self.assertRaises(AttributeError, base.response, '')
self.assertRaises(AttributeError, base.response, '[]')
response = mock.Mock(content=b'<response></response>')
self.assertEqual(base.response(response), [])
xml_mock = b"""<?xml version="1.0"?>
<response>
<lst name="responseHeader">
<int name="status">0</int>
<int name="QTime">1</int>
</lst>
<result name="response" numFound="1" start="0">
<doc>
<date name="dchdate">2000-01-01T01:01:01Z</date>
<str name="dcdocid">1</str>
<str name="dccontinent">cna</str>
<str name="dccountry">us</str>
<str name="dccollection">ftciteseerx</str>
<str name="dcprovider">CiteSeerX</str>
<str name="dctitle">Science and more</str>
<arr name="dccreator">
<str>Someone</str>
</arr>
<arr name="dcperson">
<str>Someone</str>
</arr>
<arr name="dcsubject">
<str>Science and more</str>
</arr>
<str name="dcdescription">Science, and even more.</str>
<arr name="dccontributor">
<str>The neighbour</str>
</arr>
<str name="dcdate">2001</str>
<int name="dcyear">2001</int>
<arr name="dctype">
<str>text</str>
</arr>
<arr name="dctypenorm">
<str>1</str>
</arr>
<arr name="dcformat">
<str>application/pdf</str>
</arr>
<arr name="dccontenttype">
<str>application/pdf</str>
</arr>
<arr name="dcidentifier">
<str>http://example.org/</str>
</arr>
<str name="dclink">http://example.org</str>
<str name="dcsource">http://example.org</str>
<arr name="dclanguage">
<str>en</str>
</arr>
<str name="dcrights">Under the example.org licence</str>
<int name="dcoa">1</int>
<arr name="dclang">
<str>eng</str>
</arr>
</doc>
</result>
</response>"""
response = mock.Mock(content=xml_mock)
results = base.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Science and more')
self.assertEqual(results[0]['content'], 'Science, and even more.')

178
tests/unit/engines/test_bing.py
View File

@ -1,178 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
from searx.engines import bing
from searx.testing import SearxTestCase
class TestBingEngine(SearxTestCase):
def test_request(self):
bing.supported_languages = ['en', 'fr', 'zh-CHS', 'zh-CHT', 'pt-PT', 'pt-BR']
query = u'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
dicto['language'] = 'fr-FR'
params = bing.request(query.encode('utf-8'), dicto)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('language%3AFR' in params['url'])
self.assertTrue('bing.com' in params['url'])
dicto['language'] = 'all'
params = bing.request(query.encode('utf-8'), dicto)
self.assertTrue('language' in params['url'])
def test_response(self):
dicto = defaultdict(dict)
dicto['pageno'] = 1
dicto['language'] = 'fr-FR'
self.assertRaises(AttributeError, bing.response, None)
self.assertRaises(AttributeError, bing.response, [])
self.assertRaises(AttributeError, bing.response, '')
self.assertRaises(AttributeError, bing.response, '[]')
response = mock.Mock(text='<html></html>')
response.search_params = dicto
self.assertEqual(bing.response(response), [])
response = mock.Mock(text='<html></html>')
response.search_params = dicto
self.assertEqual(bing.response(response), [])
html = """
<div>
<div id="b_tween">
<span class="sb_count" data-bm="4">23 900 000 résultats</span>
</div>
<ol id="b_results" role="main">
<div class="sa_cc" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
<div Class="sa_mc">
<div class="sb_tlst">
<h3>
<a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">
<strong>This</strong> should be the title</a>
</h3>
</div>
<div class="sb_meta"><cite><strong>this</strong>.meta.com</cite>
<span class="c_tlbxTrg">
<span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">
</span>
</span>
</div>
<p><strong>This</strong> should be the content.</p>
</div>
</div>
</ol>
</div>
"""
response = mock.Mock(text=html)
response.search_params = dicto
results = bing.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 2)
self.assertEqual(results[0]['title'], 'This should be the title')
self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
self.assertEqual(results[0]['content'], 'This should be the content.')
self.assertEqual(results[-1]['number_of_results'], 23900000)
html = """
<div>
<div id="b_tween">
<span class="sb_count" data-bm="4">9-18 résultats sur 23 900 000</span>
</div>
<ol id="b_results" role="main">
<li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
<div Class="sa_mc">
<div class="sb_tlst">
<h2>
<a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">
<strong>This</strong> should be the title</a>
</h2>
</div>
<div class="sb_meta"><cite><strong>this</strong>.meta.com</cite>
<span class="c_tlbxTrg">
<span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">
</span>
</span>
</div>
<p><strong>This</strong> should be the content.</p>
</div>
</li>
</ol>
</div>
"""
dicto['pageno'] = 2
response = mock.Mock(text=html)
response.search_params = dicto
results = bing.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 2)
self.assertEqual(results[0]['title'], 'This should be the title')
self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
self.assertEqual(results[0]['content'], 'This should be the content.')
self.assertEqual(results[-1]['number_of_results'], 23900000)
html = """
<div>
<div id="b_tween">
<span class="sb_count" data-bm="4">23 900 000 résultats</span>
</div>
<ol id="b_results" role="main">
<li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
<div Class="sa_mc">
<div class="sb_tlst">
<h2>
<a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">
<strong>This</strong> should be the title</a>
</h2>
</div>
<div class="sb_meta"><cite><strong>this</strong>.meta.com</cite>
<span class="c_tlbxTrg">
<span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">
</span>
</span>
</div>
<p><strong>This</strong> should be the content.</p>
</div>
</li>
</ol>
</div>
"""
dicto['pageno'] = 33900000
response = mock.Mock(text=html)
response.search_params = dicto
results = bing.response(response)
self.assertEqual(bing.response(response), [])
def test_fetch_supported_languages(self):
html = """<html></html>"""
response = mock.Mock(text=html)
results = bing._fetch_supported_languages(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
html = """
<html>
<body>
<form>
<div id="limit-languages">
<div>
<div><input id="es" value="es"></input></div>
</div>
<div>
<div><input id="pt_BR" value="pt_BR"></input></div>
<div><input id="pt_PT" value="pt_PT"></input></div>
</div>
</div>
</form>
</body>
</html>
"""
response = mock.Mock(text=html)
languages = bing._fetch_supported_languages(response)
self.assertEqual(type(languages), list)
self.assertEqual(len(languages), 3)
self.assertIn('es', languages)
self.assertIn('pt-BR', languages)
self.assertIn('pt-PT', languages)

132
tests/unit/engines/test_bing_images.py
View File

@ -1,132 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
from searx.engines import bing_images
from searx.testing import SearxTestCase
class TestBingImagesEngine(SearxTestCase):
def test_request(self):
bing_images.supported_languages = ['fr-FR', 'en-US']
bing_images.language_aliases = {}
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
dicto['language'] = 'fr-FR'
dicto['safesearch'] = 1
dicto['time_range'] = ''
params = bing_images.request(query, dicto)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('bing.com' in params['url'])
self.assertTrue('SRCHHPGUSR' in params['cookies'])
self.assertTrue('DEMOTE' in params['cookies']['SRCHHPGUSR'])
self.assertTrue('_EDGE_S' in params['cookies'])
self.assertTrue('fr-fr' in params['cookies']['_EDGE_S'])
dicto['language'] = 'fr'
params = bing_images.request(query, dicto)
self.assertTrue('_EDGE_S' in params['cookies'])
self.assertTrue('fr-fr' in params['cookies']['_EDGE_S'])
dicto['language'] = 'all'
params = bing_images.request(query, dicto)
self.assertTrue('_EDGE_S' in params['cookies'])
self.assertTrue('en-us' in params['cookies']['_EDGE_S'])
def test_response(self):
self.assertRaises(AttributeError, bing_images.response, None)
self.assertRaises(AttributeError, bing_images.response, [])
self.assertRaises(AttributeError, bing_images.response, '')
self.assertRaises(AttributeError, bing_images.response, '[]')
response = mock.Mock(text='<html></html>')
self.assertEqual(bing_images.response(response), [])
response = mock.Mock(text='<html></html>')
self.assertEqual(bing_images.response(response), [])
html = """
<div id="mmComponent_images_1">
<ul>
<li>
<div>
<div class="imgpt">
<a m='{"purl":"page_url","murl":"img_url","turl":"thumb_url","t":"Page 1 title"}'>
<img src="" alt="alt text" />
</a>
<div class="img_info">
<span>1 x 1 - jpeg</span>
<a>1.example.org</a>
</div>
</div>
<div></div>
</div>
<div>
<div class="imgpt">
<a m='{"purl":"page_url2","murl":"img_url2","turl":"thumb_url2","t":"Page 2 title"}'>
<img src="" alt="alt text 2" />
</a>
<div class="img_info">
<span>2 x 2 - jpeg</span>
<a>2.example.org</a>
</div>
</div>
</div>
</li>
</ul>
<ul>
<li>
<div>
<div class="imgpt">
<a m='{"purl":"page_url3","murl":"img_url3","turl":"thumb_url3","t":"Page 3 title"}'>
<img src="" alt="alt text 3" />
</a>
<div class="img_info">
<span>3 x 3 - jpeg</span>
<a>3.example.org</a>
</div>
</div>
</div>
</li>
</ul>
</div>
"""
html = html.replace('\r\n', '').replace('\n', '').replace('\r', '')
response = mock.Mock(text=html)
results = bing_images.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 3)
self.assertEqual(results[0]['title'], 'Page 1 title')
self.assertEqual(results[0]['url'], 'page_url')
self.assertEqual(results[0]['content'], '')
self.assertEqual(results[0]['thumbnail_src'], 'thumb_url')
self.assertEqual(results[0]['img_src'], 'img_url')
self.assertEqual(results[0]['img_format'], '1 x 1 - jpeg')
self.assertEqual(results[0]['source'], '1.example.org')
def test_fetch_supported_languages(self):
html = """
<div>
<div id="region-section-content">
<ul class="b_vList">
<li>
<a href="https://bing...&setmkt=de-DE&s...">Germany</a>
<a href="https://bing...&setmkt=nb-NO&s...">Norway</a>
</li>
</ul>
<ul class="b_vList">
<li>
<a href="https://bing...&setmkt=es-AR&s...">Argentina</a>
</li>
</ul>
</div>
</div>
"""
response = mock.Mock(text=html)
languages = list(bing_images._fetch_supported_languages(response))
self.assertEqual(len(languages), 3)
self.assertIn('de-DE', languages)
self.assertIn('no-NO', languages)
self.assertIn('es-AR', languages)

147
tests/unit/engines/test_bing_news.py
View File

@ -1,147 +0,0 @@
from collections import defaultdict
import mock
from searx.engines import bing_news
from searx.testing import SearxTestCase
import lxml
class TestBingNewsEngine(SearxTestCase):
def test_request(self):
bing_news.supported_languages = ['en', 'fr']
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
dicto['language'] = 'fr-FR'
dicto['time_range'] = ''
params = bing_news.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
self.assertIn('bing.com', params['url'])
self.assertIn('fr', params['url'])
dicto['language'] = 'all'
params = bing_news.request(query, dicto)
self.assertIn('en', params['url'])
def test_no_url_in_request_year_time_range(self):
dicto = defaultdict(dict)
query = 'test_query'
dicto['time_range'] = 'year'
params = bing_news.request(query, dicto)
self.assertEqual({}, params['url'])
def test_response(self):
self.assertRaises(AttributeError, bing_news.response, None)
self.assertRaises(AttributeError, bing_news.response, [])
self.assertRaises(AttributeError, bing_news.response, '')
self.assertRaises(AttributeError, bing_news.response, '[]')
response = mock.Mock(content='<html></html>')
self.assertEqual(bing_news.response(response), [])
response = mock.Mock(content='<html></html>')
self.assertEqual(bing_news.response(response), [])
html = """<?xml version="1.0" encoding="utf-8" ?>
<rss version="2.0" xmlns:News="https://www.bing.com:443/news/search?q=python&amp;setmkt=en-US&amp;first=1&amp;format=RSS">
<channel>
<title>python - Bing News</title>
<link>https://www.bing.com:443/news/search?q=python&amp;setmkt=en-US&amp;first=1&amp;format=RSS</link>
<description>Search results</description>
<image>
<url>http://10.53.64.9/rsslogo.gif</url>
<title>test</title>
<link>https://www.bing.com:443/news/search?q=test&amp;setmkt=en-US&amp;first=1&amp;format=RSS</link>
</image>
<copyright>Copyright</copyright>
<item>
<title>Title</title>
<link>https://www.bing.com/news/apiclick.aspx?ref=FexRss&amp;aid=&amp;tid=c237eccc50bd4758b106a5e3c94fce09&amp;url=http%3a%2f%2furl.of.article%2f&amp;c=xxxxxxxxx&amp;mkt=en-us</link>
<description>Article Content</description>
<pubDate>Tue, 02 Jun 2015 13:37:00 GMT</pubDate>
<News:Source>Infoworld</News:Source>
<News:Image>http://a1.bing4.com/th?id=ON.13371337133713371337133713371337&amp;pid=News</News:Image>
<News:ImageSize>w={0}&amp;h={1}&amp;c=7</News:ImageSize>
<News:ImageKeepOriginalRatio></News:ImageKeepOriginalRatio>
<News:ImageMaxWidth>620</News:ImageMaxWidth>
<News:ImageMaxHeight>413</News:ImageMaxHeight>
</item>
<item>
<title>Another Title</title>
<link>https://www.bing.com/news/apiclick.aspx?ref=FexRss&amp;aid=&amp;tid=c237eccc50bd4758b106a5e3c94fce09&amp;url=http%3a%2f%2fanother.url.of.article%2f&amp;c=xxxxxxxxx&amp;mkt=en-us</link>
<description>Another Article Content</description>
<pubDate>Tue, 02 Jun 2015 13:37:00 GMT</pubDate>
</item>
</channel>
</rss>""" # noqa
response = mock.Mock(content=html.encode('utf-8'))
results = bing_news.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 2)
self.assertEqual(results[0]['title'], 'Title')
self.assertEqual(results[0]['url'], 'http://url.of.article/')
self.assertEqual(results[0]['content'], 'Article Content')
self.assertEqual(results[0]['img_src'], 'https://www.bing.com/th?id=ON.13371337133713371337133713371337')
self.assertEqual(results[1]['title'], 'Another Title')
self.assertEqual(results[1]['url'], 'http://another.url.of.article/')
self.assertEqual(results[1]['content'], 'Another Article Content')
self.assertNotIn('img_src', results[1])
html = """<?xml version="1.0" encoding="utf-8" ?>
<rss version="2.0" xmlns:News="https://www.bing.com:443/news/search?q=python&amp;setmkt=en-US&amp;first=1&amp;format=RSS">
<channel>
<title>python - Bing News</title>
<link>https://www.bing.com:443/news/search?q=python&amp;setmkt=en-US&amp;first=1&amp;format=RSS</link>
<description>Search results</description>
<image>
<url>http://10.53.64.9/rsslogo.gif</url>
<title>test</title>
<link>https://www.bing.com:443/news/search?q=test&amp;setmkt=en-US&amp;first=1&amp;format=RSS</link>
</image>
<copyright>Copyright</copyright>
<item>
<title>Title</title>
<link>http://another.url.of.article/</link>
<description>Article Content</description>
<pubDate>garbage</pubDate>
<News:Source>Infoworld</News:Source>
<News:Image>http://another.bing.com/image</News:Image>
<News:ImageSize>w={0}&amp;h={1}&amp;c=7</News:ImageSize>
<News:ImageKeepOriginalRatio></News:ImageKeepOriginalRatio>
<News:ImageMaxWidth>620</News:ImageMaxWidth>
<News:ImageMaxHeight>413</News:ImageMaxHeight>
</item>
</channel>
</rss>""" # noqa
response = mock.Mock(content=html.encode('utf-8'))
results = bing_news.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Title')
self.assertEqual(results[0]['url'], 'http://another.url.of.article/')
self.assertEqual(results[0]['content'], 'Article Content')
self.assertEqual(results[0]['img_src'], 'http://another.bing.com/image')
html = """<?xml version="1.0" encoding="utf-8" ?>
<rss version="2.0" xmlns:News="https://www.bing.com:443/news/search?q=python&amp;setmkt=en-US&amp;first=1&amp;format=RSS">
<channel>
<title>python - Bing News</title>
<link>https://www.bing.com:443/news/search?q=python&amp;setmkt=en-US&amp;first=1&amp;format=RSS</link>
<description>Search results</description>
<image>
<url>http://10.53.64.9/rsslogo.gif</url>
<title>test</title>
<link>https://www.bing.com:443/news/search?q=test&amp;setmkt=en-US&amp;first=1&amp;format=RSS</link>
</image>
</channel>
</rss>""" # noqa
response = mock.Mock(content=html.encode('utf-8'))
results = bing_news.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
html = """<?xml version="1.0" encoding="utf-8" ?>gabarge"""
response = mock.Mock(content=html.encode('utf-8'))
self.assertRaises(lxml.etree.XMLSyntaxError, bing_news.response, response)

72
tests/unit/engines/test_bing_videos.py
View File

@ -1,72 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
from searx.engines import bing_videos
from searx.testing import SearxTestCase
class TestBingVideosEngine(SearxTestCase):
def test_request(self):
bing_videos.supported_languages = ['fr-FR', 'en-US']
bing_videos.language_aliases = {}
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
dicto['language'] = 'fr-FR'
dicto['safesearch'] = 0
dicto['time_range'] = ''
params = bing_videos.request(query, dicto)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('bing.com' in params['url'])
self.assertTrue('SRCHHPGUSR' in params['cookies'])
self.assertTrue('OFF' in params['cookies']['SRCHHPGUSR'])
self.assertTrue('_EDGE_S' in params['cookies'])
self.assertTrue('fr-fr' in params['cookies']['_EDGE_S'])
dicto['pageno'] = 2
dicto['time_range'] = 'day'
dicto['safesearch'] = 2
params = bing_videos.request(query, dicto)
self.assertTrue('first=29' in params['url'])
self.assertTrue('1440' in params['url'])
self.assertIn('SRCHHPGUSR', params['cookies'])
self.assertTrue('STRICT' in params['cookies']['SRCHHPGUSR'])
def test_response(self):
self.assertRaises(AttributeError, bing_videos.response, None)
self.assertRaises(AttributeError, bing_videos.response, [])
self.assertRaises(AttributeError, bing_videos.response, '')
self.assertRaises(AttributeError, bing_videos.response, '[]')
response = mock.Mock(text='<html></html>')
self.assertEqual(bing_videos.response(response), [])
response = mock.Mock(text='<html></html>')
self.assertEqual(bing_videos.response(response), [])
html = """
<div class="dg_u">
<div>
<a>
<div>
<div>
<div class="mc_vtvc_meta_block">
<div><span>100 views</span><span>1 year ago</span></div><div><span>ExampleTube</span><span>Channel 1<span></div> #noqa
</div>
</div>
<div class="vrhdata" vrhm='{"du":"01:11","murl":"https://www.example.com/watch?v=DEADBEEF","thid":"OVP.BINGTHUMB1","vt":"Title 1"}'></div> # noqa
</div>
</a>
</div>
</div>
"""
response = mock.Mock(text=html)
results = bing_videos.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Title 1')
self.assertEqual(results[0]['url'], 'https://www.example.com/watch?v=DEADBEEF')
self.assertEqual(results[0]['content'], '01:11 - 100 views - 1 year ago - ExampleTube - Channel 1')
self.assertEqual(results[0]['thumbnail'], 'https://www.bing.com/th?id=OVP.BINGTHUMB1')

112
tests/unit/engines/test_btdigg.py
View File

@ -1,112 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
from searx.engines import btdigg
from searx.testing import SearxTestCase
class TestBtdiggEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 0
params = btdigg.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
self.assertIn('btdig.com', params['url'])
def test_response(self):
self.assertRaises(AttributeError, btdigg.response, None)
self.assertRaises(AttributeError, btdigg.response, [])
self.assertRaises(AttributeError, btdigg.response, '')
self.assertRaises(AttributeError, btdigg.response, '[]')
response = mock.Mock(text='<html></html>')
self.assertEqual(btdigg.response(response), [])
html = u"""
<div class="one_result" style="display:table-row;background-color:#e8e8e8">
<div style="display:table-cell;color:rgb(0, 0, 0)">
<div style="display:table">
<div style="display:table-row">
<div class="torrent_name" style="display:table-cell">
<a style="color:rgb(0, 0, 204);text-decoration:underline;font-size:150%"
href="http://btdig.com/a72f35b7ee3a10928f02bb799e40ae5db701ed1c/pdf?q=pdf&amp;p=1&amp;order=0"
>3.9GBdeLibrosByHuasoFromHell(3de4)</a>
</div>
</div>
</div>
<div style="display:table">
<div style="display:table-row">
<div style="display:table-cell">
<span class="torrent_files" style="color:#666;padding-left:10px">4217</span> files <span
class="torrent_size" style="color:#666;padding-left:10px">1 GB</span><span
class="torrent_age" style="color:rgb(0, 102, 0);padding-left:10px;margin: 0px 4px"
>found 3 years ago</span>
</div>
</div>
</div>
<div style="display:table;width:100%;padding:10px">
<div style="display:table-row">
<div class="torrent_magnet" style="display:table-cell">
<div class="fa fa-magnet" style="color:#cc0000">
<a href="magnet:?xt=urn:btih:a72f35b7ee3a10928f02bb799e40ae5db701ed1c&amp;dn=3.9GBdeLibrosBy..."
title="Download via magnet-link"> magnet:?xt=urn:btih:a72f35b7ee...</a>
</div>
</div>
<div style="display:table-cell;color:rgb(0, 0, 0);text-align:right">
<span style="color:rgb(136, 136, 136);margin: 0px 0px 0px 4px"></span><span
style="color:rgb(0, 102, 0);margin: 0px 4px">found 3 years ago</span>
</div>
</div>
</div>
<div class="torrent_excerpt" style="display:table;padding:10px;white-space:nowrap">
<div class="fa fa-folder-open" style="padding-left:0em"> 3.9GBdeLibrosByHuasoFromHell(3de4)</div><br/>
<div class="fa fa-folder-open" style="padding-left:1em"> Libros H-Z</div><br/>
<div class="fa fa-folder-open" style="padding-left:2em"> H</div><br/><div class="fa fa-file-archive-o"
style="padding-left:3em"> H.H. Hollis - El truco de la espada-<b
style="color:red; background-color:yellow">pdf</b>.zip</div><span
style="color:#666;padding-left:10px">17 KB</span><br/>
<div class="fa fa-file-archive-o" style="padding-left:3em"> Hagakure - El Libro del Samurai-<b
style="color:red; background-color:yellow">pdf</b>.zip</div><span
style="color:#666;padding-left:10px">95 KB</span><br/>
<div class="fa fa-folder-open" style="padding-left:3em"> Hamsun, Knut (1859-1952)</div><br/>
<div class="fa fa-file-archive-o" style="padding-left:4em"> Hamsun, Knut - Hambre-<b
style="color:red; background-color:yellow">pdf</b>.zip</div><span
style="color:#666;padding-left:10px">786 KB</span><br/>
<div class="fa fa-plus-circle"><a
href="http://btdig.com/a72f35b7ee3a10928f02bb799e40ae5db701ed1c/pdf?q=pdf&amp;p=1&amp;order=0"
> 4214 hidden files<span style="color:#666;padding-left:10px">1 GB</span></a></div>
</div>
</div>
</div>
"""
response = mock.Mock(text=html.encode('utf-8'))
results = btdigg.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], '3.9GBdeLibrosByHuasoFromHell(3de4)')
self.assertEqual(results[0]['url'],
'http://btdig.com/a72f35b7ee3a10928f02bb799e40ae5db701ed1c/pdf?q=pdf&p=1&order=0')
self.assertEqual(results[0]['content'],
'3.9GBdeLibrosByHuasoFromHell(3de4) | ' +
'Libros H-Z | ' +
'H H.H. Hollis - El truco de la espada-pdf.zip17 KB | ' +
'Hagakure - El Libro del Samurai-pdf.zip95 KB | ' +
'Hamsun, Knut (1859-1952) | Hamsun, Knut - Hambre-pdf.zip786 KB | ' +
'4214 hidden files1 GB')
self.assertEqual(results[0]['filesize'], 1 * 1024 * 1024 * 1024)
self.assertEqual(results[0]['files'], 4217)
self.assertEqual(results[0]['magnetlink'],
'magnet:?xt=urn:btih:a72f35b7ee3a10928f02bb799e40ae5db701ed1c&dn=3.9GBdeLibrosBy...')
html = """
<div style="display:table-row;background-color:#e8e8e8">
</div>
"""
response = mock.Mock(text=html.encode('utf-8'))
results = btdigg.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)

56
tests/unit/engines/test_currency_convert.py
View File

@ -1,56 +0,0 @@
from collections import defaultdict
from datetime import datetime
import mock
from searx.engines import currency_convert
from searx.testing import SearxTestCase
class TestCurrencyConvertEngine(SearxTestCase):
def test_request(self):
query = b'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
params = currency_convert.request(query, dicto)
self.assertNotIn('url', params)
query = b'convert 10 Pound Sterlings to United States Dollars'
params = currency_convert.request(query, dicto)
self.assertIn('url', params)
self.assertIn('duckduckgo.com', params['url'])
self.assertIn('GBP', params['url'])
self.assertIn('USD', params['url'])
def test_response(self):
dicto = defaultdict(dict)
dicto['amount'] = float(10)
dicto['from'] = "GBP"
dicto['to'] = "USD"
dicto['from_name'] = "pound sterling"
dicto['to_name'] = "United States dollar"
response = mock.Mock(text='a,b,c,d', search_params=dicto)
self.assertEqual(currency_convert.response(response), [])
body = """ddg_spice_currency(
{
"conversion":{
"converted-amount": "0.5"
},
"topConversions":[
{
},
{
}
]
}
);
"""
response = mock.Mock(text=body, search_params=dicto)
results = currency_convert.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['answer'], '10.0 GBP = 5.0 USD, 1 GBP (pound sterling)' +
' = 0.5 USD (United States dollar)')
target_url = 'https://duckduckgo.com/js/spice/currency/1/{}/{}'.format(
dicto['from'], dicto['to'])
self.assertEqual(results[0]['url'], target_url)

112
tests/unit/engines/test_dailymotion.py
View File

@ -1,112 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
from searx.engines import dailymotion
from searx.testing import SearxTestCase
class TestDailymotionEngine(SearxTestCase):
def test_request(self):
dailymotion.supported_languages = ['en', 'fr']
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 0
dicto['language'] = 'fr-FR'
params = dailymotion.request(query, dicto)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('dailymotion.com' in params['url'])
self.assertTrue('fr' in params['url'])
dicto['language'] = 'all'
params = dailymotion.request(query, dicto)
self.assertTrue('en' in params['url'])
def test_response(self):
self.assertRaises(AttributeError, dailymotion.response, None)
self.assertRaises(AttributeError, dailymotion.response, [])
self.assertRaises(AttributeError, dailymotion.response, '')
self.assertRaises(AttributeError, dailymotion.response, '[]')
response = mock.Mock(text='{}')
self.assertEqual(dailymotion.response(response), [])
response = mock.Mock(text='{"data": []}')
self.assertEqual(dailymotion.response(response), [])
json = """
{
"page": 1,
"limit": 5,
"explicit": false,
"total": 289487,
"has_more": true,
"list": [
{
"created_time": 1422173451,
"title": "Title",
"description": "Description",
"duration": 81,
"url": "http://www.url",
"thumbnail_360_url": "http://thumbnail",
"id": "x2fit7q"
}
]
}
"""
response = mock.Mock(text=json)
results = dailymotion.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Title')
self.assertEqual(results[0]['url'], 'http://www.url')
self.assertEqual(results[0]['content'], 'Description')
self.assertIn('x2fit7q', results[0]['embedded'])
json = r"""
{"toto":[
{"id":200,"name":"Artist Name",
"link":"http:\/\/www.dailymotion.com\/artist\/1217","type":"artist"}
]}
"""
response = mock.Mock(text=json)
results = dailymotion.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
def test_fetch_supported_languages(self):
json = r"""
{"list":[{"code":"af","name":"Afrikaans","native_name":"Afrikaans",
"localized_name":"Afrikaans","display_name":"Afrikaans"},
{"code":"ar","name":"Arabic","native_name":"\u0627\u0644\u0639\u0631\u0628\u064a\u0629",
"localized_name":"Arabic","display_name":"Arabic"},
{"code":"la","name":"Latin","native_name":null,
"localized_name":"Latin","display_name":"Latin"}
]}
"""
response = mock.Mock(text=json)
languages = dailymotion._fetch_supported_languages(response)
self.assertEqual(type(languages), dict)
self.assertEqual(len(languages), 3)
self.assertIn('af', languages)
self.assertIn('ar', languages)
self.assertIn('la', languages)
self.assertEqual(type(languages['af']), dict)
self.assertEqual(type(languages['ar']), dict)
self.assertEqual(type(languages['la']), dict)
self.assertIn('name', languages['af'])
self.assertIn('name', languages['ar'])
self.assertNotIn('name', languages['la'])
self.assertIn('english_name', languages['af'])
self.assertIn('english_name', languages['ar'])
self.assertIn('english_name', languages['la'])
self.assertEqual(languages['af']['name'], 'Afrikaans')
self.assertEqual(languages['af']['english_name'], 'Afrikaans')
self.assertEqual(languages['ar']['name'], u'العربية')
self.assertEqual(languages['ar']['english_name'], 'Arabic')
self.assertEqual(languages['la']['english_name'], 'Latin')

57
tests/unit/engines/test_deezer.py
View File

@ -1,57 +0,0 @@
from collections import defaultdict
import mock
from searx.engines import deezer
from searx.testing import SearxTestCase
class TestDeezerEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 0
params = deezer.request(query, dicto)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('deezer.com' in params['url'])
def test_response(self):
self.assertRaises(AttributeError, deezer.response, None)
self.assertRaises(AttributeError, deezer.response, [])
self.assertRaises(AttributeError, deezer.response, '')
self.assertRaises(AttributeError, deezer.response, '[]')
response = mock.Mock(text='{}')
self.assertEqual(deezer.response(response), [])
response = mock.Mock(text='{"data": []}')
self.assertEqual(deezer.response(response), [])
json = r"""
{"data":[
{"id":100, "title":"Title of track",
"link":"https:\/\/www.deezer.com\/track\/1094042","duration":232,
"artist":{"id":200,"name":"Artist Name",
"link":"https:\/\/www.deezer.com\/artist\/1217","type":"artist"},
"album":{"id":118106,"title":"Album Title","type":"album"},"type":"track"}
]}
"""
response = mock.Mock(text=json)
results = deezer.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Title of track')
self.assertEqual(results[0]['url'], 'https://www.deezer.com/track/1094042')
self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track')
self.assertTrue('100' in results[0]['embedded'])
json = r"""
{"data":[
{"id":200,"name":"Artist Name",
"link":"https:\/\/www.deezer.com\/artist\/1217","type":"artist"}
]}
"""
response = mock.Mock(text=json)
results = deezer.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)

24
tests/unit/engines/test_deviantart.py
View File

@ -1,24 +0,0 @@
from collections import defaultdict
import mock
from searx.engines import deviantart
from searx.testing import SearxTestCase
class TestDeviantartEngine(SearxTestCase):
def test_request(self):
dicto = defaultdict(dict)
query = 'test_query'
dicto['pageno'] = 0
dicto['time_range'] = ''
params = deviantart.request(query, dicto)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('deviantart.com' in params['url'])
def test_no_url_in_request_year_time_range(self):
dicto = defaultdict(dict)
query = 'test_query'
dicto['time_range'] = 'year'
params = deviantart.request(query, dicto)
self.assertEqual({}, params['url'])

61
tests/unit/engines/test_digbt.py
View File

@ -1,61 +0,0 @@
from collections import defaultdict
import mock
from searx.engines import digbt
from searx.testing import SearxTestCase
class TestDigBTEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 0
params = digbt.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
self.assertIn('digbt.org', params['url'])
def test_response(self):
self.assertRaises(AttributeError, digbt.response, None)
self.assertRaises(AttributeError, digbt.response, [])
self.assertRaises(AttributeError, digbt.response, '')
self.assertRaises(AttributeError, digbt.response, '[]')
response = mock.Mock(text='<html></html>')
self.assertEqual(digbt.response(response), [])
html = """
<table class="table">
<tr><td class="x-item">
<div>
<a title="The Big Bang Theory" class="title" href="/The-Big-Bang-Theory-d2.html">
The Big <span class="highlight">Bang</span> Theory
</a>
<span class="ctime"><span style="color:red;">4 hours ago</span></span>
</div>
<div class="files">
<ul>
<li>The Big Bang Theory 2.9 GB</li>
<li>....</li>
</ul>
</div>
<div class="tail">
Files: 1 Size: 2.9 GB Downloads: 1 Updated: <span style="color:red;">4 hours ago</span>
&nbsp; &nbsp;
<a class="title" href="magnet:?xt=urn:btih:a&amp;dn=The+Big+Bang+Theory">
<span class="glyphicon glyphicon-magnet"></span> magnet-link
</a>
&nbsp; &nbsp;
</div>
</td></tr>
</table>
"""
response = mock.Mock(text=html.encode('utf-8'))
results = digbt.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'The Big Bang Theory')
self.assertEqual(results[0]['url'], 'https://digbt.org/The-Big-Bang-Theory-d2.html')
self.assertEqual(results[0]['content'], 'The Big Bang Theory 2.9 GB ....')
self.assertEqual(results[0]['filesize'], 3113851289)
self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:a&dn=The+Big+Bang+Theory')

16
tests/unit/engines/test_digg.py
View File

@ -1,16 +0,0 @@
from collections import defaultdict
import mock
from searx.engines import digg
from searx.testing import SearxTestCase
class TestDiggEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
params = digg.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
self.assertIn('digg.com', params['url'])

79
tests/unit/engines/test_doku.py
View File

@ -1,79 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
from searx.engines import doku
from searx.testing import SearxTestCase
class TestDokuEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
params = doku.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
def test_response(self):
self.assertRaises(AttributeError, doku.response, None)
self.assertRaises(AttributeError, doku.response, [])
self.assertRaises(AttributeError, doku.response, '')
self.assertRaises(AttributeError, doku.response, '[]')
response = mock.Mock(text='<html></html>')
self.assertEqual(doku.response(response), [])
html = u"""
<div class="search_quickresult">
<h3>Pages trouvées :</h3>
<ul class="search_quickhits">
<li> <a href="/xfconf-query" class="wikilink1" title="xfconf-query">xfconf-query</a></li>
</ul>
<div class="clearer"></div>
</div>
"""
response = mock.Mock(text=html)
results = doku.response(response)
expected = [{'content': '', 'title': 'xfconf-query', 'url': 'http://localhost:8090/xfconf-query'}]
self.assertEqual(doku.response(response), expected)
html = u"""
<dl class="search_results">
<dt><a href="/xvnc?s[]=query" class="wikilink1" title="xvnc">xvnc</a>: 40 Occurrences trouvées</dt>
<dd>er = /usr/bin/Xvnc
server_args = -inetd -<strong class="search_hit">query</strong> localhost -geometry 640x480 ... er = /usr/bin/Xvnc
server_args = -inetd -<strong class="search_hit">query</strong> localhost -geometry 800x600 ... er = /usr/bin/Xvnc
server_args = -inetd -<strong class="search_hit">query</strong> localhost -geometry 1024x768 ... er = /usr/bin/Xvnc
server_args = -inetd -<strong class="search_hit">query</strong> localhost -geometry 1280x1024 -depth 8 -Sec</dd>
<dt><a href="/postfix_mysql_tls_sasl_1404?s[]=query"
class="wikilink1"
title="postfix_mysql_tls_sasl_1404">postfix_mysql_tls_sasl_1404</a>: 14 Occurrences trouvées</dt>
<dd>tdepasse
hosts = 127.0.0.1
dbname = postfix
<strong class="search_hit">query</strong> = SELECT goto FROM alias WHERE address='%s' AND a... tdepasse
hosts = 127.0.0.1
dbname = postfix
<strong class="search_hit">query</strong> = SELECT domain FROM domain WHERE domain='%s'
#optional <strong class="search_hit">query</strong> to use when relaying for backup MX
#<strong class="search_hit">query</strong> = SELECT domain FROM domain WHERE domain='%s' and backupmx =</dd>
<dt><a href="/bind9?s[]=query" class="wikilink1" title="bind9">bind9</a>: 12 Occurrences trouvées</dt>
<dd> printcmd
;; Got answer:
;; -&gt;&gt;HEADER&lt;&lt;- opcode: <strong class="search_hit">QUERY</strong>, status: NOERROR, id: 13427
;; flags: qr aa rd ra; <strong class="search_hit">QUERY</strong>: 1, ANSWER: 1, AUTHORITY: 1, ADDITIONAL: 1
[...]
;; <strong class="search_hit">Query</strong> time: 1 msec
;; SERVER: 127.0.0.1#53(127.0.0.1)
;... par la requête (<strong class="search_hit">Query</strong> time) , entre la première et la deuxième requête.</dd>
</dl>
"""
response = mock.Mock(text=html)
results = doku.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 3)
self.assertEqual(results[0]['title'], 'xvnc')
# FIXME self.assertEqual(results[0]['url'], u'http://this.should.be.the.link/ű')
# FIXME self.assertEqual(results[0]['content'], 'This should be the content.')

106
tests/unit/engines/test_duckduckgo.py
View File

@ -1,106 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
from searx.engines import load_engine, duckduckgo
from searx.testing import SearxTestCase
class TestDuckduckgoEngine(SearxTestCase):
def test_request(self):
duckduckgo = load_engine({'engine': 'duckduckgo', 'name': 'duckduckgo'})
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
dicto['time_range'] = ''
dicto['language'] = 'de-CH'
params = duckduckgo.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
self.assertIn('duckduckgo.com', params['url'])
self.assertIn('ch-de', params['url'])
self.assertIn('s=0', params['url'])
# when ddg uses non standard codes
dicto['language'] = 'zh-HK'
params = duckduckgo.request(query, dicto)
self.assertIn('hk-tzh', params['url'])
dicto['language'] = 'en-GB'
params = duckduckgo.request(query, dicto)
self.assertIn('uk-en', params['url'])
# no country given
dicto['language'] = 'en'
params = duckduckgo.request(query, dicto)
self.assertIn('us-en', params['url'])
def test_no_url_in_request_year_time_range(self):
dicto = defaultdict(dict)
query = 'test_query'
dicto['time_range'] = 'year'
params = duckduckgo.request(query, dicto)
self.assertEqual({}, params['url'])
def test_response(self):
self.assertRaises(AttributeError, duckduckgo.response, None)
self.assertRaises(AttributeError, duckduckgo.response, [])
self.assertRaises(AttributeError, duckduckgo.response, '')
self.assertRaises(AttributeError, duckduckgo.response, '[]')
response = mock.Mock(text='<html></html>')
self.assertEqual(duckduckgo.response(response), [])
html = u"""
<div class="result results_links results_links_deep web-result result--no-result">
<div class="links_main links_deep result__body">
<h2 class="result__title">
</h2>
<div class="no-results">No results</div>
<div class="result__extras">
</div>
</div>
</div>
"""
response = mock.Mock(text=html)
results = duckduckgo.response(response)
self.assertEqual(duckduckgo.response(response), [])
html = u"""
<div class="result results_links results_links_deep web-result ">
<div class="links_main links_deep result__body">
<h2 class="result__title">
<a rel="nofollow" class="result__a" href="http://this.should.be.the.link/ű">
This <b>is</b> <b>the</b> title
</a>
</h2>
<a class="result__snippet" href="http://this.should.be.the.link/ű">
<b>This</b> should be the content.
</a>
<div class="result__extras">
</div>
</div>
</div>
"""
response = mock.Mock(text=html)
results = duckduckgo.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'This is the title')
self.assertEqual(results[0]['url'], u'http://this.should.be.the.link/ű')
self.assertEqual(results[0]['content'], 'This should be the content.')
def test_fetch_supported_languages(self):
js = """some code...regions:{
"wt-wt":"All Results","ar-es":"Argentina","au-en":"Australia","at-de":"Austria","be-fr":"Belgium (fr)"
}some more code..."""
response = mock.Mock(text=js)
languages = list(duckduckgo._fetch_supported_languages(response))
self.assertEqual(len(languages), 5)
self.assertIn('wt-WT', languages)
self.assertIn('es-AR', languages)
self.assertIn('en-AU', languages)
self.assertIn('de-AT', languages)
self.assertIn('fr-BE', languages)

255
tests/unit/engines/test_duckduckgo_definitions.py
View File

@ -1,255 +0,0 @@
from collections import defaultdict
import mock
from searx.engines import duckduckgo_definitions
from searx.testing import SearxTestCase
class TestDDGDefinitionsEngine(SearxTestCase):
def test_result_to_text(self):
url = ''
text = 'Text'
html_result = 'Html'
result = duckduckgo_definitions.result_to_text(url, text, html_result)
self.assertEqual(result, text)
html_result = '<a href="url">Text in link</a>'
result = duckduckgo_definitions.result_to_text(url, text, html_result)
self.assertEqual(result, 'Text in link')
def test_request(self):
duckduckgo_definitions.supported_languages = ['en-US', 'es-ES']
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
dicto['language'] = 'es'
params = duckduckgo_definitions.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
self.assertIn('duckduckgo.com', params['url'])
self.assertIn('headers', params)
self.assertIn('Accept-Language', params['headers'])
self.assertIn('es', params['headers']['Accept-Language'])
def test_response(self):
self.assertRaises(AttributeError, duckduckgo_definitions.response, None)
self.assertRaises(AttributeError, duckduckgo_definitions.response, [])
self.assertRaises(AttributeError, duckduckgo_definitions.response, '')
self.assertRaises(AttributeError, duckduckgo_definitions.response, '[]')
response = mock.Mock(text='{}')
self.assertEqual(duckduckgo_definitions.response(response), [])
response = mock.Mock(text='{"data": []}')
self.assertEqual(duckduckgo_definitions.response(response), [])
json = """
{
"DefinitionSource": "definition source",
"Heading": "heading",
"ImageWidth": 0,
"RelatedTopics": [
{
"Result": "Top-level domains",
"Icon": {
"URL": "",
"Height": "",
"Width": ""
},
"FirstURL": "https://first.url",
"Text": "text"
},
{
"Topics": [
{
"Result": "result topic",
"Icon": {
"URL": "",
"Height": "",
"Width": ""
},
"FirstURL": "https://duckduckgo.com/?q=2%2F2",
"Text": "result topic text"
}
],
"Name": "name"
}
],
"Entity": "Entity",
"Type": "A",
"Redirect": "",
"DefinitionURL": "http://definition.url",
"AbstractURL": "https://abstract.url",
"Definition": "this is the definition",
"AbstractSource": "abstract source",
"Infobox": {
"content": [
{
"data_type": "string",
"value": "1999",
"label": "Introduced",
"wiki_order": 0
}
],
"meta": [
{
"data_type": "string",
"value": ".test",
"label": "article_title"
}
]
},
"Image": "image.png",
"ImageIsLogo": 0,
"Abstract": "abstract",
"AbstractText": "abstract text",
"AnswerType": "",
"ImageHeight": 0,
"Results": [{
"Result" : "result title",
"Icon" : {
"URL" : "result url",
"Height" : 16,
"Width" : 16
},
"FirstURL" : "result first url",
"Text" : "result text"
}
],
"Answer": "answer"
}
"""
response = mock.Mock(text=json)
results = duckduckgo_definitions.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 4)
self.assertEqual(results[0]['answer'], 'answer')
self.assertEqual(results[1]['title'], 'heading')
self.assertEqual(results[1]['url'], 'result first url')
self.assertEqual(results[2]['suggestion'], 'text')
self.assertEqual(results[3]['infobox'], 'heading')
self.assertEqual(results[3]['id'], 'https://definition.url')
self.assertEqual(results[3]['entity'], 'Entity')
self.assertIn('abstract', results[3]['content'])
self.assertIn('this is the definition', results[3]['content'])
self.assertEqual(results[3]['img_src'], 'image.png')
self.assertIn('Introduced', results[3]['attributes'][0]['label'])
self.assertIn('1999', results[3]['attributes'][0]['value'])
self.assertIn({'url': 'https://abstract.url', 'title': 'abstract source'}, results[3]['urls'])
self.assertIn({'url': 'http://definition.url', 'title': 'definition source'}, results[3]['urls'])
self.assertIn({'name': 'name', 'suggestions': ['result topic text']}, results[3]['relatedTopics'])
json = """
{
"DefinitionSource": "definition source",
"Heading": "heading",
"ImageWidth": 0,
"RelatedTopics": [],
"Entity": "Entity",
"Type": "A",
"Redirect": "",
"DefinitionURL": "",
"AbstractURL": "https://abstract.url",
"Definition": "",
"AbstractSource": "abstract source",
"Image": "",
"ImageIsLogo": 0,
"Abstract": "",
"AbstractText": "abstract text",
"AnswerType": "",
"ImageHeight": 0,
"Results": [],
"Answer": ""
}
"""
response = mock.Mock(text=json)
results = duckduckgo_definitions.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['url'], 'https://abstract.url')
self.assertEqual(results[0]['title'], 'heading')
self.assertEqual(results[0]['content'], '')
json = """
{
"DefinitionSource": "definition source",
"Heading": "heading",
"ImageWidth": 0,
"RelatedTopics": [
{
"Result": "Top-level domains",
"Icon": {
"URL": "",
"Height": "",
"Width": ""
},
"FirstURL": "https://first.url",
"Text": "heading"
},
{
"Name": "name"
},
{
"Topics": [
{
"Result": "result topic",
"Icon": {
"URL": "",
"Height": "",
"Width": ""
},
"FirstURL": "https://duckduckgo.com/?q=2%2F2",
"Text": "heading"
}
],
"Name": "name"
}
],
"Entity": "Entity",
"Type": "A",
"Redirect": "",
"DefinitionURL": "http://definition.url",
"AbstractURL": "https://abstract.url",
"Definition": "this is the definition",
"AbstractSource": "abstract source",
"Infobox": {
"meta": [
{
"data_type": "string",
"value": ".test",
"label": "article_title"
}
]
},
"Image": "image.png",
"ImageIsLogo": 0,
"Abstract": "abstract",
"AbstractText": "abstract text",
"AnswerType": "",
"ImageHeight": 0,
"Results": [{
"Result" : "result title",
"Icon" : {
"URL" : "result url",
"Height" : 16,
"Width" : 16
},
"Text" : "result text"
}
],
"Answer": ""
}
"""
response = mock.Mock(text=json)
results = duckduckgo_definitions.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['infobox'], 'heading')
self.assertEqual(results[0]['id'], 'https://definition.url')
self.assertEqual(results[0]['entity'], 'Entity')
self.assertIn('abstract', results[0]['content'])
self.assertIn('this is the definition', results[0]['content'])
self.assertEqual(results[0]['img_src'], 'image.png')
self.assertIn({'url': 'https://abstract.url', 'title': 'abstract source'}, results[0]['urls'])
self.assertIn({'url': 'http://definition.url', 'title': 'definition source'}, results[0]['urls'])
self.assertIn({'name': 'name', 'suggestions': []}, results[0]['relatedTopics'])

75
tests/unit/engines/test_duckduckgo_images.py
View File

@ -1,75 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
from searx.engines import duckduckgo_images
from searx.testing import SearxTestCase
class TestDuckduckgoImagesEngine(SearxTestCase):
def test_request(self):
duckduckgo_images.supported_languages = ['de-CH', 'en-US']
query = 'test_query'
dicto = defaultdict(dict)
dicto['is_test'] = True
dicto['pageno'] = 1
dicto['safesearch'] = 0
dicto['language'] = 'all'
params = duckduckgo_images.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
self.assertIn('duckduckgo.com', params['url'])
self.assertIn('s=0', params['url'])
self.assertIn('p=-1', params['url'])
self.assertIn('vqd=12345', params['url'])
# test paging, safe search and language
dicto['pageno'] = 2
dicto['safesearch'] = 2
dicto['language'] = 'de'
params = duckduckgo_images.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
self.assertIn('s=50', params['url'])
self.assertIn('p=1', params['url'])
self.assertIn('ch-de', params['url'])
def test_response(self):
self.assertRaises(AttributeError, duckduckgo_images.response, None)
self.assertRaises(AttributeError, duckduckgo_images.response, [])
self.assertRaises(AttributeError, duckduckgo_images.response, '')
self.assertRaises(AttributeError, duckduckgo_images.response, '[]')
response = mock.Mock(text='If this error persists, please let us know: ops@duckduckgo.com')
self.assertRaises(Exception, duckduckgo_images.response, response)
json = u"""
{
"query": "test_query",
"results": [
{
"title": "Result 1",
"url": "https://site1.url",
"thumbnail": "https://thumb1.nail",
"image": "https://image1"
},
{
"title": "Result 2",
"url": "https://site2.url",
"thumbnail": "https://thumb2.nail",
"image": "https://image2"
}
]
}
"""
response = mock.Mock(text=json)
results = duckduckgo_images.response(response)
self.assertEqual(len(results), 2)
self.assertEqual(results[0]['title'], 'Result 1')
self.assertEqual(results[0]['url'], 'https://site1.url')
self.assertEqual(results[0]['thumbnail_src'], 'https://thumb1.nail')
self.assertEqual(results[0]['img_src'], 'https://image1')
self.assertEqual(results[1]['title'], 'Result 2')
self.assertEqual(results[1]['url'], 'https://site2.url')
self.assertEqual(results[1]['thumbnail_src'], 'https://thumb2.nail')
self.assertEqual(results[1]['img_src'], 'https://image2')

47
tests/unit/engines/test_duden.py
View File

@ -1,47 +0,0 @@
from collections import defaultdict
import mock
from searx.engines import duden
from searx.testing import SearxTestCase
from datetime import datetime
class TestDudenEngine(SearxTestCase):
def test_request(self):
query = 'Haus'
dic = defaultdict(dict)
data = [
[1, 'https://www.duden.de/suchen/dudenonline/Haus'],
[2, 'https://www.duden.de/suchen/dudenonline/Haus?search_api_fulltext=&page=1']
]
for page_no, exp_res in data:
dic['pageno'] = page_no
params = duden.request(query, dic)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('duden.de' in params['url'])
self.assertEqual(params['url'], exp_res)
def test_response(self):
resp = mock.Mock(text='<html></html>')
self.assertEqual(duden.response(resp), [])
html = """
<section class="vignette">
<h2"> <a href="/rechtschreibung/Haus">
<strong>This is the title also here</strong>
</a> </h2>
<p>This is the content</p>
</section>
"""
resp = mock.Mock(text=html)
results = duden.response(resp)
self.assertEqual(len(results), 1)
self.assertEqual(type(results), list)
# testing result (dictionary entry)
r = results[0]
self.assertEqual(r['url'], 'https://www.duden.de/rechtschreibung/Haus')
self.assertEqual(r['title'], 'This is the title also here')
self.assertEqual(r['content'], 'This is the content')

26
tests/unit/engines/test_dummy.py
View File

@ -1,26 +0,0 @@
from searx.engines import dummy
from searx.testing import SearxTestCase
class TestDummyEngine(SearxTestCase):
def test_request(self):
test_params = [
[1, 2, 3],
['a'],
[],
1
]
for params in test_params:
self.assertEqual(dummy.request(None, params), params)
def test_response(self):
responses = [
None,
[],
True,
dict(),
tuple()
]
for response in responses:
self.assertEqual(dummy.response(response), [])

113
tests/unit/engines/test_faroo.py
View File

@ -1,113 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
from searx.engines import faroo
from searx.testing import SearxTestCase
class TestFarooEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
dicto['language'] = 'fr-FR'
dicto['category'] = 'general'
params = faroo.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
self.assertIn('faroo.com', params['url'])
self.assertIn('en', params['url'])
self.assertIn('web', params['url'])
dicto['language'] = 'all'
params = faroo.request(query, dicto)
self.assertIn('en', params['url'])
dicto['language'] = 'de-DE'
params = faroo.request(query, dicto)
self.assertIn('de', params['url'])
def test_response(self):
self.assertRaises(AttributeError, faroo.response, None)
self.assertRaises(AttributeError, faroo.response, [])
self.assertRaises(AttributeError, faroo.response, '')
self.assertRaises(AttributeError, faroo.response, '[]')
response = mock.Mock(text='{}')
self.assertEqual(faroo.response(response), [])
response = mock.Mock(text='{"data": []}')
self.assertEqual(faroo.response(response), [])
response = mock.Mock(text='{"data": []}', status_code=429)
self.assertRaises(Exception, faroo.response, response)
json = """
{
"results": [
{
"title": "This is the title",
"kwic": "This is the content",
"content": "",
"url": "http://this.is.the.url/",
"iurl": "",
"domain": "css3test.com",
"author": "Jim Dalrymple",
"news": true,
"votes": "10",
"date": 1360622563000,
"related": []
},
{
"title": "This is the title2",
"kwic": "This is the content2",
"content": "",
"url": "http://this.is.the.url2/",
"iurl": "",
"domain": "css3test.com",
"author": "Jim Dalrymple",
"news": false,
"votes": "10",
"related": []
},
{
"title": "This is the title3",
"kwic": "This is the content3",
"content": "",
"url": "http://this.is.the.url3/",
"iurl": "http://upload.wikimedia.org/optimized.jpg",
"domain": "css3test.com",
"author": "Jim Dalrymple",
"news": false,
"votes": "10",
"related": []
}
],
"query": "test",
"suggestions": [],
"count": 100,
"start": 1,
"length": 10,
"time": "15"
}
"""
response = mock.Mock(text=json)
results = faroo.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 3)
self.assertEqual(results[0]['title'], 'This is the title')
self.assertEqual(results[0]['url'], 'http://this.is.the.url/')
self.assertEqual(results[0]['content'], 'This is the content')
self.assertEqual(results[1]['title'], 'This is the title2')
self.assertEqual(results[1]['url'], 'http://this.is.the.url2/')
self.assertEqual(results[1]['content'], 'This is the content2')
self.assertEqual(results[2]['thumbnail'], 'http://upload.wikimedia.org/optimized.jpg')
json = """
{}
"""
response = mock.Mock(text=json)
results = faroo.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)

60
tests/unit/engines/test_fdroid.py
View File

@ -1,60 +0,0 @@
import mock
from collections import defaultdict
from searx.engines import fdroid
from searx.testing import SearxTestCase
class TestFdroidEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dic = defaultdict(dict)
dic['pageno'] = 1
params = fdroid.request(query, dic)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('search.f-droid.org' in params['url'])
def test_response_empty(self):
resp = mock.Mock(text='<html></html>')
self.assertEqual(fdroid.response(resp), [])
def test_response_oneresult(self):
html = """
<!DOCTYPE html>
<html>
<head>
<title>test</title>
</head>
<body>
<div class="site-wrapper">
<div class="main-content">
<a class="package-header" href="https://example.com/app.url">
<img class="package-icon" src="https://example.com/appexample.logo.png" />
<div class="package-info">
<h4 class="package-name">
App Example 1
</h4>
<div class="package-desc">
<span class="package-summary">Description App Example 1</span>
<span class="package-license">GPL-3.0-only</span>
</div>
</div>
</a>
</div>
</div>
</body>
</html>
"""
resp = mock.Mock(text=html)
results = fdroid.response(resp)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['url'], 'https://example.com/app.url')
self.assertEqual(results[0]['title'], 'App Example 1')
self.assertEqual(results[0]['content'], 'Description App Example 1 - GPL-3.0-only')
self.assertEqual(results[0]['img_src'], 'https://example.com/appexample.logo.png')

142
tests/unit/engines/test_flickr.py
View File

@ -1,142 +0,0 @@
from collections import defaultdict
import mock
from searx.engines import flickr
from searx.testing import SearxTestCase
class TestFlickrEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 0
params = flickr.request(query, dicto)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('flickr.com' in params['url'])
def test_response(self):
self.assertRaises(AttributeError, flickr.response, None)
self.assertRaises(AttributeError, flickr.response, [])
self.assertRaises(AttributeError, flickr.response, '')
self.assertRaises(AttributeError, flickr.response, '[]')
response = mock.Mock(text='{}')
self.assertEqual(flickr.response(response), [])
response = mock.Mock(text='{"data": []}')
self.assertEqual(flickr.response(response), [])
json = r"""
{ "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032",
"photo": [
{ "id": "15751017054", "owner": "66847915@N08",
"secret": "69c22afc40", "server": "7285", "farm": 8,
"title": "Photo title", "ispublic": 1,
"isfriend": 0, "isfamily": 0,
"description": { "_content": "Description" },
"ownername": "Owner",
"url_o": "https:\/\/farm8.staticflickr.com\/7285\/15751017054_9178e0f963_o.jpg",
"height_o": "2100", "width_o": "2653",
"url_n": "https:\/\/farm8.staticflickr.com\/7285\/15751017054_69c22afc40_n.jpg",
"height_n": "253", "width_n": "320",
"url_z": "https:\/\/farm8.staticflickr.com\/7285\/15751017054_69c22afc40_z.jpg",
"height_z": "507", "width_z": "640" }
] }, "stat": "ok" }
"""
response = mock.Mock(text=json)
results = flickr.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Photo title')
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')
self.assertTrue('o.jpg' in results[0]['img_src'])
self.assertTrue('n.jpg' in results[0]['thumbnail_src'])
self.assertTrue('Owner' in results[0]['author'])
self.assertTrue('Description' in results[0]['content'])
json = r"""
{ "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032",
"photo": [
{ "id": "15751017054", "owner": "66847915@N08",
"secret": "69c22afc40", "server": "7285", "farm": 8,
"title": "Photo title", "ispublic": 1,
"isfriend": 0, "isfamily": 0,
"description": { "_content": "Description" },
"ownername": "Owner",
"url_z": "https:\/\/farm8.staticflickr.com\/7285\/15751017054_69c22afc40_z.jpg",
"height_z": "507", "width_z": "640" }
] }, "stat": "ok" }
"""
response = mock.Mock(text=json)
results = flickr.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Photo title')
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')
self.assertTrue('z.jpg' in results[0]['img_src'])
self.assertTrue('z.jpg' in results[0]['thumbnail_src'])
self.assertTrue('Owner' in results[0]['author'])
self.assertTrue('Description' in results[0]['content'])
json = r"""
{ "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032",
"photo": [
{ "id": "15751017054", "owner": "66847915@N08",
"secret": "69c22afc40", "server": "7285", "farm": 8,
"title": "Photo title", "ispublic": 1,
"isfriend": 0, "isfamily": 0,
"description": { "_content": "Description" },
"ownername": "Owner",
"url_o": "https:\/\/farm8.staticflickr.com\/7285\/15751017054_9178e0f963_o.jpg",
"height_o": "2100", "width_o": "2653" }
] }, "stat": "ok" }
"""
response = mock.Mock(text=json)
results = flickr.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Photo title')
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')
self.assertTrue('o.jpg' in results[0]['img_src'])
self.assertTrue('o.jpg' in results[0]['thumbnail_src'])
self.assertTrue('Owner' in results[0]['author'])
self.assertTrue('Description' in results[0]['content'])
json = r"""
{ "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032",
"photo": [
{ "id": "15751017054", "owner": "66847915@N08",
"secret": "69c22afc40", "server": "7285", "farm": 8,
"title": "Photo title", "ispublic": 1,
"isfriend": 0, "isfamily": 0,
"description": { "_content": "Description" },
"ownername": "Owner",
"url_n": "https:\/\/farm8.staticflickr.com\/7285\/15751017054_69c22afc40_n.jpg",
"height_n": "253", "width_n": "320" }
] }, "stat": "ok" }
"""
response = mock.Mock(text=json)
results = flickr.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
json = """
{ "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032",
"toto": [] }, "stat": "ok" }
"""
response = mock.Mock(text=json)
results = flickr.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
json = r"""
{"toto":[
{"id":200,"name":"Artist Name",
"link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"}
]}
"""
response = mock.Mock(text=json)
results = flickr.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)

357
tests/unit/engines/test_flickr_noapi.py
View File

@ -1,357 +0,0 @@
from collections import defaultdict
import mock
from searx.engines import flickr_noapi
from searx.testing import SearxTestCase
class TestFlickrNoapiEngine(SearxTestCase):
def test_build_flickr_url(self):
url = flickr_noapi.build_flickr_url("uid", "pid")
self.assertIn("uid", url)
self.assertIn("pid", url)
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
dicto['time_range'] = ''
params = flickr_noapi.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
self.assertIn('flickr.com', params['url'])
def test_response(self):
self.assertRaises(AttributeError, flickr_noapi.response, None)
self.assertRaises(AttributeError, flickr_noapi.response, [])
self.assertRaises(AttributeError, flickr_noapi.response, '')
self.assertRaises(AttributeError, flickr_noapi.response, '[]')
response = mock.Mock(text='"modelExport:{"legend":[],"main":{"search-photos-lite-models":[{"photos":{}}]}}')
self.assertEqual(flickr_noapi.response(response), [])
response = \
mock.Mock(text='"modelExport:{"legend":[],"main":{"search-photos-lite-models":[{"photos":{"_data":[]}}]}}')
self.assertEqual(flickr_noapi.response(response), [])
# everthing is ok test
json = """
modelExport: {
"legend": [
[
"search-photos-lite-models",
"0",
"photos",
"_data",
"0"
]
],
"main": {
"search-photos-lite-models": [
{
"photos": {
"_data": [
{
"_flickrModelRegistry": "photo-lite-models",
"title": "This%20is%20the%20title",
"username": "Owner",
"pathAlias": "klink692",
"realname": "Owner",
"license": 0,
"ownerNsid": "59729010@N00",
"canComment": false,
"commentCount": 14,
"faveCount": 21,
"id": "14001294434",
"sizes": {
"c": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg",
"width": 541,
"height": 800,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_c.jpg",
"key": "c"
},
"h": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_761d32237a_h.jpg",
"width": 1081,
"height": 1600,
"url": "//c4.staticflickr.com/8/7246/14001294434_761d32237a_h.jpg",
"key": "h"
},
"k": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_f145a2c11a_k.jpg",
"width": 1383,
"height": 2048,
"url": "//c4.staticflickr.com/8/7246/14001294434_f145a2c11a_k.jpg",
"key": "k"
},
"l": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_b.jpg",
"width": 692,
"height": 1024,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_b.jpg",
"key": "l"
},
"m": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777.jpg",
"width": 338,
"height": 500,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777.jpg",
"key": "m"
},
"n": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_n.jpg",
"width": 216,
"height": 320,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_n.jpg",
"key": "n"
},
"q": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_q.jpg",
"width": 150,
"height": 150,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_q.jpg",
"key": "q"
},
"s": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_m.jpg",
"width": 162,
"height": 240,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_m.jpg",
"key": "s"
},
"sq": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_s.jpg",
"width": 75,
"height": 75,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_s.jpg",
"key": "sq"
},
"t": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_t.jpg",
"width": 68,
"height": 100,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_t.jpg",
"key": "t"
},
"z": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg",
"width": 433,
"height": 640,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
"key": "z"
}
}
}
]
}
}
]
}
}
"""
# Flickr serves search results in a json block named 'modelExport' buried inside a script tag,
# this json is served as a single line terminating with a comma.
json = ''.join(json.split()) + ',\n'
response = mock.Mock(text=json)
results = flickr_noapi.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'This is the title')
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')
self.assertIn('k.jpg', results[0]['img_src'])
self.assertIn('n.jpg', results[0]['thumbnail_src'])
self.assertIn('Owner', results[0]['author'])
# no n size, only the z size
json = """
modelExport: {
"legend": [
[
"search-photos-lite-models",
"0",
"photos",
"_data",
"0"
]
],
"main": {
"search-photos-lite-models": [
{
"photos": {
"_data": [
{
"_flickrModelRegistry": "photo-lite-models",
"title": "This%20is%20the%20title",
"username": "Owner",
"pathAlias": "klink692",
"realname": "Owner",
"license": 0,
"ownerNsid": "59729010@N00",
"canComment": false,
"commentCount": 14,
"faveCount": 21,
"id": "14001294434",
"sizes": {
"z": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg",
"width": 433,
"height": 640,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
"key": "z"
}
}
}
]
}
}
]
}
}
"""
json = ''.join(json.split()) + ',\n'
response = mock.Mock(text=json)
results = flickr_noapi.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'This is the title')
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')
self.assertIn('z.jpg', results[0]['img_src'])
self.assertIn('z.jpg', results[0]['thumbnail_src'])
self.assertIn('Owner', results[0]['author'])
# no z or n size
json = """
modelExport: {
"legend": [
[
"search-photos-lite-models",
"0",
"photos",
"_data",
"0"
]
],
"main": {
"search-photos-lite-models": [
{
"photos": {
"_data": [
{
"_flickrModelRegistry": "photo-lite-models",
"title": "This%20is%20the%20title",
"username": "Owner",
"pathAlias": "klink692",
"realname": "Owner",
"license": 0,
"ownerNsid": "59729010@N00",
"canComment": false,
"commentCount": 14,
"faveCount": 21,
"id": "14001294434",
"sizes": {
"o": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
"width": 433,
"height": 640,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
"key": "o"
}
}
}
]
}
}
]
}
}
"""
json = ''.join(json.split()) + ',\n'
response = mock.Mock(text=json)
results = flickr_noapi.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'This is the title')
self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')
self.assertIn('o.jpg', results[0]['img_src'])
self.assertIn('o.jpg', results[0]['thumbnail_src'])
self.assertIn('Owner', results[0]['author'])
# no image test
json = """
modelExport: {
"legend": [
[
"search-photos-lite-models",
"0",
"photos",
"_data",
"0"
]
],
"main": {
"search-photos-lite-models": [
{
"photos": {
"_data": [
{
"_flickrModelRegistry": "photo-lite-models",
"title": "This is the title",
"username": "Owner",
"pathAlias": "klink692",
"realname": "Owner",
"license": 0,
"ownerNsid": "59729010@N00",
"canComment": false,
"commentCount": 14,
"faveCount": 21,
"id": "14001294434",
"sizes": {
}
}
]
}
}
]
}
}
"""
json = ''.join(json.split()) + ',\n'
response = mock.Mock(text=json)
results = flickr_noapi.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
# null test
json = """
modelExport: {
"legend": [null],
"main": {
"search-photos-lite-models": [
{
"photos": {
"_data": [null]
}
}
]
}
}
"""
json = ''.join(json.split()) + ',\n'
response = mock.Mock(text=json)
results = flickr_noapi.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
# garbage test
json = r"""
{"toto":[
{"id":200,"name":"Artist Name",
"link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"}
]}
"""
json = ''.join(json.split()) + ',\n'
response = mock.Mock(text=json)
results = flickr_noapi.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)

103
tests/unit/engines/test_framalibre.py
View File

@ -1,103 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
from searx.engines import framalibre
from searx.testing import SearxTestCase
class TestFramalibreEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 0
params = framalibre.request(query, dicto)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('framalibre.org' in params['url'])
def test_response(self):
self.assertRaises(AttributeError, framalibre.response, None)
self.assertRaises(AttributeError, framalibre.response, [])
self.assertRaises(AttributeError, framalibre.response, '')
self.assertRaises(AttributeError, framalibre.response, '[]')
response = mock.Mock(text='{}')
self.assertEqual(framalibre.response(response), [])
response = mock.Mock(text='{"data": []}')
self.assertEqual(framalibre.response(response), [])
html = u"""
<div class="nodes-list-row">
<div id="node-431"
class="node node-logiciel-annuaires node-promoted node-teaser node-teaser node-sheet clearfix nodes-list"
about="/content/gogs" typeof="sioc:Item foaf:Document">
<header class="media">
<div class="media-left">
<div class="field field-name-field-logo field-type-image field-label-hidden">
<div class="field-items">
<div class="field-item even">
<a href="/content/gogs">
<img class="media-object img-responsive" typeof="foaf:Image"
src="https://framalibre.org/sites/default/files/styles/teaser_logo/public/leslogos/gogs-lg.png?itok=rrCxKKBy"
width="70" height="70" alt="" />
</a>
</div>
</div>
</div>
</div>
<div class="media-body">
<h3 class="node-title"><a href="/content/gogs">Gogs</a></h3>
<span property="dc:title" content="Gogs" class="rdf-meta element-hidden"></span>
<div class="field field-name-field-annuaires field-type-taxonomy-term-reference field-label-hidden">
<div class="field-items">
<div class="field-item even">
<a href="/annuaires/cloudwebapps"
typeof="skos:Concept" property="rdfs:label skos:prefLabel"
datatype="" class="label label-primary">Cloud/webApps</a>
</div>
</div>
</div>
</div>
</header>
<div class="content">
<div class="field field-name-field-votre-appr-ciation field-type-fivestar field-label-hidden">
<div class="field-items">
<div class="field-item even">
</div>
</div>
</div>
<div class="field field-name-body field-type-text-with-summary field-label-hidden">
<div class="field-items">
<div class="field-item even" property="content:encoded">
<p>Gogs est une interface web basée sur git et une bonne alternative à GitHub.</p>
</div>
</div>
</div>
</div>
<footer>
<a href="/content/gogs" class="read-more btn btn-default btn-sm">Voir la notice</a>
<div class="field field-name-field-lien-officiel field-type-link-field field-label-hidden">
<div class="field-items">
<div class="field-item even">
<a href="https://gogs.io/" target="_blank" title="Voir le site officiel">
<span class="glyphicon glyphicon-globe"></span>
<span class="sr-only">Lien officiel</span>
</a>
</div>
</div>
</div>
</footer>
</div>
</div>
"""
response = mock.Mock(text=html)
results = framalibre.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'Gogs')
self.assertEqual(results[0]['url'],
'https://framalibre.org/content/gogs')
self.assertEqual(results[0]['content'],
u"Gogs est une interface web basée sur git et une bonne alternative à GitHub.")

50
tests/unit/engines/test_frinkiac.py
View File

@ -1,50 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
from searx.engines import frinkiac
from searx.testing import SearxTestCase
class TestFrinkiacEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
request_dict = defaultdict(dict)
params = frinkiac.request(query, request_dict)
self.assertTrue('url' in params)
def test_response(self):
self.assertRaises(AttributeError, frinkiac.response, None)
self.assertRaises(AttributeError, frinkiac.response, [])
self.assertRaises(AttributeError, frinkiac.response, '')
self.assertRaises(AttributeError, frinkiac.response, '[]')
text = """
[{"Id":770931,
"Episode":"S06E18",
"Timestamp":534616,
"Filename":""},
{"Id":1657080,
"Episode":"S12E14",
"Timestamp":910868,
"Filename":""},
{"Id":1943753,
"Episode":"S14E21",
"Timestamp":773439,
"Filename":""},
{"Id":107835,
"Episode":"S02E03",
"Timestamp":531709,
"Filename":""}]
"""
response = mock.Mock(text=text)
results = frinkiac.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 4)
self.assertEqual(results[0]['title'], u'S06E18')
self.assertIn('p=caption', results[0]['url'])
self.assertIn('e=S06E18', results[0]['url'])
self.assertIn('t=534616', results[0]['url'])
self.assertEqual(results[0]['thumbnail_src'], 'https://frinkiac.com/img/S06E18/534616/medium.jpg')
self.assertEqual(results[0]['img_src'], 'https://frinkiac.com/img/S06E18/534616.jpg')

231
tests/unit/engines/test_genius.py
View File

@ -1,231 +0,0 @@
from collections import defaultdict
import mock
from datetime import datetime
from searx.engines import genius
from searx.testing import SearxTestCase
class TestGeniusEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
params = genius.request(query, dicto)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('genius.com' in params['url'])
def test_response(self):
json_empty = """
{
"meta": {
"status": 200
},
"response": {
"sections": [
{
"type": "top_hit",
"hits": []
},
{
"type": "song",
"hits": []
},
{
"type": "lyric",
"hits": []
},
{
"type": "artist",
"hits": []
},
{
"type": "album",
"hits": []
},
{
"type": "tag",
"hits": []
},
{
"type": "video",
"hits": []
},
{
"type": "article",
"hits": []
},
{
"type": "user",
"hits": []
}
]
}
}
"""
resp = mock.Mock(text=json_empty)
self.assertEqual(genius.response(resp), [])
json = """
{
"meta": {
"status": 200
},
"response": {
"sections": [
{
"type": "lyric",
"hits": [
{
"highlights": [
{
"property": "lyrics",
"value": "Sample lyrics",
"snippet": true,
"ranges": []
}
],
"index": "lyric",
"type": "song",
"result": {
"_type": "song",
"annotation_count": 45,
"api_path": "/songs/52916",
"full_title": "J't'emmerde by MC Jean Gab'1",
"header_image_thumbnail_url": "https://images.genius.com/xxx.300x300x1.jpg",
"header_image_url": "https://images.genius.com/ef9f736a86df3c3b1772f3fb7fbdb21c.1000x1000x1.jpg",
"id": 52916,
"instrumental": false,
"lyrics_owner_id": 15586,
"lyrics_state": "complete",
"lyrics_updated_at": 1498744545,
"path": "/Mc-jean-gab1-jtemmerde-lyrics",
"pyongs_count": 4,
"song_art_image_thumbnail_url": "https://images.genius.com/xxx.300x300x1.jpg",
"stats": {
"hot": false,
"unreviewed_annotations": 0,
"pageviews": 62490
},
"title": "J't'emmerde",
"title_with_featured": "J't'emmerde",
"updated_by_human_at": 1498744546,
"url": "https://genius.com/Mc-jean-gab1-jtemmerde-lyrics",
"primary_artist": {
"_type": "artist",
"api_path": "/artists/12691",
"header_image_url": "https://images.genius.com/c7847662a58f8c2b0f02a6e217d60907.960x657x1.jpg",
"id": 12691,
"image_url": "https://s3.amazonaws.com/rapgenius/Mc-jean-gab1.jpg",
"index_character": "m",
"is_meme_verified": false,
"is_verified": false,
"name": "MC Jean Gab'1",
"slug": "Mc-jean-gab1",
"url": "https://genius.com/artists/Mc-jean-gab1"
}
}
}
]
},
{
"type": "artist",
"hits": [
{
"highlights": [],
"index": "artist",
"type": "artist",
"result": {
"_type": "artist",
"api_path": "/artists/191580",
"header_image_url": "https://assets.genius.com/images/default_avatar_300.png?1503090542",
"id": 191580,
"image_url": "https://assets.genius.com/images/default_avatar_300.png?1503090542",
"index_character": "a",
"is_meme_verified": false,
"is_verified": false,
"name": "ASDF Guy",
"slug": "Asdf-guy",
"url": "https://genius.com/artists/Asdf-guy"
}
}
]
},
{
"type": "album",
"hits": [
{
"highlights": [],
"index": "album",
"type": "album",
"result": {
"_type": "album",
"api_path": "/albums/132332",
"cover_art_thumbnail_url": "https://images.genius.com/xxx.300x300x1.jpg",
"cover_art_url": "https://images.genius.com/xxx.600x600x1.jpg",
"full_title": "ASD by A Skylit Drive",
"id": 132332,
"name": "ASD",
"name_with_artist": "ASD (artist: A Skylit Drive)",
"release_date_components": {
"year": 2015,
"month": null,
"day": null
},
"url": "https://genius.com/albums/A-skylit-drive/Asd",
"artist": {
"_type": "artist",
"api_path": "/artists/48712",
"header_image_url": "https://images.genius.com/814c1551293172c56306d0e310c6aa89.620x400x1.jpg",
"id": 48712,
"image_url": "https://images.genius.com/814c1551293172c56306d0e310c6aa89.620x400x1.jpg",
"index_character": "s",
"is_meme_verified": false,
"is_verified": false,
"name": "A Skylit Drive",
"slug": "A-skylit-drive",
"url": "https://genius.com/artists/A-skylit-drive"
}
}
}
]
}
]
}
}
"""
resp = mock.Mock(text=json)
results = genius.response(resp)
self.assertEqual(len(results), 3)
self.assertEqual(type(results), list)
# check lyric parsing
r = results[0]
self.assertEqual(r['url'], 'https://genius.com/Mc-jean-gab1-jtemmerde-lyrics')
self.assertEqual(r['title'], "J't'emmerde by MC Jean Gab'1")
self.assertEqual(r['content'], "Sample lyrics")
self.assertEqual(r['template'], 'videos.html')
self.assertEqual(r['thumbnail'], 'https://images.genius.com/xxx.300x300x1.jpg')
created = datetime.fromtimestamp(1498744545)
self.assertEqual(r['publishedDate'], created)
# check artist parsing
r = results[1]
self.assertEqual(r['url'], 'https://genius.com/artists/Asdf-guy')
self.assertEqual(r['title'], "ASDF Guy")
self.assertEqual(r['content'], None)
self.assertEqual(r['template'], 'videos.html')
self.assertEqual(r['thumbnail'], 'https://assets.genius.com/images/default_avatar_300.png?1503090542')
# check album parsing
r = results[2]
self.assertEqual(r['url'], 'https://genius.com/albums/A-skylit-drive/Asd')
self.assertEqual(r['title'], "ASD by A Skylit Drive")
self.assertEqual(r['content'], "Released: 2015")
self.assertEqual(r['template'], 'videos.html')
self.assertEqual(r['thumbnail'], 'https://images.genius.com/xxx.600x600x1.jpg')

119
tests/unit/engines/test_gigablast.py
View File

@ -1,119 +0,0 @@
from collections import defaultdict
import mock
from searx.engines import gigablast
from searx.testing import SearxTestCase
class TestGigablastEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 0
dicto['safesearch'] = 0
dicto['language'] = 'all'
params = gigablast.request(query, dicto)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('gigablast.com' in params['url'])
self.assertTrue('xx' in params['url'])
dicto['language'] = 'en-US'
params = gigablast.request(query, dicto)
self.assertTrue('en' in params['url'])
self.assertFalse('en-US' in params['url'])
def test_response(self):
self.assertRaises(AttributeError, gigablast.response, None)
self.assertRaises(AttributeError, gigablast.response, [])
self.assertRaises(AttributeError, gigablast.response, '')
self.assertRaises(AttributeError, gigablast.response, '[]')
response = mock.Mock(text='{"results": []}')
self.assertEqual(gigablast.response(response), [])
json = """{"results": [
{
"title":"South by Southwest 2016",
"dmozEntry":{
"dmozCatId":1041152,
"directCatId":1,
"dmozCatStr":"Top: Regional: North America: United States",
"dmozTitle":"South by Southwest (SXSW)",
"dmozSum":"Annual music, film, and interactive conference.",
"dmozAnchor":""
},
"dmozEntry":{
"dmozCatId":763945,
"directCatId":1,
"dmozCatStr":"Top: Regional: North America: United States",
"dmozTitle":"South by Southwest (SXSW)",
"dmozSum":"",
"dmozAnchor":"www.sxsw.com"
},
"dmozEntry":{
"dmozCatId":761446,
"directCatId":1,
"dmozCatStr":"Top: Regional: North America: United States",
"dmozTitle":"South by Southwest (SXSW)",
"dmozSum":"Music, film, and interactive conference and festival.",
"dmozAnchor":""
},
"indirectDmozCatId":1041152,
"indirectDmozCatId":763945,
"indirectDmozCatId":761446,
"contentType":"html",
"sum":"This should be the content.",
"url":"www.sxsw.com",
"hopCount":0,
"size":" 102k",
"sizeInBytes":104306,
"bytesUsedToComputeSummary":70000,
"docId":269411794364,
"docScore":586571136.000000,
"summaryGenTimeMS":12,
"summaryTagdbLookupTimeMS":0,
"summaryTitleRecLoadTimeMS":1,
"site":"www.sxsw.com",
"spidered":1452203608,
"firstIndexedDateUTC":1444167123,
"contentHash32":2170650347,
"language":"English",
"langAbbr":"en"
}
]}
"""
response = mock.Mock(text=json)
results = gigablast.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'South by Southwest 2016')
self.assertEqual(results[0]['url'], 'www.sxsw.com')
self.assertEqual(results[0]['content'], 'This should be the content.')
def test_fetch_supported_languages(self):
html = """<html></html>"""
response = mock.Mock(text=html)
results = gigablast._fetch_supported_languages(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
html = """
<html>
<body>
<span id="menu2">
<a href="/search?&rxikd=1&qlang=xx"></a>
<a href="/search?&rxikd=1&qlang=en"></a>
<a href="/search?&rxikd=1&prepend=gblang%3Aen"></a>
<a href="/search?&rxikd=1&qlang=zh_"></a>
<a href="/search?&rxikd=1&prepend=gblang%3Azh_tw"></a>
</span>
</body>
</html>
"""
response = mock.Mock(text=html)
languages = gigablast._fetch_supported_languages(response)
self.assertEqual(type(languages), list)
self.assertEqual(len(languages), 2)
self.assertIn('en', languages)
self.assertIn('zh-TW', languages)

61
tests/unit/engines/test_github.py
View File

@ -1,61 +0,0 @@
from collections import defaultdict
import mock
from searx.engines import github
from searx.testing import SearxTestCase
class TestGitHubEngine(SearxTestCase):
def test_request(self):
query = 'test_query'
params = github.request(query, defaultdict(dict))
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('github.com' in params['url'])
self.assertEqual(params['headers']['Accept'], github.accept_header)
def test_response(self):
self.assertRaises(AttributeError, github.response, None)
self.assertRaises(AttributeError, github.response, [])
self.assertRaises(AttributeError, github.response, '')
self.assertRaises(AttributeError, github.response, '[]')
response = mock.Mock(text='{}')
self.assertEqual(github.response(response), [])
response = mock.Mock(text='{"items": []}')
self.assertEqual(github.response(response), [])
json = """
{
"items": [
{
"name": "title",
"html_url": "url",
"description": ""
}
]
}
"""
response = mock.Mock(text=json)
results = github.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'title')
self.assertEqual(results[0]['url'], 'url')
self.assertEqual(results[0]['content'], '')
json = """
{
"items": [
{
"name": "title",
"html_url": "url",
"description": "desc"
}
]
}
"""
response = mock.Mock(text=json)
results = github.response(response)
self.assertEqual(results[0]['content'], "desc")

194
tests/unit/engines/test_google.py
View File

@ -1,194 +0,0 @@
# -*- coding: utf-8 -*-
from collections import defaultdict
import mock
import lxml
from searx.engines import google
from searx.testing import SearxTestCase
class TestGoogleEngine(SearxTestCase):
def mock_response(self, text):
response = mock.Mock(text=text, url='https://www.google.com/search?q=test&start=0&gbv=1&gws_rd=cr')
response.search_params = mock.Mock()
response.search_params.get = mock.Mock(return_value='www.google.com')
return response
def test_request(self):
google.supported_languages = ['en', 'fr', 'zh-CN', 'iw']
google.language_aliases = {'he': 'iw'}
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
dicto['language'] = 'fr-FR'
dicto['time_range'] = ''
params = google.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
self.assertIn('google.fr', params['url'])
self.assertIn('fr', params['url'])
self.assertIn('fr', params['headers']['Accept-Language'])
dicto['language'] = 'en-US'
params = google.request(query, dicto)
self.assertIn('google.com', params['url'])
self.assertIn('en', params['url'])
self.assertIn('en', params['headers']['Accept-Language'])
dicto['language'] = 'zh'
params = google.request(query, dicto)
self.assertIn('google.com', params['url'])
self.assertIn('zh-CN', params['url'])
self.assertIn('zh-CN', params['headers']['Accept-Language'])
dicto['language'] = 'he'
params = google.request(query, dicto)
self.assertIn('google.com', params['url'])
self.assertIn('iw', params['url'])
self.assertIn('iw', params['headers']['Accept-Language'])
def test_response(self):
self.assertRaises(AttributeError, google.response, None)
self.assertRaises(AttributeError, google.response, [])
self.assertRaises(AttributeError, google.response, '')
self.assertRaises(AttributeError, google.response, '[]')
response = self.mock_response('<html></html>')
self.assertEqual(google.response(response), [])
html = """
<div class="ZINbbc xpd O9g5cc uUPGi">
<div>
<div class="kCrYT">
<a href="/url?q=http://this.should.be.the.link/">
<div class="BNeawe">
<b>This</b> is <b>the</b> title
</div>
<div class="BNeawe">
http://website
</div>
</a>
</div>
<div class="kCrYT">
<div>
<div class="BNeawe">
<div>
<div class="BNeawe">
This should be the content.
</div>
</div>
</div>
</div>
</div>
</div>
</p>
<div class="ZINbbc xpd O9g5cc uUPGi">
<div>
<div class="kCrYT">
<span>
<div class="BNeawe">
Related searches
</div>
</span>
</div>
<div class="rVLSBd">
<a>
<div>
<div class="BNeawe">
suggestion title
</div>
</div>
</a>
</div>
</div>
</p>
"""
response = self.mock_response(html)
results = google.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 2)
self.assertEqual(results[0]['title'], 'This is the title')
self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
self.assertEqual(results[0]['content'], 'This should be the content.')
self.assertEqual(results[1]['suggestion'], 'suggestion title')
html = """
<li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
</li>
"""
response = self.mock_response(html)
results = google.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
response = mock.Mock(text='<html></html>', url='https://sorry.google.com')
response.search_params = mock.Mock()
response.search_params.get = mock.Mock(return_value='www.google.com')
self.assertRaises(RuntimeWarning, google.response, response)
response = mock.Mock(text='<html></html>', url='https://www.google.com/sorry/IndexRedirect')
response.search_params = mock.Mock()
response.search_params.get = mock.Mock(return_value='www.google.com')
self.assertRaises(RuntimeWarning, google.response, response)
def test_parse_images(self):
html = """
<li>
<div>
<a href="http://www.google.com/url?q=http://this.is.the.url/">
<img style="margin:3px 0;margin-right:6px;padding:0" height="90"
src="https://this.is.the.image/image.jpg" width="60" align="middle" alt="" border="0">
</a>
</div>
</li>
"""
dom = lxml.html.fromstring(html)
results = google.parse_images(dom, 'www.google.com')
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['url'], 'http://this.is.the.url/')
self.assertEqual(results[0]['title'], '')
self.assertEqual(results[0]['content'], '')
self.assertEqual(results[0]['img_src'], 'https://this.is.the.image/image.jpg')
def test_fetch_supported_languages(self):
html = """<html></html>"""
response = mock.Mock(text=html)
languages = google._fetch_supported_languages(response)
self.assertEqual(type(languages), dict)
self.assertEqual(len(languages), 0)
html = u"""
<html>
<body>
<div id="langSec">
<div>
<input name="lr" data-name="english" value="lang_en" />
<input name="lr" data-name="中文 (简体)" value="lang_zh-CN" />
<input name="lr" data-name="中文 (繁體)" value="lang_zh-TW" />
</div>
</div>
</body>
</html>
"""
response = mock.Mock(text=html)
languages = google._fetch_supported_languages(response)
self.assertEqual(type(languages), dict)
self.assertEqual(len(languages), 3)
self.assertIn('en', languages)
self.assertIn('zh-CN', languages)
self.assertIn('zh-TW', languages)
self.assertEquals(type(languages['en']), dict)
self.assertEquals(type(languages['zh-CN']), dict)
self.assertEquals(type(languages['zh-TW']), dict)
self.assertIn('name', languages['en'])
self.assertIn('name', languages['zh-CN'])
self.assertIn('name', languages['zh-TW'])
self.assertEquals(languages['en']['name'], 'English')
self.assertEquals(languages['zh-CN']['name'], u'中文 (简体)')
self.assertEquals(languages['zh-TW']['name'], u'中文 (繁體)')

Some files were not shown because too many files have changed in this diff Show More