jandieman/searxng
2
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2026-09-11 16:56:05 +00:00
Code Issues Projects Releases Wiki Activity

Compare commits

base: jandieman:3fdc6d753a339b5f4a7dc5842c94c0d8324726f1
Branches Tags
jandieman:master jandieman:gh-pages jandieman:dependabot/pip/master/minor-16b941ef60 jandieman:dependabot/pip/master/black-26.5.1 jandieman:translations jandieman:update_data_update_engine_descriptions.py
..
compare: jandieman:update_data_update_engine_descriptions.py
Branches Tags
jandieman:gh-pages jandieman:master jandieman:dependabot/pip/master/minor-16b941ef60 jandieman:dependabot/pip/master/black-26.5.1 jandieman:translations jandieman:update_data_update_engine_descriptions.py

1 Commits
3fdc6d753a ... update_dat

Author SHA1 Message Date
searxng-bot
872e838fa4 [data] update searx.data - update_engine_descriptions.py 2026-06-29 01:01:55 +00:00
361 changed files with 16187 additions and 38123 deletions
Expand all files Collapse all files

39
.github/scripts/ai_policy.cjs vendored
View File

@@ -1,39 +0,0 @@
// Closes issues and prs whose authors/agents don't accept the ai policy
// https://github.com/searxng/searxng/blob/master/AI_POLICY.rst
module.exports = async ({ github, context }) => {
const item = context.payload.pull_request || context.payload.issue;
const body = item.body || '';
const kind = context.payload.pull_request ? 'pull request' : 'issue';
// https://github.com/searxng/searxng/pull/6476#discussion_r3683782481
const hasBox = /\[[Xx]\].*AI Policy/.test(body);
const hasRef = /\[AI Policy\]:\s*https:\/\/github\.com\/searxng\/searxng\/.*AI_POLICY/.test(body);
if (hasBox && hasRef) {
return;
}
const { owner, repo } = context.repo;
await github.rest.issues.createComment({
owner,
repo,
issue_number: item.number,
body:
'Hello! Thank you for your contribution.\n\n' +
`Unfortunately your ${kind} was closed as the AI Policy has not been accepted.\n\n` +
`Please open a new ${kind} after confirming your contribution aligns with our AI Policy.`,
});
await github.rest.issues.addLabels({
owner,
repo,
issue_number: item.number,
labels: ['invalid:slop'],
});
await github.rest.issues.update({
owner,
repo,
issue_number: item.number,
state: 'closed',
state_reason: 'not_planned',
});
};

38
.github/workflows/ai-policy.yml vendored
View File

@@ -1,38 +0,0 @@
---
# yamllint disable rule:line-length
name: AI Policy
# Closes any new issues and PRs from people (or agents) who don't accept the AI Policy
# yamllint disable-line rule:truthy
on:
issues:
types: [opened]
pull_request_target:
types: [opened]
permissions:
contents: read
issues: write
pull-requests: write
jobs:
check:
name: Check AI Policy
# for issues with an author who has not contributed before
if: >-
github.event.sender.type != 'Bot' &&
contains(fromJSON('["NONE","FIRST_TIMER","FIRST_TIME_CONTRIBUTOR"]'),
github.event.issue.author_association ||
github.event.pull_request.author_association)
runs-on: ubuntu-26.04-arm
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: "false"
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const script = require('./.github/scripts/ai_policy.cjs');
await script({ github, context });

155
.github/workflows/container.yml vendored
View File

@@ -25,21 +25,25 @@ env:
jobs:
build:
if: |
github.event_name == 'workflow_dispatch'
|| (github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success')
if: github.repository_owner == 'searxng' || github.event_name == 'workflow_dispatch'
name: Build (${{ matrix.arch }})
runs-on: ${{ matrix.runner }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- runner: ubuntu-26.04
arch: amd64
- runner: ubuntu-26.04-arm
arch: arm64
- runner: ubuntu-26.04-arm
arch: armv7
- arch: amd64
march: amd64
os: ubuntu-24.04
emulation: false
- arch: arm64
march: arm64
os: ubuntu-24.04-arm
emulation: false
- arch: armv7
march: arm64
os: ubuntu-24.04-arm
emulation: true
permissions:
packages: write
@@ -49,83 +53,109 @@ jobs:
git_url: ${{ steps.build.outputs.git_url }}
steps:
- name: Login to GHCR
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: "ghcr.io"
username: "${{ github.repository_owner }}"
password: "${{ secrets.GITHUB_TOKEN }}"
# yamllint disable rule:line-length
- name: Setup podman
env:
PODMAN_VERSION: "v5.7.1"
run: |
sudo apt-get purge -y podman runc crun conmon
curl -fsSLO "https://github.com/mgoltzsche/podman-static/releases/download/${{ env.PODMAN_VERSION }}/podman-linux-${{ matrix.march }}.tar.gz"
curl -fsSLO "https://github.com/mgoltzsche/podman-static/releases/download/${{ env.PODMAN_VERSION }}/podman-linux-${{ matrix.march }}.tar.gz.asc"
gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 0CCF102C4F95D89E583FF1D4F8B5AF50344BB503
gpg --batch --verify "podman-linux-${{ matrix.march }}.tar.gz.asc" "podman-linux-${{ matrix.march }}.tar.gz"
tar -xzf "podman-linux-${{ matrix.march }}.tar.gz"
sudo cp -rfv ./podman-linux-${{ matrix.march }}/etc/. /etc/
sudo cp -rfv ./podman-linux-${{ matrix.march }}/usr/. /usr/
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
# yamllint enable rule:line-length
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Setup QEMU
uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: "${{ github.event.workflow_run.head_sha || github.sha }}"
persist-credentials: "false"
fetch-depth: "0"
- name: Setup cache Python
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0
with:
key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
restore-keys: |
python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-
path: "./local/"
- name: Get date
id: date
run: echo "date=$(date +'%Y%m%d')" >>$GITHUB_OUTPUT
- name: Setup cache container
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0
with:
key: "container-${{ matrix.arch }}-${{ hashFiles('./requirements*.txt') }}"
key: "container-${{ matrix.arch }}-${{ steps.date.outputs.date }}-${{ hashFiles('./requirements*.txt') }}"
restore-keys: |
container-${{ matrix.arch }}-${{ steps.date.outputs.date }}-
container-${{ matrix.arch }}-
path: "/var/tmp/buildah-cache-*/*"
- name: Build
id: build
env:
OVERRIDE_ARCH: "${{ matrix.arch }}"
run: make container.build
- if: ${{ matrix.emulation }}
name: Setup QEMU
uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0
test:
name: Test (${{ matrix.arch }})
runs-on: ${{ matrix.runner }}
needs: build
strategy:
fail-fast: false
matrix:
include:
- runner: ubuntu-26.04
arch: amd64
- runner: ubuntu-26.04-arm
arch: arm64
# FIXME: https://github.com/searxng/searxng/pull/6655#issuecomment-5550293085
# - runner: ubuntu-26.04-arm
# arch: armv7
steps:
- name: Login to GHCR
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
registry: "ghcr.io"
username: "${{ github.repository_owner }}"
password: "${{ secrets.GITHUB_TOKEN }}"
- name: Setup QEMU
uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0
- name: Build
id: build
env:
OVERRIDE_ARCH: "${{ matrix.arch }}"
run: make podman.build
test:
name: Test (${{ matrix.arch }})
runs-on: ${{ matrix.os }}
needs: build
strategy:
fail-fast: false
matrix:
include:
- arch: amd64
os: ubuntu-24.04
emulation: false
- arch: arm64
os: ubuntu-24.04-arm
emulation: false
- arch: armv7
os: ubuntu-24.04-arm
emulation: true
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: "${{ github.event.workflow_run.head_sha || github.sha }}"
persist-credentials: "false"
- if: ${{ matrix.emulation }}
name: Setup QEMU
uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0
- name: Login to GHCR
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
registry: "ghcr.io"
username: "${{ github.repository_owner }}"
password: "${{ secrets.GITHUB_TOKEN }}"
- name: Test
env:
OVERRIDE_ARCH: "${{ matrix.arch }}"
@@ -135,7 +165,7 @@ jobs:
release:
if: github.repository_owner == 'searxng' && github.ref_name == 'master'
name: Release
runs-on: ubuntu-26.04-arm
runs-on: ubuntu-24.04-arm
needs:
- build
- test
@@ -144,25 +174,24 @@ jobs:
packages: write
steps:
- name: Login to Docker Hub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
registry: "docker.io"
username: "${{ secrets.DOCKER_USER }}"
password: "${{ secrets.DOCKER_TOKEN }}"
persist-credentials: "false"
- name: Login to GHCR
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
registry: "ghcr.io"
username: "${{ github.repository_owner }}"
password: "${{ secrets.GITHUB_TOKEN }}"
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Login to Docker Hub
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
ref: "${{ github.event.workflow_run.head_sha || github.sha }}"
persist-credentials: "false"
registry: "docker.io"
username: "${{ secrets.DOCKER_USER }}"
password: "${{ secrets.DOCKER_TOKEN }}"
- name: Release
env:

25
.github/workflows/data-update.yml vendored
View File

@@ -21,7 +21,7 @@ jobs:
data:
if: github.repository_owner == 'searxng'
name: ${{ matrix.fetch }}
runs-on: ubuntu-26.04-arm
runs-on: ubuntu-24.04-arm
strategy:
fail-fast: false
matrix:
@@ -31,8 +31,9 @@ jobs:
- update_external_bangs.py
- update_firefox_version.py
- update_engine_traits.py
- update_wikidata.py
- update_wikidata_units.py
- update_engine_descriptions.py
- update_gsa_useragents.py
permissions:
contents: write
@@ -40,17 +41,17 @@ jobs:
steps:
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: "false"
- name: Setup cache Python
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0
with:
key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
restore-keys: |
@@ -64,17 +65,23 @@ jobs:
run: V=1 ./manage pyenv.cmd python "./searxng_extra/update/${{ matrix.fetch }}"
- name: Create PR
id: cpr
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
with:
author: "searxng-bot <searxng-bot@users.noreply.github.com>"
committer: "searxng-bot <searxng-bot@users.noreply.github.com>"
title: "[mod] data: update searx.data - ${{ matrix.fetch }}"
commit-message: "[mod] data: update searx.data - ${{ matrix.fetch }}"
branch: "ci-data-${{ matrix.fetch }}"
title: "[data] update searx.data - ${{ matrix.fetch }}"
commit-message: "[data] update searx.data - ${{ matrix.fetch }}"
branch: "update_data_${{ matrix.fetch }}"
delete-branch: "true"
draft: "false"
signoff: "false"
body: |
Update searx.data - ${{ matrix.fetch }}
[data] update searx.data - ${{ matrix.fetch }}
labels: |
data
- name: Display information
run: |
echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

12
.github/workflows/documentation.yml vendored
View File

@@ -25,25 +25,25 @@ jobs:
release:
if: github.repository_owner == 'searxng' || github.event_name == 'workflow_dispatch'
name: Release
runs-on: ubuntu-26.04-arm
runs-on: ubuntu-24.04-arm
permissions:
# for JamesIves/github-pages-deploy-action to push
contents: write
steps:
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: "false"
fetch-depth: "0"
- name: Setup cache Python
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0
with:
key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
restore-keys: |
@@ -61,11 +61,11 @@ jobs:
- if: github.ref_name == 'master'
name: Release
uses: JamesIves/github-pages-deploy-action@fa24774553152dd7873cd16ebd8d959b010c5445 # v4.9.0
uses: JamesIves/github-pages-deploy-action@d92aa235d04922e8f08b40ce78cc5442fcfbfa2f # v4.8.0
with:
folder: "dist/docs"
branch: "gh-pages"
commit-message: "[mod] docs: build from commit ${{ github.sha }}"
commit-message: "[doc] build from commit ${{ github.sha }}"
# Automatically remove deleted files from the deploy branch
clean: "true"
single-commit: "true"

41
.github/workflows/integration.yml vendored
View File

@@ -23,7 +23,7 @@ env:
jobs:
test:
name: Python ${{ matrix.python-version }}
runs-on: ubuntu-26.04
runs-on: ubuntu-24.04
strategy:
matrix:
python-version:
@@ -34,17 +34,17 @@ jobs:
steps:
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "${{ matrix.python-version }}"
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: "false"
- name: Setup cache Python
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0
with:
key: "python-${{ matrix.python-version }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
restore-keys: |
@@ -59,40 +59,37 @@ jobs:
theme:
name: Theme
runs-on: ubuntu-26.04-arm
runs-on: ubuntu-24.04-arm
steps:
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "26"
check-latest: "true"
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: "false"
- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version-file: "./.nvmrc"
- name: Setup cache Node.js
uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0
with:
key: "nodejs-${{ runner.arch }}-${{ hashFiles('./.nvmrc', './package.json') }}"
path: "./client/simple/node_modules/"
- name: Setup cache Python
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0
with:
key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
restore-keys: |
python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-
path: "./local/"
- name: Setup cache Node.js
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
key: "nodejs-${{ runner.arch }}-${{ hashFiles('**/package-lock.json') }}"
restore-keys: |
nodejs-${{ runner.arch }}-
path: "./client/simple/node_modules/"
- name: Setup venv
run: make V=1 install

34
.github/workflows/l10n.yml vendored
View File

@@ -26,27 +26,27 @@ env:
jobs:
update:
if: github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'searxng'
if: github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success'
name: Update
runs-on: ubuntu-26.04-arm
runs-on: ubuntu-24.04-arm
permissions:
# For "make V=1 weblate.push.translations"
contents: write
steps:
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}"
fetch-depth: "0"
- name: Setup cache Python
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0
with:
key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
restore-keys: |
@@ -59,7 +59,7 @@ jobs:
- name: Setup Weblate
run: |
mkdir -p ~/.config
echo "${{ secrets.WEBLATE_CONFIG }}" >~/.config/weblate
echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate
- name: Setup Git
run: |
@@ -74,7 +74,7 @@ jobs:
github.repository_owner == 'searxng'
&& (github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
name: Pull Request
runs-on: ubuntu-26.04-arm
runs-on: ubuntu-24.04-arm
permissions:
# For "make V=1 weblate.translations.commit"
contents: write
@@ -83,18 +83,18 @@ jobs:
steps:
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}"
fetch-depth: "0"
- name: Setup cache Python
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0
with:
key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
restore-keys: |
@@ -107,7 +107,7 @@ jobs:
- name: Setup Weblate
run: |
mkdir -p ~/.config
echo "${{ secrets.WEBLATE_CONFIG }}" >~/.config/weblate
echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate
- name: Setup Git
run: |
@@ -118,17 +118,23 @@ jobs:
run: make V=1 weblate.translations.commit
- name: Create PR
id: cpr
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
with:
author: "searxng-bot <searxng-bot@users.noreply.github.com>"
committer: "searxng-bot <searxng-bot@users.noreply.github.com>"
title: "[mod] i18n: update translations from Weblate"
commit-message: "[mod] i18n: update translations from Weblate"
title: "[l10n] update translations from Weblate"
commit-message: "[l10n] update translations from Weblate"
branch: "translations_update"
delete-branch: "true"
draft: "false"
signoff: "false"
body: |
Update translations from Weblate
[l10n] update translations from Weblate
labels: |
area:i18n
- name: Display information
run: |
echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

46
.github/workflows/security.yml vendored Normal file
View File

@@ -0,0 +1,46 @@
---
name: Security
# yamllint disable-line rule:truthy
on:
workflow_dispatch:
schedule:
- cron: "42 05 * * *"
concurrency:
group: ${{ github.workflow }}
cancel-in-progress: false
permissions:
contents: read
jobs:
container:
if: github.repository_owner == 'searxng'
name: Container
runs-on: ubuntu-24.04-arm
permissions:
security-events: write
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: "false"
- name: Sync GHCS from Docker Scout
uses: docker/scout-action@7520205ff60037fdc436b40b6a1d1e55a839ec2d # v1.22.0
with:
organization: "searxng"
dockerhub-user: "${{ secrets.DOCKER_USER }}"
dockerhub-password: "${{ secrets.DOCKER_TOKEN }}"
image: "registry://ghcr.io/searxng/searxng:latest"
command: "cves"
sarif-file: "./scout.sarif"
exit-code: "false"
write-comment: "false"
- name: Upload SARIFs
uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
with:
sarif_file: "./scout.sarif"

2
Makefile
View File

@@ -63,7 +63,7 @@ format: format.python format.shell
# wrap ./manage script
MANAGE += weblate.translations.commit weblate.push.translations
MANAGE += data.all data.traits data.useragents data.locales data.currencies
MANAGE += data.all data.traits data.useragents data.gsa_useragents data.locales data.currencies
MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
MANAGE += podman.build
MANAGE += docker.build docker.buildx

10
client/simple/generated/pygments.less
View File

@@ -2,7 +2,7 @@
/*
this file is generated automatically by searxng_extra/update/update_pygments.py
using pygments version 2.21.0:
using pygments version 2.20.0:
./manage templates.simple.pygments
*/
@@ -114,14 +114,14 @@
.gd { color: #FF4689 } /* Generic.Deleted */
.ge { color: #F8F8F2; font-style: italic } /* Generic.Emph */
.ges { color: #F8F8F2; font-weight: bold; font-style: italic } /* Generic.EmphStrong */
.gr { color: #FF4689 } /* Generic.Error */
.gr { color: #F8F8F2 } /* Generic.Error */
.gh { color: #F8F8F2 } /* Generic.Heading */
.gi { color: #A6E22E } /* Generic.Inserted */
.go { color: #66D9EF } /* Generic.Output */
.gp { color: #FF4689; font-weight: bold } /* Generic.Prompt */
.gs { color: #F8F8F2; font-weight: bold } /* Generic.Strong */
.gu { color: #959077 } /* Generic.Subheading */
.gt { color: #66D9EF } /* Generic.Traceback */
.gt { color: #F8F8F2 } /* Generic.Traceback */
.kc { color: #66D9EF } /* Keyword.Constant */
.kd { color: #66D9EF } /* Keyword.Declaration */
.kn { color: #FF4689 } /* Keyword.Namespace */
@@ -132,7 +132,7 @@
.m { color: #AE81FF } /* Literal.Number */
.s { color: #E6DB74 } /* Literal.String */
.na { color: #A6E22E } /* Name.Attribute */
.nb { color: #A6E22E } /* Name.Builtin */
.nb { color: #F8F8F2 } /* Name.Builtin */
.nc { color: #A6E22E } /* Name.Class */
.no { color: #66D9EF } /* Name.Constant */
.nd { color: #A6E22E } /* Name.Decorator */
@@ -166,7 +166,7 @@
.sr { color: #E6DB74 } /* Literal.String.Regex */
.s1 { color: #E6DB74 } /* Literal.String.Single */
.ss { color: #E6DB74 } /* Literal.String.Symbol */
.bp { color: #A6E22E } /* Name.Builtin.Pseudo */
.bp { color: #F8F8F2 } /* Name.Builtin.Pseudo */
.fm { color: #A6E22E } /* Name.Function.Magic */
.vc { color: #F8F8F2 } /* Name.Variable.Class */
.vg { color: #F8F8F2 } /* Name.Variable.Global */

1818
client/simple/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

24
client/simple/package.json
View File

@@ -23,27 +23,27 @@
"not dead"
],
"dependencies": {
"ionicons": "^8.1.0",
"ionicons": "^8.0.13",
"normalize.css": "8.0.1",
"ol": "^10.10.0",
"ol": "^10.9.0",
"swiped-events": "1.2.0"
},
"devDependencies": {
"@biomejs/biome": "2.5.11",
"@types/node": "^26.4.0",
"browserslist": "^4.28.8",
"@biomejs/biome": "2.5.0",
"@types/node": "^26.0.0",
"browserslist": "^4.28.2",
"browserslist-to-esbuild": "^2.1.1",
"edge.js": "^6.5.1",
"less": "^4.9.0",
"less": "^4.6.6",
"mathjs": "^15.2.0",
"sharp": "~0.35.4",
"sharp": "~0.35.1",
"sort-package-json": "^4.0.0",
"stylelint": "^17.14.1",
"stylelint": "^17.13.0",
"stylelint-config-standard-less": "^4.1.0",
"stylelint-prettier": "^5.0.3",
"svgo": "^4.1.0",
"typescript": "~7.0.2",
"vite": "^8.2.2",
"vite-bundle-analyzer": "^1.3.9"
"svgo": "^4.0.1",
"typescript": "~6.0.3",
"vite": "^8.0.16",
"vite-bundle-analyzer": "^1.3.8"
}
}

8
client/simple/src/js/main/autocomplete.ts
View File

@@ -5,7 +5,13 @@ import { assertElement } from "../util/assertElement.ts";
const fetchResults = async (qInput: HTMLInputElement, query: string): Promise<void> => {
try {
const res = await http("GET", `./autocompleter?q=${query}`);
let res: Response;
if (settings.method === "GET") {
res = await http("GET", `./autocompleter?q=${query}`);
} else {
res = await http("POST", "./autocompleter", { body: new URLSearchParams({ q: query }) });
}
const results = await res.json();

5
client/simple/src/js/main/results.ts
View File

@@ -35,9 +35,8 @@ const imageLoader = (resultElement: HTMLElement): void => {
}, 1000) as unknown as number;
};
const imageThumbnails: NodeListOf<HTMLImageElement> = document.querySelectorAll<HTMLImageElement>(
"#urls img.image_thumbnail, img.thumbnail"
);
const imageThumbnails: NodeListOf<HTMLImageElement> =
document.querySelectorAll<HTMLImageElement>("#urls img.image_thumbnail");
for (const thumbnail of imageThumbnails) {
if (thumbnail.complete && thumbnail.naturalWidth === 0) {
thumbnail.src = `${settings.theme_static_path}/img/img_load_error.svg`;

7
client/simple/src/js/plugin/Calculator.ts
View File

@@ -80,12 +80,7 @@ export default class Calculator extends Plugin {
try {
const node = Calculator.math.parse(searchInput.value);
const value = node.evaluate();
if (typeof value !== "number") {
return;
}
return `${node.toString()} = ${value}`;
return `${node.toString()} = ${node.evaluate()}`;
} catch {
// not a compatible math expression
return;

2
client/simple/src/js/util/appendAnswerElement.ts
View File

@@ -23,7 +23,7 @@ export const appendAnswerElement = (element: HTMLElement | string | number): voi
if (!(element instanceof HTMLElement)) {
const span = document.createElement("span");
span.textContent = element.toString();
span.innerHTML = element.toString();
// biome-ignore lint/style/noParameterAssign: TODO
element = span;
}

2
container/builder.dockerfile
View File

@@ -1,4 +1,4 @@
FROM docker.io/searxng/base:searxng-builder AS builder
FROM ghcr.io/searxng/base:searxng-builder AS builder
COPY ./requirements.txt ./requirements-server.txt ./

2
container/dist.dockerfile
View File

@@ -2,7 +2,7 @@ ARG CONTAINER_IMAGE_ORGANIZATION="searxng"
ARG CONTAINER_IMAGE_NAME="searxng"
FROM localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder AS builder
FROM docker.io/searxng/base:searxng AS dist
FROM ghcr.io/searxng/base:searxng AS dist
COPY --chown=977:977 --from=builder /usr/local/searxng/.venv/ ./.venv/
COPY --chown=977:977 --from=builder /usr/local/searxng/searx/ ./searx/

11
container/entrypoint.sh
View File

@@ -112,15 +112,6 @@ if [ "$(id -u)" -eq 0 ]; then
fi
# ENVs aliases
# https://github.com/searxng/searxng/issues/5934
case "${SEARXNG_PORT:-}" in
'') ;;
*[!0-9]*)
unset SEARXNG_PORT
;;
*)
export GRANIAN_PORT="$SEARXNG_PORT"
;;
esac
export GRANIAN_PORT="${SEARXNG_PORT:-$GRANIAN_PORT}"
exec /usr/local/searxng/.venv/bin/granian searx.webapp:app

12
docs/admin/installation-docker.rst
View File

@@ -283,12 +283,12 @@ container images are not officially supported):
$ make container
$ docker images
REPOSITORY TAG IMAGE ID SIZE
localhost/searxng/searxng 2026.6.19-93f66bfb4 ... 265 MB
localhost/searxng/searxng latest ... 265 MB
localhost/searxng/searxng builder ... 687 MB
docker.io/searxng/base searxng-builder ... 565 MB
docker.io/searxng/base searxng ... 143 MB
REPOSITORY TAG IMAGE ID CREATED SIZE
localhost/searxng/searxng 2025.8.1-3d96414 ... About a minute ago 183 MB
localhost/searxng/searxng latest ... About a minute ago 183 MB
localhost/searxng/searxng builder ... About a minute ago 524 MB
ghcr.io/searxng/base searxng-builder ... 2 days ago 378 MB
ghcr.io/searxng/base searxng ... 2 days ago 42.2 MB
Migrate from ``searxng-docker``
===============================

5
docs/admin/searx.favicons.rst
View File

@@ -29,11 +29,10 @@ By default and without any extensions, SearXNG serves these resolvers:
- ``duckduckgo``
- ``allesedv``
- ``google``
- ``kagi``
- ``yandex``
With the above setting favicons are displayed, the user has the option to
deactivate this feature in their settings. If the user is to have the option of
deactivate this feature in his settings. If the user is to have the option of
selecting from several *resolvers*, a further setting is required / but this
setting will be discussed :ref:`later <register resolvers>` in this article,
first we have to setup the favicons cache.
@@ -209,7 +208,6 @@ choose from, the following configuration could be used:
"duckduckgo" = "searx.favicons.resolvers.duckduckgo"
"allesedv" = "searx.favicons.resolvers.allesedv"
# "google" = "searx.favicons.resolvers.google"
# "kagi" = "searx.favicons.resolvers.kagi"
# "yandex" = "searx.favicons.resolvers.yandex"
.. note::
@@ -228,7 +226,6 @@ into the *proxy*:
- :py:obj:`searx.favicons.resolvers.duckduckgo`
- :py:obj:`searx.favicons.resolvers.allesedv`
- :py:obj:`searx.favicons.resolvers.google`
- :py:obj:`searx.favicons.resolvers.kagi`
- :py:obj:`searx.favicons.resolvers.yandex`

23
docs/admin/settings/settings_engines.rst
View File

@@ -58,9 +58,10 @@ engine is shown. Most of the options have a default value or even are optional.
# overwrite values from section 'outgoing:'
enable_http2: false
enable_http3: false
retries: 1
max_connections: 100
max_keepalive_connections: 10
keepalive_expiry: 5.0
using_tor_proxy: false
proxies:
http:
@@ -162,16 +163,6 @@ engine is shown. Most of the options have a default value or even are optional.
``enable_http`` : optional
Enable HTTP for this engine (by default only HTTPS is enabled).
``enable_http3`` : optional
Use HTTP/3 (falls back to HTTP/2). Default ``false``.
Ignored when a proxy is set.
.. hint::
HTTP/3 places demands on the IP infrastructure that are not met in every
environment. Enable this option only if you are aware of these requirements
and the extent to which they are met.
``retry_on_http_error`` : optional
Retry request on some HTTP status code.
@@ -188,12 +179,20 @@ engine is shown. Most of the options have a default value or even are optional.
Using tor proxy (``true``) or not (``false``) for this engine. The default is
taken from ``using_tor_proxy`` of the :ref:`settings outgoing`.
.. _Pool limit configuration: https://curl-cffi.readthedocs.io/en/latest/api.html#sessions
.. _Pool limit configuration: https://www.python-httpx.org/advanced/#pool-limit-configuration
``max_keepalive_connection#s`` :
`Pool limit configuration`_, overwrites value ``pool_maxsize`` from
:ref:`settings outgoing` for this engine.
``max_connections`` :
`Pool limit configuration`_, overwrites value ``pool_connections`` from
:ref:`settings outgoing` for this engine.
``keepalive_expiry`` :
`Pool limit configuration`_, overwrites value ``keepalive_expiry`` from
:ref:`settings outgoing` for this engine.
.. _private engines:

51
docs/admin/settings/settings_outgoing.rst
View File

@@ -12,12 +12,20 @@ Communication with search engines.
request_timeout: 2.0 # default timeout in seconds, can be override by engine
max_request_timeout: 10.0 # the maximum timeout in seconds
useragent_suffix: "" # information like an email address to the administrator
pool_connections: 100 # Maximum number of concurrent connections (default: 100)
enable_http2: true # Enables the use of HTTP2
pool_connections: 100 # Maximum number of allowable connections, or null
# for no limits. The default is 100.
pool_maxsize: 10 # Number of allowable keep-alive connections, or null
# to always allow. The default is 10.
enable_http2: true # See https://www.python-httpx.org/http2/
# uncomment below section if you want to use a custom server certificate
# see https://www.python-httpx.org/advanced/#changing-the-verification-defaults
# and https://www.python-httpx.org/compatibility/#ssl-configuration
# verify: ~/.mitmproxy/mitmproxy-ca-cert.cer
#
# uncomment below section if you want to use a proxy
# uncomment below section if you want to use a proxyq see: SOCKS proxies
# https://2.python-requests.org/en/latest/user/advanced/#proxies
# are also supported: see
# https://2.python-requests.org/en/latest/user/advanced/#socks
#
# proxies:
# all://:
@@ -38,26 +46,30 @@ Communication with search engines.
timeout to load). Can be override by ``timeout`` in the :ref:`settings engines`.
``useragent_suffix`` :
Suffix to add when an engine's User-Agent is set via searxng_useragent().
Contact info here may be useful to avoid an engine blocking you.
Suffix to the user-agent SearXNG uses to send requests to others engines. If an
engine wish to block you, a contact info here may be useful to avoid that.
.. _Pool limit configuration: https://curl-cffi.readthedocs.io/en/latest/api.html#sessions
.. _Pool limit configuration: https://www.python-httpx.org/advanced/#pool-limit-configuration
``pool_maxsize``:
Number of allowable keep-alive connections, or ``null`` to always allow. The
default is 10. See ``max_keepalive_connections`` `Pool limit configuration`_.
``pool_connections`` :
Maximum number of concurrent connections. The default is 100.
See ``max_clients`` `Pool limit configuration`_.
Maximum number of allowable connections, or ``null`` # for no limits. The
default is 100. See ``max_connections`` `Pool limit configuration`_.
.. _curl_cffi proxies: https://curl-cffi.readthedocs.io/en/latest/quick_start.html
``keepalive_expiry`` :
Number of seconds to keep a connection in the pool. By default 5.0 seconds.
See ``keepalive_expiry`` `Pool limit configuration`_.
.. _httpx proxies: https://www.python-httpx.org/advanced/#http-proxying
``proxies`` :
Define one or more proxies you wish to use, see `curl_cffi proxies`_.
Define one or more proxies you wish to use, see `httpx proxies`_.
If there are more than one proxy for one protocol (http, https),
requests to the engines are distributed in a round-robin fashion.
HTTP, HTTPS, SOCKS4, SOCKS5 and SOCKS5h proxies are supported
(``http://``, ``https://``, ``socks4://``, ``socks5://``, ``socks5h://``). You should
use ``socks5h://`` when using Tor so hostnames are resolved by the proxy.
``source_ips`` :
If you use multiple network interfaces, define from which IP the requests must
be made. Example:
@@ -75,15 +87,18 @@ Communication with search engines.
different proxy and source ip.
``enable_http2`` :
Enable by default (HTTP/2). Set to ``false`` to force HTTP/1.1.
HTTP/3 is opt-in per engine (``enable_http3``).
Enable by default. Set to ``false`` to disable HTTP/2.
.. _httpx verification defaults: https://www.python-httpx.org/advanced/#changing-the-verification-defaults
.. _httpx ssl configuration: https://www.python-httpx.org/compatibility/#ssl-configuration
``verify``: : ``$SSL_CERT_FILE``, ``$SSL_CERT_DIR``
HTTPS verification uses the OS's trust store by default.
Set a path to use a custom CA file.
Allow to specify a path to certificate.
see `httpx verification defaults`_.
In addition to ``verify``, SearXNG supports the ``$SSL_CERT_FILE`` (for a file) and
``$SSL_CERT_DIR`` (for a directory) OpenSSL variables.
see `httpx ssl configuration`_.
``max_redirects`` :
30 by default. Maximum redirect before it is an error.

5
docs/admin/settings/settings_search.rst
View File

@@ -8,7 +8,7 @@
search:
safe_search: 0
autocomplete: "duckduckgo"
autocomplete: ""
favicon_resolver: ""
default_lang: ""
ban_time_on_fail: 5
@@ -32,7 +32,7 @@
- ``2``: Strict
``autocomplete``:
Existing autocomplete backends, set blank to turn it off.
Existing autocomplete backends, leave blank to turn it off.
- ``360search``
- ``baidu``
@@ -41,7 +41,6 @@
- ``dbpedia``
- ``duckduckgo``
- ``google``
- ``kagi``
- ``mwmbl``
- ``naver``
- ``privacywall``

6
docs/admin/settings/settings_server.rst
View File

@@ -14,7 +14,7 @@
limiter: false
public_instance: false
image_proxy: false
method: "GET"
method: "POST"
default_http_headers:
X-Content-Type-Options : nosniff
X-Download-Options : noopen
@@ -58,8 +58,8 @@
``method`` : ``GET`` | ``POST``
HTTP method. By default, ``GET`` is used / The ``POST`` method has the
advantage with some browsers that the history is not saved, but
HTTP method. By defaults ``POST`` is used / The ``POST`` method has the
advantage with some WEB browsers that the history is not easy to read, but
there are also various disadvantages that sometimes **severely restrict the
ease of use for the end user** (e.g. back button to jump back to the previous
search page and drag & drop of search term to new tabs do not work as

4
docs/dev/engines/engine_overview.rst
View File

@@ -143,7 +143,7 @@ parameters with default value can be redefined for special purposes.
data dict ``{}``
cookies dict ``{}``
verify bool ``True``
headers.User-Agent str ``''``
headers.User-Agent str a random User-Agent
category str current category, like ``'general'``
safesearch int ``0``, between ``0`` and ``2`` (normal, moderate, strict)
time_range Optional[str] ``None``, can be ``day``, ``week``, ``month``, ``year``
@@ -229,8 +229,6 @@ following parameters can be used to specify a search request:
max_redirects int maximum redirects, hard limit
soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine
raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300
impersonate str curl_cffi impersonate target (default: chrome, none to disable)
curl_options dict Any extra libcurl options for the request
=================== =========== ==========================================================================

8
docs/dev/engines/online/cara.rst Normal file
View File

@@ -0,0 +1,8 @@
.. _cara engine:
===========
Cara Images
===========
.. automodule:: searx.engines.cara
:members:

8
docs/dev/engines/online/exaapi.rst
View File

@@ -1,8 +0,0 @@
.. _exaapi engine:
==============
Exa API Engine
==============
.. automodule:: searx.engines.exaapi
:members:

8
docs/dev/engines/online/jina.rst
View File

@@ -1,8 +0,0 @@
.. _jina engine:
===========
Jina Engine
===========
.. automodule:: searx.engines.jina
:members:

8
docs/dev/engines/online/presearch.rst Normal file
View File

@@ -0,0 +1,8 @@
.. _engine presearch:
================
Presearch Engine
================
.. automodule:: searx.engines.presearch
:members:

8
docs/dev/engines/online/yandex_api.rst
View File

@@ -1,8 +0,0 @@
.. _yandex api engine:
=================
Yandex Search API
=================
.. automodule:: searx.engines.yandex_api
:members:

4
docs/dev/quickstart.rst
View File

@@ -80,8 +80,8 @@ same environment, here are a few examples::
# to test one of the update scripts
(dev.env)$ searxng_extra/update/update_engine_traits.py --help
# to test the update of the wikidata units and property names
(dev.env)$ searxng_extra/update/update_wikidata.py
# to test the update of the wikidata units
(dev.env)$ searxng_extra/update/update_wikidata_units.py
.. sidebar:: further read

2
docs/dev/reST.rst
View File

@@ -286,7 +286,7 @@ content becomes smart.
files & folders origin :origin:`docs/dev/reST.rst` ``:origin:`docs/dev/reST.rst```
pull request :pull:`4` ``:pull:`4```
patch :patch:`af2cae6` ``:patch:`af2cae6```
PyPi package :pypi:`curl_cffi` ``:pypi:`curl_cffi```
PyPi package :pypi:`httpx` ``:pypi:`httpx```
manual page man :man:`bash` ``:man:`bash```
intersphinx_
--------------------------------------------------------------------------------------------------

98
docs/dev/search_api.rst
View File

@@ -4,32 +4,31 @@
Search API
==========
SearXNG supports querying via a simple HTTP API. Two endpoints, ``/`` and
``/search``, are supported for both GET and POST methods. The ``GET`` method
expects parameters as URL query parameters, while the POST method expects
parameters as form data (``application/x-www-form-urlencoded``).
SearXNG supports querying via a simple HTTP API.
Two endpoints, ``/`` and ``/search``, are supported for both GET and POST methods.
The GET method expects parameters as URL query parameters, while the POST method expects parameters as form data.
If you want to consume the results as JSON, CSV, or RSS, you need to set the
``format`` parameter accordingly. Supported formats are defined in
``settings.yml``, under the :ref:`settings search` section. Requesting an
unset format will return a 403 Forbidden error. Be aware that many public
instances have these formats disabled.
``format`` parameter accordingly. Supported formats are defined in ``settings.yml``, under the ``search`` section.
Requesting an unset format will return a 403 Forbidden error. Be aware that many public instances have these formats disabled.
Endpoints:
.. code::
``GET /``
``GET /search``
GET /
GET /search
POST /
POST /search
``POST /``
``POST /search``
example cURL calls:
.. code:: bash
.. code-block:: bash
curl 'https://searx.example.org/search?q=searxng&format=json'
curl -X POST 'https://searx.example.org/search' -d 'q=searxng&format=csv'
curl -L -X POST -d 'q=searxng&format=json' 'https://searx.example.org/'
Parameters
@@ -54,27 +53,90 @@ Parameters
Comma separated list, specifies the active search categories (see
:ref:`configured engines`)
``engines`` : optional
Comma separated list, specifies the active search engines (see
:ref:`configured engines`).
``language`` : default from :ref:`settings search`
Code of the language.
``pageno`` : default ``1``
Search page number.
``time_range`` : optional : [ ``day``, ``month``, ``year`` ]
``time_range`` : optional
[ ``day``, ``month``, ``year`` ]
Time range of search for engines which support it. See if an engine supports
time range search in the preferences page of an instance.
``format`` : optional : [ ``json``, ``csv``, ``rss`` ]
``format`` : optional
[ ``json``, ``csv``, ``rss`` ]
Output format of results. Format needs to be activated in :ref:`settings
search`.
``safesearch`` : default from :ref:`settings search` : [ ``0``, ``1``, ``2`` ]
``results_on_new_tab`` : default ``0``
[ ``0``, ``1`` ]
Open search results on new tab.
``image_proxy`` : default from :ref:`settings server`
[ ``True``, ``False`` ]
Proxy image results through SearXNG.
``autocomplete`` : default from :ref:`settings search`
[ ``google``, ``dbpedia``, ``duckduckgo``, ``mwmbl``, ``startpage``,
``privacywall``, ``wikipedia``, ``swisscows``, ``qwant`` ]
Service which completes words as you type.
``safesearch`` : default from :ref:`settings search`
[ ``0``, ``1``, ``2`` ]
Filter search results of engines which support safe search. See if an engine
supports safe search in the preferences page of an instance.
``theme`` : default ``simple`` : [ ``simple`` ]
``theme`` : default ``simple``
[ ``simple`` ]
Theme of instance.
Please note, available themes depend on an instance. It is possible that an
instance administrator deleted, created or renamed themes on their instance.
See the available options in the preferences page of the instance.
``enabled_plugins`` : optional
List of enabled plugins.
:default:
``Hash_plugin``, ``Self_Information``,
``Tracker_URL_remover``, ``Ahmia_blacklist``
:values:
.. enabled by default
``Hash_plugin``, ``Self_Information``,
``Tracker_URL_remover``, ``Ahmia_blacklist``,
.. disabled by default
``Hostnames_plugin``, ``Open_Access_DOI_rewrite``,
``Vim-like_hotkeys``, ``Tor_check_plugin``
``disabled_plugins``: optional
List of disabled plugins.
:default:
``Hostnames_plugin``, ``Open_Access_DOI_rewrite``,
``Vim-like_hotkeys``, ``Tor_check_plugin``
:values:
see values from ``enabled_plugins``
``enabled_engines`` : optional : *all* :origin:`engines <searx/engines>`
List of enabled engines.
``disabled_engines`` : optional : *all* :origin:`engines <searx/engines>`
List of disabled engines.

6
docs/dev/searxng_extra/update.rst
View File

@@ -90,10 +90,10 @@ Scripts to update static data in :origin:`searx/data/`
:members:
``update_wikidata.py``
``update_wikidata_units.py``
============================
:origin:`[source] <searxng_extra/update/update_wikidata.py>`
:origin:`[source] <searxng_extra/update/update_wikidata_units.py>`
.. automodule:: searxng_extra.update.update_wikidata
.. automodule:: searxng_extra.update.update_wikidata_units
:members:

6
docs/index.rst
View File

@@ -20,11 +20,15 @@ If you don't trust anyone, you can set up your own, see :ref:`installation`.
- :ref:`self hosted <installation>`
- :ref:`no user tracking / no profiling <SearXNG protect privacy>`
- javascript & cookies are optional
- script & cookies are optional
- secure, encrypted connections
- :ref:`{{engines | length}} search engines <configured engines>`
- `58 translations <https://translate.codeberg.org/projects/searxng/searxng/>`_
- about 70 `well maintained <https://uptime.searxng.org/>`__ instances on searx.space_
- :ref:`easy integration of search engines <demo online engine>`
- professional development: `CI <https://github.com/searxng/searxng/actions>`_,
`quality assurance <https://dev.searxng.org/>`_ &
`automated tested UI <https://dev.searxng.org/screenshots.html>`_
.. sidebar:: be a part

2
manage
View File

@@ -48,7 +48,7 @@ PATH="${PY_ENV}/bin:${REPO_ROOT}/node_modules/.bin:${GOROOT}/bin:${GOPATH}/bin:$
PYOBJECTS="searx"
PY_SETUP_EXTRAS='[test]'
GECKODRIVER_VERSION="v0.37.0"
GECKODRIVER_VERSION="v0.36.0"
# SPHINXOPTS=
BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization")
BLACK_TARGETS=("--exclude" "(searx/static|searx/languages.py)" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")

10
requirements-dev.txt
View File

@@ -2,9 +2,9 @@ mock==5.2.0
nose2[coverage_plugin]==0.16.0
cov-core==1.15.0
black==25.9.0
pylint==4.0.8
pylint==4.0.6
splinter==0.21.0
selenium==4.48.0
selenium==4.45.0
Sphinx==8.2.3;python_version <= "3.11"
Sphinx==9.1.0; python_version > "3.11"
sphinx-issues==6.0.0
@@ -18,11 +18,11 @@ myst-parser==5.0.0
linuxdoc==20260504
aiounittest==1.5.0
yamllint==1.38.0
wlc==2.1.1
wlc==2.0.0
coloredlogs==15.0.1
docutils>=0.21.2;python_version <= "3.11"
docutils>=0.22.4; python_version > "3.11"
parameterized==0.9.0
granian[reload]==2.8.2
basedpyright==1.39.10
granian[reload]==2.7.6
basedpyright==1.39.8
types-lxml==2026.2.16

4
requirements-server.txt
View File

@@ -1,2 +1,2 @@
granian==2.8.2
granian[pname]==2.8.2
granian==2.7.6
granian[pname]==2.7.6

14
requirements.txt
View File

@@ -1,17 +1,19 @@
certifi==2026.7.22
certifi==2026.6.17
babel==2.18.0
flask-babel==4.0.0
flask==3.1.3
jinja2==3.1.6
lxml==6.1.2
pygments==2.21.0
lxml==6.1.1
pygments==2.20.0
python-dateutil==2.9.0.post0
pyyaml==6.0.3
curl_cffi==0.16.1
httpx[http2]==0.28.1
httpx-socks[asyncio]==0.10.0
sniffio==1.3.1
valkey==6.1.1
markdown-it-py==4.2.0
msgspec==0.21.1
typer==0.27.2
typer==0.26.7
isodate==0.7.2
whitenoise==6.12.0
typing-extensions==4.16.0
typing-extensions==4.15.0

2
searx/answerers/random.py
View File

@@ -16,7 +16,7 @@ from . import Answerer, AnswererInfo
def random_characters():
random_string_letters = string.ascii_lowercase + string.digits + string.ascii_uppercase
return random.choices(random_string_letters, k=random.randint(8, 32))
return [random.choice(random_string_letters) for _ in range(random.randint(8, 32))]
def random_string():

42
searx/autocomplete.py
View File

@@ -11,7 +11,7 @@ from urllib.parse import urlencode
import lxml.etree
import lxml.html
from curl_cffi.requests.exceptions import RequestException
from httpx import HTTPError
from searx import settings
from searx.engines import (
@@ -21,8 +21,6 @@ from searx.engines import (
from searx.network import get as http_get, post as http_post
from searx.exceptions import SearxEngineResponseException
from searx.utils import extr, gen_useragent
from searx.data import ENGINE_TRAITS
from searx.enginelib.traits import EngineTraits
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
@@ -62,8 +60,8 @@ def bing(query: str, _sxng_locale: str) -> list[str]:
# bing search autocompleter
base_url = "https://www.bing.com/AS/Suggestions?"
# cvid has to be a 32 character long string consisting of numbers and uppsercase characters
cvid = ''.join(random.choices(string.ascii_uppercase + string.digits, k=32))
response = get(base_url + urlencode({'qry': query, 'csr': 1, 'cvid': cvid}), enable_http3=True)
cvid = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(32))
response = get(base_url + urlencode({'qry': query, 'csr': 1, 'cvid': cvid}))
results: list[str] = []
if response.ok:
@@ -83,7 +81,7 @@ def brave(query: str, _sxng_locale: str) -> list[str]:
url = 'https://search.brave.com/api/suggest?'
url += urlencode({'q': query})
country = 'all'
kwargs = {'cookies': {'country': country}, 'enable_http3': True}
kwargs = {'cookies': {'country': country}}
resp = get(url, **kwargs)
results: list[str] = []
@@ -127,17 +125,16 @@ def duckduckgo(query: str, sxng_locale: str) -> list[str]:
def google_complete(query: str, sxng_locale: str) -> list[str]:
"""Autocomplete from Google. Supports Google's languages
"""Autocomplete from Google. Supports Google's languages and subdomains
(:py:obj:`searx.engines.google.get_google_info`) by using the async REST
API::
https://www.google.com/complete/search?{args}
https://{subdomain}/complete/search?{args}
"""
data = ENGINE_TRAITS.get("google") or {}
traits = EngineTraits(**data)
google_info: dict[str, t.Any] = google.get_google_info({'searxng_locale': sxng_locale}, traits)
google_info: dict[str, t.Any] = google.get_google_info({'searxng_locale': sxng_locale}, engines['google'].traits)
url = 'https://{subdomain}/complete/search?{args}'
args = urlencode(
{
'q': query,
@@ -147,7 +144,7 @@ def google_complete(query: str, sxng_locale: str) -> list[str]:
)
results: list[str] = []
resp = get('https://www.google.com/complete/search?' + args, enable_http3=True)
resp = get(url.format(subdomain=google_info['subdomain'], args=args))
if resp and resp.ok:
json_txt = resp.text[resp.text.find('[') : resp.text.find(']', -3) + 1]
data = json.loads(json_txt)
@@ -156,24 +153,6 @@ def google_complete(query: str, sxng_locale: str) -> list[str]:
return results
def kagi(query: str, sxng_locale: str) -> list[str]:
"""Autocomplete from Kagi."""
args: dict[str, str] = {'q': query}
if '-' in sxng_locale:
args['r'] = sxng_locale.split('-')[1].lower()
resp = get("https://kagisuggest.com/api/autosuggest?" + urlencode(args))
results: list[str] = []
if resp.ok:
data = resp.json()
if len(data) > 1:
results = data[1]
return results
def mwmbl(query: str, _sxng_locale: str) -> list[str]:
"""Autocomplete from Mwmbl_."""
@@ -397,7 +376,6 @@ backends: dict[str, t.Callable[[str, str], list[str]]] = {
'dbpedia': dbpedia,
'duckduckgo': duckduckgo,
'google': google_complete,
'kagi': kagi,
'mwmbl': mwmbl,
'naver': naver,
'privacywall': privacywall,
@@ -418,5 +396,5 @@ def search_autocomplete(backend_name: str, query: str, sxng_locale: str) -> list
return []
try:
return backend(query, sxng_locale)
except (RequestException, SearxEngineResponseException):
except (HTTPError, SearxEngineResponseException):
return []

2
searx/botdetection/link_token.py
View File

@@ -151,6 +151,6 @@ def get_token() -> str:
if token:
token = token.decode('UTF-8') # type: ignore
else:
token = ''.join(random.choices(string.ascii_lowercase + string.digits, k=16))
token = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(16))
valkey_client.set(TOKEN_KEY, token, ex=TOKEN_LIVE_TIME)
return token

48
searx/botdetection/trusted_proxies.py
View File

@@ -63,20 +63,6 @@ class ProxyFix:
proxy_list: list[str] = cfg.get("botdetection.trusted_proxies", default=[])
return [ip_network(net, strict=False) for net in proxy_list]
def is_trusted_proxy(
self,
addr: IPv4Address | IPv6Address | None,
trusted_proxies: list[IPv4Network | IPv6Network],
) -> bool:
if addr is None:
return False
for net in trusted_proxies:
if addr.version == net.version and addr in net:
return True
return False
def trusted_remote_addr(
self,
x_forwarded_for: list[IPv4Address | IPv6Address],
@@ -84,8 +70,16 @@ class ProxyFix:
) -> str:
# always rtl
for addr in reversed(x_forwarded_for):
if not self.is_trusted_proxy(addr, trusted_proxies):
logger.debug("client address from X-Forwarded-For: %s", addr)
trust: bool = False
for net in trusted_proxies:
if addr.version == net.version and addr in net:
logger.debug("trust proxy %s (member of %s)", addr, net)
trust = True
break
# client address
if not trust:
return addr.compressed
# fallback to first address
@@ -101,21 +95,19 @@ class ProxyFix:
# in this function!
orig_remote_addr: str | None = environ.pop("REMOTE_ADDR")
orig_remote_ip: IPv4Address | IPv6Address | None = None
# Validate the IPs involved in this game and delete all invalid ones
# from the WSGI environment.
if orig_remote_addr:
try:
orig_remote_ip = ip_address(orig_remote_addr)
if orig_remote_ip.version == 6 and orig_remote_ip.ipv4_mapped:
orig_remote_ip = orig_remote_ip.ipv4_mapped
orig_remote_addr = orig_remote_ip.compressed
addr = ip_address(orig_remote_addr)
if addr.version == 6 and addr.ipv4_mapped:
addr = addr.ipv4_mapped
orig_remote_addr = addr.compressed
except ValueError as exc:
logger.error("REMOTE_ADDR: %s / discard REMOTE_ADDR from WSGI environment", exc)
orig_remote_addr = None
orig_remote_ip = None
x_real_ip: str | None = environ.get("HTTP_X_REAL_IP")
if x_real_ip:
@@ -149,13 +141,11 @@ class ProxyFix:
if not x_forwarded_for and not x_real_ip:
log_error_only_once("X-Forwarded-For nor X-Real-IP header is set!")
if x_forwarded_for or x_real_ip:
if not trusted_proxies:
log_error_only_once("missing botdetection.trusted_proxies config")
if not self.is_trusted_proxy(orig_remote_ip, trusted_proxies):
x_forwarded_for = []
x_real_ip = None
if x_forwarded_for and not trusted_proxies:
log_error_only_once("missing botdetection.trusted_proxies config")
# without trusted_proxies, this variable is useless for determining
# the real IP
x_forwarded_for = []
# securing the WSGI environment variables that are adjusted

16
searx/cache.py
View File

@@ -48,7 +48,7 @@ class ExpireCacheCfg(msgspec.Struct): # pylint: disable=too-few-public-methods
MAXHOLD_TIME: int = 60 * 60 * 24 * 7 # 7 days
"""Hold time (default in sec.), after which a value is removed from the cache."""
MAINTENANCE_PERIOD: int = 60 * 60 # 1h
MAINTENANCE_PERIOD: int = 60 * 60 # 2h
"""Maintenance period in seconds / when :py:obj:`MAINTENANCE_MODE` is set to
``auto``."""
@@ -458,22 +458,12 @@ class ExpireCacheSQLite(sqlitedb.SQLiteAppl, ExpireCache):
# Before values are taken from the table, a maintenance interval may
# need to be carried out.
self.maintenance()
sql = f"SELECT value, expire FROM {table} WHERE key = ?"
sql = f"SELECT value FROM {table} WHERE key = ?"
row = self.DB.execute(sql, (key,)).fetchone()
if row is None:
return default
# Check if value is expired. It's possible that it's expired but has not
# yet been automatically deleted by the periodic maintenance
(value, expire) = row
now = time.time()
if expire < now:
# The record is deleted during the maintenance interval. Deleting
# the record at this point offers no advantage, as a SELECT
# statement must be executed for every cache.get request anyways.
return default
return self.deserialize(value)
return self.deserialize(row[0])
def pairs(self, ctx: str) -> Iterator[tuple[str, typing.Any]]:
"""Iterate over key/value pairs from table given by argument ``ctx``.

36
searx/data/__init__.py
View File

@@ -6,7 +6,7 @@ make data.all
"""
# pylint: disable=invalid-name
__all__ = ["ahmia_blacklist_loader", "data_dir", "get_cache"]
__all__ = ["ahmia_blacklist_loader", "gsa_useragents_loader", "data_dir", "get_cache"]
import json
import typing as t
@@ -32,13 +32,6 @@ class WikiDataUnitType(t.TypedDict):
to_si_factor: float
WikiDataPropertyNameType = str | dict[str, str]
"""Name of a Wikidata property. Can be either the plain name or a dictionary of
language code to property name, e.g. ``{"en": "Date of birth"}``."""
WikiDataPropertiesType = dict[str, WikiDataPropertyNameType]
"""Dictionary from wikidata property ID to property name."""
class LocalesType(t.TypedDict):
"""Data structure of an item in ``locales.json``"""
@@ -48,7 +41,6 @@ class LocalesType(t.TypedDict):
USER_AGENTS: UserAgentType
WIKIDATA_UNITS: dict[str, WikiDataUnitType]
WIKIDATA_PROPERTIES: WikiDataPropertiesType
TRACKER_PATTERNS: TrackerPatternsDB
LOCALES: LocalesType
CURRENCIES: CurrenciesDB
@@ -60,25 +52,24 @@ ENGINE_DESCRIPTIONS: dict[str, dict[str, t.Any]]
ENGINE_TRAITS: dict[str, dict[str, t.Any]]
lazy_globals: dict[str, t.Any] = {
lazy_globals = {
"CURRENCIES": CurrenciesDB(),
"USER_AGENTS": None,
"EXTERNAL_URLS": None,
"WIKIDATA_UNITS": None,
"WIKIDATA_PROPERTIES": None,
"EXTERNAL_BANGS": None,
"OSM_KEYS_TAGS": None,
"ENGINE_DESCRIPTIONS": None,
"ENGINE_TRAITS": None,
"LOCALES": None,
"TRACKER_PATTERNS": TrackerPatternsDB(),
"GSA_USER_AGENTS": None,
}
data_json_files = {
"USER_AGENTS": "useragents.json",
"EXTERNAL_URLS": "external_urls.json",
"WIKIDATA_UNITS": "wikidata_units.json",
"WIKIDATA_PROPERTIES": "wikidata_properties.json",
"EXTERNAL_BANGS": "external_bangs.json",
"OSM_KEYS_TAGS": "osm_keys_tags.json",
"ENGINE_DESCRIPTIONS": "engine_descriptions.json",
@@ -115,3 +106,24 @@ def ahmia_blacklist_loader() -> list[str]:
"""
with open(data_dir / 'ahmia_blacklist.txt', encoding='utf-8') as f:
return f.read().split()
def gsa_useragents_loader() -> list[str]:
"""Load data from `gsa_useragents.txt` and return a list of user agents
suitable for Google. The user agents are fetched by::
searxng_extra/update/update_gsa_useragents.py
This function is used by :py:mod:`searx.engines.google`.
"""
data = lazy_globals["GSA_USER_AGENTS"]
if data is not None:
return data
log.debug("init searx.data.%s", "GSA_USER_AGENTS")
with open(data_dir / 'gsa_useragents.txt', encoding='utf-8') as f:
lazy_globals["GSA_USER_AGENTS"] = f.read().splitlines()
return lazy_globals["GSA_USER_AGENTS"]

1654
searx/data/ahmia_blacklist.txt
View File

File diff suppressed because it is too large Load Diff

141
searx/data/currencies.json
View File

@@ -288,7 +288,7 @@
"oc": "Kwanza",
"pa": "ਅੰਗੋਲਨ ਕਵਾਂਜ਼ਾ",
"pl": "Kwanza",
"pt": "kwanza",
"pt": "Kwanza",
"ru": "ангольская кванза",
"si": "ක්වන්සා",
"sr": "анголска кванза",
@@ -334,7 +334,6 @@
"ro": "Peso argentinian",
"ru": "аргентинское песо",
"sk": "Argentinské peso",
"sl": "argentinski peso",
"sr": "аргентински пезос",
"sv": "Argentinsk peso",
"ta": "ஆர்ஜென்டின பீசோ",
@@ -774,7 +773,7 @@
"nl": "Boliviaanse boliviano",
"oc": "Boliviano",
"pa": "ਬੋਲੀਵੀਆਨੋ",
"pl": "boliwiano",
"pl": "boliviano",
"pt": "Boliviano",
"ro": "boliviano",
"ru": "боливиано",
@@ -1108,7 +1107,7 @@
"fi": "Kongon frangi",
"fr": "franc congolais",
"ga": "franc an Chongó",
"gl": "franco congolés",
"gl": "Franco congolés",
"he": "פרנק קונגולזי",
"hr": "Kongoanski franak",
"hu": "kongói frank",
@@ -1637,7 +1636,7 @@
"fi": "Algerian dinaari",
"fr": "dinar algérien",
"ga": "dinar na hAilgéire",
"gl": "dinar alxeriano",
"gl": "Dinar alxeriano",
"he": "דינר אלג'ירי",
"hr": "Alžirski dinar",
"hu": "algériai dinár",
@@ -1652,7 +1651,7 @@
"pap": "dinar argelino",
"pl": "dinar algierski",
"pt": "dinar argelino",
"ro": "dinar algerian",
"ro": "Dinar algerian",
"ru": "алжирский динар",
"sk": "Alžírský dinár",
"sl": "alžirski dinar",
@@ -1787,7 +1786,7 @@
"cy": "Ewro",
"da": "Euro",
"de": "Euro",
"en": "euro",
"en": "Euro",
"eo": "eŭro",
"es": "Euro",
"et": "Euro",
@@ -1804,7 +1803,7 @@
"it": "Euro",
"ja": "ユーロ",
"ko": "유로",
"lt": "euras",
"lt": "Euras",
"lv": "eiro",
"ml": "യൂറോ",
"ms": "Euro",
@@ -2003,7 +2002,7 @@
"eo": "ganaa cedio",
"es": "cedi",
"fi": "Cedi",
"fr": "cedi",
"fr": "Cedi",
"ga": "cedi",
"gl": "Cedi",
"he": "סדי גאני",
@@ -2760,7 +2759,7 @@
"pa": "ਜਪਾਨੀ ਯੈੱਨ",
"pl": "jen",
"pt": "iene",
"ro": "yen",
"ro": "yeni",
"ru": "японская иена",
"sk": "jen",
"sl": "japonski jen",
@@ -2953,7 +2952,7 @@
"pap": "won nortkoreano",
"pl": "won północnokoreański",
"pt": "won norte-coreano",
"ro": "won nord-coreean",
"ro": "Won nord-coreean",
"ru": "вона КНДР",
"sk": "severokorejsky won",
"sl": "severnokorejski von",
@@ -3095,7 +3094,6 @@
"ca": "tenge",
"cs": "Tenge",
"cy": "tenge Casachstan",
"da": "Tenge",
"de": "Tenge",
"en": "Kazakhstani tenge",
"eo": "kazaĥa tengo",
@@ -3339,7 +3337,6 @@
"fi": "Libyan dinaari",
"fr": "dinar libyen",
"ga": "dinar na Libia",
"gl": "dinar libio",
"he": "דינר לובי ",
"hr": "Libijski dinar",
"hu": "líbiai dinár",
@@ -3542,7 +3539,6 @@
"ja": "チャット",
"ko": "미얀마 짯",
"lt": "Kijatas",
"lv": "Kjats",
"ml": "ബർമ്മീസ് ക്യാറ്റ്",
"nl": "Myanmarese kyat",
"oc": "Kyat",
@@ -4314,7 +4310,7 @@
"ar": "بيسو فلبيني",
"bg": "Филипинско песо",
"ca": "peso filipí",
"cs": "filipínské peso",
"cs": "Filipínské peso",
"de": "philippinischer Peso",
"en": "Philippine peso",
"eo": "filipina peso",
@@ -4618,7 +4614,7 @@
"fi": "Serbian dinaari",
"fr": "dinar serbe",
"ga": "Dinar na Seirbia",
"gl": "dinar serbio",
"gl": "Dinar serbio",
"he": "דינר סרבי",
"hr": "srpski dinar",
"hu": "szerb dinár",
@@ -4836,7 +4832,6 @@
"nl": "Seychelse roepie",
"pl": "Rupia seszelska",
"pt": "rupia das Seicheles",
"ro": "rupie seychelloză",
"ru": "сейшельская рупия",
"sk": "Seychelská rupia",
"sl": "sejšelska rupija",
@@ -5067,7 +5062,6 @@
"nl": "Somalische shilling",
"pl": "Szyling somalijski",
"pt": "xelim somaliano",
"ro": "șiling somalez",
"ru": "сомалийский шиллинг",
"sk": "Somálsky šiling",
"sl": "somalski šiling",
@@ -5337,7 +5331,7 @@
"pa": "ਤਾਜਿਕਿਸਤਾਨੀ ਸੋਮੋਨੀ",
"pl": "Somoni",
"pt": "Somoni",
"ro": "somoni tadjic",
"ro": "Somoni tadjic",
"ru": "таджикский сомони",
"sk": "tadžický som",
"sl": "tadžikistanski somoni",
@@ -5400,7 +5394,6 @@
"fi": "Tunisian dinaari",
"fr": "dinar tunisien",
"ga": "dinar na Túinéise",
"gl": "dinar tunisiano",
"he": "דינר תוניסאי",
"hr": "tuniski dinar",
"hu": "tunéziai dinár",
@@ -5773,7 +5766,7 @@
"fi": "Uruguayn peso",
"fr": "peso uruguayen",
"ga": "peso Uragua",
"gl": "peso uruguaio",
"gl": "Peso uruguaio",
"he": "פסו של אורוגוואי",
"hr": "Urugvajski pezo",
"hu": "uruguayi peso",
@@ -5887,7 +5880,6 @@
"ja": "ドン",
"ko": "베트남 동",
"lt": "Vietnamo dongas",
"ms": "Dồng Vietnam",
"nl": "Vietnamese dong",
"oc": "Dong",
"pa": "ਵੀਅਤਨਾਮੀ ਦੋਙ",
@@ -6127,8 +6119,7 @@
"ro": "Gulden caraibian",
"ru": "Карибский гульден",
"sk": "Karibský gulden",
"sl": "karibski goldinar",
"sv": "Karibisk gulden"
"sl": "karibski goldinar"
},
"XDR": {
"ar": "حقوق السحب الخاصة",
@@ -6156,7 +6147,7 @@
"oc": "Drechs de tiratge Especials",
"pl": "specjalne prawa ciągnienia",
"pt": "direitos especiais de saque",
"ro": "drepturi speciale de tragere",
"ro": "Drepturi speciale de tragere",
"ru": "специальные права заимствования",
"sk": "Zvláštne práva čerpania",
"sl": "posebne pravice črpanja",
@@ -6234,7 +6225,6 @@
"ja": "CFPフラン",
"ko": "CFP 프랑",
"lt": "CFP frankas",
"lv": "Klusā okeāna franks",
"ms": "Franc CFP",
"nl": "CFP-frank",
"oc": "Franc CFP",
@@ -6730,8 +6720,6 @@
"antilliaanse gulden": "ANG",
"antilski gulden": "ANG",
"aoa": "AOA",
"apvienotās karalistes ekonomika": "GBP",
"apvienotās karalistes saimniecība": "GBP",
"apvienotās karalistes sterliņu mārciņa": "GBP",
"ar": "MGA",
"arab accounting dinar": "XAD",
@@ -6844,7 +6832,6 @@
"avustralya doları": "AUD",
"awg": "AWG",
"az arany mint befektetés": "XAU",
"az egyesült királyság gazdasága": "GBP",
"azerbaidžanin manat": "AZN",
"azerbaidžano manatas": "AZN",
"azerbaidžānas manats": "AZN",
@@ -7028,7 +7015,6 @@
"bir etíope": "ETB",
"biras": "ETB",
"birleşik arap emirlikleri dirhemi": "AED",
"birleşik krallık ekonomisi": "GBP",
"birma kjato": "MMK",
"birr": "ETB",
"birr da etiópia": "ETB",
@@ -7069,7 +7055,6 @@
"bolivjano": "BOB",
"bolivya bolivianosu": "BOB",
"bolivya bolivyanosu": "BOB",
"boliwiano": "BOB",
"bolívar digital": "VED",
"bolívar soberano": "VES",
"bolívar sobirà": "VES",
@@ -7116,19 +7101,15 @@
"brit font": "GBP",
"brita pundo": "GBP",
"britaj pundoj": "GBP",
"britannian talous": "GBP",
"britanska funta": "GBP",
"britanski funt": "GBP",
"britische wirtschaft": "GBP",
"britisches pfund": "GBP",
"british economy": "GBP",
"british pound": "GBP",
"britisk pund": "GBP",
"britiske pund": "GBP",
"brits pond": "GBP",
"britse pond": "GBP",
"britská libra": "GBP",
"brittisk ekonomi": "GBP",
"brittiska pund": "GBP",
"brittiskt pund": "GBP",
"brunei doları": "BND",
@@ -7212,7 +7193,6 @@
"cedi du ghana": "GHS",
"cedi ghana": "GHS",
"cedi ghanese": "GHS",
"cedi ghanéen": "GHS",
"centr afrika franko": "XAF",
"central african cfa franc": "XAF",
"centralafrikansk cfa franc": "XAF",
@@ -7315,6 +7295,7 @@
"colón costa ricense": "CRC",
"colón costa riquenho": "CRC",
"colón costa riquense": "CRC",
"colón costa riqueny": "CRC",
"colón costaricain": "CRC",
"colón costaricano": "CRC",
"colón costaricien": "CRC",
@@ -8427,7 +8408,6 @@
"dólares canadenses": "CAD",
"dólares estadounidenses": "USD",
"dólares neozelandeses": "NZD",
"dồng vietnam": "VND",
"dram": "AMD",
"dram armean": "AMD",
"dram armenia": "AMD",
@@ -8445,13 +8425,11 @@
"drame arménio": "AMD",
"dramm": "AMD",
"drechs de tiratge especials": "XDR",
"drept special de tragere": "XDR",
"drepturi speciale de tragere": "XDR",
"drets especials de gir": "XDR",
"droits de tirage speciaux": "XDR",
"droits de tirage spéciaux": "XDR",
"dschibuti franc": "DJF",
"dvn": "VND",
"dzd": "DZD",
"dzsibuti frank": "DJF",
"džibučio frankas": "DJF",
@@ -8465,21 +8443,6 @@
"eastern caribbean currency union": "XCD",
"eastern caribbean dollar": "XCD",
"ec$": "XCD",
"economi'r deyrnas unedig": "GBP",
"economia": "GBP",
"economia del regne unit": "GBP",
"economia del regno unito": "GBP",
"economia del reialme unit": "GBP",
"economia del reino unido": "GBP",
"economia do reino unido": "GBP",
"economia regatului unit": "GBP",
"economie du royaume uni": "GBP",
"economie van het verenigd koninkrijk": "GBP",
"economía del reino unido": "GBP",
"economía do reino unido": "GBP",
"economy": "GBP",
"economy of the uk": "GBP",
"economy of the united kingdom": "GBP",
"egipatska funta": "EGP",
"egipta pundo": "EGP",
"egipto svaras": "EGP",
@@ -8499,12 +8462,6 @@
"einr": "INR",
"eiro": "EUR",
"ekialdeko karibeko dolar": "XCD",
"ekonomi britania raya": "GBP",
"ekonomi united kingdom": "GBP",
"ekonomie van die verenigde koninkryk": "GBP",
"ekonomika spojeného království": "GBP",
"ekonomika v spojenom kráľovstve": "GBP",
"ekonomio de britujo": "GBP",
"el peso": "GTQ",
"emalangeni": "SZL",
"emas sebagai pelaburan": "XAU",
@@ -8536,7 +8493,6 @@
"ermenistan dramı": "AMD",
"ern": "ERN",
"erreal brasildar": "BRL",
"erresuma batuko ekonomia": "GBP",
"errublo": "RUB",
"errublo errusiar": "RUB",
"errupia indiar": "INR",
@@ -8607,8 +8563,6 @@
"eyrir": "ISK",
"e£": "EGP",
"èuro": "EUR",
"économie britannique": "GBP",
"économie du royaume uni": "GBP",
"észak ír font": "GBP",
"észak koreai von": "KPW",
"e₹": "INR",
@@ -8742,9 +8696,6 @@
"forintti": "HUF",
"forinți": "HUF",
"fòrint": "HUF",
"förenade konungariket storbritannien och irlands ekonomi": "GBP",
"förenade konungariket storbritannien och nordirlands ekonomi": "GBP",
"förenade kungarikets ekonomi": "GBP",
"franak cfp": "XPF",
"franc": [
"XPF",
@@ -8997,9 +8948,6 @@
"gold als kapitalanlage": "XAU",
"gold as an investment": "XAU",
"gold as currency": "XAU",
"gospodarka wielkiej brytanii": "GBP",
"gospodarstvo ujedinjenog kraljevstva": "GBP",
"gospodarstvo združenega kraljestva": "GBP",
"gourde": "HTG",
"gourde haiti": "HTG",
"gourde haitiano": "HTG",
@@ -9419,7 +9367,6 @@
"juaņs": "CNY",
"juhokoréjsky won": "KRW",
"juhosudánska libra": "SSP",
"jungtinės karalystės ekonomika": "GBP",
"jungtinių arabų emyratų dirhamas": "AED",
"jungtinių valstijų doleris": "USD",
"južnoafrički rand": "ZAR",
@@ -9484,7 +9431,6 @@
"karibi forint": "XCG",
"karibia guldeno": "XCG",
"karibischer gulden": "XCG",
"karibisk gulden": "XCG",
"karibski goldinar": "XCG",
"karibský gulden": "XCG",
"karipski gulden": "XCG",
@@ -9545,9 +9491,6 @@
"kina papua nugini": "PGK",
"kina papuana": "PGK",
"kina papuásia": "PGK",
"kinh tế anh": "GBP",
"kinh tế vương quốc anh": "GBP",
"kinh tế vương quốc liên hiệp anh và bắc ireland": "GBP",
"kip": "LAK",
"kip laos": "LAK",
"kip laosiano": "LAK",
@@ -9575,8 +9518,6 @@
"kíp lào": "LAK",
"kīp": "LAK",
"kjat": "MMK",
"kjats": "MMK",
"klusā okeāna franks": "XPF",
"km": "BAM",
"kmf": "KMF",
"koeweitse dinar": "KWD",
@@ -10082,7 +10023,6 @@
"lire sterline": "GBP",
"lire turque": "TRY",
"lisente": "LSL",
"list of syrian coins": "SYP",
"liura de gibartar": "GIP",
"liura egipciana": "EGP",
"liura esterlina": "GBP",
@@ -11192,7 +11132,6 @@
"põhja korea won": "KPW",
"põhja makedoonia denaar": "MKD",
"prata como investimento": "XAG",
"produits agricole de l'angleterre": "GBP",
"pula": "BWP",
"pula botswana": "BWP",
"pula botswanais": "BWP",
@@ -11243,7 +11182,6 @@
"qatarisk rial": "QAR",
"qäpik": "AZN",
"qindarka": "ALL",
"quanza": "AOA",
"quetzal": "GTQ",
"quetzal guatemala": "GTQ",
"quetzal guatemalteco": "GTQ",
@@ -11375,6 +11313,7 @@
"riel camboxano": "KHR",
"riel camboyano": "KHR",
"riel campuchia": "KHR",
"riel kambodżański": "KHR",
"riel kamboja": "KHR",
"riel na cambóide": "KHR",
"rietumāfrikas franks": "XOF",
@@ -11569,7 +11508,6 @@
"rupia del pakistan": "PKR",
"rupia dell'india": "INR",
"rupia delle seychelles": "SCR",
"rupia din seychelles": "SCR",
"rupia do nepal": "NPR",
"rupia do paquistão": "PKR",
"rupia do seri lanca": "LKR",
@@ -11625,7 +11563,6 @@
],
"rupie indiană": "INR",
"rupie indiane": "INR",
"rupie seychelloză": "SCR",
"rupies índies": "INR",
"rupija": [
"NPR",
@@ -12055,10 +11992,6 @@
"sterliņu mārciņa": "GBP",
"stērliņu mārciņa": "GBP",
"stn": "STN",
"storbritannien och irlands ekonomi": "GBP",
"storbritannien och nordirlands ekonomi": "GBP",
"storbritanniens ekonomi": "GBP",
"storbritanniens økonomi": "GBP",
"stredoafrický frank": "XAF",
"středoafrický frank": "XAF",
"sucre": "XSU",
@@ -12108,7 +12041,6 @@
"suriye lirası": "SYP",
"suudi arabistan riyali": "SAR",
"suudi riyali": "SAR",
"suurbritannia majandus": "GBP",
"suurbritannia nael": "GBP",
"suurbritannia naelsterling": "GBP",
"suvereni bolivar": "VES",
@@ -12215,7 +12147,6 @@
"švicarski frank": "CHF",
"švýcarský frank": "CHF",
"șekel nou": "ILS",
"șiling somalez": "SOS",
"şekel": "ILS",
"şili pesosu": "CLP",
"s₣": "CHF",
@@ -12326,7 +12257,6 @@
"thaise baht": "THB",
"thajský baht": "THB",
"thb": "THB",
"the australian dollar": "AUD",
"thebe": "BWP",
"third belarusian ruble": "BYN",
"tical": "THB",
@@ -12558,8 +12488,6 @@
"uguiya": "MRU",
"ugx": "UGX",
"ui": "UYI",
"uk economy": "GBP",
"uk's economy": "GBP",
"ukl": "GBP",
"ukraina grivna": "UAH",
"ukraina hrivno": "UAH",
@@ -12600,8 +12528,6 @@
"unidades de inversion": "MXV",
"unidades de inversión": "MXV",
"united arab emirates dirham": "AED",
"united kingdom economy": "GBP",
"united kingdom's economy": "GBP",
"united states dollar": [
"USN",
"USD"
@@ -12703,7 +12629,6 @@
"venemaa rubla": "RUB",
"venezuelai bolívar": "VES",
"venezuelan digital bolívar": "VED",
"verenigd koninkrijk economie": "GBP",
"verenigde arabiese emirate dirham": "AED",
"verenigde arabische emiraten dirham": "AED",
"ves": "VES",
@@ -12735,12 +12660,6 @@
"wir euro": "CHE",
"wir franc": "CHW",
"wir franken": "CHW",
"wirtschaft": "GBP",
"wirtschaft des vereinigten königreichs": "GBP",
"wirtschaft im vereinigten königreich": "GBP",
"wirtschaft in dem vereinigten königreich": "GBP",
"wirtschaft vom vereinigten königreich": "GBP",
"wirtschaft von dem vereinigten königreich": "GBP",
"wit russische roebel": "BYN",
"won": "KRW",
"won bắc triều tiên": "KPW",
@@ -12821,7 +12740,6 @@
"yen": "JPY",
"yen giapponese": "JPY",
"yen japones": "JPY",
"yen japonez": "JPY",
"yen japonés": "JPY",
"yeni": "JPY",
"yeni i̇srail şekeli": "ILS",
@@ -12834,7 +12752,6 @@
"yeşil burun adaları eskudosu": "CVE",
"yên nhật": "JPY",
"yhdistyneen kuningaskunnan punta": "GBP",
"yhdistyneen kuningaskunnan talous": "GBP",
"yhdistyneiden arabiemiraattien dirhami": "AED",
"yhdysvaltain dollari": "USD",
"ytl": "TRY",
@@ -12889,6 +12806,7 @@
"zimbabwe zig": "ZWG",
"zimbabwean dollar": "ZWL",
"zimbabwean gold": "ZWG",
"zimbabwean zig": "ZWG",
"zimbabwen kulta": "ZWG",
"zimbabwiansky zlatý": "ZWG",
"zimbabwský dolar": "ZWL",
@@ -13047,7 +12965,6 @@
"FKP",
"EGP"
],
"£S": "SYP",
"£e": "EGP",
"£s": "SYP",
"¥": [
@@ -13584,8 +13501,6 @@
"египетский фунт": "EGP",
"единая система региональных взаиморасчётов": "XSU",
"единая система региональных взаиморасчетов": "XSU",
"економіка великобританії": "GBP",
"економіка великої британії": "GBP",
"енглеска фунта": "GBP",
"еритрейська накфа": "ERN",
"еритрејска накфа": "ERN",
@@ -13642,8 +13557,6 @@
"израелски шекел": "ILS",
"израильский новый шекель": "ILS",
"източнокарибски долар": "XCD",
"икономика на великобритания": "GBP",
"икономика на обединеното кралство": "GBP",
"индийска рупия": "INR",
"индийская рупия": "INR",
"индијска рупија": "INR",
@@ -14057,7 +13970,6 @@
"PLZ",
"PLN"
],
"привреда уједињеног краљевства": "GBP",
"пула": "BWP",
"південно африканський ранд": "ZAR",
"південнокорейська вона": "KRW",
@@ -14145,7 +14057,6 @@
"севернокорејски вон": "KPW",
"северо корейская вона": "KPW",
"северокорейская вона": "KPW",
"седі": "GHS",
"сейшел рупиясе": "SCR",
"сейшелска рупия": "SCR",
"сейшельская рупия": "SCR",
@@ -14198,8 +14109,6 @@
"старый румынский лей": "RON",
"стерлинг фунты": "GBP",
"стерлиң фунты": "GBP",
"стопанство на великобритания": "GBP",
"стопанство на обединеното кралство": "GBP",
"суверен боливар": "VES",
"суверенний болівар": "VES",
"суверенный боливар": "VES",
@@ -14450,7 +14359,6 @@
"шриланкийска рупия": "LKR",
"шриланчанска рупија": "LKR",
"щатски долар": "USD",
"экономика великобритании": "GBP",
"эритрейская накфа": "ERN",
"эритрея накфасы": "ERN",
"эсватини лилангение": "SZL",
@@ -14600,8 +14508,6 @@
"יואן סיני": "CNY",
"ין יפני": "JPY",
"כארתולי לארי": "GEL",
"כלכלת בריטניה": "GBP",
"כלכלת הממלכה המאוחדת": "GBP",
"כתר דני": "DKK",
"כתר נורבגי": "NOK",
"כתר נורווגי": "NOK",
@@ -14749,7 +14655,6 @@
"استثمار البلاتين": "XPT",
"استثمار الذهب": "XAU",
"استثمار الفضة": "XAG",
"اقتصاد المملكة المتحدة": "GBP",
"الاستثمار في الذهب": "XAU",
"الأوقية الموريتانية": "MRU",
"البات": "THB",
@@ -14803,7 +14708,6 @@
"أوقية": "MRU",
"أوقية موريتانية": "MRU",
"أوقيه موريتانيه": "MRU",
"إقتصاد بريطانى": "GBP",
"إيسكودو جزر الرأس الأخضر": "CVE",
"بات": "THB",
"بات تايلاندي": "THB",
@@ -15055,8 +14959,6 @@
"فورنت مجري": "HUF",
"فورينت مجري": "HUF",
"فِرَنْكٌ رُوَنْدِيٌّ": "RWF",
"قائمة النقود المعدنية السورية": "SYP",
"قائمة عملات سوريا المعدنية": "SYP",
"ك": "KWD",
"كتزال غواتيمالي": "GTQ",
"كرونة آيسلندية": "ISK",
@@ -15186,7 +15088,6 @@
"মালদ্বীপীয় রুফিয়াহ": "MVR",
"মিয়ানমার ক্যত": "MMK",
"মিশরীয় পাউন্ড": "EGP",
"যুক্তরাজ্যের অর্থনীতি": "GBP",
"রুশ রুবল": "RUB",
"রেনমিনবি": "CNY",
"রেন্মিন্বি": "CNY",
@@ -15818,7 +15719,6 @@
"엔": "JPY",
"엔화": "JPY",
"영국 파운드": "GBP",
"영국의 경제": "GBP",
"예멘 리알": "YER",
"예멘 리얄": "YER",
"예멘리얄": "YER",
@@ -16026,11 +15926,9 @@
"イエメン・リアル": "YER",
"イエメン・リヤル": "YER",
"イエメン・リヤール": "YER",
"イギリスの経済": "GBP",
"イギリスの通貨": "GBP",
"イギリスポンド": "GBP",
"イギリス・ポンド": "GBP",
"イギリス経済": "GBP",
"イラクの通貨": "IQD",
"イラク・ディナール": "IQD",
"イランの通貨": "IRR",
@@ -16332,7 +16230,6 @@
"英ポンド": "GBP",
"西アフリカcfaフラン": "XOF",
"豪ドル": "AUD",
"財政・経済政策": "GBP",
"越南銅": "VND",
"金投資": "XAU",
"韓国ウォン": "KRW",

2872
searx/data/engine_descriptions.json
View File

File diff suppressed because it is too large Load Diff

1759
searx/data/engine_traits.json
View File

File diff suppressed because it is too large Load Diff

7684
searx/data/external_bangs.json
View File

File diff suppressed because it is too large Load Diff

4502
searx/data/gsa_useragents.txt Normal file
View File

File diff suppressed because it is too large Load Diff

12
searx/data/tracker_patterns.py
View File

@@ -10,7 +10,7 @@ import re
from collections.abc import Iterator
from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
from curl_cffi.requests.exceptions import RequestException
from httpx import HTTPError
from searx.data.core import get_cache, log
from searx.network import get as http_get
@@ -28,11 +28,11 @@ class TrackerPatternsDB:
ctx_name = "data_tracker_patterns"
# ClearURL rule lists, the first one that responds HTTP 200 is used
CLEAR_LIST_URL = [
"https://cdn.jsdelivr.net/gh/clearurls/rules@refs/heads/gh-pages/data.minify.json",
"https://rules2.clearurls.xyz/data.minify.json",
# ClearURL rule lists, the first one that responds HTTP 200 is used
"https://rules1.clearurls.xyz/data.minify.json",
"https://rules2.clearurls.xyz/data.minify.json",
"https://raw.githubusercontent.com/ClearURLs/Rules/refs/heads/master/data.min.json",
]
class Fields:
@@ -87,8 +87,8 @@ class TrackerPatternsDB:
try:
resp = http_get(url, timeout=3)
except RequestException as exc:
log.warning("TRACKER_PATTERNS: RequestException while fetching %s: %s", url, exc)
except HTTPError as exc:
log.warning("TRACKER_PATTERNS: HTTPError (%s) occured while fetching %s", url, exc)
continue
if resp.status_code != 200:

4
searx/data/useragents.json
View File

@@ -5,7 +5,7 @@
],
"ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}",
"versions": [
"154.0",
"153.0"
"151.0",
"150.0"
]
}

23618
searx/data/wikidata_properties.json
View File

File diff suppressed because it is too large Load Diff

32
searx/data/wikidata_units.json
View File

@@ -3272,7 +3272,7 @@
"Q128822": {
"si_name": "Q182429",
"symbol": "kn",
"to_si_factor": 0.514
"to_si_factor": 0.5144444444444445
},
"Q12912288": {
"si_name": null,
@@ -3474,6 +3474,11 @@
"symbol": "mm⁻²",
"to_si_factor": 1e-06
},
"Q136039973": {
"si_name": "Q6137407",
"symbol": "FPS",
"to_si_factor": 1.0
},
"Q1361854": {
"si_name": "Q11570",
"symbol": "dwt",
@@ -3516,7 +3521,7 @@
},
"Q1377741": {
"si_name": "Q25250",
"symbol": "V<sub>P</sub>",
"symbol": "V_P",
"to_si_factor": 1.0429e+27
},
"Q1386162": {
@@ -3689,11 +3694,6 @@
"symbol": "apc",
"to_si_factor": 0.0308568
},
"Q16068": {
"si_name": null,
"symbol": "DM",
"to_si_factor": null
},
"Q160857": {
"si_name": "Q25236",
"symbol": "hp",
@@ -3872,11 +3872,11 @@
"Q180892": {
"si_name": "Q11570",
"symbol": "M☉",
"to_si_factor": 1.988416e+30
"to_si_factor": 1.9884e+30
},
"Q1811": {
"si_name": "Q11573",
"symbol": "au",
"symbol": "AU",
"to_si_factor": 149597870700.0
},
"Q1815100": {
@@ -4454,11 +4454,6 @@
"symbol": "ng",
"to_si_factor": 1e-12
},
"Q2285395": {
"si_name": null,
"symbol": "dBW",
"to_si_factor": null
},
"Q22934083": {
"si_name": "Q25406",
"symbol": "nC",
@@ -5249,11 +5244,6 @@
"symbol": "μA",
"to_si_factor": 1e-06
},
"Q31274648": {
"si_name": "Q6137407",
"symbol": "FPS",
"to_si_factor": 1.0
},
"Q3186734": {
"si_name": "Q3186734",
"symbol": "J/(m³ K)",
@@ -6326,7 +6316,7 @@
},
"Q536785": {
"si_name": "Q844211",
"symbol": "ρ<sub>P</sub>",
"symbol": "ρ_P",
"to_si_factor": 5.155e+96
},
"Q53679433": {
@@ -6981,7 +6971,7 @@
},
"Q685662": {
"si_name": "Q44395",
"symbol": "p<sub>P</sub>",
"symbol": "p_P",
"to_si_factor": 4.633e+113
},
"Q686163": {

47
searx/enginelib/__init__.py
View File

@@ -47,7 +47,7 @@ ENGINES_CACHE: ExpireCacheSQLite = ExpireCacheSQLite.build_cache(
ExpireCacheCfg(
name="ENGINES_CACHE",
MAXHOLD_TIME=60 * 60 * 24 * 7, # 7 days
MAINTENANCE_PERIOD=60 * 60, # 1h
MAINTENANCE_PERIOD=60 * 60, # 2h
MAX_VALUE_LEN=1024 * 1024 * 1024, # 1MB
)
)
@@ -305,7 +305,7 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
region: str = ""
"""For an engine, when there is ``region: ...`` in the YAML settings the engine
does support only this one region:
does support only this one region::
.. code:: yaml
@@ -317,9 +317,6 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
enable_http: bool
"""Enable HTTP (by default only HTTPS is enabled)."""
enable_http3: bool = False
"""Enables the use of HTTP/3 if available"""
shortcut: str
"""Code used to execute bang requests (``!foo``)"""
@@ -364,43 +361,37 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
https: socks5://proxy:port
"""
def setup(self, engine_settings: dict[str, t.Any]) -> bool | None: # pylint: disable=unused-argument
def setup(self, engine_settings: dict[str, t.Any]) -> bool: # pylint: disable=unused-argument
"""Dynamic setup of the engine settings.
With this method, the engine's setup is carried out. For example, to
check or dynamically adapt the values handed over in the parameter
``engine_settings``.
``engine_settings``. The return value (True/False) indicates whether
the setup was successful and the engine can be built or rejected.
Whether the initialization was successful can be indicated by the return
value ``True`` or even ``False``.
- If no return value (``None`` ) is given from this method , this is
equivalent to ``True``.
- If an exception is thrown as part of the initialization, this is
equivalent to ``False``.
The method is optional and is called **synchronously** as part of the
The method is optional and is called synchronously as part of the
initialization of the service and is therefore only suitable for simple
(local) exams/changes at the engine setting.
The :py:obj:`Engine.init` method must be used for longer tasks in which
values of a remote must be determined, for example.
(local) exams/changes at the engine setting. The :py:obj:`Engine.init`
method must be used for longer tasks in which values of a remote must be
determined, for example.
"""
return True
def init(self, engine_settings: dict[str, t.Any]) -> bool | None: # pylint: disable=unused-argument
"""Initialization of the engine.
The method is optional and called **asynchronous** (in a thread). The
method is comparable to :py:obj:`Engine.setup`, it is suitable, for
caching data that first needs to be requested from a remote.
The method is optional and asynchronous (in a thread). It is suitable,
for example, for setting up a cache (for the engine) or for querying
values (required by the engine) from a remote.
The method is optional and runs **asynchronously** (in a thread), it is
comparable to :py:obj:`Engine.setup`. For instance, it is suitable for
caching data that first needs to be requested from a remote source.
Whether the initialization was successful can be indicated by the return
value ``True`` or even ``False``.
The evaluation of the return value is analogous to :py:obj:`Engine.setup`.
- If no return value is given from this init method (``None``), this is
equivalent to ``True``.
- If an exception is thrown as part of the initialization, this is
equivalent to ``False``.
"""
return True

28
searx/engines/360search_videos.py
View File

@@ -6,8 +6,7 @@ from urllib.parse import urlencode
from datetime import datetime
from searx.exceptions import SearxEngineAPIException
from searx.result_types import EngineResults
from searx.utils import html_to_text
from searx.utils import html_to_text, get_embeded_stream_url
about = {
"website": "https://tv.360kan.com/",
@@ -30,12 +29,12 @@ def request(query, params):
return params
def response(resp) -> EngineResults:
def response(resp):
try:
data = resp.json()
except Exception as e:
raise SearxEngineAPIException(f"Invalid response: {e}") from e
res = EngineResults()
results = []
if "data" not in data or "result" not in data["data"]:
raise SearxEngineAPIException("Invalid response")
@@ -51,15 +50,16 @@ def response(resp) -> EngineResults:
except (ValueError, TypeError):
published_date = None
res.add(
res.types.LegacyResult(
url=entry["play_url"],
title=html_to_text(entry["title"]),
content=html_to_text(entry["description"]),
template='videos.html',
publishedDate=published_date,
thumbnail=entry["cover_img"],
)
results.append(
{
'url': entry["play_url"],
'title': html_to_text(entry["title"]),
'content': html_to_text(entry["description"]),
'template': 'videos.html',
'publishedDate': published_date,
'thumbnail': entry["cover_img"],
"iframe_src": get_embeded_stream_url(entry["play_url"]),
}
)
return res
return results

2
searx/engines/500px.py
View File

@@ -82,7 +82,7 @@ fragment SXNG_query on Query {
def setup(_) -> bool:
global SXNG_query # pylint: disable=global-statement
rand_str: str = "".join(random.choices(string.ascii_letters, k=5))
rand_str: str = "".join(random.choice(string.ascii_letters) for _ in range(5))
SXNG_query = SXNG_query.replace("SXNG_query", "PhotoSearchPaginationContainer_query_1" + rand_str)
return True

1
searx/engines/__builtins__.pyi
View File

@@ -26,7 +26,6 @@ categories: list[str]
disabled: bool
display_error_messages: bool
enable_http: bool
enable_http3: bool
engine_type: str
inactive: bool
max_page: int

21
searx/engines/__init__.py
View File

@@ -187,9 +187,8 @@ def set_loggers(engine: "Engine|types.ModuleType", engine_name: str):
def update_engine_attributes(engine: "Engine | types.ModuleType", engine_data: dict[str, t.Any]):
# pylint: disable=too-many-branches
# set / update engine attributes from engine_data
# set engine attributes from engine_data
kvargs: dict[str, t.Any]
engine.about = getattr(engine, "about", EngineAbout())
if isinstance(engine.about, EngineAbout):
kvargs = {**msgspec.to_builtins(engine.about), **engine_data.get("about", {})}
else:
@@ -270,27 +269,21 @@ def is_engine_active(engine: "Engine | types.ModuleType"):
def call_engine_setup(engine: "Engine | types.ModuleType", engine_data: dict[str, t.Any]) -> bool:
setup_ok: bool | None = False
setup_ok = False
setup_func = getattr(engine, "setup", None)
if setup_func is None:
setup_ok = True
elif not callable(setup_func):
logger.error(f"engine's setup method isn't a callable (is of type: {type(setup_func)})")
logger.error("engine's setup method isn't a callable (is of type: %s)", type(setup_func))
else:
try:
setup_ok = engine.setup(engine_data)
except Exception as e: # pylint: disable=broad-except
logger.exception(f"(PID {os.getpid()}) {engine.name}: engine SETUP failed, exception: {e}")
setup_ok = False
# The evaluation of the return value is analogous to Engine.init
if setup_ok is None:
setup_ok = True
logger.exception('exception : {0}'.format(e))
if not setup_ok:
logger.error(f"(PID {os.getpid()}) {engine.name}: engine setup was not successful")
logger.error("%s: Engine setup was not successful, engine is set to inactive.", engine.name)
return setup_ok
@@ -318,16 +311,14 @@ def load_engines(engine_list: list[dict[str, t.Any]]):
for engine_data in engine_list:
if engine_data.get("inactive") is True:
continue
engine = load_engine(engine_data)
if engine:
register_engine(engine)
else:
# if an engine can't be loaded (if for example the engine is missing
# tor or some other requirements) its set to inactive!
logger.error(
f"(PID {os.getpid()}) {engine_data.get('name', '???')}: can't register engine (loading engine failed)"
f"(PID {os.getpid()}) loading engine %s failed: set engine to inactive!", engine_data.get("name", "???")
)
engine_data["inactive"] = True
return engines

2
searx/engines/acfun.py
View File

@@ -83,7 +83,7 @@ def extract_video_data(video_block):
published_date = None
if create_time:
try:
published_date = datetime.fromisoformat(create_time.strip())
published_date = datetime.strptime(create_time.strip(), "%Y-%m-%d")
except (ValueError, TypeError):
pass

10
searx/engines/adobe_stock.py
View File

@@ -35,7 +35,7 @@ Implementation
==============
"""
import typing as t
from datetime import datetime, timedelta
from urllib.parse import urlencode
@@ -85,7 +85,7 @@ Additional subcategories:
# Do we need support for "free_collection" and "include_stock_enterprise"?
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if not categories:
raise ValueError("adobe_stock engine: categories is unset")
@@ -100,9 +100,9 @@ def setup(_: dict[str, t.Any]) -> bool | None:
raise ValueError("adobe_stock engine: adobe_content_types is unset")
if isinstance(adobe_content_types, list):
for content_type in adobe_content_types:
if content_type not in ADOBE_VALID_TYPES:
raise ValueError("adobe_stock engine: adobe_content_types: '%s' is invalid" % content_type)
for t in adobe_content_types:
if t not in ADOBE_VALID_TYPES:
raise ValueError("adobe_stock engine: adobe_content_types: '%s' is invalid" % t)
else:
raise ValueError(
"adobe_stock engine: adobe_content_types must be a list of strings not %s" % type(adobe_content_types)

2
searx/engines/arxiv.py
View File

@@ -109,7 +109,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
comments_elements = eval_xpath_getindex(entry, xpath_comment, 0, default=None)
comments: str = "" if comments_elements is None else comments_elements.text
publishedDate = datetime.fromisoformat(eval_xpath_getindex(entry, xpath_published, 0).text.rstrip("Z"))
publishedDate = datetime.strptime(eval_xpath_getindex(entry, xpath_published, 0).text, "%Y-%m-%dT%H:%M:%SZ")
res.add(
res.types.Paper(

38
searx/engines/baidu.py
View File

@@ -7,17 +7,13 @@
# There exits a https://github.com/ohblue/baidu-serp-api/
# but we don't use it here (may we can learn from).
import typing as t
from urllib.parse import urlencode
from datetime import datetime
from html import unescape
import time
import json
from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException, SearxEngineAccessDeniedException
from searx.enginelib import EngineCache
from searx.network import get as http_get
from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException
from searx.utils import html_to_text
about = {
@@ -39,34 +35,11 @@ baidu_category = 'general'
time_range_support = True
time_range_dict = {"day": 86400, "week": 604800, "month": 2592000, "year": 31536000}
image_base_url = "https://image.baidu.com/"
COOKIE_CACHE_KEY = "cookie"
COOKIE_CACHE_EXPIRATION_SECONDS = 3600
CACHE: EngineCache
"""Stores cookies from Baidu image search warmup."""
def setup(engine_settings: dict[str, t.Any]) -> bool:
def init(_):
if baidu_category not in ('general', 'images', 'it'):
raise SearxEngineAPIException(f"Unsupported category: {baidu_category}")
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"])
return True
def get_image_cookies(headers: dict[str, str]) -> dict[str, str]:
cookies: dict[str, str] | None = CACHE.get(COOKIE_CACHE_KEY)
if cookies:
return cookies
warmup = http_get(image_base_url, headers=headers, timeout=10)
cookies = dict(warmup.cookies.items())
CACHE.set(key=COOKIE_CACHE_KEY, value=cookies, expire=COOKIE_CACHE_EXPIRATION_SECONDS)
return cookies
def request(query, params):
page_num = params["pageno"]
@@ -115,9 +88,6 @@ def request(query, params):
if baidu_category == 'it':
query_params["paramList"] += f",timestamp_range={past}-{now}"
if baidu_category == 'images':
params["cookies"] = get_image_cookies(params["headers"])
params["url"] = f"{query_url}?{urlencode(query_params)}"
params["allow_redirects"] = False
return params
@@ -133,8 +103,6 @@ def response(resp):
# baidu's JSON encoder wrongly quotes / and ' characters by \\ and \'
text = text.replace(r"\/", "/").replace(r"\'", "'")
data = json.loads(text, strict=False)
if data.get("antiFlag") == 1:
raise SearxEngineAccessDeniedException(data.get("message", "Forbid spider access"))
parsers = {'general': parse_general, 'images': parse_images, 'it': parse_it}
return parsers[baidu_category](data)
@@ -184,7 +152,7 @@ def parse_images(data):
img_date = item.get("bdImgnewsDate")
publishedDate = None
if img_date:
publishedDate = datetime.fromisoformat(img_date)
publishedDate = datetime.strptime(img_date, "%Y-%m-%d %H:%M")
results.append(
{
"template": "images.html",

34
searx/engines/bilibili.py
View File

@@ -8,7 +8,6 @@ import random
import string
from urllib.parse import urlencode
from datetime import datetime, timedelta
from zoneinfo import ZoneInfo
from searx import utils
@@ -32,7 +31,7 @@ base_url = "https://api.bilibili.com/x/web-interface/search/type"
cookie = {
"innersign": "0",
"buvid3": "".join(random.choices(string.hexdigits, k=16)) + "infoc",
"buvid3": "".join(random.choice(string.hexdigits) for _ in range(16)) + "infoc",
"i-wanna-go-back": "-1",
"b_ut": "7",
"FEED_LIVE_VERSION": "V8",
@@ -40,32 +39,6 @@ cookie = {
"home_feed_column": "4",
}
_CN_TZ = ZoneInfo("Asia/Shanghai")
# Calendar-day time filter (Asia/Shanghai); dict values are days to subtract from today.
time_range_support = True
time_range_dict = {"day": 0, "week": 6, "month": 29, "year": 364}
def _pubtime_range(time_range: str) -> tuple[int, int]:
"""Return ``(pubtime_begin_s, pubtime_end_s)`` for Bilibili's search API.
Time ranges follow Bilibili's website semantics: they are counted in
**calendar days** in China Standard Time (``Asia/Shanghai``), not as
sliding 24-hour windows. For example, ``day`` means from 00:00:00 to
23:59:59 of the current local day; ``week`` spans from 00:00:00 on the
calendar day six days ago through the end of today, and so on.
The returned Unix timestamps (seconds) map to Bilibili's
``pubtime_begin_s`` and ``pubtime_end_s`` query parameters.
"""
now = datetime.now(_CN_TZ)
pubtime_end_s = int(now.replace(hour=23, minute=59, second=59, microsecond=0).timestamp())
begin_day = now - timedelta(days=time_range_dict[time_range])
pubtime_begin_s = int(begin_day.replace(hour=0, minute=0, second=0, microsecond=0).timestamp())
return pubtime_begin_s, pubtime_end_s
def request(query, params):
query_params = {
@@ -77,11 +50,6 @@ def request(query, params):
"search_type": "video",
}
if params.get("time_range") in time_range_dict:
pubtime_begin_s, pubtime_end_s = _pubtime_range(params["time_range"])
query_params["pubtime_begin_s"] = pubtime_begin_s
query_params["pubtime_end_s"] = pubtime_end_s
params["url"] = f"{base_url}?{urlencode(query_params)}"
params["headers"]["Referer"] = "https://www.bilibili.com/"
params["headers"]["Accept"] = "application/json, text/javascript, */*; q=0.01"

1
searx/engines/bing.py
View File

@@ -40,7 +40,6 @@ about: dict[str, t.Any] = {
# engine dependent config
categories = ["general", "web"]
safesearch = True
enable_http3 = True
_safesearch_map: dict[int, str] = {
0: "off",
1: "moderate",

1
searx/engines/bing_images.py
View File

@@ -25,7 +25,6 @@ about = {
# engine dependent config
categories = ["images", "web"]
paging = True
enable_http3 = True
safesearch = True
time_range_support = True
time_map = {

1
searx/engines/bing_news.py
View File

@@ -33,7 +33,6 @@ categories = ["news"]
paging = True
"""If go through the pages and there are actually no new results for another
page, then bing returns the results from the last page again."""
enable_http3 = True
time_range_support = True
time_map = {

1
searx/engines/bing_videos.py
View File

@@ -26,7 +26,6 @@ about = {
# engine dependent config
categories = ["videos", "web"]
paging = True
enable_http3 = True
safesearch = True
time_range_support = True

2
searx/engines/bitchute.py
View File

@@ -44,7 +44,7 @@ def response(resp):
"url": 'https://www.bitchute.com/video/' + item['video_id'],
"content": html_to_text(item['description']),
"author": item['channel']['channel_name'],
"publishedDate": datetime.fromisoformat(item["date_published"].rstrip("Z")),
"publishedDate": datetime.strptime(item["date_published"], "%Y-%m-%dT%H:%M:%S.%fZ"),
"length": item['duration'],
"views": item['view_count'],
"thumbnail": item['thumbnail_url'],

4
searx/engines/boardreader.py
View File

@@ -45,7 +45,7 @@ CACHE_SESSION_ID_KEY = "session_id_key"
KEYWORD_RE = re.compile(r"\[\/?Keyword\]")
def setup(engine_settings: dict[str, t.Any]) -> bool:
def init(engine_settings: dict[str, t.Any]) -> bool:
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_name=engine_settings["name"])
return True
@@ -104,7 +104,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
title=_remove_keyword_marker(result["Subject"]),
content=_remove_keyword_marker(result["Text"]),
url=result["Url"],
publishedDate=datetime.fromisoformat(result["Published"]),
publishedDate=datetime.strptime(result["Published"], "%Y-%m-%d %H:%M:%S"),
metadata=gettext.gettext("Posted by {author}").format(author=result["Author"]),
)
)

30
searx/engines/brave.py
View File

@@ -135,7 +135,7 @@ from searx.utils import (
eval_xpath_getindex,
eval_xpath_list,
extract_text,
get_embedded_stream_url,
get_embeded_stream_url,
js_obj_str_to_json_str,
js_obj_str_to_python,
)
@@ -151,7 +151,6 @@ about = {
base_url = "https://search.brave.com/"
categories = []
enable_http3 = True
brave_category: t.Literal["search", "videos", "images", "news", "goggles"] = "search"
"""Brave supports common web-search, videos, images, news, and goggles search.
@@ -248,13 +247,13 @@ def extract_json_data(text: str) -> dict[str, t.Any]:
# node_ids: [0, 19],
# data: [{type:"data",data: .... ["q","goggles_id"],route:1,url:1}}]
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# form: null,
# error: null
# });
text = text[text.index("<script") : text.index("</script")]
if not text:
raise ValueError("can't find JS/JSON data in the given text")
start = text.index("data: [{")
newline = text.index("\n", start)
end = text.rindex("}}]", start, newline)
js_obj_str = "{" + text[start:end] + "}}]}"
end = text.rindex("}}]")
js_obj_str = text[start:end]
js_obj_str = "{" + js_obj_str + "}}]}"
# js_obj_str = js_obj_str.replace("\xa0", "") # remove ASCII for &nbsp;
# js_obj_str = js_obj_str.replace(r"\u003C", "<").replace(r"\u003c", "<") # fix broken HTML tags in strings
json_str = js_obj_str_to_json_str(js_obj_str)
@@ -339,7 +338,7 @@ def _parse_search(resp: SXNG_Response) -> EngineResults:
if len(video_tag):
# In my tests a video tag in the WEB search was most often not a
# video, except the ones from youtube ..
iframe_src = get_embedded_stream_url(url)
iframe_src = get_embeded_stream_url(url)
if iframe_src:
item["iframe_src"] = iframe_src
item["template"] = "videos.html"
@@ -354,14 +353,14 @@ def _parse_news(resp: SXNG_Response) -> EngineResults:
res = EngineResults()
dom = html.fromstring(resp.text)
for result in eval_xpath_list(dom, "//div[@data-type='news']"):
url = eval_xpath_getindex(result, ".//a/@href", 0, default=None)
for result in eval_xpath_list(dom, "//div[contains(@class, 'results')]//div[@data-type='news']"):
url = eval_xpath_getindex(result, ".//a[contains(@class, 'result-header')]/@href", 0, default=None)
if url is None:
continue
title = eval_xpath_list(result, ".//div[contains(@class, 'title')]")
content = eval_xpath_list(result, ".//div[contains(@class, 'description')]")
thumbnail = eval_xpath_getindex(result, ".//a[contains(@class, 'thumbnail')]//img/@src", 0, default="")
title = eval_xpath_list(result, ".//span[contains(@class, 'snippet-title')]")
content = eval_xpath_list(result, ".//p[contains(@class, 'desc')]")
thumbnail = eval_xpath_getindex(result, ".//div[contains(@class, 'image-wrapper')]//img/@src", 0, default="")
item = res.types.LegacyResult(
template="default.html",
@@ -407,6 +406,9 @@ def _parse_videos(json_resp: dict[str, t.Any]) -> EngineResults:
)
if result["thumbnail"] is not None:
item["thumbnail"] = result["thumbnail"]["src"]
iframe_src = get_embeded_stream_url(result["url"])
if iframe_src:
item["iframe_src"] = iframe_src
res.add(item)

24
searx/engines/braveapi.py
View File

@@ -31,7 +31,6 @@ from dateutil import parser
from searx.exceptions import SearxEngineAPIException
from searx.result_types import EngineResults
from searx.utils import html_to_text
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
@@ -40,7 +39,7 @@ if t.TYPE_CHECKING:
about = {
"website": "https://api.search.brave.com/",
"wikidata_id": None,
"official_api_documentation": "https://api-dashboard.search.brave.com/api-reference/web/search/get",
"official_api_documentation": "https://api-dashboard.search.brave.com/documentation",
"use_official_api": True,
"require_api_key": True,
"results": "JSON",
@@ -63,10 +62,8 @@ base_url = "https://api.search.brave.com/res/v1/web/search"
time_range_map = {"day": "past_day", "week": "past_week", "month": "past_month", "year": "past_year"}
"""Mapping of SearXNG time ranges to Brave API time ranges."""
max_page = 10
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
"""Initialize the engine."""
if not api_key:
raise SearxEngineAPIException("No API key provided")
@@ -77,8 +74,7 @@ def request(query: str, params: "OnlineParams") -> None:
search_args: dict[str, str | int | None] = {
"q": query,
"count": results_per_page,
"offset": params["pageno"] - 1,
"text_decorations": False,
"offset": (params["pageno"] - 1) * results_per_page,
}
# Apply time filter if specified
@@ -91,7 +87,6 @@ def request(query: str, params: "OnlineParams") -> None:
params["url"] = f"{base_url}?{urlencode(search_args)}"
params["headers"]["X-Subscription-Token"] = api_key
params["headers"]["Accept"] = "application/json"
def _extract_published_date(published_date_raw: str):
@@ -117,19 +112,14 @@ def response(resp: "SXNG_Response") -> EngineResults:
res = EngineResults()
data = resp.json()
for result in (data.get("web") or {}).get("results", []):
thumbnail_obj = result.get("thumbnail")
thumbnail = ""
if thumbnail_obj and not thumbnail_obj.get("logo", False):
thumbnail = thumbnail_obj.get("src") or ""
for result in data.get("web", {}).get("results", []):
res.add(
res.types.MainResult(
url=result["url"],
title=html_to_text(result["title"]),
content=html_to_text(result.get("description", "")),
title=result["title"],
content=result.get("description", ""),
publishedDate=_extract_published_date(result.get("age")),
thumbnail=thumbnail,
thumbnail=result.get("thumbnail", {}).get("src"),
),
)

85
searx/engines/cara.py Normal file
View File

@@ -0,0 +1,85 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
# pylint: disable=invalid-name
"""Cara_ is a social media and portfolio-sharing platform for artists and art
enthusiasts.
With the widespread use of generative AI, Cara_ decided to build a place that
filters out gen AI images so that people searching for authentic creatives and
images can do so easily.
.. _Cara: https://cara.app/about
"""
from urllib.parse import urlencode
import typing as t
from searx.result_types import EngineResults
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
about = {
"website": "https://cara.app",
"official_api_documentation": None,
"use_official_api": False,
"require_api_key": False,
"results": "JSON",
}
base_url = "https://cara.app"
images_url = "https://images.cara.app"
categories = ["images"]
paging = True
results_per_page = 24
# if using HTTP2, we get blocked immediately
enable_http2 = False
def request(query: str, params: "OnlineParams") -> None:
args = {
"q": query,
"sortBy": "Top",
"take": results_per_page,
"skip": (params["pageno"] - 1) * results_per_page,
}
params["url"] = f"{base_url}/api/search/portfolio-posts?{urlencode(args)}"
def response(resp: "SXNG_Response"):
res = EngineResults()
json_data: list[dict[str, t.Any]] = resp.json()
for result in json_data:
thumbnail, img = None, None
i: dict[str, str]
for i in result["images"]:
if thumbnail is None or i["isCoverImg"]:
thumbnail = i
if img is None or not i["isCoverImg"]:
img = i
if not thumbnail or not img:
continue
res.add(
res.types.LegacyResult(
{
"template": "images.html",
"url": f"{base_url}/post/{result['id']}",
"thumbnail_src": f"{images_url}/{thumbnail['src']}?height=256",
"img_src": f"{images_url}/{img['src']}",
"title": result["title"],
"content": result["content"],
"author": result["name"],
}
)
)
return res

10
searx/engines/chatnoir.py
View File

@@ -41,7 +41,7 @@ search_index = "cw22"
<https://www.chatnoir.eu/docs/api-general>`_ for a full list."""
def _obtain_api_key() -> tuple[str, str]:
def _obtain_api_key() -> tuple[str, str, str]:
home_resp = get(base_url)
if not home_resp.ok:
raise SearxEngineAPIException("failed to obtain api key")
@@ -58,9 +58,10 @@ def _obtain_api_key() -> tuple[str, str]:
)
if not token_resp.ok:
raise SearxEngineAPIException("failed to obtain api key")
session_id = token_resp.cookies["sessionid"]
scraped_api_key = token_resp.json()["token"]["token"]
return csrf_token, scraped_api_key
return csrf_token, session_id, scraped_api_key
def request(query: str, params: "OnlineParams"):
@@ -72,7 +73,7 @@ def request(query: str, params: "OnlineParams"):
params["headers"].update(headers)
else:
csrf_token, scraped_api_key = _obtain_api_key()
csrf_token, session_id, scraped_api_key = _obtain_api_key()
headers = {
"Authorization": f"Bearer {scraped_api_key}",
@@ -80,11 +81,10 @@ def request(query: str, params: "OnlineParams"):
}
params["headers"].update(headers)
params["cookies"] = {"csrftoken": csrf_token}
params["cookies"] = {"csrftoken": session_id, "sessionid": session_id}
params["url"] = f"{base_url}/api/v1/_search"
params["method"] = "POST"
params["impersonate"] = "none"
json_data = {
"query": query,

2
searx/engines/chefkoch.py
View File

@@ -43,7 +43,7 @@ def response(resp):
publishedDate = None
if recipe['submissionDate']:
publishedDate = datetime.fromisoformat(result['recipe']['submissionDate'][:19])
publishedDate = datetime.strptime(result['recipe']['submissionDate'][:19], "%Y-%m-%dT%H:%M:%S")
content = [
f"Schwierigkeitsstufe (1-3): {recipe['difficulty']}",

2
searx/engines/chinaso.py
View File

@@ -78,7 +78,7 @@ time_range_dict = {'day': '24h', 'week': '1w', 'month': '1m', 'year': '1y'}
base_url = "https://www.chinaso.com"
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if chinaso_news_source not in t.get_args(ChinasoNewsSourceType):
raise ValueError(f"Unsupported news source: {chinaso_news_source}")

3
searx/engines/command.py
View File

@@ -74,7 +74,6 @@ Implementations
===============
"""
import typing as t
import re
from os.path import expanduser, isabs, realpath, commonprefix
from shlex import split as shlex_split
@@ -101,7 +100,7 @@ _command_logger = logger.getChild('command')
_compiled_parse_regex = {}
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings):
check_parsing_options(engine_settings)
if 'command' not in engine_settings:

82
searx/engines/deviantart.py
View File

@@ -1,18 +1,11 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Deviantart (Images)"""
import typing as t
import urllib.parse
from lxml import html
from searx.result_types import EngineResults
from searx.utils import extract_text, eval_xpath, eval_xpath_list
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
# about
about = {
"website": 'https://www.deviantart.com/',
@@ -30,62 +23,63 @@ paging = True
# search-url
base_url = 'https://www.deviantart.com'
results_xpath = '//div[@data-testid="content_row"]//a[.//*[@data-testid="thumb"]]'
img_src_xpath = './/img/@srcset'
thumbnail_src_xpath = './/img/@src'
author_xpath = './/*[@property="schema:name"]/@content'
cursor_xpath = '//a[contains(@href, "cursor=") and contains(., "Next")]/@href'
results_xpath = '//div[@class="V_S0t_"]/div/div/a'
url_xpath = './@href'
thumbnail_src_xpath = './div/img/@src'
img_src_xpath = './div/img/@srcset'
title_xpath = './@aria-label'
premium_xpath = '../div/div/div/text()'
premium_keytext = 'Watch the artist to view this deviation'
cursor_xpath = '(//a[@class="vQ2brP"]/@href)[last()]'
def request(query: str, params: "OnlineParams"):
def request(query, params):
# https://www.deviantart.com/search?q=foo
args = {'q': query}
if params['pageno'] > 1:
cursor = params['engine_data'].get('cursor')
if cursor:
args['cursor'] = cursor
nextpage_url = params['engine_data'].get('nextpage')
# don't use nextpage when user selected to jump back to page 1
if params['pageno'] > 1 and nextpage_url is not None:
params['url'] = nextpage_url
else:
params['url'] = f"{base_url}/search?{urllib.parse.urlencode({'q': query})}"
params['url'] = f"{base_url}/search?{urllib.parse.urlencode(args)}"
return params
def response(resp: "SXNG_Response") -> EngineResults:
def response(resp):
res = EngineResults()
results = []
dom = html.fromstring(resp.text)
for result in eval_xpath_list(dom, results_xpath):
thumbnail_src = extract_text(eval_xpath(result, thumbnail_src_xpath))
img_src = extract_text(eval_xpath(result, img_src_xpath))
# mature locked thumbs have blur transform (blur_15, blur_30 etc..)
if ',blur_' in f'{thumbnail_src}{img_src}':
# skip images that are blurred
_text = extract_text(eval_xpath(result, premium_xpath))
if _text and premium_keytext in _text:
continue
img_src = extract_text(eval_xpath(result, img_src_xpath))
if img_src:
img_src = img_src.split(' ')[0]
parsed_url = urllib.parse.urlparse(img_src)
img_src = parsed_url._replace(path=parsed_url.path.split('/v1')[0]).geturl()
author = extract_text(eval_xpath(result, author_xpath))
res.add(
res.types.Image(
template='images.html',
url=result.get('href'),
img_src=img_src or "",
thumbnail_src=thumbnail_src or "",
title=result.get('aria-label'),
author=author or "",
)
results.append(
{
'template': 'images.html',
'url': extract_text(eval_xpath(result, url_xpath)),
'img_src': img_src,
'thumbnail_src': extract_text(eval_xpath(result, thumbnail_src_xpath)),
'title': extract_text(eval_xpath(result, title_xpath)),
}
)
nextpage_url = extract_text(eval_xpath(dom, cursor_xpath))
cursor = urllib.parse.parse_qs(urllib.parse.urlparse(nextpage_url or '').query).get('cursor', [None])[0]
if cursor:
res.add(
res.types.LegacyResult(
engine_data=cursor,
key='cursor',
)
if nextpage_url:
results.append(
{
'engine_data': nextpage_url.replace("http://", "https://"),
'key': 'nextpage',
}
)
return res
return results

29
searx/engines/dogpile.py
View File

@@ -8,9 +8,6 @@ import typing as t
from datetime import datetime, timezone
import html
from searx.enginelib import EngineCache
from searx.exceptions import SearxEngineAPIException
from searx.network import post
from searx.utils import format_duration, html_to_text, humanize_number
from searx.result_types import EngineResults
@@ -38,39 +35,17 @@ dogpile_categ = "search"
base_url = "https://www.dogpile.com"
safe_search_map = {0: "none", 1: "moderate", 2: "heavy"}
CACHE: EngineCache
"""Cache for the API token from dogpile"""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if dogpile_categ not in ("search", "images", "videos", "news"):
raise ValueError("invalid search type: %s" % dogpile_categ)
global CACHE # pylint: disable=global-statement
CACHE = EngineCache("dogpile") # one token for images/videos/news
return True
def _obtain_token() -> str:
token = CACHE.get("token")
if token:
return token
resp = post(f"{base_url}/api/token/refresh", headers={"Origin": base_url}, cookies={"dp_api_token": "1"})
if not resp.ok:
raise SearxEngineAPIException("failed to obtain dogpile token")
token = resp.json()["token"]
CACHE.set("token", token, expire=240) # 300s ttl
return token
def request(query: str, params: "OnlineParams"):
params["url"] = f"{base_url}/api/{dogpile_categ}"
params["headers"]["Origin"] = base_url
params["cookies"]["dp_api_token"] = "1"
if dogpile_categ != "search": # web doesnt need token
params["headers"]["x-dogpile-token"] = _obtain_token()
params["method"] = "POST"
params["json"] = {"q": query, "qadf": safe_search_map[params["safesearch"]], "page": params["pageno"]}
return params
def response(resp: "SXNG_Response"):

71
searx/engines/duckduckgo_extra.py
View File

@@ -10,8 +10,7 @@ from datetime import datetime
from urllib.parse import urlencode
from urllib.parse import quote_plus
from searx.result_types import EngineResults, MainResult, LegacyResult, Image
from searx.utils import html_to_text, gen_useragent, extr
from searx.utils import get_embeded_stream_url, html_to_text, gen_useragent, extr
from searx.network import get # see https://github.com/searxng/searxng/issues/762
from searx.engines.duckduckgo import fetch_traits # pylint: disable=unused-import
@@ -48,7 +47,7 @@ _HTTP_User_Agent: str = gen_useragent()
send_accept_language_header = False
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings: dict[str, t.Any]):
if engine_settings["ddg_category"] not in ["images", "videos", "news"]:
raise ValueError(f"Unsupported DuckDuckGo category: {engine_settings['ddg_category']}")
@@ -98,7 +97,6 @@ def request(query: str, params: "OnlineParams") -> None:
# The vqd value is generated from the query and the UA header. To be able to
# reuse the vqd value, the UA header must be static.
headers["User-Agent"] = _HTTP_User_Agent
params["impersonate"] = "none"
vqd = get_vqd(query=query, params=params) or fetch_vqd(query=query, params=params)
headers["Accept"] = "*/*"
@@ -150,51 +148,54 @@ def request(query: str, params: "OnlineParams") -> None:
def _image_result(result):
return Image(
url=result['url'],
title=result['title'],
content='',
thumbnail_src=result['thumbnail'],
img_src=result['image'],
resolution='%s x %s' % (result['width'], result['height']),
source=result['source'],
)
return {
'template': 'images.html',
'url': result['url'],
'title': result['title'],
'content': '',
'thumbnail_src': result['thumbnail'],
'img_src': result['image'],
'resolution': '%s x %s' % (result['width'], result['height']),
'source': result['source'],
}
def _video_result(result):
return LegacyResult(
template='videos.html',
url=result['content'],
title=result['title'],
content=result['description'],
thumbnail=result['images'].get('small') or result['images'].get('medium'),
source=result['provider'],
length=result['duration'],
metadata=result.get('uploader'),
)
return {
'template': 'videos.html',
'url': result['content'],
'title': result['title'],
'content': result['description'],
'thumbnail': result['images'].get('small') or result['images'].get('medium'),
'iframe_src': get_embeded_stream_url(result['content']),
'source': result['provider'],
'length': result['duration'],
'metadata': result.get('uploader'),
}
def _news_result(result):
return MainResult(
url=result['url'],
title=result['title'],
content=html_to_text(result['excerpt']),
publishedDate=datetime.fromtimestamp(result['date']),
)
return {
'url': result['url'],
'title': result['title'],
'content': html_to_text(result['excerpt']),
'source': result['source'],
'publishedDate': datetime.fromtimestamp(result['date']),
}
def response(resp: "SXNG_Response") -> EngineResults:
res = EngineResults()
def response(resp):
results = []
res_json = resp.json()
for result in res_json['results']:
if ddg_category == 'images':
res.add(_image_result(result))
results.append(_image_result(result))
elif ddg_category == 'videos':
res.add(_video_result(result))
results.append(_video_result(result))
elif ddg_category == 'news':
res.add(_news_result(result))
results.append(_news_result(result))
else:
raise ValueError(f"Invalid duckduckgo category: {ddg_category}")
return res
return results

14
searx/engines/duckduckgo_weather.py
View File

@@ -109,19 +109,7 @@ def response(resp: SXNG_Response):
json_data = loads(resp.text[resp.text.find('\n') + 1 : resp.text.rfind('\n') - 2])
location = json_data.get("location")
if not location:
return res
metadata = json_data.get("weatherAlerts", {}).get("metadata", {})
geoloc = weather.GeoLocation(
name=location,
latitude=metadata.get("latitude"),
longitude=metadata.get("longitude"),
elevation=0,
country_code=metadata.get("language").split("-")[-1],
timezone=json_data.get("location"),
)
geoloc = weather.GeoLocation.by_query(resp.search_params["query"])
weather_answer = EngineResults.types.WeatherAnswer(
current=_weather_data(geoloc, json_data["currentWeather"]),

3
searx/engines/elasticsearch.py
View File

@@ -41,7 +41,6 @@ authentication configured to read from ``my-index`` index.
"""
import typing as t
from json import loads, dumps
from searx.exceptions import SearxEngineAPIException
from searx.result_types import EngineResults
@@ -69,7 +68,7 @@ show_metadata = False
page_size = 10
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings):
if 'query_type' in engine_settings and engine_settings['query_type'] not in _available_query_types:
raise ValueError('unsupported query type', engine_settings['query_type'])

168
searx/engines/exaapi.py
View File

@@ -1,168 +0,0 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Engine to search using the official `Exa Search API`_. Exa is a search engine for AI agents.
.. _Exa Search API: https://exa.ai/docs/reference/search
Configuration
=============
The engine has the following mandatory setting:
- :py:obj:`api_key`
You can obtain an API key from the `API Key section <https://dashboard.exa.ai/api-keys>`_ in the Exa dashboard.
Optional settings are:
- :py:obj:`results_per_page`
- :py:obj:`search_type`
- :py:obj:`content_mode`
- :py:obj:`content_max_characters`
.. code:: yaml
- name: exaapi
engine: exaapi
shortcut: exa
api_key: "..."
results_per_page: 10
search_type: auto
content_mode: highlights
inactive: false
The API supports SafeSearch and region-aware results.
"""
import typing as t
from dateutil import parser
from searx.exceptions import SearxEngineAPIException
from searx.result_types import EngineResults
from searx.utils import html_to_text
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
SearchType = t.Literal["fast", "auto", "instant", "deep", "deep-lite", "deep-reasoning"]
ContentMode = t.Literal["highlights", "text"]
about = {
"website": "https://exa.ai",
"wikidata_id": None,
"official_api_documentation": "https://exa.ai/docs/reference/search",
"use_official_api": True,
"require_api_key": True,
"results": "JSON",
}
api_key: str = ""
"""API key for Exa Search API (required)."""
categories = ["general", "web"]
safesearch = True
base_url = "https://api.exa.ai/search"
results_per_page: int = 10
"""Maximum number of results per request. Value must be between 1 and 100, default is 10."""
search_type: SearchType = "auto"
"""Search type. Default is auto, see documentation for more information."""
content_mode: ContentMode = "highlights"
"""Content to request from the API: ``highlights`` (excerpts) or ``text`` (page text)."""
content_max_characters: int = 500
"""Maximum characters for the requested content."""
def setup(_: dict[str, t.Any]) -> bool | None:
if not api_key:
raise SearxEngineAPIException("No API key provided")
if not 1 <= results_per_page <= 100:
raise ValueError("results_per_page must be between 1 and 100")
if search_type not in t.get_args(SearchType):
raise ValueError(f"Unsupported search type: {search_type}")
if content_mode not in t.get_args(ContentMode):
raise ValueError(f"Unsupported content mode: {content_mode}")
if content_max_characters < 1:
raise ValueError("content_max_characters must be at least 1")
def _contents_payload() -> dict[str, t.Any]:
if content_mode == "text":
return {"text": {"maxCharacters": content_max_characters, "stripLinks": True}}
return {"highlights": {"maxCharacters": content_max_characters}}
def _extract_content(result: dict[str, t.Any]) -> str:
if content_mode == "text":
return html_to_text(result.get("text") or "")
return html_to_text(" ".join(result.get("highlights") or []))
def request(query: str, params: "OnlineParams") -> None:
"""Create the API request."""
body: dict[str, t.Any] = {
"query": query,
"type": search_type,
"numResults": results_per_page,
"contents": _contents_payload(),
}
# Apply SafeSearch if enabled
if params["safesearch"]:
body["moderation"] = True
# Apply region-aware results if specified
locale_parts = params["searxng_locale"].split("-")
region = locale_parts[-1]
if len(locale_parts) > 1:
body["userLocation"] = region.upper()
params["url"] = base_url
params["method"] = "POST"
params["headers"]["x-api-key"] = api_key
params["json"] = body
def _extract_published_date(value: str | None):
"""Extract and parse the published date from the API response.
Args:
value: Raw date string from the API
Returns:
Parsed datetime object or None if parsing fails
"""
if not value:
return None
try:
return parser.parse(value)
except (parser.ParserError, TypeError, OverflowError):
return None
def response(resp: "SXNG_Response") -> EngineResults:
"""Process the API response and return results."""
res = EngineResults()
for result in resp.json().get("results", []):
url = result.get("url")
if not url:
continue
res.add(
res.types.MainResult(
url=url,
title=html_to_text(result.get("title") or url),
content=_extract_content(result),
thumbnail=result.get("image") or "",
publishedDate=_extract_published_date(result.get("publishedDate")),
author=result.get("author") or "",
)
)
return res

2
searx/engines/findthatmeme.py
View File

@@ -37,7 +37,7 @@ def response(resp):
for item in search_res:
img = 'https://s3.thehackerblog.com/findthatmeme/' + item['image_path']
thumb = 'https://s3.thehackerblog.com/findthatmeme/thumb/' + item.get('thumbnail', '')
date = datetime.fromisoformat(item["updated_at"].split("T")[0])
date = datetime.strptime(item["updated_at"].split("T")[0], "%Y-%m-%d")
formatted_date = datetime.fromtimestamp(date.timestamp())
results.append(

2
searx/engines/fireball.py
View File

@@ -47,7 +47,7 @@ CACHE_VALID_DURATION = 30 * 24 * 3600 # one month, same as website
"""Duration how long settings cookies are valid."""
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings: dict[str, t.Any]):
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"])

2
searx/engines/fyyd.py
View File

@@ -47,7 +47,7 @@ def response(resp: "SXNG_Response"):
title=result["title"],
content=result["description"],
thumbnail=result["smallImageURL"],
publishedDate=datetime.fromisoformat(result["status_since"]),
publishedDate=datetime.strptime(result["status_since"], "%Y-%m-%d %H:%M:%S"),
metadata=f"Rank: {result['rank']} || {result['episode_count']} episodes",
)
)

3
searx/engines/gitea.py
View File

@@ -38,7 +38,6 @@ Implementation
"""
import typing as t
from urllib.parse import urlencode
from dateutil import parser
@@ -78,7 +77,7 @@ page_size: int = 10
"""Maximum number of results per page (default 10)."""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if not base_url:
raise ValueError('gitea engine: base_url is unset')

268
searx/engines/google.py
View File

@@ -9,15 +9,12 @@ engines:
- :ref:`google scholar engine`
- :ref:`google autocomplete`
This implementation uses Nokia user agents to request an XML layout from Google.
The normal web version requires executing JavaScript to load the results and
therefore is currently not used here. See `Google discussion`_ for more
information on that topic.
.. _Google discussion: https://github.com/searxng/searxng/issues/6359
"""
import random
import re
import string
import time
import typing as t
from urllib.parse import unquote, urlencode
@@ -35,6 +32,7 @@ from searx.utils import (
eval_xpath_getindex,
eval_xpath_list,
extract_text,
gen_gsa_useragent,
)
if t.TYPE_CHECKING:
@@ -47,16 +45,16 @@ about = {
"official_api_documentation": "https://developers.google.com/custom-search/",
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": "HTML",
}
# engine dependent config
categories = ["general", "web"]
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
"""`Google max 50 pages`_
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
"""
time_range_support = True
language_support = True
@@ -67,23 +65,38 @@ time_range_dict = {"day": "d", "week": "w", "month": "m", "year": "y"}
# Filter results. 0: None, 1: Moderate, 2: Strict
filter_mapping = {0: "off", 1: "medium", 2: "high"}
# https://github.com/searxng/searxng/issues/6359
nokia_useragents = (
"Nokia7610/2.0 (5.0509.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
"Nokia7610/2.0 (7.0642.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
"Nokia6230/2.0 (05.50) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"Nokia6230i/2.0 (03.80) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"Nokia6280/2.0 (03.60) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"NokiaN72/2.0617.1.0.3 Series60/2.8 Profile/MIDP-2.0 Configuration/CLDC-1.1",
)
# specific xpath variables
# ------------------------
# Suggestions are links placed in a *card-section*, we extract only the text
# from the links not the links itself.
suggestion_xpath = '//table[contains(@class, "HExoMb")]//a[contains(@class, "ZWRArf")]'
suggestion_xpath = '//div[contains(@class, "gGQDvd iIWm4b")]//a'
_arcid_range = string.ascii_letters + string.digits + "_-"
_arcid_random: tuple[str, int] | None = None
def ui_async(start: int) -> str:
"""Format of the response from UI's async request.
- ``arc_id:<...>,use_ac:true,_fmt:prog``
The arc_id is random generated every hour.
"""
global _arcid_random # pylint: disable=global-statement
use_ac = "use_ac:true"
# _fmt:html returns a HTTP 500 when user search for celebrities like
# '!google natasha allegri' or '!google chris evans'
_fmt = "_fmt:prog"
# create a new random arc_id every hour
if not _arcid_random or (int(time.time()) - _arcid_random[1]) > 3600:
_arcid_random = ("".join(random.choices(_arcid_range, k=23)), int(time.time()))
arc_id = f"arc_id:srp_{_arcid_random[0]}_1{start:02}"
return ",".join([arc_id, use_ac, _fmt])
def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[str, t.Any]:
@@ -115,11 +128,19 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
A instance of :py:obj:`babel.core.Locale` build from the
``searxng_locale`` value.
subdomain:
Google subdomain :py:obj:`google_domains` that fits to the country
code.
params:
Py-Dictionary with additional request arguments (can be passed to
:py:func:`urllib.parse.urlencode`).
- ``hl`` parameter: specifies the interface language of user interface.
- ``lr`` parameter: restricts search results to documents written in
a particular language.
- ``cr`` parameter: restricts search results to documents
originating in a particular country.
- ``ie`` parameter: sets the character encoding scheme that should
be used to interpret the query string ('utf8').
- ``oe`` parameter: sets the character encoding scheme that should
@@ -136,6 +157,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
ret_val: dict[str, t.Any] = {
"language": None,
"country": None,
"subdomain": None,
"params": {},
"headers": {},
"cookies": {},
@@ -148,7 +170,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
except babel.core.UnknownLocaleError:
locale = None
eng_lang = eng_traits.get_language(sxng_locale) or "lang_en"
eng_lang = eng_traits.get_language(sxng_locale, "lang_en")
lang_code = eng_lang.split("_")[-1] # lang_zh-TW --> zh-TW / lang_en --> en
country = eng_traits.get_region(sxng_locale, eng_traits.all_locale)
@@ -163,6 +185,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
ret_val["language"] = eng_lang
ret_val["country"] = country
ret_val["locale"] = locale
ret_val["subdomain"] = eng_traits.custom["supported_domains"].get(country.upper(), "www.google.com")
# hl parameter:
# The hl parameter specifies the interface language (host language) of
@@ -174,7 +197,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
# https://developers.google.com/custom-search/docs/xml_results_appendices#interfaceLanguages
# https://github.com/searxng/searxng/issues/2515#issuecomment-1607150817
ret_val["params"]["hl"] = f"{lang_code}"
ret_val["params"]["hl"] = f"{lang_code}-{country}"
# lr parameter:
# The lr (language restrict) parameter restricts search results to
@@ -201,11 +224,9 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
# specify a region (country) only if a region is given in the selected
# locale --> https://github.com/searxng/searxng/issues/2672
if country is not None:
ret_val["params"]["cr"] = ""
if len(sxng_locale.split("-")) > 1:
ret_val["params"]["cr"] = "country" + country
ret_val["params"]["cr"] = ""
if len(sxng_locale.split("-")) > 1:
ret_val["params"]["cr"] = "country" + country
# gl parameter: (mandatory by Google News)
# The gl parameter value is a two-letter country code. For WebSearch
@@ -247,6 +268,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
# HTTP headers
ret_val["headers"]["Accept"] = "*/*"
ret_val["headers"]["User-Agent"] = gen_gsa_useragent()
# Cookies
@@ -280,78 +302,88 @@ def detect_google_sorry(resp: "SXNG_Response"):
raise SearxEngineCaptchaException()
def unwrap_google_url(raw_url: str) -> str:
# remove redirector from url
if raw_url.startswith("/url?q="):
return unquote(raw_url[7:].split("&sa=U")[0])
return raw_url
def wml_dom(resp: "SXNG_Response"):
detect_google_sorry(resp)
text = resp.text
if text.lstrip().startswith("<?xml"):
text = text.split("?>", 1)[-1]
return html.fromstring(text)
def google_request(
query: str,
params: "OnlineParams",
extra_args: dict[str, t.Any] | None = None,
*,
eng_traits: EngineTraits | None = None,
use_time_range: bool = True,
use_safesearch: bool = True,
safesearch_map: dict[int, str] | None = None,
use_locales: bool = True,
) -> None:
google_info = get_google_info(params, eng_traits or traits)
if not use_locales:
google_info["params"].pop("lr")
google_info["params"].pop("cr")
start = (params["pageno"] - 1) * 10
args: dict[str, t.Any] = {
"q": query,
"sca_esv": "1",
**google_info["params"],
**(extra_args or {}),
}
if start:
args["start"] = start
if use_time_range and params["time_range"] in time_range_dict:
args["tbs"] = "qdr:" + time_range_dict[params["time_range"]]
if use_safesearch and params["safesearch"]:
args["safe"] = (safesearch_map or filter_mapping)[params["safesearch"]]
params["url"] = f"https://www.google.com/wml/search?{urlencode(args)}"
params["headers"]["User-Agent"] = random.choice(nokia_useragents)
params["impersonate"] = "chrome99_android"
def request(query: str, params: "OnlineParams") -> None:
google_request(query, params)
"""Google search request"""
# pylint: disable=line-too-long
start = (params["pageno"] - 1) * 10
google_info = get_google_info(params, traits)
# https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
query_url = (
"https://"
+ google_info["subdomain"]
+ "/search"
+ "?"
+ urlencode(
{
"q": query,
**google_info["params"],
"filter": "0",
"start": start,
# 'vet': '12ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0QxK8CegQIARAC..i',
# 'ved': '2ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0Q_skCegQIARAG',
# 'cs' : 1,
# 'sa': 'N',
# 'yv': 3,
# 'prmd': 'vin',
# 'ei': 'GASaY6TxOcy_xc8PtYeY6AE',
# 'sa': 'N',
# 'sstk': 'AcOHfVkD7sWCSAheZi-0tx_09XDO55gTWY0JNq3_V26cNN-c8lfD45aZYPI8s_Bqp8s57AHz5pxchDtAGCA_cikAWSjy9kw3kgg'
# formally known as use_mobile_ui
# "asearch": "arc",
# "async": str_async,
}
)
)
if params["time_range"] in time_range_dict:
query_url += "&" + urlencode({"tbs": "qdr:" + time_range_dict[params["time_range"]]})
if params["safesearch"]:
query_url += "&" + urlencode({"safe": filter_mapping[params["safesearch"]]})
params["url"] = query_url
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])
def response(resp: "SXNG_Response") -> EngineResults:
# regex match to get image map that is found inside the returned javascript:
# (function(){var s='...';var i=['...'] ...}
RE_DATA_IMAGE = re.compile(r"(data:image[^']*?)'[^']*?'((?:dimg|pimg|tsuid)[^']*)")
def parse_url_images(text: str):
data_image_map = {}
for image_url, img_id in RE_DATA_IMAGE.findall(text):
data_image_map[img_id] = image_url.encode('utf-8').decode("unicode-escape")
logger.debug("data:image objects --> %s", list(data_image_map.keys()))
return data_image_map
def response(resp: "SXNG_Response"):
"""Get response from google's search request"""
# pylint: disable=too-many-branches, too-many-statements
detect_google_sorry(resp)
data_image_map = parse_url_images(resp.text)
results = EngineResults()
dom = wml_dom(resp)
# convert the text to dom
dom = html.fromstring(resp.text)
# parse results
for result in eval_xpath_list(dom, '//div[contains(@class, "zMzFAb")]'):
for result in eval_xpath_list(dom, '//a[@data-ved and not(@class)]'):
# pylint: disable=too-many-nested-blocks
try:
title_tag = eval_xpath_getindex(
result, './/a[contains(@class, "fuLhoc")]//span[contains(@class, "CVA68e")]', 0, default=None
)
title_tag = eval_xpath_getindex(result, './/div[@style]', 0, default=None)
if title_tag is None:
# this not one of the common google results *section*
logger.debug("ignoring item from the result_xpath list: missing title")
continue
title = extract_text(title_tag)
raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
raw_url = result.get("href")
if raw_url is None:
logger.debug(
'ignoring item from the result_xpath list: missing url of title "%s"',
@@ -359,19 +391,30 @@ def response(resp: "SXNG_Response") -> EngineResults:
)
continue
url = unwrap_google_url(raw_url)
content = extract_text(
eval_xpath(result, './/div[contains(@class, "taTFJ")]//span[contains(@class, "FrIlee")]')
)
thumbnail = eval_xpath_getindex(result, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
results.add(
results.types.MainResult(
url=url,
title=title or "",
content=content or "",
thumbnail=thumbnail or "",
)
)
if raw_url.startswith('/url?q='):
url = unquote(raw_url[7:].split("&sa=U")[0]) # remove the google redirector
else:
url = raw_url
content_nodes = eval_xpath(result, '../..//div[contains(@class, "ilUpNd H66NU aSRlid")]')
for item in content_nodes:
for script in item.xpath(".//script"):
script.getparent().remove(script)
content = extract_text(content_nodes[0])
# Images that are NOT the favicon
xpath_image = eval_xpath_getindex(result, './/img', index=0, default=None)
thumbnail = None
if xpath_image is not None:
thumbnail = xpath_image.get("src")
if thumbnail.startswith("data:image"):
img_id = xpath_image.get("id")
if img_id:
thumbnail = data_image_map.get(img_id)
results.append({"url": url, "title": title, "content": content or '', "thumbnail": thumbnail})
except Exception as e: # pylint: disable=broad-except
logger.error(e, exc_info=True)
@@ -379,8 +422,10 @@ def response(resp: "SXNG_Response") -> EngineResults:
# parse suggestion
for suggestion in eval_xpath_list(dom, suggestion_xpath):
results.add(results.types.LegacyResult(suggestion=extract_text(suggestion)))
# append suggestion
results.append({"suggestion": extract_text(suggestion)})
# return results
return results
@@ -413,12 +458,14 @@ skip_countries = [
]
def fetch_traits(engine_traits: EngineTraits):
def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True):
"""Fetch languages from Google."""
# pylint: disable=import-outside-toplevel, too-many-branches
from searx.network import get # see https://github.com/searxng/searxng/issues/762
engine_traits.custom["supported_domains"] = {}
resp = get("https://www.google.com/preferences", timeout=5)
if not resp.ok:
raise RuntimeError("Response from Google preferences is not OK.")
@@ -469,3 +516,22 @@ def fetch_traits(engine_traits: EngineTraits):
# alias regions
engine_traits.regions["zh-CN"] = "HK"
# supported domains
if add_domains:
resp = get("https://www.google.com/supported_domains", timeout=5)
if not resp.ok:
raise RuntimeError("Response from Google supported domains is not OK.")
for domain in resp.text.split():
domain = domain.strip()
if not domain or domain in [
".google.com",
]:
continue
region = domain.split(".")[-1].upper()
engine_traits.custom["supported_domains"][region] = "www" + domain
if region == "HK":
# There is no google.cn, we use .com.hk for zh-CN
engine_traits.custom["supported_domains"]["CN"] = "www" + domain

186
searx/engines/google_cse.py
View File

@@ -1,186 +0,0 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google Custom Search Engine"""
import datetime
import typing as t
from json import loads
from urllib.parse import urlencode
from searx.enginelib import EngineCache
from searx.exceptions import SearxEngineAPIException, SearxEngineTooManyRequestsException
from searx.network import get
from searx.result_types import EngineResults, Result, MainResult, Image
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import filter_mapping, get_google_info
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
about = {
"website": "https://www.google.com",
"wikidata_id": "Q2233943",
"official_api_documentation": "https://developers.google.com/custom-search/docs/element",
"use_official_api": False,
"require_api_key": False,
"results": "JSONP",
"description": "Platform for creating custom search engines based on Google Search.",
}
categories = ["general", "web"]
paging = True
enable_http3 = True
max_page = 5
page_size = 20
time_range_support = True
language_support = True
safesearch = True
GoogleCategType = t.Literal["", "image"]
google_categ: GoogleCategType = ""
"""Google CSE category. Set to ``""`` for web search."""
CX = "partner-pub-8993703457585266:4862972284" # blackle.com
CACHE: EngineCache
def setup(engine_settings: dict[str, t.Any]) -> bool:
global CACHE # pylint: disable=global-statement
if google_categ not in t.get_args(GoogleCategType):
raise ValueError("invalid google cse category: %s" % google_categ)
CACHE = EngineCache(engine_settings["name"])
return True
def _cse_token() -> dict[str, str]:
token: dict[str, str] = CACHE.get(CX)
if token:
return token
resp = get(f"https://www.google.com/cse/cse.js?cx={CX}", timeout=10)
if not resp.ok:
raise SearxEngineAPIException("failed to obtain cse token")
end = resp.text.rfind("});")
start = resp.text.rfind("({")
opts: dict[str, str] = loads(resp.text[start + 1 : end + 1])
cse_tok = opts.get("cse_token")
if not cse_tok:
raise SearxEngineAPIException("failed to obtain cse token")
exp = opts.get("exp")
token = {
"cse_tok": cse_tok,
"cselibv": opts.get("cselibVersion", ""),
"exp": ",".join(exp) if exp else "",
}
CACHE.set(CX, token, expire=3600)
return token
def _get_start_and_end_date_str(time_range: str) -> tuple[str, str]:
time_range_map = {"day": 1, "week": 7, "month": 30, "year": 365}
end_date = datetime.datetime.now()
start_date = end_date - datetime.timedelta(days=time_range_map[time_range])
return start_date.strftime("%Y%m%d"), end_date.strftime("%Y%m%d")
def request(query: str, params: "OnlineParams") -> None:
token = _cse_token()
google_info = get_google_info(params, traits)
args = {
"rsz": "filtered_cse",
"num": str(page_size),
"hl": google_info["params"]["hl"],
"cselibv": token["cselibv"],
"cx": CX,
"q": query,
"safe": filter_mapping[params["safesearch"]],
"cse_tok": token["cse_tok"],
"callback": "_",
"rurl": "",
"searchtype": google_categ,
}
if params["time_range"]:
start_date, end_date = _get_start_and_end_date_str(params["time_range"])
args["sort"] = f"date:r:{start_date}:{end_date}"
if google_info["country"] not in (None, "ZZ"):
args["gl"] = google_info["country"]
if token["exp"]:
args["exp"] = token["exp"]
start = (params["pageno"] - 1) * page_size
if start:
args["start"] = str(start)
params["url"] = "https://cse.google.com/cse/element/v1?" + urlencode(args)
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])
params["headers"]["Referer"] = "https://cse.google.com/"
def response(resp: "SXNG_Response") -> EngineResults:
json_resp = resp.text[resp.text.find("{") : resp.text.rfind("}") + 1]
data = loads(json_resp)
# not the real types, but a sufficient approximation
item: dict[str, str]
error: dict[str, str | int]
if error := data.get("error"):
message = error.get("message", "unknown error")
if error.get("code") == 429:
raise SearxEngineTooManyRequestsException(message=f"google cse: {message}")
raise SearxEngineAPIException(f"google cse: {message}")
results = EngineResults()
for item in data.get("results", []):
res: Result | None
if google_categ == "":
res = web_item(item)
elif google_categ == "image":
res = img_item(item)
if res is not None:
results.add(res)
return results
def web_item(item: dict[str, str]) -> MainResult | None:
url = item.get("unescapedUrl")
if not url:
return None
return MainResult(
url=url,
title=item.get("titleNoFormatting", ""),
content=item.get("contentNoFormatting", ""),
thumbnail=item.get("richSnippet", {}).get("cseThumbnail", {}).get("src", ""), # type: ignore
)
def img_item(item: dict[str, str]) -> Image | None:
resolution = ""
if item.get("height") and item.get("width"):
resolution = f"{item['width']}x{item['height']}"
return Image(
url=item["originalContextUrl"],
title=item.get("titleNoFormatting", ""),
content=item.get("contentNoFormatting", ""),
img_src=item["unescapedUrl"],
thumbnail_src=item["tbUrl"],
resolution=resolution,
img_format=item["fileFormat"].split("/")[-1],
)

140
searx/engines/google_images.py
View File

@@ -1,76 +1,122 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google Images: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google Images engine using the internal
Google API used by the Google Go Android app.
import typing as t
from urllib.parse import parse_qs, unquote, urlparse
This internal API offer results in
- JSON (``_fmt:json``)
- Protobuf_ (``_fmt:pb``)
- Protobuf_ compressed? (``_fmt:pc``)
- HTML (``_fmt:html``)
- Protobuf_ encoded in JSON (``_fmt:jspb``).
.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers
"""
from urllib.parse import urlencode
from json import loads
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, wml_dom
from searx.result_types import EngineResults
from searx.utils import eval_xpath_list
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
from searx.engines.google import (
get_google_info,
time_range_dict,
detect_google_sorry,
)
# about
about = {
"website": "https://images.google.com",
"wikidata_id": "Q521550",
"official_api_documentation": "https://developers.google.com/custom-search",
"website": 'https://images.google.com',
"wikidata_id": 'Q521550',
"official_api_documentation": 'https://developers.google.com/custom-search',
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": 'JSON',
}
# engine dependent config
categories = ["images", "web"]
categories = ['images', 'web']
paging = True
enable_http3 = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
"""`Google max 50 pages`_
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
"""
time_range_support = True
language_support = True
safesearch = True
filter_mapping = {0: "images", 1: "active", 2: "active"}
filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "isch"},
eng_traits=traits,
safesearch_map=filter_mapping,
use_locales=False,
def request(query, params):
"""Google-Image search request"""
google_info = get_google_info(params, traits)
query_url = (
'https://'
+ google_info['subdomain']
+ '/search'
+ '?'
+ urlencode({'q': query, 'tbm': "isch", **google_info['params'], 'asearch': 'isch'})
# don't urlencode this because wildly different AND bad results
# pagination uses Zero-based numbering
+ f'&async=_fmt:json,p:1,ijn:{params["pageno"] - 1}'
)
if params['time_range'] in time_range_dict:
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
if params['safesearch']:
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
params['url'] = query_url
params['cookies'] = google_info['cookies']
params['headers'].update(google_info['headers'])
# this ua will allow getting ~50 results instead of 10. #1641
params['headers']['User-Agent'] = (
'NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12;' f' {google_info.get("country", "US")}) gzip'
)
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
dom = wml_dom(resp)
return params
for link in eval_xpath_list(dom, '//a[contains(@href, "/imgres?")]'):
qs = parse_qs(urlparse(link.get("href", "")).query)
img_src = qs.get("imgurl", [""])[0]
url = qs.get("imgrefurl", [""])[0]
if not img_src or not url:
continue
width, height = qs.get("w", [""])[0], qs.get("h", [""])[0]
tbnid = qs.get("tbnid", [""])[0]
results.add(
results.types.Image(
url=url,
title=unquote(urlparse(img_src).path.rsplit("/", 1)[-1]) or urlparse(url).netloc,
img_src=img_src,
thumbnail_src=f"https://encrypted-tbn0.gstatic.com/images?q=tbn:{tbnid}",
resolution=f"{width} x {height}" if width and height else "",
)
)
def response(resp):
"""Get response from google's search request"""
results = []
detect_google_sorry(resp)
json_start = resp.text.find('{"ischj":')
json_data = loads(resp.text[json_start:])
for item in json_data["ischj"].get("metadata", []):
result_item = {
'url': item["result"]["referrer_url"],
'title': item["result"]["page_title"],
'content': item["text_in_grid"]["snippet"],
'source': item["result"]["site_title"],
'resolution': f'{item["original_image"]["width"]} x {item["original_image"]["height"]}',
'img_src': item["original_image"]["url"],
'thumbnail_src': item["thumbnail"]["url"],
'template': 'images.html',
}
author = item["result"].get('iptc', {}).get('creator')
if author:
result_item['author'] = ', '.join(author)
copyright_notice = item["result"].get('iptc', {}).get('copyright_notice')
if copyright_notice:
result_item['source'] += ' | ' + copyright_notice
freshness_date = item["result"].get("freshness_date")
if freshness_date:
result_item['source'] += ' | ' + freshness_date
file_size = item.get('gsa', {}).get('file_size')
if file_size:
result_item['source'] += ' (%s)' % file_size
results.append(result_item)
return results

329
searx/engines/google_news.py
View File

@@ -1,91 +1,324 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google News: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google News engine.
Google News has a different region handling compared to Google WEB.
- the ``ceid`` argument has to be set (:py:obj:`ceid_list`)
- the hl_ argument has to be set correctly (and different to Google WEB)
- the gl_ argument is mandatory
If one of this argument is not set correctly, the request is redirected to
CONSENT dialog::
https://consent.google.com/m?continue=
The google news API ignores some parameters from the common :ref:`google API`:
- num_ : the number of search results is ignored / there is no paging all
results for a query term are in the first response.
- save_ : is ignored / Google-News results are always *SafeSearch*
.. _hl: https://developers.google.com/custom-search/docs/xml_results#hlsp
.. _gl: https://developers.google.com/custom-search/docs/xml_results#glsp
.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
"""
import typing as t
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, unwrap_google_url, wml_dom
from searx.result_types import EngineResults
import json
import base64
from urllib.parse import urlencode
from lxml import html
import babel
from searx import locales
from searx.utils import (
eval_xpath_getindex,
eval_xpath,
eval_xpath_list,
eval_xpath_getindex,
extract_text,
)
from searx.engines.google import fetch_traits as _fetch_traits # pylint: disable=unused-import
from searx.engines.google import (
get_google_info,
detect_google_sorry,
)
from searx.enginelib.traits import EngineTraits
from searx.result_types import EngineResults
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
# about
about = {
"website": "https://www.google.com",
"website": "https://news.google.com",
"wikidata_id": "Q12020",
"official_api_documentation": "https://developers.google.com/custom-search",
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": "HTML",
}
# engine dependent config
categories = ["news"]
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
"""
paging = False
time_range_support = False
language_support = True
safesearch = False
# Google-News results are always *SafeSearch*. Option 'safesearch' is set to
# False here.
#
# safesearch : results are identical for safesearch=0 and safesearch=2
safesearch = True
base_url: str = "https://news.google.com"
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "nws"},
eng_traits=traits,
use_time_range=False,
use_safesearch=False,
use_locales=False,
"""Google-News search request"""
sxng_locale = params.get("searxng_locale", "en-US")
ceid: str = locales.get_engine_locale(
sxng_locale, traits.custom["ceid"], default="US:en"
) # pyright: ignore[reportAssignmentType]
google_info = get_google_info(params, traits)
google_info["subdomain"] = "news.google.com" # google news has only one domain
ceid_region, ceid_lang = ceid.split(":")
ceid_lang, ceid_suffix = (
ceid_lang.split(":")
+ [
"",
]
)[:2]
google_info["params"]["hl"] = ceid_lang
if ceid_suffix and ceid_suffix not in ["Hans", "Hant"]:
if ceid_region.lower() == ceid_lang:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
else:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_suffix
elif ceid_region.lower() != ceid_lang:
if ceid_region in ["AT", "BE", "CH", "IL", "SA", "IN", "BD", "PT"]:
google_info["params"]["hl"] = ceid_lang
else:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
google_info["params"]["lr"] = "lang_" + ceid_lang.split("-")[0]
google_info["params"]["gl"] = ceid_region
query_url = (
"https://"
+ google_info["subdomain"]
+ "/search?"
+ urlencode(
{"q": query, **google_info["params"]},
)
# ceid includes a ':' character which must not be urlencoded
+ ("&ceid=%s" % ceid)
)
def _span_text(link, css_class: str):
return extract_text(
eval_xpath_getindex(link, f'.//span[contains(@class, "{css_class}")]', 0, default=None),
allow_none=True,
)
params["url"] = query_url
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
seen = set()
for link in eval_xpath_list(wml_dom(resp), '//a[contains(@href, "/url?q=")]'):
href = link.get("href")
if not href:
"""Get response from google's search request"""
res = EngineResults()
detect_google_sorry(resp)
# convert the text to dom
dom = html.fromstring(resp.text)
for result in eval_xpath_list(dom, "//div[@jslog and @data-n-tid and @jsdata]"):
url: str = eval_xpath_getindex(result, "./a[@target='_blank']/@href", 0, default=0)
if not url:
continue
if url.startswith("./"):
url = base_url + url[1:]
# The real URL is often encoded in the "jslog" attribute
jslog: str | None = eval_xpath_getindex(result, "./a[@target='_blank']/@jslog", 0, default=None)
# Try to extract the real URL from jslog
real_url: str | None = None
if jslog:
# jslog format is usually: "95014; 5:<base64>; track:click,vis". We
# want the second part (index 1) after splitting by ";"
parts: list[str] = jslog.split(";")
if len(parts) > 1:
b64_data: str = parts[1].split(":")[-1].strip()
# Pad base64 if necessary
b64_data += "=" * (-len(b64_data) % 4)
decoded_data: list[str | None] = json.loads(base64.b64decode(b64_data).decode("utf-8"))
# The URL is typically the last element in the decoded array
if (
isinstance(decoded_data, list)
and isinstance(decoded_data[-1], str)
and decoded_data[-1].startswith("http")
):
real_url = decoded_data[-1]
if real_url:
url = real_url
else:
logger.error(f"no real-url found: {url}")
continue
url = unwrap_google_url(href)
if url in seen or "google.com/search" in url:
continue
title = extract_text(eval_xpath(result, "./h4")) or ""
title = _span_text(link, "M3vVJe") or _span_text(link, "fuLhoc")
if not title:
continue
# The pub_date is mostly a string like 'yesterday', not a real timezone
# date or time. Therefore we can't use publishedDate and place the
# *pub* sting into the content.
source = _span_text(link, "dXDvrc")
pub_date = _span_text(link, "YVIcad")
thumbnail = eval_xpath_getindex(link, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
pub_date = extract_text(eval_xpath(result, ".//time"))
pub_origin = extract_text(eval_xpath(result, ".//div[contains(@class, 'vr1PYe')]"))
content = " / ".join([x for x in [pub_origin, pub_date] if x])
seen.add(url)
results.add(
results.types.MainResult(
thumbnail: str = eval_xpath_getindex(result, ".//figure/img/@src", 0, default="")
if thumbnail and thumbnail.startswith("/"):
thumbnail = base_url + thumbnail
res.add(
res.types.MainResult(
url=url,
title=title,
content=" / ".join(x for x in [source, pub_date] if x),
thumbnail=thumbnail or "",
content=content,
thumbnail=thumbnail,
)
)
return results
return res
ceid_list = [
"AE:ar",
"AR:es-419",
"AT:de",
"AU:en",
"BD:bn",
"BE:fr",
"BE:nl",
"BG:bg",
"BR:pt-419",
"BW:en",
"CA:en",
"CA:fr",
"CH:de",
"CH:fr",
"CL:es-419",
"CN:zh-Hans",
"CO:es-419",
"CU:es-419",
"CZ:cs",
"DE:de",
"EE:et",
"EG:ar",
"ES:ca",
"ES:es",
"ET:en",
"FI:fi",
"FR:fr",
"GB:en",
"GH:en",
"GR:el",
"HK:zh-Hant",
"HU:hu",
"ID:en",
"ID:id",
"IE:en",
"IL:en",
"IL:he",
"IN:bn",
"IN:en",
"IN:gu",
"IN:hi",
"IN:ml",
"IN:mr",
"IN:pa",
"IN:ta",
"IN:te",
"IT:it",
"JP:ja",
"KE:en",
"KR:ko",
"LB:ar",
"LT:lt",
"LV:en",
"LV:lv",
"MA:fr",
"MY:en",
"MY:ms",
"NA:en",
"NG:en",
"NL:nl",
"NO:no",
"NZ:en",
"PH:en",
"PK:en",
"PL:pl",
"RO:ro",
"RS:sr",
"RU:ru",
"SA:ar",
"SE:sv",
"SG:en",
"SI:sl",
"SK:sk",
"SN:fr",
"TH:th",
"TR:tr",
"TZ:en",
"UA:ru",
"UA:uk",
"UG:en",
"US:en",
"VN:vi",
"ZA:en",
"ZW:en",
]
"""List of region/language combinations supported by Google News. Values of the
``ceid`` argument of the Google News REST API."""
_skip_values = [
"ET:en", # english (ethiopia)
"ID:en", # english (indonesia)
"LV:en", # english (latvia)
]
_ceid_locale_map = {"NO:no": "nb-NO"}
def fetch_traits(engine_traits: EngineTraits):
_fetch_traits(engine_traits, add_domains=False)
engine_traits.custom["ceid"] = {}
for ceid in ceid_list:
if ceid in _skip_values:
continue
region, lang = ceid.split(":")
x = lang.split("-")
if len(x) > 1:
if x[1] not in ["Hant", "Hans"]:
lang = x[0]
sxng_locale = _ceid_locale_map.get(ceid, lang + "-" + region)
try:
locale = babel.Locale.parse(sxng_locale, sep="-")
except babel.UnknownLocaleError:
print("ERROR: %s -> %s is unknown by babel" % (ceid, sxng_locale))
continue
engine_traits.custom["ceid"][locales.region_tag(locale)] = ceid

1
searx/engines/google_play.py
View File

@@ -20,7 +20,6 @@ about = {
}
play_categ = None # apps|movies
enable_http3 = True
base_url = 'https://play.google.com'
search_url = base_url + "/store/search?{query}&c={play_categ}"

9
searx/engines/google_scholar.py
View File

@@ -27,7 +27,7 @@ import typing as t
from urllib.parse import urlencode
from datetime import datetime
from lxml import html
from curl_cffi.requests.exceptions import TooManyRedirects
import httpx
from searx.utils import (
eval_xpath,
@@ -63,7 +63,6 @@ about = {
# engine dependent config
categories = ["science", "scientific publications"]
paging = True
enable_http3 = True
max_page = 50
"""`Google max 50 pages`_
@@ -78,6 +77,8 @@ def request(query: str, params: "OnlineParams") -> None:
"""Google-Scholar search request"""
google_info = get_google_info(params, traits)
# subdomain is: scholar.google.xy
google_info["subdomain"] = google_info["subdomain"].replace("www.", "scholar.")
args = {
"q": query,
@@ -88,7 +89,7 @@ def request(query: str, params: "OnlineParams") -> None:
}
args.update(time_range_args(params))
params["url"] = "https://scholar.google.com/scholar?" + urlencode(args)
params["url"] = "https://" + google_info["subdomain"] + "/scholar?" + urlencode(args)
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])
@@ -103,7 +104,7 @@ def response(resp: "SXNG_Response") -> EngineResults: # pylint: disable=too-man
raise SearxEngineAccessDeniedException(
message="google_scholar: unusual traffic detected",
)
raise TooManyRedirects(f"location {resp.headers['Location'].split('?')[0]}")
raise httpx.TooManyRedirects(f"location {resp.headers['Location'].split('?')[0]}")
res = EngineResults()
dom = html.fromstring(resp.text)

204
searx/engines/google_videos.py
View File

@@ -1,85 +1,185 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google Videos: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google Videos engine.
import typing as t
.. admonition:: Content-Security-Policy (CSP)
This engine needs to allow images from the `data URLs`_ (prefixed with the
``data:`` scheme)::
Header set Content-Security-Policy "img-src 'self' data: ;"
.. _data URLs:
https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
"""
import re
from urllib.parse import urlencode, urlparse, parse_qs, unquote
from lxml import html
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, unwrap_google_url, wml_dom
from searx.result_types import EngineResults
from searx.utils import (
eval_xpath_getindex,
eval_xpath_list,
eval_xpath_getindex,
extract_text,
parse_duration_string,
)
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import (
get_google_info,
time_range_dict,
filter_mapping,
suggestion_xpath,
detect_google_sorry,
ui_async,
)
from searx.utils import get_embeded_stream_url
# about
about = {
"website": "https://www.google.com",
"wikidata_id": "Q219885",
"official_api_documentation": "https://developers.google.com/custom-search",
"website": 'https://www.google.com',
"wikidata_id": 'Q219885',
"official_api_documentation": 'https://developers.google.com/custom-search',
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": 'HTML',
}
# engine dependent config
categories = ["videos", "web"]
categories = ['videos', 'web']
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
"""
language_support = True
time_range_support = True
safesearch = True
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "vid"},
eng_traits=traits,
use_locales=False,
# =26;[3,"dimg_ZNMiZPCqE4apxc8P3a2tuAQ_137"]a87;data:image/jpeg;base64,/9j/4AAQSkZJRgABA
# ...6T+9Nl4cnD+gr9OK8I56/tX3l86nWYw//2Q==26;
RE_DATA_IMAGE = re.compile(r'"(dimg_[^"]*)"[^;]*;(data:image[^;]*;[^;]*);?')
def parse_data_images(text: str):
data_image_map = {}
for img_id, data_image in RE_DATA_IMAGE.findall(text):
end_pos = data_image.rfind("=")
if end_pos > 0:
data_image = data_image[: end_pos + 1]
data_image_map[img_id] = data_image
logger.debug("data:image objects --> %s", list(data_image_map.keys()))
return data_image_map
def request(query, params):
"""Google-Video search request"""
google_info = get_google_info(params, traits)
start = (params['pageno'] - 1) * 10
query_url = (
'https://'
+ google_info['subdomain']
+ '/search'
+ "?"
+ urlencode(
{
'q': query,
'tbm': "vid",
'start': start,
**google_info['params'],
'asearch': 'arc',
'async': ui_async(start),
}
)
)
if params['time_range'] in time_range_dict:
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
if 'safesearch' in params:
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
params['url'] = query_url
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
params['cookies'] = google_info['cookies']
params['headers'].update(google_info['headers'])
return params
for result in eval_xpath_list(wml_dom(resp), '//div[contains(@class, "zMzFAb")]'):
def response(resp):
"""Get response from google's search request"""
results = []
detect_google_sorry(resp)
data_image_map = parse_data_images(resp.text)
# convert the text to dom
dom = html.fromstring(resp.text)
result_divs = eval_xpath_list(dom, '//div[contains(@class, "MjjYud")]')
# parse results
for result in result_divs:
title = extract_text(
eval_xpath_getindex(result, './/span[contains(@class, "CVA68e")]', 0, default=None),
eval_xpath_getindex(result, './/h3[contains(@class, "LC20lb")] | .//div[@role="heading"]', 0, default=None),
allow_none=True,
)
raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
if not title or not raw_url:
continue
url = unwrap_google_url(raw_url)
thumbnail = eval_xpath_getindex(result, './/img[contains(@class, "SygO9d")]/@src', 0, default="")
if "/default.jpg" in thumbnail:
thumbnail = thumbnail.split("?")[0].replace("/default.jpg", "/hqdefault.jpg")
length = None
for span in eval_xpath_list(result, './/span[contains(@class, "YVIcad")]'):
length = parse_duration_string(extract_text(span) or "")
if length:
break
results.add(
results.types.MainResult(
url=url,
title=title,
thumbnail=thumbnail,
length=length,
template="videos.html",
)
url = eval_xpath_getindex(
result, './/a[@jsname="UWckNb"]/@href | .//a[contains(@href, "/url?q=")]/@href', 0, default=None
)
if url and url.startswith('/url?q='):
url = unquote(url[7:].split('&sa=U')[0])
content = extract_text(
eval_xpath_getindex(result, './/div[contains(@class, "ITZIwc")]', 0, default=None), allow_none=True
)
pub_info = extract_text(
eval_xpath_getindex(
result, './/div[contains(@class, "gqF9jc")] | .//div[contains(@class, "WRu9Cd")]', 0, default=None
),
allow_none=True,
)
# Broader XPath to find any <img> element
thumbnail = eval_xpath_getindex(result, './/img/@src', 0, default=None)
duration = extract_text(
eval_xpath_getindex(result, './/span[contains(@class, "k1U36b")]', 0, default=None), allow_none=True
)
video_id = eval_xpath_getindex(result, './/div[@jscontroller="rTuANe"]/@data-vid', 0, default=None)
# Fallback for video_id from URL if not found via XPath
if not video_id and url and 'youtube.com' in url:
parsed_url = urlparse(url)
video_id = parse_qs(parsed_url.query).get('v', [None])[0]
# Handle thumbnail
if thumbnail and thumbnail.startswith('data:image'):
img_id = eval_xpath_getindex(result, './/img/@id', 0, default=None)
if img_id and img_id in data_image_map:
thumbnail = data_image_map[img_id]
else:
thumbnail = None
if not thumbnail and video_id:
thumbnail = f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg"
# Handle video embed URL
embed_url = None
if video_id:
embed_url = get_embeded_stream_url(f"https://www.youtube.com/watch?v={video_id}")
elif url:
embed_url = get_embeded_stream_url(url)
# Only append results with valid title and url
if title and url:
results.append(
{
'url': url,
'title': title,
'content': content or '',
'author': pub_info,
'thumbnail': thumbnail,
'length': duration,
'iframe_src': embed_url,
'template': 'videos.html',
}
)
# parse suggestion
for suggestion in eval_xpath_list(dom, suggestion_xpath):
results.append({'suggestion': extract_text(suggestion)})
return results

90
searx/engines/heexy.py Normal file
View File

@@ -0,0 +1,90 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Heexy_ is a minimalist search engine that focuses on privacy.
Although it also supports news and videos, these are not implemented here
because they usually return no result to very few irrelevant ones.
It seems to use Bing internally, as the image thumbnails are loaded from Bing.
.. _Heexy: https://docs.heexy.org/introduction
"""
from urllib.parse import urlencode
import typing as t
from searx.exceptions import SearxEngineAccessDeniedException
from searx.result_types import EngineResults
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
about = {
"website": "https://heexy.org",
"wikidata_id": None,
"official_api_documentation": None,
"use_official_api": False,
"require_api_key": False,
"results": "JSON",
}
paging = True
safesearch = True
categories = ["general"]
heexy_categ = "web"
"""Category to search in. Can be either "web" or "image"."""
base_url = "https://seapi.heexy.org"
safe_search_map = {0: "off", 1: "on", 2: "on"}
def init(_):
if heexy_categ not in ("web", "image"):
raise ValueError("invalid search category: %s" % heexy_categ)
def request(query: str, params: "OnlineParams") -> None:
args = {
"q": query,
"page": params["pageno"],
"safe": safe_search_map[params["safesearch"]],
}
if params["searxng_locale"] != "all":
args["lang"] = params["searxng_locale"].split("-")[0]
params["url"] = f"{base_url}/search/{heexy_categ}?{urlencode(args)}"
params["headers"]["Origin"] = base_url
def response(resp: "SXNG_Response"):
res = EngineResults()
json_resp = resp.json()
if not json_resp["success"]:
raise SearxEngineAccessDeniedException()
result: dict[str, str]
for result in json_resp["results"]:
if heexy_categ == "web":
res.add(
res.types.MainResult(
url=result["url"],
title=result["title"],
content=result["description"],
)
)
elif heexy_categ == "image":
res.add(
res.types.Image(
title=result["description"],
url=result["url"],
thumbnail_src=result["image"],
img_src=result["rawImage"],
)
)
return res

5
searx/engines/huggingface.py
View File

@@ -33,7 +33,6 @@ Implementations
"""
import typing as t
from urllib.parse import urlencode
from datetime import datetime
@@ -63,7 +62,7 @@ huggingface_endpoint = 'models'
"""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if huggingface_endpoint not in ('datasets', 'models', 'spaces'):
raise SearxEngineAPIException(f"Unsupported Hugging Face endpoint: {huggingface_endpoint}")
@@ -92,7 +91,7 @@ def response(resp) -> EngineResults:
published_date = None
try:
published_date = datetime.fromisoformat(entry["createdAt"].rstrip("Z"))
published_date = datetime.strptime(entry["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
except (ValueError, TypeError):
pass

3
searx/engines/invidious.py
View File

@@ -8,7 +8,6 @@ No public instance offer a public API now
"""
import typing as t
import time
import random
from urllib.parse import quote_plus, urlparse
@@ -35,7 +34,7 @@ time_range_support = True
base_url: list[str] | str = []
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if not base_url:
raise ValueError("missing invidious base_url")

2
searx/engines/iqiyi.py
View File

@@ -43,7 +43,7 @@ def _result(video: dict[str, typing.Any], album_info: dict[str, typing.Any]):
release_time = album_info.get("releaseTime", {}).get("value")
if release_time:
try:
published_date = datetime.fromisoformat(release_time)
published_date = datetime.strptime(release_time, "%Y-%m-%d")
except (ValueError, TypeError):
pass

Some files were not shown because too many files have changed in this diff Show More