1 Commits

Author SHA1 Message Date
searxng-bot
b1fc49e357 [l10n] update translations from Weblate
493c1d2a7 - 2026-07-07 - lukisko <lukisko@noreply.codeberg.org>
a67b467e5 - 2026-07-06 - marc-lopez <marc-lopez@noreply.codeberg.org>
2026-07-10 08:12:43 +00:00
336 changed files with 5265 additions and 28700 deletions

View File

@@ -1,39 +0,0 @@
// Closes issues and prs whose authors/agents don't accept the ai policy
// https://github.com/searxng/searxng/blob/master/AI_POLICY.rst
module.exports = async ({ github, context }) => {
const item = context.payload.pull_request || context.payload.issue;
const body = item.body || '';
const kind = context.payload.pull_request ? 'pull request' : 'issue';
// https://github.com/searxng/searxng/pull/6476#discussion_r3683782481
const hasBox = /\[[Xx]\].*AI Policy/.test(body);
const hasRef = /\[AI Policy\]:\s*https:\/\/github\.com\/searxng\/searxng\/.*AI_POLICY/.test(body);
if (hasBox && hasRef) {
return;
}
const { owner, repo } = context.repo;
await github.rest.issues.createComment({
owner,
repo,
issue_number: item.number,
body:
'Hello! Thank you for your contribution.\n\n' +
`Unfortunately your ${kind} was closed as the AI Policy has not been accepted.\n\n` +
`Please open a new ${kind} after confirming your contribution aligns with our AI Policy.`,
});
await github.rest.issues.addLabels({
owner,
repo,
issue_number: item.number,
labels: ['invalid:slop'],
});
await github.rest.issues.update({
owner,
repo,
issue_number: item.number,
state: 'closed',
state_reason: 'not_planned',
});
};

View File

@@ -1,38 +0,0 @@
---
# yamllint disable rule:line-length
name: AI Policy
# Closes any new issues and PRs from people (or agents) who don't accept the AI Policy
# yamllint disable-line rule:truthy
on:
issues:
types: [opened]
pull_request_target:
types: [opened]
permissions:
contents: read
issues: write
pull-requests: write
jobs:
check:
name: Check AI Policy
# for issues with an author who has not contributed before
if: >-
github.event.sender.type != 'Bot' &&
contains(fromJSON('["NONE","FIRST_TIMER","FIRST_TIME_CONTRIBUTOR"]'),
github.event.issue.author_association ||
github.event.pull_request.author_association)
runs-on: ubuntu-26.04-arm
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: "false"
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const script = require('./.github/scripts/ai_policy.cjs');
await script({ github, context });

View File

@@ -25,21 +25,25 @@ env:
jobs:
build:
if: |
github.event_name == 'workflow_dispatch'
|| (github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success')
if: github.repository_owner == 'searxng' || github.event_name == 'workflow_dispatch'
name: Build (${{ matrix.arch }})
runs-on: ${{ matrix.runner }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- runner: ubuntu-26.04
arch: amd64
- runner: ubuntu-26.04-arm
arch: arm64
- runner: ubuntu-26.04-arm
arch: armv7
- arch: amd64
march: amd64
os: ubuntu-24.04
emulation: false
- arch: arm64
march: arm64
os: ubuntu-24.04-arm
emulation: false
- arch: armv7
march: arm64
os: ubuntu-24.04-arm
emulation: true
permissions:
packages: write
@@ -49,25 +53,33 @@ jobs:
git_url: ${{ steps.build.outputs.git_url }}
steps:
- name: Login to GHCR
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: "ghcr.io"
username: "${{ github.repository_owner }}"
password: "${{ secrets.GITHUB_TOKEN }}"
# yamllint disable rule:line-length
- name: Setup podman
env:
PODMAN_VERSION: "v5.7.1"
run: |
sudo apt-get purge -y podman runc crun conmon
curl -fsSLO "https://github.com/mgoltzsche/podman-static/releases/download/${{ env.PODMAN_VERSION }}/podman-linux-${{ matrix.march }}.tar.gz"
curl -fsSLO "https://github.com/mgoltzsche/podman-static/releases/download/${{ env.PODMAN_VERSION }}/podman-linux-${{ matrix.march }}.tar.gz.asc"
gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 0CCF102C4F95D89E583FF1D4F8B5AF50344BB503
gpg --batch --verify "podman-linux-${{ matrix.march }}.tar.gz.asc" "podman-linux-${{ matrix.march }}.tar.gz"
tar -xzf "podman-linux-${{ matrix.march }}.tar.gz"
sudo cp -rfv ./podman-linux-${{ matrix.march }}/etc/. /etc/
sudo cp -rfv ./podman-linux-${{ matrix.march }}/usr/. /usr/
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
# yamllint enable rule:line-length
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Setup QEMU
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: "${{ github.event.workflow_run.head_sha || github.sha }}"
persist-credentials: "false"
fetch-depth: "0"
@@ -79,51 +91,70 @@ jobs:
python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-
path: "./local/"
- name: Get date
id: date
run: echo "date=$(date +'%Y%m%d')" >>$GITHUB_OUTPUT
- name: Setup cache container
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
key: "container-${{ matrix.arch }}-${{ hashFiles('./requirements*.txt') }}"
key: "container-${{ matrix.arch }}-${{ steps.date.outputs.date }}-${{ hashFiles('./requirements*.txt') }}"
restore-keys: |
container-${{ matrix.arch }}-${{ steps.date.outputs.date }}-
container-${{ matrix.arch }}-
path: "/var/tmp/buildah-cache-*/*"
- name: Build
id: build
env:
OVERRIDE_ARCH: "${{ matrix.arch }}"
run: make container.build
- if: ${{ matrix.emulation }}
name: Setup QEMU
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
test:
name: Test (${{ matrix.arch }})
runs-on: ${{ matrix.runner }}
needs: build
strategy:
fail-fast: false
matrix:
include:
- runner: ubuntu-26.04
arch: amd64
- runner: ubuntu-26.04-arm
arch: arm64
- runner: ubuntu-26.04-arm
arch: armv7
steps:
- name: Login to GHCR
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 # v4.3.0
with:
registry: "ghcr.io"
username: "${{ github.repository_owner }}"
password: "${{ secrets.GITHUB_TOKEN }}"
- name: Setup QEMU
- name: Build
id: build
env:
OVERRIDE_ARCH: "${{ matrix.arch }}"
run: make podman.build
test:
name: Test (${{ matrix.arch }})
runs-on: ${{ matrix.os }}
needs: build
strategy:
fail-fast: false
matrix:
include:
- arch: amd64
os: ubuntu-24.04
emulation: false
- arch: arm64
os: ubuntu-24.04-arm
emulation: false
- arch: armv7
os: ubuntu-24.04-arm
emulation: true
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: "false"
- if: ${{ matrix.emulation }}
name: Setup QEMU
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Login to GHCR
uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 # v4.3.0
with:
ref: "${{ github.event.workflow_run.head_sha || github.sha }}"
persist-credentials: "false"
registry: "ghcr.io"
username: "${{ github.repository_owner }}"
password: "${{ secrets.GITHUB_TOKEN }}"
- name: Test
env:
@@ -134,7 +165,7 @@ jobs:
release:
if: github.repository_owner == 'searxng' && github.ref_name == 'master'
name: Release
runs-on: ubuntu-26.04-arm
runs-on: ubuntu-24.04-arm
needs:
- build
- test
@@ -143,25 +174,24 @@ jobs:
packages: write
steps:
- name: Login to Docker Hub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
registry: "docker.io"
username: "${{ secrets.DOCKER_USER }}"
password: "${{ secrets.DOCKER_TOKEN }}"
persist-credentials: "false"
- name: Login to GHCR
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 # v4.3.0
with:
registry: "ghcr.io"
username: "${{ github.repository_owner }}"
password: "${{ secrets.GITHUB_TOKEN }}"
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Login to Docker Hub
uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 # v4.3.0
with:
ref: "${{ github.event.workflow_run.head_sha || github.sha }}"
persist-credentials: "false"
registry: "docker.io"
username: "${{ secrets.DOCKER_USER }}"
password: "${{ secrets.DOCKER_TOKEN }}"
- name: Release
env:

View File

@@ -21,7 +21,7 @@ jobs:
data:
if: github.repository_owner == 'searxng'
name: ${{ matrix.fetch }}
runs-on: ubuntu-26.04-arm
runs-on: ubuntu-24.04-arm
strategy:
fail-fast: false
matrix:
@@ -31,7 +31,7 @@ jobs:
- update_external_bangs.py
- update_firefox_version.py
- update_engine_traits.py
- update_wikidata.py
- update_wikidata_units.py
- update_engine_descriptions.py
permissions:
@@ -40,12 +40,12 @@ jobs:
steps:
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: "false"
@@ -64,17 +64,23 @@ jobs:
run: V=1 ./manage pyenv.cmd python "./searxng_extra/update/${{ matrix.fetch }}"
- name: Create PR
id: cpr
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
with:
author: "searxng-bot <searxng-bot@users.noreply.github.com>"
committer: "searxng-bot <searxng-bot@users.noreply.github.com>"
title: "[mod] data: update searx.data - ${{ matrix.fetch }}"
commit-message: "[mod] data: update searx.data - ${{ matrix.fetch }}"
branch: "ci-data-${{ matrix.fetch }}"
title: "[data] update searx.data - ${{ matrix.fetch }}"
commit-message: "[data] update searx.data - ${{ matrix.fetch }}"
branch: "update_data_${{ matrix.fetch }}"
delete-branch: "true"
draft: "false"
signoff: "false"
body: |
Update searx.data - ${{ matrix.fetch }}
[data] update searx.data - ${{ matrix.fetch }}
labels: |
data
- name: Display information
run: |
echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

View File

@@ -25,19 +25,19 @@ jobs:
release:
if: github.repository_owner == 'searxng' || github.event_name == 'workflow_dispatch'
name: Release
runs-on: ubuntu-26.04-arm
runs-on: ubuntu-24.04-arm
permissions:
# for JamesIves/github-pages-deploy-action to push
contents: write
steps:
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: "false"
fetch-depth: "0"
@@ -61,11 +61,11 @@ jobs:
- if: github.ref_name == 'master'
name: Release
uses: JamesIves/github-pages-deploy-action@fa24774553152dd7873cd16ebd8d959b010c5445 # v4.9.0
uses: JamesIves/github-pages-deploy-action@d92aa235d04922e8f08b40ce78cc5442fcfbfa2f # v4.8.0
with:
folder: "dist/docs"
branch: "gh-pages"
commit-message: "[mod] docs: build from commit ${{ github.sha }}"
commit-message: "[doc] build from commit ${{ github.sha }}"
# Automatically remove deleted files from the deploy branch
clean: "true"
single-commit: "true"

View File

@@ -23,7 +23,7 @@ env:
jobs:
test:
name: Python ${{ matrix.python-version }}
runs-on: ubuntu-26.04
runs-on: ubuntu-24.04
strategy:
matrix:
python-version:
@@ -34,12 +34,12 @@ jobs:
steps:
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "${{ matrix.python-version }}"
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: "false"
@@ -59,24 +59,29 @@ jobs:
theme:
name: Theme
runs-on: ubuntu-26.04-arm
runs-on: ubuntu-24.04-arm
steps:
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "26"
check-latest: "true"
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: "false"
- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version-file: "./.nvmrc"
- name: Setup cache Node.js
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
key: "nodejs-${{ runner.arch }}-${{ hashFiles('./.nvmrc', './package.json') }}"
path: "./client/simple/node_modules/"
- name: Setup cache Python
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
@@ -85,14 +90,6 @@ jobs:
python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-
path: "./local/"
- name: Setup cache Node.js
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
key: "nodejs-${{ runner.arch }}-${{ hashFiles('**/package-lock.json') }}"
restore-keys: |
nodejs-${{ runner.arch }}-
path: "./client/simple/node_modules/"
- name: Setup venv
run: make V=1 install

View File

@@ -26,21 +26,21 @@ env:
jobs:
update:
if: github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'searxng'
if: github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success'
name: Update
runs-on: ubuntu-26.04-arm
runs-on: ubuntu-24.04-arm
permissions:
# For "make V=1 weblate.push.translations"
contents: write
steps:
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}"
fetch-depth: "0"
@@ -59,7 +59,7 @@ jobs:
- name: Setup Weblate
run: |
mkdir -p ~/.config
echo "${{ secrets.WEBLATE_CONFIG }}" >~/.config/weblate
echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate
- name: Setup Git
run: |
@@ -74,7 +74,7 @@ jobs:
github.repository_owner == 'searxng'
&& (github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
name: Pull Request
runs-on: ubuntu-26.04-arm
runs-on: ubuntu-24.04-arm
permissions:
# For "make V=1 weblate.translations.commit"
contents: write
@@ -83,12 +83,12 @@ jobs:
steps:
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}"
fetch-depth: "0"
@@ -107,7 +107,7 @@ jobs:
- name: Setup Weblate
run: |
mkdir -p ~/.config
echo "${{ secrets.WEBLATE_CONFIG }}" >~/.config/weblate
echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate
- name: Setup Git
run: |
@@ -118,17 +118,23 @@ jobs:
run: make V=1 weblate.translations.commit
- name: Create PR
id: cpr
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
with:
author: "searxng-bot <searxng-bot@users.noreply.github.com>"
committer: "searxng-bot <searxng-bot@users.noreply.github.com>"
title: "[mod] i18n: update translations from Weblate"
commit-message: "[mod] i18n: update translations from Weblate"
title: "[l10n] update translations from Weblate"
commit-message: "[l10n] update translations from Weblate"
branch: "translations_update"
delete-branch: "true"
draft: "false"
signoff: "false"
body: |
Update translations from Weblate
[l10n] update translations from Weblate
labels: |
area:i18n
- name: Display information
run: |
echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

46
.github/workflows/security.yml vendored Normal file
View File

@@ -0,0 +1,46 @@
---
name: Security
# yamllint disable-line rule:truthy
on:
workflow_dispatch:
schedule:
- cron: "42 05 * * *"
concurrency:
group: ${{ github.workflow }}
cancel-in-progress: false
permissions:
contents: read
jobs:
container:
if: github.repository_owner == 'searxng'
name: Container
runs-on: ubuntu-24.04-arm
permissions:
security-events: write
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: "false"
- name: Sync GHCS from Docker Scout
uses: docker/scout-action@ce97ec1bb85613e8eb35d086fad0c77a6cedf983 # v1.23.0
with:
organization: "searxng"
dockerhub-user: "${{ secrets.DOCKER_USER }}"
dockerhub-password: "${{ secrets.DOCKER_TOKEN }}"
image: "registry://ghcr.io/searxng/searxng:latest"
command: "cves"
sarif-file: "./scout.sarif"
exit-code: "false"
write-comment: "false"
- name: Upload SARIFs
uses: github/codeql-action/upload-sarif@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3
with:
sarif_file: "./scout.sarif"

View File

@@ -2,7 +2,7 @@
/*
this file is generated automatically by searxng_extra/update/update_pygments.py
using pygments version 2.21.0:
using pygments version 2.20.0:
./manage templates.simple.pygments
*/
@@ -114,14 +114,14 @@
.gd { color: #FF4689 } /* Generic.Deleted */
.ge { color: #F8F8F2; font-style: italic } /* Generic.Emph */
.ges { color: #F8F8F2; font-weight: bold; font-style: italic } /* Generic.EmphStrong */
.gr { color: #FF4689 } /* Generic.Error */
.gr { color: #F8F8F2 } /* Generic.Error */
.gh { color: #F8F8F2 } /* Generic.Heading */
.gi { color: #A6E22E } /* Generic.Inserted */
.go { color: #66D9EF } /* Generic.Output */
.gp { color: #FF4689; font-weight: bold } /* Generic.Prompt */
.gs { color: #F8F8F2; font-weight: bold } /* Generic.Strong */
.gu { color: #959077 } /* Generic.Subheading */
.gt { color: #66D9EF } /* Generic.Traceback */
.gt { color: #F8F8F2 } /* Generic.Traceback */
.kc { color: #66D9EF } /* Keyword.Constant */
.kd { color: #66D9EF } /* Keyword.Declaration */
.kn { color: #FF4689 } /* Keyword.Namespace */
@@ -132,7 +132,7 @@
.m { color: #AE81FF } /* Literal.Number */
.s { color: #E6DB74 } /* Literal.String */
.na { color: #A6E22E } /* Name.Attribute */
.nb { color: #A6E22E } /* Name.Builtin */
.nb { color: #F8F8F2 } /* Name.Builtin */
.nc { color: #A6E22E } /* Name.Class */
.no { color: #66D9EF } /* Name.Constant */
.nd { color: #A6E22E } /* Name.Decorator */
@@ -166,7 +166,7 @@
.sr { color: #E6DB74 } /* Literal.String.Regex */
.s1 { color: #E6DB74 } /* Literal.String.Single */
.ss { color: #E6DB74 } /* Literal.String.Symbol */
.bp { color: #A6E22E } /* Name.Builtin.Pseudo */
.bp { color: #F8F8F2 } /* Name.Builtin.Pseudo */
.fm { color: #A6E22E } /* Name.Function.Magic */
.vc { color: #F8F8F2 } /* Name.Variable.Class */
.vg { color: #F8F8F2 } /* Name.Variable.Global */

File diff suppressed because it is too large Load Diff

View File

@@ -23,27 +23,27 @@
"not dead"
],
"dependencies": {
"ionicons": "^8.1.0",
"ionicons": "^8.0.13",
"normalize.css": "8.0.1",
"ol": "^10.10.0",
"ol": "^10.9.0",
"swiped-events": "1.2.0"
},
"devDependencies": {
"@biomejs/biome": "2.5.9",
"@types/node": "^26.2.0",
"browserslist": "^4.28.8",
"@biomejs/biome": "2.5.2",
"@types/node": "^26.1.0",
"browserslist": "^4.28.4",
"browserslist-to-esbuild": "^2.1.1",
"edge.js": "^6.5.1",
"less": "^4.9.0",
"less": "^4.6.7",
"mathjs": "^15.2.0",
"sharp": "~0.35.3",
"sort-package-json": "^4.0.0",
"stylelint": "^17.14.1",
"stylelint": "^17.14.0",
"stylelint-config-standard-less": "^4.1.0",
"stylelint-prettier": "^5.0.3",
"svgo": "^4.0.2",
"typescript": "~7.0.2",
"vite": "^8.2.1",
"vite-bundle-analyzer": "^1.3.9"
"svgo": "^4.0.1",
"typescript": "~6.0.3",
"vite": "^8.1.3",
"vite-bundle-analyzer": "^1.3.8"
}
}

View File

@@ -5,7 +5,13 @@ import { assertElement } from "../util/assertElement.ts";
const fetchResults = async (qInput: HTMLInputElement, query: string): Promise<void> => {
try {
const res = await http("GET", `./autocompleter?q=${query}`);
let res: Response;
if (settings.method === "GET") {
res = await http("GET", `./autocompleter?q=${query}`);
} else {
res = await http("POST", "./autocompleter", { body: new URLSearchParams({ q: query }) });
}
const results = await res.json();

View File

@@ -80,12 +80,7 @@ export default class Calculator extends Plugin {
try {
const node = Calculator.math.parse(searchInput.value);
const value = node.evaluate();
if (typeof value !== "number") {
return;
}
return `${node.toString()} = ${value}`;
return `${node.toString()} = ${node.evaluate()}`;
} catch {
// not a compatible math expression
return;

View File

@@ -23,7 +23,7 @@ export const appendAnswerElement = (element: HTMLElement | string | number): voi
if (!(element instanceof HTMLElement)) {
const span = document.createElement("span");
span.textContent = element.toString();
span.innerHTML = element.toString();
// biome-ignore lint/style/noParameterAssign: TODO
element = span;
}

View File

@@ -112,15 +112,6 @@ if [ "$(id -u)" -eq 0 ]; then
fi
# ENVs aliases
# https://github.com/searxng/searxng/issues/5934
case "${SEARXNG_PORT:-}" in
'') ;;
*[!0-9]*)
unset SEARXNG_PORT
;;
*)
export GRANIAN_PORT="$SEARXNG_PORT"
;;
esac
export GRANIAN_PORT="${SEARXNG_PORT:-$GRANIAN_PORT}"
exec /usr/local/searxng/.venv/bin/granian searx.webapp:app

View File

@@ -29,11 +29,10 @@ By default and without any extensions, SearXNG serves these resolvers:
- ``duckduckgo``
- ``allesedv``
- ``google``
- ``kagi``
- ``yandex``
With the above setting favicons are displayed, the user has the option to
deactivate this feature in their settings. If the user is to have the option of
deactivate this feature in his settings. If the user is to have the option of
selecting from several *resolvers*, a further setting is required / but this
setting will be discussed :ref:`later <register resolvers>` in this article,
first we have to setup the favicons cache.
@@ -209,7 +208,6 @@ choose from, the following configuration could be used:
"duckduckgo" = "searx.favicons.resolvers.duckduckgo"
"allesedv" = "searx.favicons.resolvers.allesedv"
# "google" = "searx.favicons.resolvers.google"
# "kagi" = "searx.favicons.resolvers.kagi"
# "yandex" = "searx.favicons.resolvers.yandex"
.. note::
@@ -228,7 +226,6 @@ into the *proxy*:
- :py:obj:`searx.favicons.resolvers.duckduckgo`
- :py:obj:`searx.favicons.resolvers.allesedv`
- :py:obj:`searx.favicons.resolvers.google`
- :py:obj:`searx.favicons.resolvers.kagi`
- :py:obj:`searx.favicons.resolvers.yandex`

View File

@@ -8,7 +8,7 @@
search:
safe_search: 0
autocomplete: "duckduckgo"
autocomplete: ""
favicon_resolver: ""
default_lang: ""
ban_time_on_fail: 5
@@ -32,7 +32,7 @@
- ``2``: Strict
``autocomplete``:
Existing autocomplete backends, set blank to turn it off.
Existing autocomplete backends, leave blank to turn it off.
- ``360search``
- ``baidu``
@@ -41,7 +41,6 @@
- ``dbpedia``
- ``duckduckgo``
- ``google``
- ``kagi``
- ``mwmbl``
- ``naver``
- ``privacywall``

View File

@@ -14,7 +14,7 @@
limiter: false
public_instance: false
image_proxy: false
method: "GET"
method: "POST"
default_http_headers:
X-Content-Type-Options : nosniff
X-Download-Options : noopen
@@ -58,8 +58,8 @@
``method`` : ``GET`` | ``POST``
HTTP method. By default, ``GET`` is used / The ``POST`` method has the
advantage with some browsers that the history is not saved, but
HTTP method. By defaults ``POST`` is used / The ``POST`` method has the
advantage with some WEB browsers that the history is not easy to read, but
there are also various disadvantages that sometimes **severely restrict the
ease of use for the end user** (e.g. back button to jump back to the previous
search page and drag & drop of search term to new tabs do not work as

View File

@@ -1,8 +0,0 @@
.. _exaapi engine:
==============
Exa API Engine
==============
.. automodule:: searx.engines.exaapi
:members:

View File

@@ -1,8 +0,0 @@
.. _jina engine:
===========
Jina Engine
===========
.. automodule:: searx.engines.jina
:members:

View File

@@ -0,0 +1,8 @@
.. _engine presearch:
================
Presearch Engine
================
.. automodule:: searx.engines.presearch
:members:

View File

@@ -1,8 +0,0 @@
.. _yandex api engine:
=================
Yandex Search API
=================
.. automodule:: searx.engines.yandex_api
:members:

View File

@@ -80,8 +80,8 @@ same environment, here are a few examples::
# to test one of the update scripts
(dev.env)$ searxng_extra/update/update_engine_traits.py --help
# to test the update of the wikidata units and property names
(dev.env)$ searxng_extra/update/update_wikidata.py
# to test the update of the wikidata units
(dev.env)$ searxng_extra/update/update_wikidata_units.py
.. sidebar:: further read

View File

@@ -90,10 +90,10 @@ Scripts to update static data in :origin:`searx/data/`
:members:
``update_wikidata.py``
``update_wikidata_units.py``
============================
:origin:`[source] <searxng_extra/update/update_wikidata.py>`
:origin:`[source] <searxng_extra/update/update_wikidata_units.py>`
.. automodule:: searxng_extra.update.update_wikidata
.. automodule:: searxng_extra.update.update_wikidata_units
:members:

View File

@@ -20,11 +20,15 @@ If you don't trust anyone, you can set up your own, see :ref:`installation`.
- :ref:`self hosted <installation>`
- :ref:`no user tracking / no profiling <SearXNG protect privacy>`
- javascript & cookies are optional
- script & cookies are optional
- secure, encrypted connections
- :ref:`{{engines | length}} search engines <configured engines>`
- `58 translations <https://translate.codeberg.org/projects/searxng/searxng/>`_
- about 70 `well maintained <https://uptime.searxng.org/>`__ instances on searx.space_
- :ref:`easy integration of search engines <demo online engine>`
- professional development: `CI <https://github.com/searxng/searxng/actions>`_,
`quality assurance <https://dev.searxng.org/>`_ &
`automated tested UI <https://dev.searxng.org/screenshots.html>`_
.. sidebar:: be a part

2
manage
View File

@@ -48,7 +48,7 @@ PATH="${PY_ENV}/bin:${REPO_ROOT}/node_modules/.bin:${GOROOT}/bin:${GOPATH}/bin:$
PYOBJECTS="searx"
PY_SETUP_EXTRAS='[test]'
GECKODRIVER_VERSION="v0.37.0"
GECKODRIVER_VERSION="v0.36.0"
# SPHINXOPTS=
BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization")
BLACK_TARGETS=("--exclude" "(searx/static|searx/languages.py)" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")

View File

@@ -1,10 +1,10 @@
mock==5.2.0
nose2[coverage_plugin]==0.16.0
cov-core==1.15.0
black==26.5.1
pylint==4.0.7
black==25.9.0
pylint==4.0.6
splinter==0.21.0
selenium==4.47.0
selenium==4.45.0
Sphinx==8.2.3;python_version <= "3.11"
Sphinx==9.1.0; python_version > "3.11"
sphinx-issues==6.0.0
@@ -18,11 +18,11 @@ myst-parser==5.0.0
linuxdoc==20260504
aiounittest==1.5.0
yamllint==1.38.0
wlc==2.1.1
wlc==2.1.0
coloredlogs==15.0.1
docutils>=0.21.2;python_version <= "3.11"
docutils>=0.22.4; python_version > "3.11"
parameterized==0.9.0
granian[reload]==2.8.2
basedpyright==1.39.10
granian[reload]==2.7.8
basedpyright==1.39.9
types-lxml==2026.2.16

View File

@@ -1,2 +1,2 @@
granian==2.8.2
granian[pname]==2.8.2
granian==2.7.8
granian[pname]==2.7.8

View File

@@ -1,19 +1,19 @@
certifi==2026.7.22
certifi==2026.6.17
babel==2.18.0
flask-babel==4.0.0
flask==3.1.3
jinja2==3.1.6
lxml==6.1.2
pygments==2.21.0
lxml==6.1.1
pygments==2.20.0
python-dateutil==2.9.0.post0
pyyaml==6.0.3
httpx[http2]==0.28.1
httpx-socks[asyncio]==0.13.1
httpx-socks[asyncio]==0.10.0
sniffio==1.3.1
valkey==6.1.1
markdown-it-py==4.2.0
msgspec==0.21.1
typer==0.27.1
typer==0.26.8
isodate==0.7.2
whitenoise==6.12.0
typing-extensions==4.16.0

View File

@@ -1,6 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Implementation of the :py:obj:`preference <searx.preference>` settings."""
# pylint: disable = too-few-public-methods
import typing as t

View File

@@ -38,6 +38,7 @@ area:
"""
__all__ = ["AnswererInfo", "Answerer", "AnswerStorage"]

View File

@@ -13,6 +13,7 @@ from dataclasses import dataclass
from searx.utils import load_module
from searx.result_types.answer import BaseAnswer
_default = pathlib.Path(__file__).parent
log: logging.Logger = logging.getLogger("searx.answerers")

View File

@@ -16,7 +16,7 @@ from . import Answerer, AnswererInfo
def random_characters():
random_string_letters = string.ascii_lowercase + string.digits + string.ascii_uppercase
return random.choices(random_string_letters, k=random.randint(8, 32))
return [random.choice(random_string_letters) for _ in range(random.randint(8, 32))]
def random_string():

View File

@@ -62,7 +62,7 @@ def bing(query: str, _sxng_locale: str) -> list[str]:
# bing search autocompleter
base_url = "https://www.bing.com/AS/Suggestions?"
# cvid has to be a 32 character long string consisting of numbers and uppsercase characters
cvid = ''.join(random.choices(string.ascii_uppercase + string.digits, k=32))
cvid = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(32))
response = get(base_url + urlencode({'qry': query, 'csr': 1, 'cvid': cvid}))
results: list[str] = []
@@ -127,17 +127,18 @@ def duckduckgo(query: str, sxng_locale: str) -> list[str]:
def google_complete(query: str, sxng_locale: str) -> list[str]:
"""Autocomplete from Google. Supports Google's languages
"""Autocomplete from Google. Supports Google's languages and subdomains
(:py:obj:`searx.engines.google.get_google_info`) by using the async REST
API::
https://www.google.com/complete/search?{args}
https://{subdomain}/complete/search?{args}
"""
data = ENGINE_TRAITS.get("google") or {}
traits = EngineTraits(**data)
google_info: dict[str, t.Any] = google.get_google_info({'searxng_locale': sxng_locale}, traits)
url = 'https://{subdomain}/complete/search?{args}'
args = urlencode(
{
'q': query,
@@ -147,7 +148,7 @@ def google_complete(query: str, sxng_locale: str) -> list[str]:
)
results: list[str] = []
resp = get('https://www.google.com/complete/search?' + args)
resp = get(url.format(subdomain=google_info['subdomain'], args=args))
if resp and resp.ok:
json_txt = resp.text[resp.text.find('[') : resp.text.find(']', -3) + 1]
data = json.loads(json_txt)
@@ -156,24 +157,6 @@ def google_complete(query: str, sxng_locale: str) -> list[str]:
return results
def kagi(query: str, sxng_locale: str) -> list[str]:
"""Autocomplete from Kagi."""
args: dict[str, str] = {'q': query}
if '-' in sxng_locale:
args['r'] = sxng_locale.split('-')[1].lower()
resp = get("https://kagisuggest.com/api/autosuggest?" + urlencode(args))
results: list[str] = []
if resp.ok:
data = resp.json()
if len(data) > 1:
results = data[1]
return results
def mwmbl(query: str, _sxng_locale: str) -> list[str]:
"""Autocomplete from Mwmbl_."""
@@ -397,7 +380,6 @@ backends: dict[str, t.Callable[[str, str], list[str]]] = {
'dbpedia': dbpedia,
'duckduckgo': duckduckgo,
'google': google_complete,
'kagi': kagi,
'mwmbl': mwmbl,
'naver': naver,
'privacywall': privacywall,

View File

@@ -5,6 +5,7 @@ Implementations used for bot detection.
"""
__all__ = ["init", "dump_request", "get_network", "too_many_requests", "ProxyFix"]

View File

@@ -182,7 +182,7 @@ class Config:
if default is UNSET:
raise KeyError(name)
return default
modulename, name = str(fqn).rsplit('.', 1)
(modulename, name) = str(fqn).rsplit('.', 1)
m = __import__(modulename, {}, {}, [name], 0)
return getattr(m, name)

View File

@@ -13,6 +13,7 @@ Accept_ header ..
"""
from ipaddress import (
IPv4Network,
IPv6Network,

View File

@@ -14,6 +14,7 @@ bot if the Accept-Encoding_ header ..
"""
from ipaddress import (
IPv4Network,
IPv6Network,

View File

@@ -11,6 +11,7 @@ if the Accept-Language_ header is unset.
"""
from ipaddress import (
IPv4Network,
IPv6Network,

View File

@@ -11,6 +11,7 @@ the Connection_ header is set to ``close``.
"""
from ipaddress import (
IPv4Network,
IPv6Network,

View File

@@ -20,7 +20,6 @@ Metadata`_. A request is filtered out in case of:
"""
# pylint: disable=unused-argument

View File

@@ -12,6 +12,7 @@ the User-Agent_ header is unset or matches the regular expression
"""
import re
from ipaddress import (
IPv4Network,
@@ -24,6 +25,7 @@ import flask
from . import config
from ._helpers import too_many_requests
USER_AGENT = (
r'('
+ r'unknown'

View File

@@ -55,6 +55,7 @@ from ._helpers import (
logger,
)
logger = logger.getChild('ip_limit')
BURST_WINDOW = 20

View File

@@ -23,7 +23,6 @@ The ``ip_lists`` method implements :py:obj:`block-list <block_ip>` and
]
"""
# pylint: disable=unused-argument

View File

@@ -151,6 +151,6 @@ def get_token() -> str:
if token:
token = token.decode('UTF-8') # type: ignore
else:
token = ''.join(random.choices(string.ascii_lowercase + string.digits, k=16))
token = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(16))
valkey_client.set(TOKEN_KEY, token, ex=TOKEN_LIVE_TIME)
return token

View File

@@ -1,7 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Implementation of a middleware to determine the real IP of an HTTP request
(:py:obj:`flask.request.remote_addr`) behind a proxy chain."""
# pylint: disable=too-many-branches
@@ -64,20 +63,6 @@ class ProxyFix:
proxy_list: list[str] = cfg.get("botdetection.trusted_proxies", default=[])
return [ip_network(net, strict=False) for net in proxy_list]
def is_trusted_proxy(
self,
addr: IPv4Address | IPv6Address | None,
trusted_proxies: list[IPv4Network | IPv6Network],
) -> bool:
if addr is None:
return False
for net in trusted_proxies:
if addr.version == net.version and addr in net:
return True
return False
def trusted_remote_addr(
self,
x_forwarded_for: list[IPv4Address | IPv6Address],
@@ -85,8 +70,16 @@ class ProxyFix:
) -> str:
# always rtl
for addr in reversed(x_forwarded_for):
if not self.is_trusted_proxy(addr, trusted_proxies):
logger.debug("client address from X-Forwarded-For: %s", addr)
trust: bool = False
for net in trusted_proxies:
if addr.version == net.version and addr in net:
logger.debug("trust proxy %s (member of %s)", addr, net)
trust = True
break
# client address
if not trust:
return addr.compressed
# fallback to first address
@@ -102,21 +95,19 @@ class ProxyFix:
# in this function!
orig_remote_addr: str | None = environ.pop("REMOTE_ADDR")
orig_remote_ip: IPv4Address | IPv6Address | None = None
# Validate the IPs involved in this game and delete all invalid ones
# from the WSGI environment.
if orig_remote_addr:
try:
orig_remote_ip = ip_address(orig_remote_addr)
if orig_remote_ip.version == 6 and orig_remote_ip.ipv4_mapped:
orig_remote_ip = orig_remote_ip.ipv4_mapped
orig_remote_addr = orig_remote_ip.compressed
addr = ip_address(orig_remote_addr)
if addr.version == 6 and addr.ipv4_mapped:
addr = addr.ipv4_mapped
orig_remote_addr = addr.compressed
except ValueError as exc:
logger.error("REMOTE_ADDR: %s / discard REMOTE_ADDR from WSGI environment", exc)
orig_remote_addr = None
orig_remote_ip = None
x_real_ip: str | None = environ.get("HTTP_X_REAL_IP")
if x_real_ip:
@@ -150,13 +141,11 @@ class ProxyFix:
if not x_forwarded_for and not x_real_ip:
log_error_only_once("X-Forwarded-For nor X-Real-IP header is set!")
if x_forwarded_for or x_real_ip:
if not trusted_proxies:
log_error_only_once("missing botdetection.trusted_proxies config")
if not self.is_trusted_proxy(orig_remote_ip, trusted_proxies):
x_forwarded_for = []
x_real_ip = None
if x_forwarded_for and not trusted_proxies:
log_error_only_once("missing botdetection.trusted_proxies config")
# without trusted_proxies, this variable is useless for determining
# the real IP
x_forwarded_for = []
# securing the WSGI environment variables that are adjusted

View File

@@ -1,6 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Providing a Valkey database for the botdetection methods."""
import valkey
__all__ = ["set_valkey_client", "get_valkey_client"]

View File

@@ -1,6 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Implementations needed for a branding of SearXNG."""
# pylint: disable=too-few-public-methods
# Struct fields aren't discovered in Python 3.14

View File

@@ -48,7 +48,7 @@ class ExpireCacheCfg(msgspec.Struct): # pylint: disable=too-few-public-methods
MAXHOLD_TIME: int = 60 * 60 * 24 * 7 # 7 days
"""Hold time (default in sec.), after which a value is removed from the cache."""
MAINTENANCE_PERIOD: int = 60 * 60 # 1h
MAINTENANCE_PERIOD: int = 60 * 60 # 2h
"""Maintenance period in seconds / when :py:obj:`MAINTENANCE_MODE` is set to
``auto``."""
@@ -458,22 +458,12 @@ class ExpireCacheSQLite(sqlitedb.SQLiteAppl, ExpireCache):
# Before values are taken from the table, a maintenance interval may
# need to be carried out.
self.maintenance()
sql = f"SELECT value, expire FROM {table} WHERE key = ?"
sql = f"SELECT value FROM {table} WHERE key = ?"
row = self.DB.execute(sql, (key,)).fetchone()
if row is None:
return default
# Check if value is expired. It's possible that it's expired but has not
# yet been automatically deleted by the periodic maintenance
value, expire = row
now = time.time()
if expire < now:
# The record is deleted during the maintenance interval. Deleting
# the record at this point offers no advantage, as a SELECT
# statement must be executed for every cache.get request anyways.
return default
return self.deserialize(value)
return self.deserialize(row[0])
def pairs(self, ctx: str) -> Iterator[tuple[str, typing.Any]]:
"""Iterate over key/value pairs from table given by argument ``ctx``.

View File

@@ -3,6 +3,7 @@
import warnings
# limiter backward compatibility
# ------------------------------

View File

@@ -4,7 +4,6 @@
make data.all
"""
# pylint: disable=invalid-name
__all__ = ["ahmia_blacklist_loader", "data_dir", "get_cache"]
@@ -33,13 +32,6 @@ class WikiDataUnitType(t.TypedDict):
to_si_factor: float
WikiDataPropertyNameType = str | dict[str, str]
"""Name of a Wikidata property. Can be either the plain name or a dictionary of
language code to property name, e.g. ``{"en": "Date of birth"}``."""
WikiDataPropertiesType = dict[str, WikiDataPropertyNameType]
"""Dictionary from wikidata property ID to property name."""
class LocalesType(t.TypedDict):
"""Data structure of an item in ``locales.json``"""
@@ -49,7 +41,6 @@ class LocalesType(t.TypedDict):
USER_AGENTS: UserAgentType
WIKIDATA_UNITS: dict[str, WikiDataUnitType]
WIKIDATA_PROPERTIES: WikiDataPropertiesType
TRACKER_PATTERNS: TrackerPatternsDB
LOCALES: LocalesType
CURRENCIES: CurrenciesDB
@@ -61,12 +52,11 @@ ENGINE_DESCRIPTIONS: dict[str, dict[str, t.Any]]
ENGINE_TRAITS: dict[str, dict[str, t.Any]]
lazy_globals: dict[str, t.Any] = {
lazy_globals = {
"CURRENCIES": CurrenciesDB(),
"USER_AGENTS": None,
"EXTERNAL_URLS": None,
"WIKIDATA_UNITS": None,
"WIKIDATA_PROPERTIES": None,
"EXTERNAL_BANGS": None,
"OSM_KEYS_TAGS": None,
"ENGINE_DESCRIPTIONS": None,
@@ -79,7 +69,6 @@ data_json_files = {
"USER_AGENTS": "useragents.json",
"EXTERNAL_URLS": "external_urls.json",
"WIKIDATA_UNITS": "wikidata_units.json",
"WIKIDATA_PROPERTIES": "wikidata_properties.json",
"EXTERNAL_BANGS": "external_bangs.json",
"OSM_KEYS_TAGS": "osm_keys_tags.json",
"ENGINE_DESCRIPTIONS": "engine_descriptions.json",

File diff suppressed because it is too large Load Diff

View File

@@ -288,7 +288,7 @@
"oc": "Kwanza",
"pa": "ਅੰਗੋਲਨ ਕਵਾਂਜ਼ਾ",
"pl": "Kwanza",
"pt": "kwanza",
"pt": "Kwanza",
"ru": "ангольская кванза",
"si": "ක්වන්සා",
"sr": "анголска кванза",
@@ -334,7 +334,6 @@
"ro": "Peso argentinian",
"ru": "аргентинское песо",
"sk": "Argentinské peso",
"sl": "argentinski peso",
"sr": "аргентински пезос",
"sv": "Argentinsk peso",
"ta": "ஆர்ஜென்டின பீசோ",
@@ -2003,7 +2002,7 @@
"eo": "ganaa cedio",
"es": "cedi",
"fi": "Cedi",
"fr": "cedi",
"fr": "Cedi",
"ga": "cedi",
"gl": "Cedi",
"he": "סדי גאני",
@@ -2953,7 +2952,7 @@
"pap": "won nortkoreano",
"pl": "won północnokoreański",
"pt": "won norte-coreano",
"ro": "won nord-coreean",
"ro": "Won nord-coreean",
"ru": "вона КНДР",
"sk": "severokorejsky won",
"sl": "severnokorejski von",
@@ -3095,7 +3094,6 @@
"ca": "tenge",
"cs": "Tenge",
"cy": "tenge Casachstan",
"da": "Tenge",
"de": "Tenge",
"en": "Kazakhstani tenge",
"eo": "kazaĥa tengo",
@@ -4836,7 +4834,6 @@
"nl": "Seychelse roepie",
"pl": "Rupia seszelska",
"pt": "rupia das Seicheles",
"ro": "rupie seychelloză",
"ru": "сейшельская рупия",
"sk": "Seychelská rupia",
"sl": "sejšelska rupija",
@@ -5067,7 +5064,6 @@
"nl": "Somalische shilling",
"pl": "Szyling somalijski",
"pt": "xelim somaliano",
"ro": "șiling somalez",
"ru": "сомалийский шиллинг",
"sk": "Somálsky šiling",
"sl": "somalski šiling",
@@ -5887,7 +5883,6 @@
"ja": "ドン",
"ko": "베트남 동",
"lt": "Vietnamo dongas",
"ms": "Dồng Vietnam",
"nl": "Vietnamese dong",
"oc": "Dong",
"pa": "ਵੀਅਤਨਾਮੀ ਦੋਙ",
@@ -6127,8 +6122,7 @@
"ro": "Gulden caraibian",
"ru": "Карибский гульден",
"sk": "Karibský gulden",
"sl": "karibski goldinar",
"sv": "Karibisk gulden"
"sl": "karibski goldinar"
},
"XDR": {
"ar": "حقوق السحب الخاصة",
@@ -6730,8 +6724,6 @@
"antilliaanse gulden": "ANG",
"antilski gulden": "ANG",
"aoa": "AOA",
"apvienotās karalistes ekonomika": "GBP",
"apvienotās karalistes saimniecība": "GBP",
"apvienotās karalistes sterliņu mārciņa": "GBP",
"ar": "MGA",
"arab accounting dinar": "XAD",
@@ -6844,7 +6836,6 @@
"avustralya doları": "AUD",
"awg": "AWG",
"az arany mint befektetés": "XAU",
"az egyesült királyság gazdasága": "GBP",
"azerbaidžanin manat": "AZN",
"azerbaidžano manatas": "AZN",
"azerbaidžānas manats": "AZN",
@@ -7028,7 +7019,6 @@
"bir etíope": "ETB",
"biras": "ETB",
"birleşik arap emirlikleri dirhemi": "AED",
"birleşik krallık ekonomisi": "GBP",
"birma kjato": "MMK",
"birr": "ETB",
"birr da etiópia": "ETB",
@@ -7116,19 +7106,15 @@
"brit font": "GBP",
"brita pundo": "GBP",
"britaj pundoj": "GBP",
"britannian talous": "GBP",
"britanska funta": "GBP",
"britanski funt": "GBP",
"britische wirtschaft": "GBP",
"britisches pfund": "GBP",
"british economy": "GBP",
"british pound": "GBP",
"britisk pund": "GBP",
"britiske pund": "GBP",
"brits pond": "GBP",
"britse pond": "GBP",
"britská libra": "GBP",
"brittisk ekonomi": "GBP",
"brittiska pund": "GBP",
"brittiskt pund": "GBP",
"brunei doları": "BND",
@@ -7212,7 +7198,6 @@
"cedi du ghana": "GHS",
"cedi ghana": "GHS",
"cedi ghanese": "GHS",
"cedi ghanéen": "GHS",
"centr afrika franko": "XAF",
"central african cfa franc": "XAF",
"centralafrikansk cfa franc": "XAF",
@@ -7315,6 +7300,7 @@
"colón costa ricense": "CRC",
"colón costa riquenho": "CRC",
"colón costa riquense": "CRC",
"colón costa riqueny": "CRC",
"colón costaricain": "CRC",
"colón costaricano": "CRC",
"colón costaricien": "CRC",
@@ -8427,7 +8413,6 @@
"dólares canadenses": "CAD",
"dólares estadounidenses": "USD",
"dólares neozelandeses": "NZD",
"dồng vietnam": "VND",
"dram": "AMD",
"dram armean": "AMD",
"dram armenia": "AMD",
@@ -8451,7 +8436,6 @@
"droits de tirage speciaux": "XDR",
"droits de tirage spéciaux": "XDR",
"dschibuti franc": "DJF",
"dvn": "VND",
"dzd": "DZD",
"dzsibuti frank": "DJF",
"džibučio frankas": "DJF",
@@ -8465,21 +8449,6 @@
"eastern caribbean currency union": "XCD",
"eastern caribbean dollar": "XCD",
"ec$": "XCD",
"economi'r deyrnas unedig": "GBP",
"economia": "GBP",
"economia del regne unit": "GBP",
"economia del regno unito": "GBP",
"economia del reialme unit": "GBP",
"economia del reino unido": "GBP",
"economia do reino unido": "GBP",
"economia regatului unit": "GBP",
"economie du royaume uni": "GBP",
"economie van het verenigd koninkrijk": "GBP",
"economía del reino unido": "GBP",
"economía do reino unido": "GBP",
"economy": "GBP",
"economy of the uk": "GBP",
"economy of the united kingdom": "GBP",
"egipatska funta": "EGP",
"egipta pundo": "EGP",
"egipto svaras": "EGP",
@@ -8499,12 +8468,6 @@
"einr": "INR",
"eiro": "EUR",
"ekialdeko karibeko dolar": "XCD",
"ekonomi britania raya": "GBP",
"ekonomi united kingdom": "GBP",
"ekonomie van die verenigde koninkryk": "GBP",
"ekonomika spojeného království": "GBP",
"ekonomika v spojenom kráľovstve": "GBP",
"ekonomio de britujo": "GBP",
"el peso": "GTQ",
"emalangeni": "SZL",
"emas sebagai pelaburan": "XAU",
@@ -8536,7 +8499,6 @@
"ermenistan dramı": "AMD",
"ern": "ERN",
"erreal brasildar": "BRL",
"erresuma batuko ekonomia": "GBP",
"errublo": "RUB",
"errublo errusiar": "RUB",
"errupia indiar": "INR",
@@ -8607,8 +8569,6 @@
"eyrir": "ISK",
"e£": "EGP",
"èuro": "EUR",
"économie britannique": "GBP",
"économie du royaume uni": "GBP",
"észak ír font": "GBP",
"észak koreai von": "KPW",
"e₹": "INR",
@@ -8742,9 +8702,6 @@
"forintti": "HUF",
"forinți": "HUF",
"fòrint": "HUF",
"förenade konungariket storbritannien och irlands ekonomi": "GBP",
"förenade konungariket storbritannien och nordirlands ekonomi": "GBP",
"förenade kungarikets ekonomi": "GBP",
"franak cfp": "XPF",
"franc": [
"XPF",
@@ -8997,9 +8954,6 @@
"gold als kapitalanlage": "XAU",
"gold as an investment": "XAU",
"gold as currency": "XAU",
"gospodarka wielkiej brytanii": "GBP",
"gospodarstvo ujedinjenog kraljevstva": "GBP",
"gospodarstvo združenega kraljestva": "GBP",
"gourde": "HTG",
"gourde haiti": "HTG",
"gourde haitiano": "HTG",
@@ -9419,7 +9373,6 @@
"juaņs": "CNY",
"juhokoréjsky won": "KRW",
"juhosudánska libra": "SSP",
"jungtinės karalystės ekonomika": "GBP",
"jungtinių arabų emyratų dirhamas": "AED",
"jungtinių valstijų doleris": "USD",
"južnoafrički rand": "ZAR",
@@ -9484,7 +9437,6 @@
"karibi forint": "XCG",
"karibia guldeno": "XCG",
"karibischer gulden": "XCG",
"karibisk gulden": "XCG",
"karibski goldinar": "XCG",
"karibský gulden": "XCG",
"karipski gulden": "XCG",
@@ -9545,9 +9497,6 @@
"kina papua nugini": "PGK",
"kina papuana": "PGK",
"kina papuásia": "PGK",
"kinh tế anh": "GBP",
"kinh tế vương quốc anh": "GBP",
"kinh tế vương quốc liên hiệp anh và bắc ireland": "GBP",
"kip": "LAK",
"kip laos": "LAK",
"kip laosiano": "LAK",
@@ -11192,7 +11141,6 @@
"põhja korea won": "KPW",
"põhja makedoonia denaar": "MKD",
"prata como investimento": "XAG",
"produits agricole de l'angleterre": "GBP",
"pula": "BWP",
"pula botswana": "BWP",
"pula botswanais": "BWP",
@@ -11243,7 +11191,6 @@
"qatarisk rial": "QAR",
"qäpik": "AZN",
"qindarka": "ALL",
"quanza": "AOA",
"quetzal": "GTQ",
"quetzal guatemala": "GTQ",
"quetzal guatemalteco": "GTQ",
@@ -11569,7 +11516,6 @@
"rupia del pakistan": "PKR",
"rupia dell'india": "INR",
"rupia delle seychelles": "SCR",
"rupia din seychelles": "SCR",
"rupia do nepal": "NPR",
"rupia do paquistão": "PKR",
"rupia do seri lanca": "LKR",
@@ -11625,7 +11571,6 @@
],
"rupie indiană": "INR",
"rupie indiane": "INR",
"rupie seychelloză": "SCR",
"rupies índies": "INR",
"rupija": [
"NPR",
@@ -12055,10 +12000,6 @@
"sterliņu mārciņa": "GBP",
"stērliņu mārciņa": "GBP",
"stn": "STN",
"storbritannien och irlands ekonomi": "GBP",
"storbritannien och nordirlands ekonomi": "GBP",
"storbritanniens ekonomi": "GBP",
"storbritanniens økonomi": "GBP",
"stredoafrický frank": "XAF",
"středoafrický frank": "XAF",
"sucre": "XSU",
@@ -12108,7 +12049,6 @@
"suriye lirası": "SYP",
"suudi arabistan riyali": "SAR",
"suudi riyali": "SAR",
"suurbritannia majandus": "GBP",
"suurbritannia nael": "GBP",
"suurbritannia naelsterling": "GBP",
"suvereni bolivar": "VES",
@@ -12215,7 +12155,6 @@
"švicarski frank": "CHF",
"švýcarský frank": "CHF",
"șekel nou": "ILS",
"șiling somalez": "SOS",
"şekel": "ILS",
"şili pesosu": "CLP",
"s₣": "CHF",
@@ -12558,8 +12497,6 @@
"uguiya": "MRU",
"ugx": "UGX",
"ui": "UYI",
"uk economy": "GBP",
"uk's economy": "GBP",
"ukl": "GBP",
"ukraina grivna": "UAH",
"ukraina hrivno": "UAH",
@@ -12600,8 +12537,6 @@
"unidades de inversion": "MXV",
"unidades de inversión": "MXV",
"united arab emirates dirham": "AED",
"united kingdom economy": "GBP",
"united kingdom's economy": "GBP",
"united states dollar": [
"USN",
"USD"
@@ -12703,7 +12638,6 @@
"venemaa rubla": "RUB",
"venezuelai bolívar": "VES",
"venezuelan digital bolívar": "VED",
"verenigd koninkrijk economie": "GBP",
"verenigde arabiese emirate dirham": "AED",
"verenigde arabische emiraten dirham": "AED",
"ves": "VES",
@@ -12735,12 +12669,6 @@
"wir euro": "CHE",
"wir franc": "CHW",
"wir franken": "CHW",
"wirtschaft": "GBP",
"wirtschaft des vereinigten königreichs": "GBP",
"wirtschaft im vereinigten königreich": "GBP",
"wirtschaft in dem vereinigten königreich": "GBP",
"wirtschaft vom vereinigten königreich": "GBP",
"wirtschaft von dem vereinigten königreich": "GBP",
"wit russische roebel": "BYN",
"won": "KRW",
"won bắc triều tiên": "KPW",
@@ -12834,7 +12762,6 @@
"yeşil burun adaları eskudosu": "CVE",
"yên nhật": "JPY",
"yhdistyneen kuningaskunnan punta": "GBP",
"yhdistyneen kuningaskunnan talous": "GBP",
"yhdistyneiden arabiemiraattien dirhami": "AED",
"yhdysvaltain dollari": "USD",
"ytl": "TRY",
@@ -13584,8 +13511,6 @@
"египетский фунт": "EGP",
"единая система региональных взаиморасчётов": "XSU",
"единая система региональных взаиморасчетов": "XSU",
"економіка великобританії": "GBP",
"економіка великої британії": "GBP",
"енглеска фунта": "GBP",
"еритрейська накфа": "ERN",
"еритрејска накфа": "ERN",
@@ -13642,8 +13567,6 @@
"израелски шекел": "ILS",
"израильский новый шекель": "ILS",
"източнокарибски долар": "XCD",
"икономика на великобритания": "GBP",
"икономика на обединеното кралство": "GBP",
"индийска рупия": "INR",
"индийская рупия": "INR",
"индијска рупија": "INR",
@@ -14057,7 +13980,6 @@
"PLZ",
"PLN"
],
"привреда уједињеног краљевства": "GBP",
"пула": "BWP",
"південно африканський ранд": "ZAR",
"південнокорейська вона": "KRW",
@@ -14145,7 +14067,6 @@
"севернокорејски вон": "KPW",
"северо корейская вона": "KPW",
"северокорейская вона": "KPW",
"седі": "GHS",
"сейшел рупиясе": "SCR",
"сейшелска рупия": "SCR",
"сейшельская рупия": "SCR",
@@ -14198,8 +14119,6 @@
"старый румынский лей": "RON",
"стерлинг фунты": "GBP",
"стерлиң фунты": "GBP",
"стопанство на великобритания": "GBP",
"стопанство на обединеното кралство": "GBP",
"суверен боливар": "VES",
"суверенний болівар": "VES",
"суверенный боливар": "VES",
@@ -14450,7 +14369,6 @@
"шриланкийска рупия": "LKR",
"шриланчанска рупија": "LKR",
"щатски долар": "USD",
"экономика великобритании": "GBP",
"эритрейская накфа": "ERN",
"эритрея накфасы": "ERN",
"эсватини лилангение": "SZL",
@@ -14600,8 +14518,6 @@
"יואן סיני": "CNY",
"ין יפני": "JPY",
"כארתולי לארי": "GEL",
"כלכלת בריטניה": "GBP",
"כלכלת הממלכה המאוחדת": "GBP",
"כתר דני": "DKK",
"כתר נורבגי": "NOK",
"כתר נורווגי": "NOK",
@@ -14749,7 +14665,6 @@
"استثمار البلاتين": "XPT",
"استثمار الذهب": "XAU",
"استثمار الفضة": "XAG",
"اقتصاد المملكة المتحدة": "GBP",
"الاستثمار في الذهب": "XAU",
"الأوقية الموريتانية": "MRU",
"البات": "THB",
@@ -14803,7 +14718,6 @@
"أوقية": "MRU",
"أوقية موريتانية": "MRU",
"أوقيه موريتانيه": "MRU",
"إقتصاد بريطانى": "GBP",
"إيسكودو جزر الرأس الأخضر": "CVE",
"بات": "THB",
"بات تايلاندي": "THB",
@@ -15186,7 +15100,6 @@
"মালদ্বীপীয় রুফিয়াহ": "MVR",
"মিয়ানমার ক্যত": "MMK",
"মিশরীয় পাউন্ড": "EGP",
"যুক্তরাজ্যের অর্থনীতি": "GBP",
"রুশ রুবল": "RUB",
"রেনমিনবি": "CNY",
"রেন্মিন্বি": "CNY",
@@ -15818,7 +15731,6 @@
"엔": "JPY",
"엔화": "JPY",
"영국 파운드": "GBP",
"영국의 경제": "GBP",
"예멘 리알": "YER",
"예멘 리얄": "YER",
"예멘리얄": "YER",
@@ -16026,11 +15938,9 @@
"イエメン・リアル": "YER",
"イエメン・リヤル": "YER",
"イエメン・リヤール": "YER",
"イギリスの経済": "GBP",
"イギリスの通貨": "GBP",
"イギリスポンド": "GBP",
"イギリス・ポンド": "GBP",
"イギリス経済": "GBP",
"イラクの通貨": "IQD",
"イラク・ディナール": "IQD",
"イランの通貨": "IRR",
@@ -16332,7 +16242,6 @@
"英ポンド": "GBP",
"西アフリカcfaフラン": "XOF",
"豪ドル": "AUD",
"財政・経済政策": "GBP",
"越南銅": "VND",
"金投資": "XAU",
"韓国ウォン": "KRW",

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Simple implementation to store TrackerPatterns data in a SQL database."""
# pylint: disable=too-many-branches
import typing as t

View File

@@ -5,7 +5,7 @@
],
"ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}",
"versions": [
"154.0",
"153.0"
"152.0",
"151.0"
]
}

File diff suppressed because it is too large Load Diff

View File

@@ -3474,6 +3474,11 @@
"symbol": "mm⁻²",
"to_si_factor": 1e-06
},
"Q136039973": {
"si_name": "Q6137407",
"symbol": "FPS",
"to_si_factor": 1.0
},
"Q1361854": {
"si_name": "Q11570",
"symbol": "dwt",
@@ -3516,7 +3521,7 @@
},
"Q1377741": {
"si_name": "Q25250",
"symbol": "V<sub>P</sub>",
"symbol": "V_P",
"to_si_factor": 1.0429e+27
},
"Q1386162": {
@@ -3689,11 +3694,6 @@
"symbol": "apc",
"to_si_factor": 0.0308568
},
"Q16068": {
"si_name": null,
"symbol": "DM",
"to_si_factor": null
},
"Q160857": {
"si_name": "Q25236",
"symbol": "hp",
@@ -3872,11 +3872,11 @@
"Q180892": {
"si_name": "Q11570",
"symbol": "M☉",
"to_si_factor": 1.988416e+30
"to_si_factor": 1.9884e+30
},
"Q1811": {
"si_name": "Q11573",
"symbol": "au",
"symbol": "AU",
"to_si_factor": 149597870700.0
},
"Q1815100": {
@@ -4454,11 +4454,6 @@
"symbol": "ng",
"to_si_factor": 1e-12
},
"Q2285395": {
"si_name": null,
"symbol": "dBW",
"to_si_factor": null
},
"Q22934083": {
"si_name": "Q25406",
"symbol": "nC",
@@ -5249,11 +5244,6 @@
"symbol": "μA",
"to_si_factor": 1e-06
},
"Q31274648": {
"si_name": "Q6137407",
"symbol": "FPS",
"to_si_factor": 1.0
},
"Q3186734": {
"si_name": "Q3186734",
"symbol": "J/(m³ K)",
@@ -6326,7 +6316,7 @@
},
"Q536785": {
"si_name": "Q844211",
"symbol": "ρ<sub>P</sub>",
"symbol": "ρ_P",
"to_si_factor": 5.155e+96
},
"Q53679433": {
@@ -6981,7 +6971,7 @@
},
"Q685662": {
"si_name": "Q44395",
"symbol": "p<sub>P</sub>",
"symbol": "p_P",
"to_si_factor": 4.633e+113
},
"Q686163": {

View File

@@ -47,7 +47,7 @@ ENGINES_CACHE: ExpireCacheSQLite = ExpireCacheSQLite.build_cache(
ExpireCacheCfg(
name="ENGINES_CACHE",
MAXHOLD_TIME=60 * 60 * 24 * 7, # 7 days
MAINTENANCE_PERIOD=60 * 60, # 1h
MAINTENANCE_PERIOD=60 * 60, # 2h
MAX_VALUE_LEN=1024 * 1024 * 1024, # 1MB
)
)

View File

@@ -82,7 +82,7 @@ fragment SXNG_query on Query {
def setup(_) -> bool:
global SXNG_query # pylint: disable=global-statement
rand_str: str = "".join(random.choices(string.ascii_letters, k=5))
rand_str: str = "".join(random.choice(string.ascii_letters) for _ in range(5))
SXNG_query = SXNG_query.replace("SXNG_query", "PhotoSearchPaginationContainer_query_1" + rand_str)
return True

View File

@@ -83,7 +83,7 @@ def extract_video_data(video_block):
published_date = None
if create_time:
try:
published_date = datetime.fromisoformat(create_time.strip())
published_date = datetime.strptime(create_time.strip(), "%Y-%m-%d")
except (ValueError, TypeError):
pass

View File

@@ -109,7 +109,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
comments_elements = eval_xpath_getindex(entry, xpath_comment, 0, default=None)
comments: str = "" if comments_elements is None else comments_elements.text
publishedDate = datetime.fromisoformat(eval_xpath_getindex(entry, xpath_published, 0).text.rstrip("Z"))
publishedDate = datetime.strptime(eval_xpath_getindex(entry, xpath_published, 0).text, "%Y-%m-%dT%H:%M:%SZ")
res.add(
res.types.Paper(

View File

@@ -25,7 +25,6 @@ To use this engine, add an entry similar to the following to your engine list in
https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app
"""
import typing as t
from searx.enginelib import EngineCache

View File

@@ -186,7 +186,7 @@ def parse_images(data):
img_date = item.get("bdImgnewsDate")
publishedDate = None
if img_date:
publishedDate = datetime.fromisoformat(img_date)
publishedDate = datetime.strptime(img_date, "%Y-%m-%d %H:%M")
results.append(
{
"template": "images.html",

View File

@@ -1,6 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""BASE (Scholar publications)"""
from datetime import datetime
import re

View File

@@ -32,7 +32,7 @@ base_url = "https://api.bilibili.com/x/web-interface/search/type"
cookie = {
"innersign": "0",
"buvid3": "".join(random.choices(string.hexdigits, k=16)) + "infoc",
"buvid3": "".join(random.choice(string.hexdigits) for _ in range(16)) + "infoc",
"i-wanna-go-back": "-1",
"b_ut": "7",
"FEED_LIVE_VERSION": "V8",

View File

@@ -44,7 +44,7 @@ def response(resp):
"url": 'https://www.bitchute.com/video/' + item['video_id'],
"content": html_to_text(item['description']),
"author": item['channel']['channel_name'],
"publishedDate": datetime.fromisoformat(item["date_published"].rstrip("Z")),
"publishedDate": datetime.strptime(item["date_published"], "%Y-%m-%dT%H:%M:%S.%fZ"),
"length": item['duration'],
"views": item['view_count'],
"thumbnail": item['thumbnail_url'],

View File

@@ -104,7 +104,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
title=_remove_keyword_marker(result["Subject"]),
content=_remove_keyword_marker(result["Text"]),
url=result["Url"],
publishedDate=datetime.fromisoformat(result["Published"]),
publishedDate=datetime.strptime(result["Published"], "%Y-%m-%d %H:%M:%S"),
metadata=gettext.gettext("Posted by {author}").format(author=result["Author"]),
)
)

View File

@@ -43,7 +43,7 @@ def response(resp):
publishedDate = None
if recipe['submissionDate']:
publishedDate = datetime.fromisoformat(result['recipe']['submissionDate'][:19])
publishedDate = datetime.strptime(result['recipe']['submissionDate'][:19], "%Y-%m-%dT%H:%M:%S")
content = [
f"Schwierigkeitsstufe (1-3): {recipe['difficulty']}",

View File

@@ -83,6 +83,7 @@ from threading import Thread
from searx import logger
from searx.result_types import EngineResults
engine_type = 'offline'
paging = True
command = []

View File

@@ -1,18 +1,11 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Deviantart (Images)"""
import typing as t
import urllib.parse
from lxml import html
from searx.result_types import EngineResults
from searx.utils import extract_text, eval_xpath, eval_xpath_list
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
# about
about = {
"website": 'https://www.deviantart.com/',
@@ -30,62 +23,63 @@ paging = True
# search-url
base_url = 'https://www.deviantart.com'
results_xpath = '//div[@data-testid="content_row"]//a[.//*[@data-testid="thumb"]]'
img_src_xpath = './/img/@srcset'
thumbnail_src_xpath = './/img/@src'
author_xpath = './/*[@property="schema:name"]/@content'
cursor_xpath = '//a[contains(@href, "cursor=") and contains(., "Next")]/@href'
results_xpath = '//div[@class="V_S0t_"]/div/div/a'
url_xpath = './@href'
thumbnail_src_xpath = './div/img/@src'
img_src_xpath = './div/img/@srcset'
title_xpath = './@aria-label'
premium_xpath = '../div/div/div/text()'
premium_keytext = 'Watch the artist to view this deviation'
cursor_xpath = '(//a[@class="vQ2brP"]/@href)[last()]'
def request(query: str, params: "OnlineParams"):
def request(query, params):
# https://www.deviantart.com/search?q=foo
args = {'q': query}
if params['pageno'] > 1:
cursor = params['engine_data'].get('cursor')
if cursor:
args['cursor'] = cursor
nextpage_url = params['engine_data'].get('nextpage')
# don't use nextpage when user selected to jump back to page 1
if params['pageno'] > 1 and nextpage_url is not None:
params['url'] = nextpage_url
else:
params['url'] = f"{base_url}/search?{urllib.parse.urlencode({'q': query})}"
params['url'] = f"{base_url}/search?{urllib.parse.urlencode(args)}"
return params
def response(resp: "SXNG_Response") -> EngineResults:
def response(resp):
res = EngineResults()
results = []
dom = html.fromstring(resp.text)
for result in eval_xpath_list(dom, results_xpath):
thumbnail_src = extract_text(eval_xpath(result, thumbnail_src_xpath))
img_src = extract_text(eval_xpath(result, img_src_xpath))
# mature locked thumbs have blur transform (blur_15, blur_30 etc..)
if ',blur_' in f'{thumbnail_src}{img_src}':
# skip images that are blurred
_text = extract_text(eval_xpath(result, premium_xpath))
if _text and premium_keytext in _text:
continue
img_src = extract_text(eval_xpath(result, img_src_xpath))
if img_src:
img_src = img_src.split(' ')[0]
parsed_url = urllib.parse.urlparse(img_src)
img_src = parsed_url._replace(path=parsed_url.path.split('/v1')[0]).geturl()
author = extract_text(eval_xpath(result, author_xpath))
res.add(
res.types.Image(
template='images.html',
url=result.get('href'),
img_src=img_src or "",
thumbnail_src=thumbnail_src or "",
title=result.get('aria-label'),
author=author or "",
)
results.append(
{
'template': 'images.html',
'url': extract_text(eval_xpath(result, url_xpath)),
'img_src': img_src,
'thumbnail_src': extract_text(eval_xpath(result, thumbnail_src_xpath)),
'title': extract_text(eval_xpath(result, title_xpath)),
}
)
nextpage_url = extract_text(eval_xpath(dom, cursor_xpath))
cursor = urllib.parse.parse_qs(urllib.parse.urlparse(nextpage_url or '').query).get('cursor', [None])[0]
if cursor:
res.add(
res.types.LegacyResult(
engine_data=cursor,
key='cursor',
)
if nextpage_url:
results.append(
{
'engine_data': nextpage_url.replace("http://", "https://"),
'key': 'nextpage',
}
)
return res
return results

View File

@@ -1,6 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Docker Hub (IT)"""
# pylint: disable=use-dict-literal
from urllib.parse import urlencode

View File

@@ -43,10 +43,9 @@ def init(_):
def request(query: str, params: "OnlineParams"):
params["url"] = f"{base_url}/api/{dogpile_categ}"
params["headers"]["Origin"] = base_url
params["method"] = "POST"
params["json"] = {"q": query, "qadf": safe_search_map[params["safesearch"]], "page": params["pageno"]}
return params
def response(resp: "SXNG_Response"):

View File

@@ -164,7 +164,6 @@ Terms / phrases that you keep coming across:
https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Accept-Language
"""
# pylint: disable=global-statement
import json

View File

@@ -12,7 +12,6 @@ least we could not find out how language support should work. It seems that
most of the features are based on English terms.
"""
import typing as t
from urllib.parse import urlencode, urlparse, urljoin

View File

@@ -17,6 +17,7 @@ from searx.result_types import EngineResults
from searx.extended_types import SXNG_Response
from searx import weather
about = {
"website": 'https://duckduckgo.com/',
"wikidata_id": 'Q12805',

View File

@@ -2,6 +2,7 @@
# pylint: disable=invalid-name
"""Dummy Offline"""
# about
about = {
"wikidata_id": None,

View File

@@ -1,168 +0,0 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Engine to search using the official `Exa Search API`_. Exa is a search engine for AI agents.
.. _Exa Search API: https://exa.ai/docs/reference/search
Configuration
=============
The engine has the following mandatory setting:
- :py:obj:`api_key`
You can obtain an API key from the `API Key section <https://dashboard.exa.ai/api-keys>`_ in the Exa dashboard.
Optional settings are:
- :py:obj:`results_per_page`
- :py:obj:`search_type`
- :py:obj:`content_mode`
- :py:obj:`content_max_characters`
.. code:: yaml
- name: exaapi
engine: exaapi
shortcut: exa
api_key: "..."
results_per_page: 10
search_type: auto
content_mode: highlights
inactive: false
The API supports SafeSearch and region-aware results.
"""
import typing as t
from dateutil import parser
from searx.exceptions import SearxEngineAPIException
from searx.result_types import EngineResults
from searx.utils import html_to_text
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
SearchType = t.Literal["fast", "auto", "instant", "deep", "deep-lite", "deep-reasoning"]
ContentMode = t.Literal["highlights", "text"]
about = {
"website": "https://exa.ai",
"wikidata_id": None,
"official_api_documentation": "https://exa.ai/docs/reference/search",
"use_official_api": True,
"require_api_key": True,
"results": "JSON",
}
api_key: str = ""
"""API key for Exa Search API (required)."""
categories = ["general", "web"]
safesearch = True
base_url = "https://api.exa.ai/search"
results_per_page: int = 10
"""Maximum number of results per request. Value must be between 1 and 100, default is 10."""
search_type: SearchType = "auto"
"""Search type. Default is auto, see documentation for more information."""
content_mode: ContentMode = "highlights"
"""Content to request from the API: ``highlights`` (excerpts) or ``text`` (page text)."""
content_max_characters: int = 500
"""Maximum characters for the requested content."""
def init(_):
if not api_key:
raise SearxEngineAPIException("No API key provided")
if not 1 <= results_per_page <= 100:
raise ValueError("results_per_page must be between 1 and 100")
if search_type not in t.get_args(SearchType):
raise ValueError(f"Unsupported search type: {search_type}")
if content_mode not in t.get_args(ContentMode):
raise ValueError(f"Unsupported content mode: {content_mode}")
if content_max_characters < 1:
raise ValueError("content_max_characters must be at least 1")
def _contents_payload() -> dict[str, t.Any]:
if content_mode == "text":
return {"text": {"maxCharacters": content_max_characters, "stripLinks": True}}
return {"highlights": {"maxCharacters": content_max_characters}}
def _extract_content(result: dict[str, t.Any]) -> str:
if content_mode == "text":
return html_to_text(result.get("text") or "")
return html_to_text(" ".join(result.get("highlights") or []))
def request(query: str, params: "OnlineParams") -> None:
"""Create the API request."""
body: dict[str, t.Any] = {
"query": query,
"type": search_type,
"numResults": results_per_page,
"contents": _contents_payload(),
}
# Apply SafeSearch if enabled
if params["safesearch"]:
body["moderation"] = True
# Apply region-aware results if specified
locale_parts = params["searxng_locale"].split("-")
region = locale_parts[-1]
if len(locale_parts) > 1:
body["userLocation"] = region.upper()
params["url"] = base_url
params["method"] = "POST"
params["headers"]["x-api-key"] = api_key
params["json"] = body
def _extract_published_date(value: str | None):
"""Extract and parse the published date from the API response.
Args:
value: Raw date string from the API
Returns:
Parsed datetime object or None if parsing fails
"""
if not value:
return None
try:
return parser.parse(value)
except (parser.ParserError, TypeError, OverflowError):
return None
def response(resp: "SXNG_Response") -> EngineResults:
"""Process the API response and return results."""
res = EngineResults()
for result in resp.json().get("results", []):
url = result.get("url")
if not url:
continue
res.add(
res.types.MainResult(
url=url,
title=html_to_text(result.get("title") or url),
content=_extract_content(result),
thumbnail=result.get("image") or "",
publishedDate=_extract_published_date(result.get("publishedDate")),
author=result.get("author") or "",
)
)
return res

View File

@@ -37,7 +37,7 @@ def response(resp):
for item in search_res:
img = 'https://s3.thehackerblog.com/findthatmeme/' + item['image_path']
thumb = 'https://s3.thehackerblog.com/findthatmeme/thumb/' + item.get('thumbnail', '')
date = datetime.fromisoformat(item["updated_at"].split("T")[0])
date = datetime.strptime(item["updated_at"].split("T")[0], "%Y-%m-%d")
formatted_date = datetime.fromtimestamp(date.timestamp())
results.append(

View File

@@ -47,7 +47,7 @@ def response(resp: "SXNG_Response"):
title=result["title"],
content=result["description"],
thumbnail=result["smallImageURL"],
publishedDate=datetime.fromisoformat(result["status_since"]),
publishedDate=datetime.strptime(result["status_since"], "%Y-%m-%d %H:%M:%S"),
metadata=f"Rank: {result['rank']} || {result['episode_count']} episodes",
)
)

View File

@@ -65,6 +65,7 @@ code lines are just relabeled (starting from 1) and appended (a disjoint set of
code blocks in a single file might be returned from the API).
"""
import typing as t
from urllib.parse import urlencode

View File

@@ -9,15 +9,12 @@ engines:
- :ref:`google scholar engine`
- :ref:`google autocomplete`
This implementation uses Nokia user agents to request an XML layout from Google.
The normal web version requires executing JavaScript to load the results and
therefore is currently not used here. See `Google discussion`_ for more
information on that topic.
.. _Google discussion: https://github.com/searxng/searxng/issues/6359
"""
import random
import re
import string
import time
import typing as t
from urllib.parse import unquote, urlencode
@@ -47,16 +44,16 @@ about = {
"official_api_documentation": "https://developers.google.com/custom-search/",
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": "HTML",
}
# engine dependent config
categories = ["general", "web"]
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
"""`Google max 50 pages`_
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
"""
time_range_support = True
language_support = True
@@ -67,23 +64,38 @@ time_range_dict = {"day": "d", "week": "w", "month": "m", "year": "y"}
# Filter results. 0: None, 1: Moderate, 2: Strict
filter_mapping = {0: "off", 1: "medium", 2: "high"}
# https://github.com/searxng/searxng/issues/6359
nokia_useragents = (
"Nokia7610/2.0 (5.0509.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
"Nokia7610/2.0 (7.0642.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
"Nokia6230/2.0 (05.50) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"Nokia6230i/2.0 (03.80) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"Nokia6280/2.0 (03.60) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"NokiaN72/2.0617.1.0.3 Series60/2.8 Profile/MIDP-2.0 Configuration/CLDC-1.1",
)
# specific xpath variables
# ------------------------
# Suggestions are links placed in a *card-section*, we extract only the text
# from the links not the links itself.
suggestion_xpath = '//table[contains(@class, "HExoMb")]//a[contains(@class, "ZWRArf")]'
suggestion_xpath = '//div[contains(@class, "gGQDvd iIWm4b")]//a'
_arcid_range = string.ascii_letters + string.digits + "_-"
_arcid_random: tuple[str, int] | None = None
def ui_async(start: int) -> str:
"""Format of the response from UI's async request.
- ``arc_id:<...>,use_ac:true,_fmt:prog``
The arc_id is random generated every hour.
"""
global _arcid_random # pylint: disable=global-statement
use_ac = "use_ac:true"
# _fmt:html returns a HTTP 500 when user search for celebrities like
# '!google natasha allegri' or '!google chris evans'
_fmt = "_fmt:prog"
# create a new random arc_id every hour
if not _arcid_random or (int(time.time()) - _arcid_random[1]) > 3600:
_arcid_random = ("".join(random.choices(_arcid_range, k=23)), int(time.time()))
arc_id = f"arc_id:srp_{_arcid_random[0]}_1{start:02}"
return ",".join([arc_id, use_ac, _fmt])
def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[str, t.Any]:
@@ -115,11 +127,19 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
A instance of :py:obj:`babel.core.Locale` build from the
``searxng_locale`` value.
subdomain:
Google subdomain :py:obj:`google_domains` that fits to the country
code.
params:
Py-Dictionary with additional request arguments (can be passed to
:py:func:`urllib.parse.urlencode`).
- ``hl`` parameter: specifies the interface language of user interface.
- ``lr`` parameter: restricts search results to documents written in
a particular language.
- ``cr`` parameter: restricts search results to documents
originating in a particular country.
- ``ie`` parameter: sets the character encoding scheme that should
be used to interpret the query string ('utf8').
- ``oe`` parameter: sets the character encoding scheme that should
@@ -136,6 +156,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
ret_val: dict[str, t.Any] = {
"language": None,
"country": None,
"subdomain": None,
"params": {},
"headers": {},
"cookies": {},
@@ -148,7 +169,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
except babel.core.UnknownLocaleError:
locale = None
eng_lang = eng_traits.get_language(sxng_locale) or "lang_en"
eng_lang = eng_traits.get_language(sxng_locale, "lang_en")
lang_code = eng_lang.split("_")[-1] # lang_zh-TW --> zh-TW / lang_en --> en
country = eng_traits.get_region(sxng_locale, eng_traits.all_locale)
@@ -163,6 +184,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
ret_val["language"] = eng_lang
ret_val["country"] = country
ret_val["locale"] = locale
ret_val["subdomain"] = eng_traits.custom["supported_domains"].get(country.upper(), "www.google.com")
# hl parameter:
# The hl parameter specifies the interface language (host language) of
@@ -201,11 +223,9 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
# specify a region (country) only if a region is given in the selected
# locale --> https://github.com/searxng/searxng/issues/2672
if country is not None:
ret_val["params"]["cr"] = ""
if len(sxng_locale.split("-")) > 1:
ret_val["params"]["cr"] = "country" + country
ret_val["params"]["cr"] = ""
if len(sxng_locale.split("-")) > 1:
ret_val["params"]["cr"] = "country" + country
# gl parameter: (mandatory by Google News)
# The gl parameter value is a two-letter country code. For WebSearch
@@ -280,77 +300,88 @@ def detect_google_sorry(resp: "SXNG_Response"):
raise SearxEngineCaptchaException()
def unwrap_google_url(raw_url: str) -> str:
# remove redirector from url
if raw_url.startswith("/url?q="):
return unquote(raw_url[7:].split("&sa=U")[0])
return raw_url
def wml_dom(resp: "SXNG_Response"):
detect_google_sorry(resp)
text = resp.text
if text.lstrip().startswith("<?xml"):
text = text.split("?>", 1)[-1]
return html.fromstring(text)
def google_request(
query: str,
params: "OnlineParams",
extra_args: dict[str, t.Any] | None = None,
*,
eng_traits: EngineTraits | None = None,
use_time_range: bool = True,
use_safesearch: bool = True,
safesearch_map: dict[int, str] | None = None,
use_locales: bool = True,
) -> None:
google_info = get_google_info(params, eng_traits or traits)
if not use_locales:
google_info["params"].pop("lr")
google_info["params"].pop("cr")
start = (params["pageno"] - 1) * 10
args: dict[str, t.Any] = {
"q": query,
"sca_esv": "1",
**google_info["params"],
**(extra_args or {}),
}
if start:
args["start"] = start
if use_time_range and params["time_range"] in time_range_dict:
args["tbs"] = "qdr:" + time_range_dict[params["time_range"]]
if use_safesearch and params["safesearch"]:
args["safe"] = (safesearch_map or filter_mapping)[params["safesearch"]]
params["url"] = f"https://www.google.com/wml/search?{urlencode(args)}"
params["headers"]["User-Agent"] = random.choice(nokia_useragents)
def request(query: str, params: "OnlineParams") -> None:
google_request(query, params)
"""Google search request"""
# pylint: disable=line-too-long
start = (params["pageno"] - 1) * 10
google_info = get_google_info(params, traits)
# https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
query_url = (
"https://"
+ google_info["subdomain"]
+ "/search"
+ "?"
+ urlencode(
{
"q": query,
**google_info["params"],
"filter": "0",
"start": start,
# 'vet': '12ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0QxK8CegQIARAC..i',
# 'ved': '2ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0Q_skCegQIARAG',
# 'cs' : 1,
# 'sa': 'N',
# 'yv': 3,
# 'prmd': 'vin',
# 'ei': 'GASaY6TxOcy_xc8PtYeY6AE',
# 'sa': 'N',
# 'sstk': 'AcOHfVkD7sWCSAheZi-0tx_09XDO55gTWY0JNq3_V26cNN-c8lfD45aZYPI8s_Bqp8s57AHz5pxchDtAGCA_cikAWSjy9kw3kgg'
# formally known as use_mobile_ui
# "asearch": "arc",
# "async": str_async,
}
)
)
if params["time_range"] in time_range_dict:
query_url += "&" + urlencode({"tbs": "qdr:" + time_range_dict[params["time_range"]]})
if params["safesearch"]:
query_url += "&" + urlencode({"safe": filter_mapping[params["safesearch"]]})
params["url"] = query_url
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])
def response(resp: "SXNG_Response") -> EngineResults:
# regex match to get image map that is found inside the returned javascript:
# (function(){var s='...';var i=['...'] ...}
RE_DATA_IMAGE = re.compile(r"(data:image[^']*?)'[^']*?'((?:dimg|pimg|tsuid)[^']*)")
def parse_url_images(text: str):
data_image_map = {}
for image_url, img_id in RE_DATA_IMAGE.findall(text):
data_image_map[img_id] = image_url.encode('utf-8').decode("unicode-escape")
logger.debug("data:image objects --> %s", list(data_image_map.keys()))
return data_image_map
def response(resp: "SXNG_Response"):
"""Get response from google's search request"""
# pylint: disable=too-many-branches, too-many-statements
detect_google_sorry(resp)
data_image_map = parse_url_images(resp.text)
results = EngineResults()
dom = wml_dom(resp)
# convert the text to dom
dom = html.fromstring(resp.text)
# parse results
for result in eval_xpath_list(dom, '//div[contains(@class, "zMzFAb")]'):
for result in eval_xpath_list(dom, '//a[@data-ved and not(@class)]'):
# pylint: disable=too-many-nested-blocks
try:
title_tag = eval_xpath_getindex(
result, './/a[contains(@class, "fuLhoc")]//span[contains(@class, "CVA68e")]', 0, default=None
)
title_tag = eval_xpath_getindex(result, './/div[@style]', 0, default=None)
if title_tag is None:
# this not one of the common google results *section*
logger.debug("ignoring item from the result_xpath list: missing title")
continue
title = extract_text(title_tag)
raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
raw_url = result.get("href")
if raw_url is None:
logger.debug(
'ignoring item from the result_xpath list: missing url of title "%s"',
@@ -358,19 +389,30 @@ def response(resp: "SXNG_Response") -> EngineResults:
)
continue
url = unwrap_google_url(raw_url)
content = extract_text(
eval_xpath(result, './/div[contains(@class, "taTFJ")]//span[contains(@class, "FrIlee")]')
)
thumbnail = eval_xpath_getindex(result, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
results.add(
results.types.MainResult(
url=url,
title=title or "",
content=content or "",
thumbnail=thumbnail or "",
)
)
if raw_url.startswith('/url?q='):
url = unquote(raw_url[7:].split("&sa=U")[0]) # remove the google redirector
else:
url = raw_url
content_nodes = eval_xpath(result, '../..//div[contains(@class, "ilUpNd H66NU aSRlid")]')
for item in content_nodes:
for script in item.xpath(".//script"):
script.getparent().remove(script)
content = extract_text(content_nodes[0])
# Images that are NOT the favicon
xpath_image = eval_xpath_getindex(result, './/img', index=0, default=None)
thumbnail = None
if xpath_image is not None:
thumbnail = xpath_image.get("src")
if thumbnail.startswith("data:image"):
img_id = xpath_image.get("id")
if img_id:
thumbnail = data_image_map.get(img_id)
results.append({"url": url, "title": title, "content": content or '', "thumbnail": thumbnail})
except Exception as e: # pylint: disable=broad-except
logger.error(e, exc_info=True)
@@ -378,8 +420,10 @@ def response(resp: "SXNG_Response") -> EngineResults:
# parse suggestion
for suggestion in eval_xpath_list(dom, suggestion_xpath):
results.add(results.types.LegacyResult(suggestion=extract_text(suggestion)))
# append suggestion
results.append({"suggestion": extract_text(suggestion)})
# return results
return results
@@ -412,12 +456,14 @@ skip_countries = [
]
def fetch_traits(engine_traits: EngineTraits):
def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True):
"""Fetch languages from Google."""
# pylint: disable=import-outside-toplevel, too-many-branches
from searx.network import get # see https://github.com/searxng/searxng/issues/762
engine_traits.custom["supported_domains"] = {}
resp = get("https://www.google.com/preferences", timeout=5)
if not resp.ok:
raise RuntimeError("Response from Google preferences is not OK.")
@@ -468,3 +514,22 @@ def fetch_traits(engine_traits: EngineTraits):
# alias regions
engine_traits.regions["zh-CN"] = "HK"
# supported domains
if add_domains:
resp = get("https://www.google.com/supported_domains", timeout=5)
if not resp.ok:
raise RuntimeError("Response from Google supported domains is not OK.")
for domain in resp.text.split():
domain = domain.strip()
if not domain or domain in [
".google.com",
]:
continue
region = domain.split(".")[-1].upper()
engine_traits.custom["supported_domains"][region] = "www" + domain
if region == "HK":
# There is no google.cn, we use .com.hk for zh-CN
engine_traits.custom["supported_domains"]["CN"] = "www" + domain

View File

@@ -95,11 +95,12 @@ def request(query: str, params: "OnlineParams") -> None:
token = _cse_token()
google_info = get_google_info(params, traits)
info: dict[str, str] = google_info["params"]
args = {
"rsz": "filtered_cse",
"num": str(page_size),
"hl": google_info["params"]["hl"],
"hl": info["hl"],
"cselibv": token["cselibv"],
"cx": CX,
"q": query,
@@ -113,6 +114,10 @@ def request(query: str, params: "OnlineParams") -> None:
start_date, end_date = _get_start_and_end_date_str(params["time_range"])
args["sort"] = f"date:r:{start_date}:{end_date}"
if info.get("lr"):
args["lr"] = info["lr"]
if info.get("cr"):
args["cr"] = info["cr"]
if google_info["country"] not in (None, "ZZ"):
args["gl"] = google_info["country"]
if token["exp"]:

View File

@@ -1,75 +1,122 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google Images: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google Images engine using the internal
Google API used by the Google Go Android app.
import typing as t
from urllib.parse import parse_qs, unquote, urlparse
This internal API offer results in
- JSON (``_fmt:json``)
- Protobuf_ (``_fmt:pb``)
- Protobuf_ compressed? (``_fmt:pc``)
- HTML (``_fmt:html``)
- Protobuf_ encoded in JSON (``_fmt:jspb``).
.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers
"""
from urllib.parse import urlencode
from json import loads
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, wml_dom
from searx.result_types import EngineResults
from searx.utils import eval_xpath_list
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
from searx.engines.google import (
get_google_info,
time_range_dict,
detect_google_sorry,
)
# about
about = {
"website": "https://images.google.com",
"wikidata_id": "Q521550",
"official_api_documentation": "https://developers.google.com/custom-search",
"website": 'https://images.google.com',
"wikidata_id": 'Q521550',
"official_api_documentation": 'https://developers.google.com/custom-search',
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": 'JSON',
}
# engine dependent config
categories = ["images", "web"]
categories = ['images', 'web']
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
"""`Google max 50 pages`_
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
"""
time_range_support = True
language_support = True
safesearch = True
filter_mapping = {0: "images", 1: "active", 2: "active"}
filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "isch"},
eng_traits=traits,
safesearch_map=filter_mapping,
use_locales=False,
def request(query, params):
"""Google-Image search request"""
google_info = get_google_info(params, traits)
query_url = (
'https://'
+ google_info['subdomain']
+ '/search'
+ '?'
+ urlencode({'q': query, 'tbm': "isch", **google_info['params'], 'asearch': 'isch'})
# don't urlencode this because wildly different AND bad results
# pagination uses Zero-based numbering
+ f'&async=_fmt:json,p:1,ijn:{params["pageno"] - 1}'
)
if params['time_range'] in time_range_dict:
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
if params['safesearch']:
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
params['url'] = query_url
params['cookies'] = google_info['cookies']
params['headers'].update(google_info['headers'])
# this ua will allow getting ~50 results instead of 10. #1641
params['headers']['User-Agent'] = (
'NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12;' f' {google_info.get("country", "US")}) gzip'
)
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
dom = wml_dom(resp)
return params
for link in eval_xpath_list(dom, '//a[contains(@href, "/imgres?")]'):
qs = parse_qs(urlparse(link.get("href", "")).query)
img_src = qs.get("imgurl", [""])[0]
url = qs.get("imgrefurl", [""])[0]
if not img_src or not url:
continue
width, height = qs.get("w", [""])[0], qs.get("h", [""])[0]
tbnid = qs.get("tbnid", [""])[0]
results.add(
results.types.Image(
url=url,
title=unquote(urlparse(img_src).path.rsplit("/", 1)[-1]) or urlparse(url).netloc,
img_src=img_src,
thumbnail_src=f"https://encrypted-tbn0.gstatic.com/images?q=tbn:{tbnid}",
resolution=f"{width} x {height}" if width and height else "",
)
)
def response(resp):
"""Get response from google's search request"""
results = []
detect_google_sorry(resp)
json_start = resp.text.find('{"ischj":')
json_data = loads(resp.text[json_start:])
for item in json_data["ischj"].get("metadata", []):
result_item = {
'url': item["result"]["referrer_url"],
'title': item["result"]["page_title"],
'content': item["text_in_grid"]["snippet"],
'source': item["result"]["site_title"],
'resolution': f'{item["original_image"]["width"]} x {item["original_image"]["height"]}',
'img_src': item["original_image"]["url"],
'thumbnail_src': item["thumbnail"]["url"],
'template': 'images.html',
}
author = item["result"].get('iptc', {}).get('creator')
if author:
result_item['author'] = ', '.join(author)
copyright_notice = item["result"].get('iptc', {}).get('copyright_notice')
if copyright_notice:
result_item['source'] += ' | ' + copyright_notice
freshness_date = item["result"].get("freshness_date")
if freshness_date:
result_item['source'] += ' | ' + freshness_date
file_size = item.get('gsa', {}).get('file_size')
if file_size:
result_item['source'] += ' (%s)' % file_size
results.append(result_item)
return results

View File

@@ -1,91 +1,324 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google News: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google News engine.
Google News has a different region handling compared to Google WEB.
- the ``ceid`` argument has to be set (:py:obj:`ceid_list`)
- the hl_ argument has to be set correctly (and different to Google WEB)
- the gl_ argument is mandatory
If one of this argument is not set correctly, the request is redirected to
CONSENT dialog::
https://consent.google.com/m?continue=
The google news API ignores some parameters from the common :ref:`google API`:
- num_ : the number of search results is ignored / there is no paging all
results for a query term are in the first response.
- save_ : is ignored / Google-News results are always *SafeSearch*
.. _hl: https://developers.google.com/custom-search/docs/xml_results#hlsp
.. _gl: https://developers.google.com/custom-search/docs/xml_results#glsp
.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
"""
import typing as t
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, unwrap_google_url, wml_dom
from searx.result_types import EngineResults
import json
import base64
from urllib.parse import urlencode
from lxml import html
import babel
from searx import locales
from searx.utils import (
eval_xpath_getindex,
eval_xpath,
eval_xpath_list,
eval_xpath_getindex,
extract_text,
)
from searx.engines.google import fetch_traits as _fetch_traits # pylint: disable=unused-import
from searx.engines.google import (
get_google_info,
detect_google_sorry,
)
from searx.enginelib.traits import EngineTraits
from searx.result_types import EngineResults
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
# about
about = {
"website": "https://www.google.com",
"website": "https://news.google.com",
"wikidata_id": "Q12020",
"official_api_documentation": "https://developers.google.com/custom-search",
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": "HTML",
}
# engine dependent config
categories = ["news"]
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
"""
paging = False
time_range_support = False
language_support = True
safesearch = False
# Google-News results are always *SafeSearch*. Option 'safesearch' is set to
# False here.
#
# safesearch : results are identical for safesearch=0 and safesearch=2
safesearch = True
base_url: str = "https://news.google.com"
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "nws"},
eng_traits=traits,
use_time_range=False,
use_safesearch=False,
use_locales=False,
"""Google-News search request"""
sxng_locale = params.get("searxng_locale", "en-US")
ceid: str = locales.get_engine_locale(
sxng_locale, traits.custom["ceid"], default="US:en"
) # pyright: ignore[reportAssignmentType]
google_info = get_google_info(params, traits)
google_info["subdomain"] = "news.google.com" # google news has only one domain
ceid_region, ceid_lang = ceid.split(":")
ceid_lang, ceid_suffix = (
ceid_lang.split(":")
+ [
"",
]
)[:2]
google_info["params"]["hl"] = ceid_lang
if ceid_suffix and ceid_suffix not in ["Hans", "Hant"]:
if ceid_region.lower() == ceid_lang:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
else:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_suffix
elif ceid_region.lower() != ceid_lang:
if ceid_region in ["AT", "BE", "CH", "IL", "SA", "IN", "BD", "PT"]:
google_info["params"]["hl"] = ceid_lang
else:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
google_info["params"]["lr"] = "lang_" + ceid_lang.split("-")[0]
google_info["params"]["gl"] = ceid_region
query_url = (
"https://"
+ google_info["subdomain"]
+ "/search?"
+ urlencode(
{"q": query, **google_info["params"]},
)
# ceid includes a ':' character which must not be urlencoded
+ ("&ceid=%s" % ceid)
)
def _span_text(link, css_class: str):
return extract_text(
eval_xpath_getindex(link, f'.//span[contains(@class, "{css_class}")]', 0, default=None),
allow_none=True,
)
params["url"] = query_url
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
seen = set()
for link in eval_xpath_list(wml_dom(resp), '//a[contains(@href, "/url?q=")]'):
href = link.get("href")
if not href:
"""Get response from google's search request"""
res = EngineResults()
detect_google_sorry(resp)
# convert the text to dom
dom = html.fromstring(resp.text)
for result in eval_xpath_list(dom, "//div[@jslog and @data-n-tid and @jsdata]"):
url: str = eval_xpath_getindex(result, "./a[@target='_blank']/@href", 0, default=0)
if not url:
continue
if url.startswith("./"):
url = base_url + url[1:]
# The real URL is often encoded in the "jslog" attribute
jslog: str | None = eval_xpath_getindex(result, "./a[@target='_blank']/@jslog", 0, default=None)
# Try to extract the real URL from jslog
real_url: str | None = None
if jslog:
# jslog format is usually: "95014; 5:<base64>; track:click,vis". We
# want the second part (index 1) after splitting by ";"
parts: list[str] = jslog.split(";")
if len(parts) > 1:
b64_data: str = parts[1].split(":")[-1].strip()
# Pad base64 if necessary
b64_data += "=" * (-len(b64_data) % 4)
decoded_data: list[str | None] = json.loads(base64.b64decode(b64_data).decode("utf-8"))
# The URL is typically the last element in the decoded array
if (
isinstance(decoded_data, list)
and isinstance(decoded_data[-1], str)
and decoded_data[-1].startswith("http")
):
real_url = decoded_data[-1]
if real_url:
url = real_url
else:
logger.error(f"no real-url found: {url}")
continue
url = unwrap_google_url(href)
if url in seen or "google.com/search" in url:
continue
title = extract_text(eval_xpath(result, "./h4")) or ""
title = _span_text(link, "M3vVJe") or _span_text(link, "fuLhoc")
if not title:
continue
# The pub_date is mostly a string like 'yesterday', not a real timezone
# date or time. Therefore we can't use publishedDate and place the
# *pub* sting into the content.
source = _span_text(link, "dXDvrc")
pub_date = _span_text(link, "YVIcad")
thumbnail = eval_xpath_getindex(link, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
pub_date = extract_text(eval_xpath(result, ".//time"))
pub_origin = extract_text(eval_xpath(result, ".//div[contains(@class, 'vr1PYe')]"))
content = " / ".join([x for x in [pub_origin, pub_date] if x])
seen.add(url)
results.add(
results.types.MainResult(
thumbnail: str = eval_xpath_getindex(result, ".//figure/img/@src", 0, default="")
if thumbnail and thumbnail.startswith("/"):
thumbnail = base_url + thumbnail
res.add(
res.types.MainResult(
url=url,
title=title,
content=" / ".join(x for x in [source, pub_date] if x),
thumbnail=thumbnail or "",
content=content,
thumbnail=thumbnail,
)
)
return results
return res
ceid_list = [
"AE:ar",
"AR:es-419",
"AT:de",
"AU:en",
"BD:bn",
"BE:fr",
"BE:nl",
"BG:bg",
"BR:pt-419",
"BW:en",
"CA:en",
"CA:fr",
"CH:de",
"CH:fr",
"CL:es-419",
"CN:zh-Hans",
"CO:es-419",
"CU:es-419",
"CZ:cs",
"DE:de",
"EE:et",
"EG:ar",
"ES:ca",
"ES:es",
"ET:en",
"FI:fi",
"FR:fr",
"GB:en",
"GH:en",
"GR:el",
"HK:zh-Hant",
"HU:hu",
"ID:en",
"ID:id",
"IE:en",
"IL:en",
"IL:he",
"IN:bn",
"IN:en",
"IN:gu",
"IN:hi",
"IN:ml",
"IN:mr",
"IN:pa",
"IN:ta",
"IN:te",
"IT:it",
"JP:ja",
"KE:en",
"KR:ko",
"LB:ar",
"LT:lt",
"LV:en",
"LV:lv",
"MA:fr",
"MY:en",
"MY:ms",
"NA:en",
"NG:en",
"NL:nl",
"NO:no",
"NZ:en",
"PH:en",
"PK:en",
"PL:pl",
"RO:ro",
"RS:sr",
"RU:ru",
"SA:ar",
"SE:sv",
"SG:en",
"SI:sl",
"SK:sk",
"SN:fr",
"TH:th",
"TR:tr",
"TZ:en",
"UA:ru",
"UA:uk",
"UG:en",
"US:en",
"VN:vi",
"ZA:en",
"ZW:en",
]
"""List of region/language combinations supported by Google News. Values of the
``ceid`` argument of the Google News REST API."""
_skip_values = [
"ET:en", # english (ethiopia)
"ID:en", # english (indonesia)
"LV:en", # english (latvia)
]
_ceid_locale_map = {"NO:no": "nb-NO"}
def fetch_traits(engine_traits: EngineTraits):
_fetch_traits(engine_traits, add_domains=False)
engine_traits.custom["ceid"] = {}
for ceid in ceid_list:
if ceid in _skip_values:
continue
region, lang = ceid.split(":")
x = lang.split("-")
if len(x) > 1:
if x[1] not in ["Hant", "Hans"]:
lang = x[0]
sxng_locale = _ceid_locale_map.get(ceid, lang + "-" + region)
try:
locale = babel.Locale.parse(sxng_locale, sep="-")
except babel.UnknownLocaleError:
print("ERROR: %s -> %s is unknown by babel" % (ceid, sxng_locale))
continue
engine_traits.custom["ceid"][locales.region_tag(locale)] = ceid

View File

@@ -77,6 +77,8 @@ def request(query: str, params: "OnlineParams") -> None:
"""Google-Scholar search request"""
google_info = get_google_info(params, traits)
# subdomain is: scholar.google.xy
google_info["subdomain"] = google_info["subdomain"].replace("www.", "scholar.")
args = {
"q": query,
@@ -87,7 +89,7 @@ def request(query: str, params: "OnlineParams") -> None:
}
args.update(time_range_args(params))
params["url"] = "https://scholar.google.com/scholar?" + urlencode(args)
params["url"] = "https://" + google_info["subdomain"] + "/scholar?" + urlencode(args)
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])

View File

@@ -1,87 +1,185 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google Videos: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google Videos engine.
import typing as t
.. admonition:: Content-Security-Policy (CSP)
This engine needs to allow images from the `data URLs`_ (prefixed with the
``data:`` scheme)::
Header set Content-Security-Policy "img-src 'self' data: ;"
.. _data URLs:
https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
"""
import re
from urllib.parse import urlencode, urlparse, parse_qs, unquote
from lxml import html
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, unwrap_google_url, wml_dom
from searx.result_types import EngineResults
from searx.utils import (
eval_xpath_getindex,
eval_xpath_list,
eval_xpath_getindex,
extract_text,
get_embeded_stream_url,
parse_duration_string,
)
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import (
get_google_info,
time_range_dict,
filter_mapping,
suggestion_xpath,
detect_google_sorry,
ui_async,
)
from searx.utils import get_embeded_stream_url
# about
about = {
"website": "https://www.google.com",
"wikidata_id": "Q219885",
"official_api_documentation": "https://developers.google.com/custom-search",
"website": 'https://www.google.com',
"wikidata_id": 'Q219885',
"official_api_documentation": 'https://developers.google.com/custom-search',
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": 'HTML',
}
# engine dependent config
categories = ["videos", "web"]
categories = ['videos', 'web']
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
"""
language_support = True
time_range_support = True
safesearch = True
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "vid"},
eng_traits=traits,
use_locales=False,
# =26;[3,"dimg_ZNMiZPCqE4apxc8P3a2tuAQ_137"]a87;data:image/jpeg;base64,/9j/4AAQSkZJRgABA
# ...6T+9Nl4cnD+gr9OK8I56/tX3l86nWYw//2Q==26;
RE_DATA_IMAGE = re.compile(r'"(dimg_[^"]*)"[^;]*;(data:image[^;]*;[^;]*);?')
def parse_data_images(text: str):
data_image_map = {}
for img_id, data_image in RE_DATA_IMAGE.findall(text):
end_pos = data_image.rfind("=")
if end_pos > 0:
data_image = data_image[: end_pos + 1]
data_image_map[img_id] = data_image
logger.debug("data:image objects --> %s", list(data_image_map.keys()))
return data_image_map
def request(query, params):
"""Google-Video search request"""
google_info = get_google_info(params, traits)
start = (params['pageno'] - 1) * 10
query_url = (
'https://'
+ google_info['subdomain']
+ '/search'
+ "?"
+ urlencode(
{
'q': query,
'tbm': "vid",
'start': start,
**google_info['params'],
'asearch': 'arc',
'async': ui_async(start),
}
)
)
if params['time_range'] in time_range_dict:
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
if 'safesearch' in params:
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
params['url'] = query_url
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
params['cookies'] = google_info['cookies']
params['headers'].update(google_info['headers'])
return params
for result in eval_xpath_list(wml_dom(resp), '//div[contains(@class, "zMzFAb")]'):
def response(resp):
"""Get response from google's search request"""
results = []
detect_google_sorry(resp)
data_image_map = parse_data_images(resp.text)
# convert the text to dom
dom = html.fromstring(resp.text)
result_divs = eval_xpath_list(dom, '//div[contains(@class, "MjjYud")]')
# parse results
for result in result_divs:
title = extract_text(
eval_xpath_getindex(result, './/span[contains(@class, "CVA68e")]', 0, default=None),
eval_xpath_getindex(result, './/h3[contains(@class, "LC20lb")] | .//div[@role="heading"]', 0, default=None),
allow_none=True,
)
raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
if not title or not raw_url:
continue
url = unwrap_google_url(raw_url)
thumbnail = eval_xpath_getindex(result, './/img[contains(@class, "SygO9d")]/@src', 0, default="")
if "/default.jpg" in thumbnail:
thumbnail = thumbnail.split("?")[0].replace("/default.jpg", "/hqdefault.jpg")
length = None
for span in eval_xpath_list(result, './/span[contains(@class, "YVIcad")]'):
length = parse_duration_string(extract_text(span) or "")
if length:
break
results.add(
results.types.MainResult(
url=url,
title=title,
thumbnail=thumbnail,
length=length,
iframe_src=get_embeded_stream_url(url) or "",
template="videos.html",
)
url = eval_xpath_getindex(
result, './/a[@jsname="UWckNb"]/@href | .//a[contains(@href, "/url?q=")]/@href', 0, default=None
)
if url and url.startswith('/url?q='):
url = unquote(url[7:].split('&sa=U')[0])
content = extract_text(
eval_xpath_getindex(result, './/div[contains(@class, "ITZIwc")]', 0, default=None), allow_none=True
)
pub_info = extract_text(
eval_xpath_getindex(
result, './/div[contains(@class, "gqF9jc")] | .//div[contains(@class, "WRu9Cd")]', 0, default=None
),
allow_none=True,
)
# Broader XPath to find any <img> element
thumbnail = eval_xpath_getindex(result, './/img/@src', 0, default=None)
duration = extract_text(
eval_xpath_getindex(result, './/span[contains(@class, "k1U36b")]', 0, default=None), allow_none=True
)
video_id = eval_xpath_getindex(result, './/div[@jscontroller="rTuANe"]/@data-vid', 0, default=None)
# Fallback for video_id from URL if not found via XPath
if not video_id and url and 'youtube.com' in url:
parsed_url = urlparse(url)
video_id = parse_qs(parsed_url.query).get('v', [None])[0]
# Handle thumbnail
if thumbnail and thumbnail.startswith('data:image'):
img_id = eval_xpath_getindex(result, './/img/@id', 0, default=None)
if img_id and img_id in data_image_map:
thumbnail = data_image_map[img_id]
else:
thumbnail = None
if not thumbnail and video_id:
thumbnail = f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg"
# Handle video embed URL
embed_url = None
if video_id:
embed_url = get_embeded_stream_url(f"https://www.youtube.com/watch?v={video_id}")
elif url:
embed_url = get_embeded_stream_url(url)
# Only append results with valid title and url
if title and url:
results.append(
{
'url': url,
'title': title,
'content': content or '',
'author': pub_info,
'thumbnail': thumbnail,
'length': duration,
'iframe_src': embed_url,
'template': 'videos.html',
}
)
# parse suggestion
for suggestion in eval_xpath_list(dom, suggestion_xpath):
results.append({'suggestion': extract_text(suggestion)})
return results

View File

@@ -68,9 +68,7 @@ def _get_api_token(query: str) -> str:
if cached_token:
return cached_token
resp = get(
f"{base_url}/search?q={query}", headers={"User-Agent": gen_useragent(), "Accept-Language": "en-US,en:q=0.9"}
)
resp = get(f"{base_url}/search?q={query}", headers={"User-Agent": gen_useragent()})
if not resp.ok:
raise SearxEngineAPIException("failed to obtain request token: invalid response code")
@@ -78,7 +76,7 @@ def _get_api_token(query: str) -> str:
if not token:
raise SearxEngineAPIException("failed to obtain request token: no token found")
CACHE.set("token", token, expire=3 * 60)
CACHE.set("token", token)
return token

View File

@@ -4,6 +4,7 @@
from urllib.parse import urlencode
from dateutil import parser
about = {
# pylint: disable=line-too-long
"website": "https://hex.pm/",

View File

@@ -91,7 +91,7 @@ def response(resp) -> EngineResults:
published_date = None
try:
published_date = datetime.fromisoformat(entry["createdAt"].rstrip("Z"))
published_date = datetime.strptime(entry["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
except (ValueError, TypeError):
pass

View File

@@ -43,7 +43,7 @@ def _result(video: dict[str, typing.Any], album_info: dict[str, typing.Any]):
release_time = album_info.get("releaseTime", {}).get("value")
if release_time:
try:
published_date = datetime.fromisoformat(release_time)
published_date = datetime.strptime(release_time, "%Y-%m-%d")
except (ValueError, TypeError):
pass

View File

@@ -1,89 +0,0 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Jina is a search AI and part of Elastic, the company behind ElasticSearch.
The engine requires an API key, you can get one from the
`API dashboard <https://jina.ai/api-dashboard/>`_ without signup.
.. code:: yaml
- name: jina
engine: jina
shortcut: ji
api_key: "jina_..."
jina_engine: reader
inactive: false
By default, Jina's own index is used. You can change that by setting a different :py:obj:`jina_engine`.
"""
import typing as t
from urllib.parse import urlencode
from dateutil import parser
from searx.result_types import EngineResults
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
about = {
"website": "https://jina.ai",
"wikidata_id": None,
"official_api_documentation": "https://s.jina.ai/docs",
"use_official_api": True,
"require_api_key": True,
"results": "JSON",
}
categories = ["general"]
paging = True
jina_engine = "reader"
"""Search mode. Currently supported values are 'reader', 'google' and 'bing'."""
base_url = "https://s.jina.ai"
api_key: str | None = None
def setup(_):
if not api_key:
raise ValueError("missing api key")
def request(query: str, params: "OnlineParams"):
# setting 'no-content' pushes the response time down to a third
args = {"q": query, "page": params["pageno"], "engine": jina_engine, "respondWith": "no-content"}
params["url"] = f"{base_url}/?{urlencode(args)}"
params["headers"].update(
{
"Accept": "application/json",
"Authorization": f"Bearer {api_key}",
}
)
def response(resp: "SXNG_Response"):
res = EngineResults()
json_resp: dict[str, t.Any] = resp.json()
result: dict[str, str]
for result in json_resp["data"]:
published_date = None
if result.get("date"):
try:
published_date = parser.parse(result["date"])
except parser.ParserError:
pass
res.add(
res.types.MainResult(
url=result["url"],
title=result["title"],
content=result["description"],
publishedDate=published_date,
)
)
return res

View File

@@ -108,12 +108,14 @@ def get_infobox(alt_forms, result_url, definitions):
infobox_content.append(f'<p><i>Other forms:</i> {", ".join(alt_forms[1:])}</p>')
# definitions
infobox_content.append('''
infobox_content.append(
'''
<small><a href="https://www.edrdg.org/wiki/index.php/JMdict-EDICT_Dictionary_Project">JMdict</a>
and <a href="https://www.edrdg.org/enamdict/enamdict_doc.html">JMnedict</a>
by <a href="https://www.edrdg.org/edrdg/licence.html">EDRDG</a>, CC BY-SA 3.0.</small>
<ul>
''')
'''
)
for pos, engdef, extra in definitions:
if pos == 'Wikipedia definition':
infobox_content.append('</ul><small>Wikipedia, CC BY-SA 3.0.</small><ul>')

View File

@@ -1,66 +0,0 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Keenable is a fast web search with keyless mode support"""
import typing as t
from datetime import datetime
from searx.extended_types import SXNG_Response
from searx.result_types import EngineResults
from searx.utils import searxng_useragent
if t.TYPE_CHECKING:
from searx.search.processors import OnlineParams
about = {
"website": "https://keenable.ai",
"official_api_documentation": "https://docs.keenable.ai",
"use_official_api": True,
"require_api_key": False,
"results": "JSON",
}
api_key = ""
""" Optional API Key. You can create a key at `the official website
<https://keenable.ai/signup>'_ if you need higher rate limits."""
categories = ["general"]
base_url = "https://api.keenable.ai"
keenable_mode = "pro"
def request(query: str, params: "OnlineParams"):
if api_key:
params["url"] = f"{base_url}/v1/search"
params["headers"]["X-API-KEY"] = api_key
else:
params["url"] = f"{base_url}/v1/search/public"
params["method"] = "POST"
params["headers"]["X-Keenable-Title"] = searxng_useragent()
params["json"] = {"query": query, "mode": keenable_mode}
def response(resp: "SXNG_Response") -> EngineResults:
res = EngineResults()
results: list[dict[str, str]] = resp.json()["results"] # type: ignore[reportAny]
for result in results:
published = None
pub = result.get("published_at")
if pub:
try:
published = datetime.fromisoformat(pub.rstrip("Z"))
except ValueError:
pass
res.add(
res.types.MainResult(
url=result["url"],
title=result["title"],
content=result["description"] or result["snippet"],
publishedDate=published,
)
)
return res

View File

@@ -92,7 +92,7 @@ def _get_communities(json):
'title': result['community']['title'],
'content': markdown_to_text(result['community'].get('description', '')),
'thumbnail': result['community'].get('icon', result['community'].get('banner')),
'publishedDate': datetime.fromisoformat(counts['published'][:19]),
'publishedDate': datetime.strptime(counts['published'][:19], '%Y-%m-%dT%H:%M:%S'),
'metadata': metadata,
}
)
@@ -141,7 +141,7 @@ def _get_posts(json):
'title': result['post']['name'],
'content': content,
'thumbnail': thumbnail,
'publishedDate': datetime.fromisoformat(result['post']['published'][:19]),
'publishedDate': datetime.strptime(result['post']['published'][:19], '%Y-%m-%dT%H:%M:%S'),
'metadata': metadata,
}
)
@@ -170,7 +170,7 @@ def _get_comments(json):
'url': result['comment']['ap_id'],
'title': result['post']['name'],
'content': markdown_to_text(result['comment']['content']),
'publishedDate': datetime.fromisoformat(result['comment']['published'][:19]),
'publishedDate': datetime.strptime(result['comment']['published'][:19], '%Y-%m-%dT%H:%M:%S'),
'metadata': metadata,
}
)

View File

@@ -6,27 +6,6 @@ Lofgren .
.. _Marginalia Search:
https://about.marginalia-search.com/
.. _marginalia filters:
Marginalia Filters
=================
Custom filters enable server-side customization of Marginalia search results.
Filter definitions are written in XML and scoped to an API key. Filters can
not be used with the public API key ``public``. The
`Marginalia Filter Editor`_ can be used to create custom filters with a GUI.
Alternatively, filters can be written manually in XML. To associate a filter
definition with an API key, upload the XML data to the ``/filter/<NAME>`` API
endpoint, where ``<NAME>`` is the name for the newly created filter. For more
information, see the `Marginalia filters announcement blogpost`_ and the
official `Marginalia API documentation`_.
.. _Marginalia Filter Editor: https://marginalia-search.com/filters
.. _Marginalia filters announcement blogpost: https://www.marginalia.nu/log/a_127_index_filtering/
.. _Marginalia API documentation: https://about.marginalia-search.com/article/api/
Configuration
=============
@@ -34,10 +13,6 @@ The engine has the following required settings:
- :py:obj:`api_key`
The engine has the following optional settings:
- :py:obj:`filter_name`
You can configure a Marginalia engine by:
.. code:: yaml
@@ -46,7 +21,6 @@ You can configure a Marginalia engine by:
engine: marginalia
shortcut: mar
api_key: ...
filter_name: ...
Implementations
===============
@@ -55,8 +29,6 @@ Implementations
import typing as t
from urllib.parse import urlencode
from searx.network import get
from searx.utils import searxng_useragent
from searx.result_types import EngineResults
from searx.extended_types import SXNG_Response
@@ -82,8 +54,6 @@ api_key = None
https://about.marginalia-search.com/article/api/
"""
filter_name: str | None = None
"""The name of the custom filter to apply to each search."""
class ApiSearchResult(t.TypedDict):
@@ -113,25 +83,6 @@ class ApiSearchResults(t.TypedDict):
results: list[ApiSearchResult]
def _marginalia_headers() -> dict[str, t.Any]:
return {
"User-Agent": searxng_useragent(),
"API-Key": api_key,
}
def _get_filter_names() -> list[str]:
resp = get(f"{base_url}/filter", headers=_marginalia_headers())
if resp.ok:
filter_names = resp.json()
else:
filter_names = []
if not isinstance(filter_names, list):
raise TypeError("marginalia api returned invalid filter list format")
return filter_names
def request(query: str, params: dict[str, t.Any]):
query_params = {
@@ -140,11 +91,10 @@ def request(query: str, params: dict[str, t.Any]):
"nsfw": min(params["safesearch"], 1),
"query": query,
}
if filter_name:
query_params["filter"] = filter_name
params["url"] = f"{base_url}/search?{urlencode(query_params)}"
params["headers"].update(_marginalia_headers())
params["headers"]["User-Agent"] = searxng_useragent()
params["headers"]["API-Key"] = api_key
def response(resp: SXNG_Response):
@@ -164,18 +114,14 @@ def response(resp: SXNG_Response):
return res
def init(_: dict[str, t.Any]):
def init(engine_settings: dict[str, t.Any]):
if not api_key:
_api_key = engine_settings.get("api_key")
if not _api_key:
logger.error("missing api_key: see https://about.marginalia-search.com/article/api")
return False
if api_key == "public":
if _api_key == "public":
logger.error("invalid api_key (%s): see https://about.marginalia-search.com/article/api", api_key)
elif filter_name:
filter_names: list[str] = _get_filter_names()
if filter_name not in filter_names:
logger.error(f"invalid value for filter_name: '{filter_name}'")
return False
return True

View File

@@ -60,7 +60,7 @@ def response(resp):
'title': result['username'] + f" ({result['followers_count']} followers)",
'content': result['note'],
'thumbnail': result.get('avatar'),
'publishedDate': datetime.fromisoformat(result['created_at'][:10]),
'publishedDate': datetime.strptime(result['created_at'][:10], "%Y-%m-%d"),
}
)
elif mastodon_type == "hashtags":

View File

@@ -49,6 +49,7 @@ except ImportError:
from searx.result_types import EngineResults
engine_type = 'offline'
# mongodb connection variables

View File

@@ -4,6 +4,7 @@
from urllib.parse import urlencode
from dateutil import parser
about = {
"website": "https://npms.io/",
"wikidata_id": "Q7067518",

View File

@@ -44,7 +44,7 @@ def response(resp) -> EngineResults:
cve_id = item["cve"]["id"]
description = item["cve"]["descriptions"][0]["value"]
date = datetime.fromisoformat(item["cve"]["published"])
date = datetime.strptime(item["cve"]["published"], "%Y-%m-%dT%H:%M:%S.%f")
# Extract severity (Low, Medium, High, or Critical) and CVSS score, if available
info = item["cve"].get("metrics", {}).get("cvssMetricV31", [{}])[0].get("cvssData", {})

Some files were not shown because too many files have changed in this diff Show More