mirror of
https://github.com/searxng/searxng.git
synced 2026-07-22 16:01:24 +00:00
Compare commits
107 Commits
b3e08f2a44
...
translatio
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c129bda190 | ||
|
|
9f9c00819e | ||
|
|
b72a87676f | ||
|
|
f2432e33d6 | ||
|
|
7b2199ecdf | ||
|
|
4a9c19d7bf | ||
|
|
5cb4cb2bc5 | ||
|
|
5a448596ab | ||
|
|
9c49b7e0d7 | ||
|
|
58e02a01ae | ||
|
|
7fa9f16225 | ||
|
|
9e25585aec | ||
|
|
c19d86faa3 | ||
|
|
21fa7b0be1 | ||
|
|
74b4e7c8d1 | ||
|
|
39f4dd24a5 | ||
|
|
62a1ab7edd | ||
|
|
4abac08de5 | ||
|
|
6a4d5148d6 | ||
|
|
799086874d | ||
|
|
83139c26b3 | ||
|
|
8456831a04 | ||
|
|
b512eaa272 | ||
|
|
1412926f5c | ||
|
|
3b573e0f89 | ||
|
|
da6a230413 | ||
|
|
f69b22c45c | ||
|
|
f930443726 | ||
|
|
d58ced8f71 | ||
|
|
556d08c395 | ||
|
|
1017631800 | ||
|
|
b64e6ee44a | ||
|
|
a6438586a5 | ||
|
|
fd5eb84a37 | ||
|
|
888364c1ce | ||
|
|
1cdf01a719 | ||
|
|
747cec4c23 | ||
|
|
4ef70e9451 | ||
|
|
80c9806de1 | ||
|
|
9d7ca4febc | ||
|
|
c5cd510d82 | ||
|
|
73a0219ab8 | ||
|
|
7ed7adfb05 | ||
|
|
d7367e0897 | ||
|
|
21773bbb2d | ||
|
|
67973783de | ||
|
|
0f9e30e3f9 | ||
|
|
640f88c9bd | ||
|
|
c5d8d05f03 | ||
|
|
d115c61a70 | ||
|
|
c5b1d066e5 | ||
|
|
774616ada6 | ||
|
|
6b2ec018e2 | ||
|
|
bfeaad6c37 | ||
|
|
28d3885764 | ||
|
|
b084194c09 | ||
|
|
8c4df4cf3f | ||
|
|
a0e594e16e | ||
|
|
cb4bfbe129 | ||
|
|
a5660bcc4f | ||
|
|
1396265774 | ||
|
|
0fd40d5f29 | ||
|
|
13a5ace8b5 | ||
|
|
a6831797ba | ||
|
|
0e990f78a3 | ||
|
|
990c63b709 | ||
|
|
e535e4c61a | ||
|
|
79506441c5 | ||
|
|
6cb1c077e9 | ||
|
|
8605230eb8 | ||
|
|
a54722e692 | ||
|
|
7e9a6c2861 | ||
|
|
357662d86d | ||
|
|
bfa76e4cd9 | ||
|
|
a9d49a3349 | ||
|
|
f8ffbf36f9 | ||
|
|
e3126b89e6 | ||
|
|
f9f3d089ec | ||
|
|
e3713717f2 | ||
|
|
952896d29e | ||
|
|
4cc32b2457 | ||
|
|
cce0957f54 | ||
|
|
9375c0a6b6 | ||
|
|
a702741e4e | ||
|
|
aeced67249 | ||
|
|
199e03de1d | ||
|
|
9cd2439e5e | ||
|
|
9f4d8bca02 | ||
|
|
de76a4a39b | ||
|
|
a85a5e2794 | ||
|
|
92abd98a55 | ||
|
|
93e867c6b1 | ||
|
|
75c1b1dade | ||
|
|
097ab64c70 | ||
|
|
0e9f513efc | ||
|
|
fd42d4fda1 | ||
|
|
5c38d2feab | ||
|
|
38b678c493 | ||
|
|
fe1848673f | ||
|
|
8b10095e8a | ||
|
|
b5ef7ec8f3 | ||
|
|
bd73cc09ea | ||
|
|
4dfdc822cf | ||
|
|
502c820a25 | ||
|
|
4fb49b4498 | ||
|
|
cf1410af8d | ||
|
|
6c9dcd4242 |
@@ -1,5 +1,6 @@
|
||||
*
|
||||
|
||||
!container/*.template.*
|
||||
!container/entrypoint.sh
|
||||
!searx/**
|
||||
!requirements*.txt
|
||||
|
||||
138
.github/workflows/container.yml
vendored
138
.github/workflows/container.yml
vendored
@@ -25,25 +25,21 @@ env:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository_owner == 'searxng' || github.event_name == 'workflow_dispatch'
|
||||
if: |
|
||||
github.event_name == 'workflow_dispatch'
|
||||
|| (github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success')
|
||||
name: Build (${{ matrix.arch }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- arch: amd64
|
||||
march: amd64
|
||||
os: ubuntu-24.04
|
||||
emulation: false
|
||||
- arch: arm64
|
||||
march: arm64
|
||||
os: ubuntu-24.04-arm
|
||||
emulation: false
|
||||
- arch: armv7
|
||||
march: arm64
|
||||
os: ubuntu-24.04-arm
|
||||
emulation: true
|
||||
- runner: ubuntu-26.04
|
||||
arch: amd64
|
||||
- runner: ubuntu-26.04-arm
|
||||
arch: arm64
|
||||
- runner: ubuntu-26.04-arm
|
||||
arch: armv7
|
||||
|
||||
permissions:
|
||||
packages: write
|
||||
@@ -53,109 +49,82 @@ jobs:
|
||||
git_url: ${{ steps.build.outputs.git_url }}
|
||||
|
||||
steps:
|
||||
# yamllint disable rule:line-length
|
||||
- name: Setup podman
|
||||
env:
|
||||
PODMAN_VERSION: "v5.7.1"
|
||||
run: |
|
||||
sudo apt-get purge -y podman runc crun conmon
|
||||
|
||||
curl -fsSLO "https://github.com/mgoltzsche/podman-static/releases/download/${{ env.PODMAN_VERSION }}/podman-linux-${{ matrix.march }}.tar.gz"
|
||||
curl -fsSLO "https://github.com/mgoltzsche/podman-static/releases/download/${{ env.PODMAN_VERSION }}/podman-linux-${{ matrix.march }}.tar.gz.asc"
|
||||
gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 0CCF102C4F95D89E583FF1D4F8B5AF50344BB503
|
||||
gpg --batch --verify "podman-linux-${{ matrix.march }}.tar.gz.asc" "podman-linux-${{ matrix.march }}.tar.gz"
|
||||
|
||||
tar -xzf "podman-linux-${{ matrix.march }}.tar.gz"
|
||||
sudo cp -rfv ./podman-linux-${{ matrix.march }}/etc/. /etc/
|
||||
sudo cp -rfv ./podman-linux-${{ matrix.march }}/usr/. /usr/
|
||||
|
||||
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
||||
# yamllint enable rule:line-length
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
registry: "ghcr.io"
|
||||
username: "${{ github.repository_owner }}"
|
||||
password: "${{ secrets.GITHUB_TOKEN }}"
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version: "${{ env.PYTHON_VERSION }}"
|
||||
|
||||
- name: Setup QEMU
|
||||
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: "${{ github.event.workflow_run.head_sha || github.sha }}"
|
||||
persist-credentials: "false"
|
||||
fetch-depth: "0"
|
||||
|
||||
- name: Setup cache Python
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
|
||||
restore-keys: |
|
||||
python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-
|
||||
path: "./local/"
|
||||
|
||||
- name: Get date
|
||||
id: date
|
||||
run: echo "date=$(date +'%Y%m%d')" >>$GITHUB_OUTPUT
|
||||
|
||||
- name: Setup cache container
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
key: "container-${{ matrix.arch }}-${{ steps.date.outputs.date }}-${{ hashFiles('./requirements*.txt') }}"
|
||||
key: "container-${{ matrix.arch }}-${{ hashFiles('./requirements*.txt') }}"
|
||||
restore-keys: |
|
||||
container-${{ matrix.arch }}-${{ steps.date.outputs.date }}-
|
||||
container-${{ matrix.arch }}-
|
||||
path: "/var/tmp/buildah-cache-*/*"
|
||||
|
||||
- if: ${{ matrix.emulation }}
|
||||
name: Setup QEMU
|
||||
uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
registry: "ghcr.io"
|
||||
username: "${{ github.repository_owner }}"
|
||||
password: "${{ secrets.GITHUB_TOKEN }}"
|
||||
|
||||
- name: Build
|
||||
id: build
|
||||
env:
|
||||
OVERRIDE_ARCH: "${{ matrix.arch }}"
|
||||
run: make podman.build
|
||||
run: make container.build
|
||||
|
||||
test:
|
||||
name: Test (${{ matrix.arch }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
needs: build
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- arch: amd64
|
||||
os: ubuntu-24.04
|
||||
emulation: false
|
||||
- arch: arm64
|
||||
os: ubuntu-24.04-arm
|
||||
emulation: false
|
||||
- arch: armv7
|
||||
os: ubuntu-24.04-arm
|
||||
emulation: true
|
||||
- runner: ubuntu-26.04
|
||||
arch: amd64
|
||||
- runner: ubuntu-26.04-arm
|
||||
arch: arm64
|
||||
- runner: ubuntu-26.04-arm
|
||||
arch: armv7
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
persist-credentials: "false"
|
||||
|
||||
- if: ${{ matrix.emulation }}
|
||||
name: Setup QEMU
|
||||
uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
registry: "ghcr.io"
|
||||
username: "${{ github.repository_owner }}"
|
||||
password: "${{ secrets.GITHUB_TOKEN }}"
|
||||
|
||||
- name: Setup QEMU
|
||||
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: "${{ github.event.workflow_run.head_sha || github.sha }}"
|
||||
persist-credentials: "false"
|
||||
|
||||
- name: Test
|
||||
env:
|
||||
OVERRIDE_ARCH: "${{ matrix.arch }}"
|
||||
@@ -165,7 +134,7 @@ jobs:
|
||||
release:
|
||||
if: github.repository_owner == 'searxng' && github.ref_name == 'master'
|
||||
name: Release
|
||||
runs-on: ubuntu-24.04-arm
|
||||
runs-on: ubuntu-26.04-arm
|
||||
needs:
|
||||
- build
|
||||
- test
|
||||
@@ -174,24 +143,25 @@ jobs:
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
persist-credentials: "false"
|
||||
registry: "docker.io"
|
||||
username: "${{ secrets.DOCKER_USER }}"
|
||||
password: "${{ secrets.DOCKER_TOKEN }}"
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
registry: "ghcr.io"
|
||||
username: "${{ github.repository_owner }}"
|
||||
password: "${{ secrets.GITHUB_TOKEN }}"
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
registry: "docker.io"
|
||||
username: "${{ secrets.DOCKER_USER }}"
|
||||
password: "${{ secrets.DOCKER_TOKEN }}"
|
||||
ref: "${{ github.event.workflow_run.head_sha || github.sha }}"
|
||||
persist-credentials: "false"
|
||||
|
||||
- name: Release
|
||||
env:
|
||||
|
||||
23
.github/workflows/data-update.yml
vendored
23
.github/workflows/data-update.yml
vendored
@@ -21,7 +21,7 @@ jobs:
|
||||
data:
|
||||
if: github.repository_owner == 'searxng'
|
||||
name: ${{ matrix.fetch }}
|
||||
runs-on: ubuntu-24.04-arm
|
||||
runs-on: ubuntu-26.04-arm
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -33,7 +33,6 @@ jobs:
|
||||
- update_engine_traits.py
|
||||
- update_wikidata_units.py
|
||||
- update_engine_descriptions.py
|
||||
- update_gsa_useragents.py
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -41,17 +40,17 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version: "${{ env.PYTHON_VERSION }}"
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: "false"
|
||||
|
||||
- name: Setup cache Python
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
|
||||
restore-keys: |
|
||||
@@ -65,23 +64,17 @@ jobs:
|
||||
run: V=1 ./manage pyenv.cmd python "./searxng_extra/update/${{ matrix.fetch }}"
|
||||
|
||||
- name: Create PR
|
||||
id: cpr
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
author: "searxng-bot <searxng-bot@users.noreply.github.com>"
|
||||
committer: "searxng-bot <searxng-bot@users.noreply.github.com>"
|
||||
title: "[data] update searx.data - ${{ matrix.fetch }}"
|
||||
commit-message: "[data] update searx.data - ${{ matrix.fetch }}"
|
||||
branch: "update_data_${{ matrix.fetch }}"
|
||||
title: "[mod] data: update searx.data - ${{ matrix.fetch }}"
|
||||
commit-message: "[mod] data: update searx.data - ${{ matrix.fetch }}"
|
||||
branch: "ci-data-${{ matrix.fetch }}"
|
||||
delete-branch: "true"
|
||||
draft: "false"
|
||||
signoff: "false"
|
||||
body: |
|
||||
[data] update searx.data - ${{ matrix.fetch }}
|
||||
Update searx.data - ${{ matrix.fetch }}
|
||||
labels: |
|
||||
data
|
||||
|
||||
- name: Display information
|
||||
run: |
|
||||
echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
|
||||
echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
|
||||
|
||||
10
.github/workflows/documentation.yml
vendored
10
.github/workflows/documentation.yml
vendored
@@ -25,25 +25,25 @@ jobs:
|
||||
release:
|
||||
if: github.repository_owner == 'searxng' || github.event_name == 'workflow_dispatch'
|
||||
name: Release
|
||||
runs-on: ubuntu-24.04-arm
|
||||
runs-on: ubuntu-26.04-arm
|
||||
permissions:
|
||||
# for JamesIves/github-pages-deploy-action to push
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version: "${{ env.PYTHON_VERSION }}"
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: "false"
|
||||
fetch-depth: "0"
|
||||
|
||||
- name: Setup cache Python
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
|
||||
restore-keys: |
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
with:
|
||||
folder: "dist/docs"
|
||||
branch: "gh-pages"
|
||||
commit-message: "[doc] build from commit ${{ github.sha }}"
|
||||
commit-message: "[mod] docs: build from commit ${{ github.sha }}"
|
||||
# Automatically remove deleted files from the deploy branch
|
||||
clean: "true"
|
||||
single-commit: "true"
|
||||
|
||||
37
.github/workflows/integration.yml
vendored
37
.github/workflows/integration.yml
vendored
@@ -23,7 +23,7 @@ env:
|
||||
jobs:
|
||||
test:
|
||||
name: Python ${{ matrix.python-version }}
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-26.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
@@ -34,17 +34,17 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version: "${{ matrix.python-version }}"
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: "false"
|
||||
|
||||
- name: Setup cache Python
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
key: "python-${{ matrix.python-version }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
|
||||
restore-keys: |
|
||||
@@ -59,37 +59,40 @@ jobs:
|
||||
|
||||
theme:
|
||||
name: Theme
|
||||
runs-on: ubuntu-24.04-arm
|
||||
runs-on: ubuntu-26.04-arm
|
||||
steps:
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version: "${{ env.PYTHON_VERSION }}"
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
persist-credentials: "false"
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version-file: "./.nvmrc"
|
||||
node-version: "26"
|
||||
check-latest: "true"
|
||||
|
||||
- name: Setup cache Node.js
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
key: "nodejs-${{ runner.arch }}-${{ hashFiles('./.nvmrc', './package.json') }}"
|
||||
path: "./client/simple/node_modules/"
|
||||
persist-credentials: "false"
|
||||
|
||||
- name: Setup cache Python
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
|
||||
restore-keys: |
|
||||
python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-
|
||||
path: "./local/"
|
||||
|
||||
- name: Setup cache Node.js
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
key: "nodejs-${{ runner.arch }}-${{ hashFiles('**/package-lock.json') }}"
|
||||
restore-keys: |
|
||||
nodejs-${{ runner.arch }}-
|
||||
path: "./client/simple/node_modules/"
|
||||
|
||||
- name: Setup venv
|
||||
run: make V=1 install
|
||||
|
||||
|
||||
34
.github/workflows/l10n.yml
vendored
34
.github/workflows/l10n.yml
vendored
@@ -26,27 +26,27 @@ env:
|
||||
|
||||
jobs:
|
||||
update:
|
||||
if: github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success'
|
||||
if: github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'searxng'
|
||||
name: Update
|
||||
runs-on: ubuntu-24.04-arm
|
||||
runs-on: ubuntu-26.04-arm
|
||||
permissions:
|
||||
# For "make V=1 weblate.push.translations"
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version: "${{ env.PYTHON_VERSION }}"
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}"
|
||||
fetch-depth: "0"
|
||||
|
||||
- name: Setup cache Python
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
|
||||
restore-keys: |
|
||||
@@ -59,7 +59,7 @@ jobs:
|
||||
- name: Setup Weblate
|
||||
run: |
|
||||
mkdir -p ~/.config
|
||||
echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate
|
||||
echo "${{ secrets.WEBLATE_CONFIG }}" >~/.config/weblate
|
||||
|
||||
- name: Setup Git
|
||||
run: |
|
||||
@@ -74,7 +74,7 @@ jobs:
|
||||
github.repository_owner == 'searxng'
|
||||
&& (github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
|
||||
name: Pull Request
|
||||
runs-on: ubuntu-24.04-arm
|
||||
runs-on: ubuntu-26.04-arm
|
||||
permissions:
|
||||
# For "make V=1 weblate.translations.commit"
|
||||
contents: write
|
||||
@@ -83,18 +83,18 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version: "${{ env.PYTHON_VERSION }}"
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}"
|
||||
fetch-depth: "0"
|
||||
|
||||
- name: Setup cache Python
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}"
|
||||
restore-keys: |
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
- name: Setup Weblate
|
||||
run: |
|
||||
mkdir -p ~/.config
|
||||
echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate
|
||||
echo "${{ secrets.WEBLATE_CONFIG }}" >~/.config/weblate
|
||||
|
||||
- name: Setup Git
|
||||
run: |
|
||||
@@ -118,23 +118,17 @@ jobs:
|
||||
run: make V=1 weblate.translations.commit
|
||||
|
||||
- name: Create PR
|
||||
id: cpr
|
||||
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
|
||||
with:
|
||||
author: "searxng-bot <searxng-bot@users.noreply.github.com>"
|
||||
committer: "searxng-bot <searxng-bot@users.noreply.github.com>"
|
||||
title: "[l10n] update translations from Weblate"
|
||||
commit-message: "[l10n] update translations from Weblate"
|
||||
title: "[mod] i18n: update translations from Weblate"
|
||||
commit-message: "[mod] i18n: update translations from Weblate"
|
||||
branch: "translations_update"
|
||||
delete-branch: "true"
|
||||
draft: "false"
|
||||
signoff: "false"
|
||||
body: |
|
||||
[l10n] update translations from Weblate
|
||||
Update translations from Weblate
|
||||
labels: |
|
||||
area:i18n
|
||||
|
||||
- name: Display information
|
||||
run: |
|
||||
echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
|
||||
echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
|
||||
|
||||
46
.github/workflows/security.yml
vendored
46
.github/workflows/security.yml
vendored
@@ -1,46 +0,0 @@
|
||||
---
|
||||
name: Security
|
||||
|
||||
# yamllint disable-line rule:truthy
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "42 05 * * *"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
container:
|
||||
if: github.repository_owner == 'searxng'
|
||||
name: Container
|
||||
runs-on: ubuntu-24.04-arm
|
||||
permissions:
|
||||
security-events: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
persist-credentials: "false"
|
||||
|
||||
- name: Sync GHCS from Docker Scout
|
||||
uses: docker/scout-action@cd72f264beff1cd72735de31148b9d3244a0234a # v1.21.0
|
||||
with:
|
||||
organization: "searxng"
|
||||
dockerhub-user: "${{ secrets.DOCKER_USER }}"
|
||||
dockerhub-password: "${{ secrets.DOCKER_TOKEN }}"
|
||||
image: "registry://ghcr.io/searxng/searxng:latest"
|
||||
command: "cves"
|
||||
sarif-file: "./scout.sarif"
|
||||
exit-code: "false"
|
||||
write-comment: "false"
|
||||
|
||||
- name: Upload SARIFs
|
||||
uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
|
||||
with:
|
||||
sarif_file: "./scout.sarif"
|
||||
2
Makefile
2
Makefile
@@ -63,7 +63,7 @@ format: format.python format.shell
|
||||
# wrap ./manage script
|
||||
|
||||
MANAGE += weblate.translations.commit weblate.push.translations
|
||||
MANAGE += data.all data.traits data.useragents data.gsa_useragents data.locales data.currencies
|
||||
MANAGE += data.all data.traits data.useragents data.locales data.currencies
|
||||
MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
|
||||
MANAGE += podman.build
|
||||
MANAGE += docker.build docker.buildx
|
||||
|
||||
1177
client/simple/package-lock.json
generated
1177
client/simple/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -29,21 +29,21 @@
|
||||
"swiped-events": "1.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "2.5.0",
|
||||
"@types/node": "^25.9.3",
|
||||
"browserslist": "^4.28.2",
|
||||
"@biomejs/biome": "2.5.3",
|
||||
"@types/node": "^26.1.1",
|
||||
"browserslist": "^4.28.6",
|
||||
"browserslist-to-esbuild": "^2.1.1",
|
||||
"edge.js": "^6.5.1",
|
||||
"less": "^4.6.4",
|
||||
"less": "^4.6.7",
|
||||
"mathjs": "^15.2.0",
|
||||
"sharp": "~0.35.1",
|
||||
"sharp": "~0.35.3",
|
||||
"sort-package-json": "^4.0.0",
|
||||
"stylelint": "^17.13.0",
|
||||
"stylelint": "^17.14.0",
|
||||
"stylelint-config-standard-less": "^4.1.0",
|
||||
"stylelint-prettier": "^5.0.3",
|
||||
"svgo": "^4.0.1",
|
||||
"typescript": "~6.0.3",
|
||||
"vite": "^8.0.16",
|
||||
"svgo": "^4.0.2",
|
||||
"typescript": "~7.0.2",
|
||||
"vite": "^8.1.4",
|
||||
"vite-bundle-analyzer": "^1.3.8"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,8 +35,9 @@ const imageLoader = (resultElement: HTMLElement): void => {
|
||||
}, 1000) as unknown as number;
|
||||
};
|
||||
|
||||
const imageThumbnails: NodeListOf<HTMLImageElement> =
|
||||
document.querySelectorAll<HTMLImageElement>("#urls img.image_thumbnail");
|
||||
const imageThumbnails: NodeListOf<HTMLImageElement> = document.querySelectorAll<HTMLImageElement>(
|
||||
"#urls img.image_thumbnail, img.thumbnail"
|
||||
);
|
||||
for (const thumbnail of imageThumbnails) {
|
||||
if (thumbnail.complete && thumbnail.naturalWidth === 0) {
|
||||
thumbnail.src = `${settings.theme_static_path}/img/img_load_error.svg`;
|
||||
|
||||
@@ -77,9 +77,9 @@ export default class Calculator extends Plugin {
|
||||
|
||||
protected async run(): Promise<string | undefined> {
|
||||
const searchInput = getElement<HTMLInputElement>("q");
|
||||
const node = Calculator.math.parse(searchInput.value);
|
||||
|
||||
try {
|
||||
const node = Calculator.math.parse(searchInput.value);
|
||||
return `${node.toString()} = ${node.evaluate()}`;
|
||||
} catch {
|
||||
// not a compatible math expression
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM ghcr.io/searxng/base:searxng-builder AS builder
|
||||
FROM docker.io/searxng/base:searxng-builder AS builder
|
||||
|
||||
COPY ./requirements.txt ./requirements-server.txt ./
|
||||
|
||||
@@ -21,8 +21,6 @@ RUN --mount=type=cache,id=uv,target=/root/.cache/uv set -eux -o pipefail; \
|
||||
|
||||
COPY --exclude=./searx/version_frozen.py ./searx/ ./searx/
|
||||
|
||||
ARG TIMESTAMP_SETTINGS="0"
|
||||
|
||||
RUN set -eux -o pipefail; \
|
||||
python -m compileall -q -f -j 0 --invalidation-mode=unchecked-hash ./searx/; \
|
||||
find ./searx/static/ -type f \
|
||||
@@ -30,5 +28,4 @@ RUN set -eux -o pipefail; \
|
||||
-exec gzip -9 -k {} + \
|
||||
-exec brotli -9 -k {} + \
|
||||
-exec gzip --test {}.gz + \
|
||||
-exec brotli --test {}.br +; \
|
||||
touch -c --date="@$TIMESTAMP_SETTINGS" ./searx/settings.yml
|
||||
-exec brotli --test {}.br +
|
||||
|
||||
@@ -2,7 +2,7 @@ ARG CONTAINER_IMAGE_ORGANIZATION="searxng"
|
||||
ARG CONTAINER_IMAGE_NAME="searxng"
|
||||
|
||||
FROM localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder AS builder
|
||||
FROM ghcr.io/searxng/base:searxng AS dist
|
||||
FROM docker.io/searxng/base:searxng AS dist
|
||||
|
||||
COPY --chown=977:977 --from=builder /usr/local/searxng/.venv/ ./.venv/
|
||||
COPY --chown=977:977 --from=builder /usr/local/searxng/searx/ ./searx/
|
||||
|
||||
@@ -77,43 +77,23 @@ volume_handler() {
|
||||
setup_ownership "$target" "directory"
|
||||
}
|
||||
|
||||
# Handle configuration file updates
|
||||
config_handler() {
|
||||
local target="$1"
|
||||
local template="$2"
|
||||
local new_template_target="$target.new"
|
||||
setup() {
|
||||
local template_settings="/usr/local/searxng/settings.template.yml"
|
||||
local target_settings="$__SEARXNG_CONFIG_PATH/settings.yml"
|
||||
|
||||
# Create/Update the configuration file
|
||||
if [ -f "$target" ]; then
|
||||
setup_ownership "$target" "file"
|
||||
|
||||
if [ "$template" -nt "$target" ]; then
|
||||
cp -pfT "$template" "$new_template_target"
|
||||
|
||||
cat <<EOF
|
||||
...
|
||||
... INFORMATION
|
||||
... Update available for "$target"
|
||||
... It is recommended to update the configuration file to ensure proper functionality
|
||||
...
|
||||
... New version placed at "$new_template_target"
|
||||
... Please review and merge changes
|
||||
...
|
||||
EOF
|
||||
fi
|
||||
else
|
||||
if [ ! -f "$target_settings" ]; then
|
||||
cat <<EOF
|
||||
...
|
||||
... INFORMATION
|
||||
... "$target" does not exist, creating from template...
|
||||
... "$target_settings" does not exist, creating from template...
|
||||
...
|
||||
EOF
|
||||
cp -pfT "$template" "$target"
|
||||
cp -pfT "$template_settings" "$target_settings"
|
||||
|
||||
sed -i "s/ultrasecretkey/$(head -c 24 /dev/urandom | base64 | tr -dc 'a-zA-Z0-9')/g" "$target"
|
||||
sed -i "s/ultrasecretkey/$(head -c 24 /dev/urandom | base64 | tr -dc 'a-zA-Z0-9')/g" "$target_settings"
|
||||
fi
|
||||
|
||||
check_file "$target"
|
||||
check_file "$target_settings"
|
||||
}
|
||||
|
||||
cat <<EOF
|
||||
@@ -124,8 +104,7 @@ EOF
|
||||
volume_handler "$__SEARXNG_CONFIG_PATH"
|
||||
volume_handler "$__SEARXNG_DATA_PATH"
|
||||
|
||||
# Check for files
|
||||
config_handler "$__SEARXNG_SETTINGS_PATH" "/usr/local/searxng/searx/settings.yml"
|
||||
setup
|
||||
|
||||
# root only features
|
||||
if [ "$(id -u)" -eq 0 ]; then
|
||||
|
||||
8
container/settings.template.yml
Normal file
8
container/settings.template.yml
Normal file
@@ -0,0 +1,8 @@
|
||||
# Read the documentation before extending the defaults:
|
||||
# https://docs.searxng.org/admin/settings/
|
||||
|
||||
use_default_settings: true
|
||||
|
||||
server:
|
||||
secret_key: "ultrasecretkey"
|
||||
image_proxy: true
|
||||
@@ -283,12 +283,12 @@ container images are not officially supported):
|
||||
$ make container
|
||||
|
||||
$ docker images
|
||||
REPOSITORY TAG IMAGE ID CREATED SIZE
|
||||
localhost/searxng/searxng 2025.8.1-3d96414 ... About a minute ago 183 MB
|
||||
localhost/searxng/searxng latest ... About a minute ago 183 MB
|
||||
localhost/searxng/searxng builder ... About a minute ago 524 MB
|
||||
ghcr.io/searxng/base searxng-builder ... 2 days ago 378 MB
|
||||
ghcr.io/searxng/base searxng ... 2 days ago 42.2 MB
|
||||
REPOSITORY TAG IMAGE ID SIZE
|
||||
localhost/searxng/searxng 2026.6.19-93f66bfb4 ... 265 MB
|
||||
localhost/searxng/searxng latest ... 265 MB
|
||||
localhost/searxng/searxng builder ... 687 MB
|
||||
docker.io/searxng/base searxng-builder ... 565 MB
|
||||
docker.io/searxng/base searxng ... 143 MB
|
||||
|
||||
Migrate from ``searxng-docker``
|
||||
===============================
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
.. _aol engine:
|
||||
|
||||
===
|
||||
AOL
|
||||
===
|
||||
|
||||
.. automodule:: searx.engines.aol
|
||||
:members:
|
||||
@@ -4,31 +4,32 @@
|
||||
Search API
|
||||
==========
|
||||
|
||||
SearXNG supports querying via a simple HTTP API.
|
||||
Two endpoints, ``/`` and ``/search``, are supported for both GET and POST methods.
|
||||
The GET method expects parameters as URL query parameters, while the POST method expects parameters as form data.
|
||||
SearXNG supports querying via a simple HTTP API. Two endpoints, ``/`` and
|
||||
``/search``, are supported for both GET and POST methods. The ``GET`` method
|
||||
expects parameters as URL query parameters, while the POST method expects
|
||||
parameters as form data (``application/x-www-form-urlencoded``).
|
||||
|
||||
If you want to consume the results as JSON, CSV, or RSS, you need to set the
|
||||
``format`` parameter accordingly. Supported formats are defined in ``settings.yml``, under the ``search`` section.
|
||||
Requesting an unset format will return a 403 Forbidden error. Be aware that many public instances have these formats disabled.
|
||||
|
||||
``format`` parameter accordingly. Supported formats are defined in
|
||||
``settings.yml``, under the :ref:`settings search` section. Requesting an
|
||||
unset format will return a 403 Forbidden error. Be aware that many public
|
||||
instances have these formats disabled.
|
||||
|
||||
Endpoints:
|
||||
|
||||
``GET /``
|
||||
``GET /search``
|
||||
.. code::
|
||||
|
||||
``POST /``
|
||||
``POST /search``
|
||||
GET /
|
||||
GET /search
|
||||
POST /
|
||||
POST /search
|
||||
|
||||
example cURL calls:
|
||||
|
||||
.. code-block:: bash
|
||||
.. code:: bash
|
||||
|
||||
curl 'https://searx.example.org/search?q=searxng&format=json'
|
||||
|
||||
curl -X POST 'https://searx.example.org/search' -d 'q=searxng&format=csv'
|
||||
|
||||
curl -L -X POST -d 'q=searxng&format=json' 'https://searx.example.org/'
|
||||
|
||||
Parameters
|
||||
@@ -53,90 +54,27 @@ Parameters
|
||||
Comma separated list, specifies the active search categories (see
|
||||
:ref:`configured engines`)
|
||||
|
||||
``engines`` : optional
|
||||
Comma separated list, specifies the active search engines (see
|
||||
:ref:`configured engines`).
|
||||
|
||||
``language`` : default from :ref:`settings search`
|
||||
Code of the language.
|
||||
|
||||
``pageno`` : default ``1``
|
||||
Search page number.
|
||||
|
||||
``time_range`` : optional
|
||||
[ ``day``, ``month``, ``year`` ]
|
||||
|
||||
``time_range`` : optional : [ ``day``, ``month``, ``year`` ]
|
||||
Time range of search for engines which support it. See if an engine supports
|
||||
time range search in the preferences page of an instance.
|
||||
|
||||
``format`` : optional
|
||||
[ ``json``, ``csv``, ``rss`` ]
|
||||
|
||||
``format`` : optional : [ ``json``, ``csv``, ``rss`` ]
|
||||
Output format of results. Format needs to be activated in :ref:`settings
|
||||
search`.
|
||||
|
||||
``results_on_new_tab`` : default ``0``
|
||||
[ ``0``, ``1`` ]
|
||||
|
||||
Open search results on new tab.
|
||||
|
||||
``image_proxy`` : default from :ref:`settings server`
|
||||
[ ``True``, ``False`` ]
|
||||
|
||||
Proxy image results through SearXNG.
|
||||
|
||||
``autocomplete`` : default from :ref:`settings search`
|
||||
[ ``google``, ``dbpedia``, ``duckduckgo``, ``mwmbl``, ``startpage``,
|
||||
``privacywall``, ``wikipedia``, ``swisscows``, ``qwant`` ]
|
||||
|
||||
Service which completes words as you type.
|
||||
|
||||
``safesearch`` : default from :ref:`settings search`
|
||||
[ ``0``, ``1``, ``2`` ]
|
||||
|
||||
``safesearch`` : default from :ref:`settings search` : [ ``0``, ``1``, ``2`` ]
|
||||
Filter search results of engines which support safe search. See if an engine
|
||||
supports safe search in the preferences page of an instance.
|
||||
|
||||
``theme`` : default ``simple``
|
||||
[ ``simple`` ]
|
||||
|
||||
``theme`` : default ``simple`` : [ ``simple`` ]
|
||||
Theme of instance.
|
||||
|
||||
Please note, available themes depend on an instance. It is possible that an
|
||||
instance administrator deleted, created or renamed themes on their instance.
|
||||
See the available options in the preferences page of the instance.
|
||||
|
||||
``enabled_plugins`` : optional
|
||||
List of enabled plugins.
|
||||
|
||||
:default:
|
||||
``Hash_plugin``, ``Self_Information``,
|
||||
``Tracker_URL_remover``, ``Ahmia_blacklist``
|
||||
|
||||
:values:
|
||||
.. enabled by default
|
||||
|
||||
``Hash_plugin``, ``Self_Information``,
|
||||
``Tracker_URL_remover``, ``Ahmia_blacklist``,
|
||||
|
||||
.. disabled by default
|
||||
|
||||
``Hostnames_plugin``, ``Open_Access_DOI_rewrite``,
|
||||
``Vim-like_hotkeys``, ``Tor_check_plugin``
|
||||
|
||||
``disabled_plugins``: optional
|
||||
List of disabled plugins.
|
||||
|
||||
:default:
|
||||
``Hostnames_plugin``, ``Open_Access_DOI_rewrite``,
|
||||
``Vim-like_hotkeys``, ``Tor_check_plugin``
|
||||
|
||||
:values:
|
||||
see values from ``enabled_plugins``
|
||||
|
||||
``enabled_engines`` : optional : *all* :origin:`engines <searx/engines>`
|
||||
List of enabled engines.
|
||||
|
||||
``disabled_engines`` : optional : *all* :origin:`engines <searx/engines>`
|
||||
List of disabled engines.
|
||||
|
||||
|
||||
@@ -58,8 +58,8 @@ Configured Engines
|
||||
{% for mod in engines %}
|
||||
|
||||
* - `{{mod.name}} <{{mod.about and mod.about.website}}>`_
|
||||
{%- if mod.about and mod.about.language %}
|
||||
({{mod.about.language | upper}})
|
||||
{%- if mod.language %}
|
||||
({{mod.language | upper}})
|
||||
{%- endif %}
|
||||
- ``!{{mod.shortcut}}``
|
||||
- {%- if 'searx.engines.' + mod.__name__ in documented_modules %}
|
||||
|
||||
@@ -2,27 +2,27 @@ mock==5.2.0
|
||||
nose2[coverage_plugin]==0.16.0
|
||||
cov-core==1.15.0
|
||||
black==25.9.0
|
||||
pylint==4.0.5
|
||||
pylint==4.0.6
|
||||
splinter==0.21.0
|
||||
selenium==4.44.0
|
||||
selenium==4.45.0
|
||||
Sphinx==8.2.3;python_version <= "3.11"
|
||||
Sphinx==9.1.0; python_version > "3.11"
|
||||
sphinx-issues==6.0.0
|
||||
sphinx-jinja==2.0.2
|
||||
sphinx-tabs==3.5.0
|
||||
furo==2025.12.19
|
||||
sphinxcontrib-programoutput==0.19
|
||||
sphinxcontrib-programoutput==0.20
|
||||
sphinx-autobuild==2025.8.25
|
||||
sphinx-notfound-page==1.1.0
|
||||
myst-parser==5.0.0
|
||||
linuxdoc==20260504
|
||||
aiounittest==1.5.0
|
||||
yamllint==1.38.0
|
||||
wlc==2.0.0
|
||||
wlc==2.1.0
|
||||
coloredlogs==15.0.1
|
||||
docutils>=0.21.2;python_version <= "3.11"
|
||||
docutils>=0.22.4; python_version > "3.11"
|
||||
parameterized==0.9.0
|
||||
granian[reload]==2.7.6
|
||||
basedpyright==1.39.7
|
||||
granian[reload]==2.7.9
|
||||
basedpyright==1.39.9
|
||||
types-lxml==2026.2.16
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
granian==2.7.6
|
||||
granian[pname]==2.7.6
|
||||
granian==2.7.9
|
||||
granian[pname]==2.7.9
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
certifi==2026.5.20
|
||||
certifi==2026.6.17
|
||||
babel==2.18.0
|
||||
flask-babel==4.0.0
|
||||
flask==3.1.3
|
||||
@@ -13,7 +13,7 @@ sniffio==1.3.1
|
||||
valkey==6.1.1
|
||||
markdown-it-py==4.2.0
|
||||
msgspec==0.21.1
|
||||
typer==0.26.7
|
||||
typer==0.26.8
|
||||
isodate==0.7.2
|
||||
whitenoise==6.12.0
|
||||
typing-extensions==4.15.0
|
||||
typing-extensions==4.16.0
|
||||
|
||||
@@ -21,6 +21,8 @@ from searx.engines import (
|
||||
from searx.network import get as http_get, post as http_post
|
||||
from searx.exceptions import SearxEngineResponseException
|
||||
from searx.utils import extr, gen_useragent
|
||||
from searx.data import ENGINE_TRAITS
|
||||
from searx.enginelib.traits import EngineTraits
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
@@ -133,7 +135,9 @@ def google_complete(query: str, sxng_locale: str) -> list[str]:
|
||||
|
||||
"""
|
||||
|
||||
google_info: dict[str, t.Any] = google.get_google_info({'searxng_locale': sxng_locale}, engines['google'].traits)
|
||||
data = ENGINE_TRAITS.get("google") or {}
|
||||
traits = EngineTraits(**data)
|
||||
google_info: dict[str, t.Any] = google.get_google_info({'searxng_locale': sxng_locale}, traits)
|
||||
url = 'https://{subdomain}/complete/search?{args}'
|
||||
args = urlencode(
|
||||
{
|
||||
|
||||
@@ -458,12 +458,22 @@ class ExpireCacheSQLite(sqlitedb.SQLiteAppl, ExpireCache):
|
||||
# Before values are taken from the table, a maintenance interval may
|
||||
# need to be carried out.
|
||||
self.maintenance()
|
||||
sql = f"SELECT value FROM {table} WHERE key = ?"
|
||||
sql = f"SELECT value, expire FROM {table} WHERE key = ?"
|
||||
row = self.DB.execute(sql, (key,)).fetchone()
|
||||
if row is None:
|
||||
return default
|
||||
|
||||
return self.deserialize(row[0])
|
||||
# Check if value is expired. It's possible that it's expired but has not
|
||||
# yet been automatically deleted by the periodic maintenance
|
||||
(value, expire) = row
|
||||
now = time.time()
|
||||
if expire < now:
|
||||
# The record is deleted during the maintenance interval. Deleting
|
||||
# the record at this point offers no advantage, as a SELECT
|
||||
# statement must be executed for every cache.get request anyways.
|
||||
return default
|
||||
|
||||
return self.deserialize(value)
|
||||
|
||||
def pairs(self, ctx: str) -> Iterator[tuple[str, typing.Any]]:
|
||||
"""Iterate over key/value pairs from table given by argument ``ctx``.
|
||||
|
||||
@@ -6,7 +6,7 @@ make data.all
|
||||
"""
|
||||
# pylint: disable=invalid-name
|
||||
|
||||
__all__ = ["ahmia_blacklist_loader", "gsa_useragents_loader", "data_dir", "get_cache"]
|
||||
__all__ = ["ahmia_blacklist_loader", "data_dir", "get_cache"]
|
||||
|
||||
import json
|
||||
import typing as t
|
||||
@@ -63,7 +63,6 @@ lazy_globals = {
|
||||
"ENGINE_TRAITS": None,
|
||||
"LOCALES": None,
|
||||
"TRACKER_PATTERNS": TrackerPatternsDB(),
|
||||
"GSA_USER_AGENTS": None,
|
||||
}
|
||||
|
||||
data_json_files = {
|
||||
@@ -106,24 +105,3 @@ def ahmia_blacklist_loader() -> list[str]:
|
||||
"""
|
||||
with open(data_dir / 'ahmia_blacklist.txt', encoding='utf-8') as f:
|
||||
return f.read().split()
|
||||
|
||||
|
||||
def gsa_useragents_loader() -> list[str]:
|
||||
"""Load data from `gsa_useragents.txt` and return a list of user agents
|
||||
suitable for Google. The user agents are fetched by::
|
||||
|
||||
searxng_extra/update/update_gsa_useragents.py
|
||||
|
||||
This function is used by :py:mod:`searx.engines.google`.
|
||||
|
||||
"""
|
||||
data = lazy_globals["GSA_USER_AGENTS"]
|
||||
if data is not None:
|
||||
return data
|
||||
|
||||
log.debug("init searx.data.%s", "GSA_USER_AGENTS")
|
||||
|
||||
with open(data_dir / 'gsa_useragents.txt', encoding='utf-8') as f:
|
||||
lazy_globals["GSA_USER_AGENTS"] = f.read().splitlines()
|
||||
|
||||
return lazy_globals["GSA_USER_AGENTS"]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -773,7 +773,7 @@
|
||||
"nl": "Boliviaanse boliviano",
|
||||
"oc": "Boliviano",
|
||||
"pa": "ਬੋਲੀਵੀਆਨੋ",
|
||||
"pl": "boliviano",
|
||||
"pl": "boliwiano",
|
||||
"pt": "Boliviano",
|
||||
"ro": "boliviano",
|
||||
"ru": "боливиано",
|
||||
@@ -1107,7 +1107,7 @@
|
||||
"fi": "Kongon frangi",
|
||||
"fr": "franc congolais",
|
||||
"ga": "franc an Chongó",
|
||||
"gl": "Franco congolés",
|
||||
"gl": "franco congolés",
|
||||
"he": "פרנק קונגולזי",
|
||||
"hr": "Kongoanski franak",
|
||||
"hu": "kongói frank",
|
||||
@@ -1636,7 +1636,7 @@
|
||||
"fi": "Algerian dinaari",
|
||||
"fr": "dinar algérien",
|
||||
"ga": "dinar na hAilgéire",
|
||||
"gl": "Dinar alxeriano",
|
||||
"gl": "dinar alxeriano",
|
||||
"he": "דינר אלג'ירי",
|
||||
"hr": "Alžirski dinar",
|
||||
"hu": "algériai dinár",
|
||||
@@ -1651,7 +1651,7 @@
|
||||
"pap": "dinar argelino",
|
||||
"pl": "dinar algierski",
|
||||
"pt": "dinar argelino",
|
||||
"ro": "Dinar algerian",
|
||||
"ro": "dinar algerian",
|
||||
"ru": "алжирский динар",
|
||||
"sk": "Alžírský dinár",
|
||||
"sl": "alžirski dinar",
|
||||
@@ -1786,7 +1786,7 @@
|
||||
"cy": "Ewro",
|
||||
"da": "Euro",
|
||||
"de": "Euro",
|
||||
"en": "Euro",
|
||||
"en": "euro",
|
||||
"eo": "eŭro",
|
||||
"es": "Euro",
|
||||
"et": "Euro",
|
||||
@@ -1803,7 +1803,7 @@
|
||||
"it": "Euro",
|
||||
"ja": "ユーロ",
|
||||
"ko": "유로",
|
||||
"lt": "Euras",
|
||||
"lt": "euras",
|
||||
"lv": "eiro",
|
||||
"ml": "യൂറോ",
|
||||
"ms": "Euro",
|
||||
@@ -2759,7 +2759,7 @@
|
||||
"pa": "ਜਪਾਨੀ ਯੈੱਨ",
|
||||
"pl": "jen",
|
||||
"pt": "iene",
|
||||
"ro": "yeni",
|
||||
"ro": "yen",
|
||||
"ru": "японская иена",
|
||||
"sk": "jen",
|
||||
"sl": "japonski jen",
|
||||
@@ -3337,6 +3337,7 @@
|
||||
"fi": "Libyan dinaari",
|
||||
"fr": "dinar libyen",
|
||||
"ga": "dinar na Libia",
|
||||
"gl": "dinar libio",
|
||||
"he": "דינר לובי ",
|
||||
"hr": "Libijski dinar",
|
||||
"hu": "líbiai dinár",
|
||||
@@ -3539,6 +3540,7 @@
|
||||
"ja": "チャット",
|
||||
"ko": "미얀마 짯",
|
||||
"lt": "Kijatas",
|
||||
"lv": "Kjats",
|
||||
"ml": "ബർമ്മീസ് ക്യാറ്റ്",
|
||||
"nl": "Myanmarese kyat",
|
||||
"oc": "Kyat",
|
||||
@@ -4310,7 +4312,7 @@
|
||||
"ar": "بيسو فلبيني",
|
||||
"bg": "Филипинско песо",
|
||||
"ca": "peso filipí",
|
||||
"cs": "Filipínské peso",
|
||||
"cs": "filipínské peso",
|
||||
"de": "philippinischer Peso",
|
||||
"en": "Philippine peso",
|
||||
"eo": "filipina peso",
|
||||
@@ -4614,7 +4616,7 @@
|
||||
"fi": "Serbian dinaari",
|
||||
"fr": "dinar serbe",
|
||||
"ga": "Dinar na Seirbia",
|
||||
"gl": "Dinar serbio",
|
||||
"gl": "dinar serbio",
|
||||
"he": "דינר סרבי",
|
||||
"hr": "srpski dinar",
|
||||
"hu": "szerb dinár",
|
||||
@@ -5331,7 +5333,7 @@
|
||||
"pa": "ਤਾਜਿਕਿਸਤਾਨੀ ਸੋਮੋਨੀ",
|
||||
"pl": "Somoni",
|
||||
"pt": "Somoni",
|
||||
"ro": "Somoni tadjic",
|
||||
"ro": "somoni tadjic",
|
||||
"ru": "таджикский сомони",
|
||||
"sk": "tadžický som",
|
||||
"sl": "tadžikistanski somoni",
|
||||
@@ -5394,6 +5396,7 @@
|
||||
"fi": "Tunisian dinaari",
|
||||
"fr": "dinar tunisien",
|
||||
"ga": "dinar na Túinéise",
|
||||
"gl": "dinar tunisiano",
|
||||
"he": "דינר תוניסאי",
|
||||
"hr": "tuniski dinar",
|
||||
"hu": "tunéziai dinár",
|
||||
@@ -5766,7 +5769,7 @@
|
||||
"fi": "Uruguayn peso",
|
||||
"fr": "peso uruguayen",
|
||||
"ga": "peso Uragua",
|
||||
"gl": "Peso uruguaio",
|
||||
"gl": "peso uruguaio",
|
||||
"he": "פסו של אורוגוואי",
|
||||
"hr": "Urugvajski pezo",
|
||||
"hu": "uruguayi peso",
|
||||
@@ -6147,7 +6150,7 @@
|
||||
"oc": "Drechs de tiratge Especials",
|
||||
"pl": "specjalne prawa ciągnienia",
|
||||
"pt": "direitos especiais de saque",
|
||||
"ro": "Drepturi speciale de tragere",
|
||||
"ro": "drepturi speciale de tragere",
|
||||
"ru": "специальные права заимствования",
|
||||
"sk": "Zvláštne práva čerpania",
|
||||
"sl": "posebne pravice črpanja",
|
||||
@@ -6225,6 +6228,7 @@
|
||||
"ja": "CFPフラン",
|
||||
"ko": "CFP 프랑",
|
||||
"lt": "CFP frankas",
|
||||
"lv": "Klusā okeāna franks",
|
||||
"ms": "Franc CFP",
|
||||
"nl": "CFP-frank",
|
||||
"oc": "Franc CFP",
|
||||
@@ -7055,6 +7059,7 @@
|
||||
"bolivjano": "BOB",
|
||||
"bolivya bolivianosu": "BOB",
|
||||
"bolivya bolivyanosu": "BOB",
|
||||
"boliwiano": "BOB",
|
||||
"bolívar digital": "VED",
|
||||
"bolívar soberano": "VES",
|
||||
"bolívar sobirà": "VES",
|
||||
@@ -8425,6 +8430,7 @@
|
||||
"drame arménio": "AMD",
|
||||
"dramm": "AMD",
|
||||
"drechs de tiratge especials": "XDR",
|
||||
"drept special de tragere": "XDR",
|
||||
"drepturi speciale de tragere": "XDR",
|
||||
"drets especials de gir": "XDR",
|
||||
"droits de tirage speciaux": "XDR",
|
||||
@@ -9518,6 +9524,8 @@
|
||||
"kíp lào": "LAK",
|
||||
"kīp": "LAK",
|
||||
"kjat": "MMK",
|
||||
"kjats": "MMK",
|
||||
"klusā okeāna franks": "XPF",
|
||||
"km": "BAM",
|
||||
"kmf": "KMF",
|
||||
"koeweitse dinar": "KWD",
|
||||
@@ -10023,6 +10031,7 @@
|
||||
"lire sterline": "GBP",
|
||||
"lire turque": "TRY",
|
||||
"lisente": "LSL",
|
||||
"list of syrian coins": "SYP",
|
||||
"liura de gibartar": "GIP",
|
||||
"liura egipciana": "EGP",
|
||||
"liura esterlina": "GBP",
|
||||
@@ -11313,7 +11322,6 @@
|
||||
"riel camboxano": "KHR",
|
||||
"riel camboyano": "KHR",
|
||||
"riel campuchia": "KHR",
|
||||
"riel kambodżański": "KHR",
|
||||
"riel kamboja": "KHR",
|
||||
"riel na cambóide": "KHR",
|
||||
"rietumāfrikas franks": "XOF",
|
||||
@@ -12257,6 +12265,7 @@
|
||||
"thaise baht": "THB",
|
||||
"thajský baht": "THB",
|
||||
"thb": "THB",
|
||||
"the australian dollar": "AUD",
|
||||
"thebe": "BWP",
|
||||
"third belarusian ruble": "BYN",
|
||||
"tical": "THB",
|
||||
@@ -12740,6 +12749,7 @@
|
||||
"yen": "JPY",
|
||||
"yen giapponese": "JPY",
|
||||
"yen japones": "JPY",
|
||||
"yen japonez": "JPY",
|
||||
"yen japonés": "JPY",
|
||||
"yeni": "JPY",
|
||||
"yeni i̇srail şekeli": "ILS",
|
||||
@@ -12806,7 +12816,6 @@
|
||||
"zimbabwe zig": "ZWG",
|
||||
"zimbabwean dollar": "ZWL",
|
||||
"zimbabwean gold": "ZWG",
|
||||
"zimbabwean zig": "ZWG",
|
||||
"zimbabwen kulta": "ZWG",
|
||||
"zimbabwiansky zlatý": "ZWG",
|
||||
"zimbabwský dolar": "ZWL",
|
||||
@@ -12965,6 +12974,7 @@
|
||||
"FKP",
|
||||
"EGP"
|
||||
],
|
||||
"£S": "SYP",
|
||||
"£e": "EGP",
|
||||
"£s": "SYP",
|
||||
"¥": [
|
||||
@@ -14959,6 +14969,8 @@
|
||||
"فورنت مجري": "HUF",
|
||||
"فورينت مجري": "HUF",
|
||||
"فِرَنْكٌ رُوَنْدِيٌّ": "RWF",
|
||||
"قائمة النقود المعدنية السورية": "SYP",
|
||||
"قائمة عملات سوريا المعدنية": "SYP",
|
||||
"ك": "KWD",
|
||||
"كتزال غواتيمالي": "GTQ",
|
||||
"كرونة آيسلندية": "ISK",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -5,7 +5,7 @@
|
||||
],
|
||||
"ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}",
|
||||
"versions": [
|
||||
"151.0",
|
||||
"150.0"
|
||||
"152.0",
|
||||
"151.0"
|
||||
]
|
||||
}
|
||||
@@ -3272,7 +3272,7 @@
|
||||
"Q128822": {
|
||||
"si_name": "Q182429",
|
||||
"symbol": "kn",
|
||||
"to_si_factor": 0.5144444444444445
|
||||
"to_si_factor": 0.514
|
||||
},
|
||||
"Q12912288": {
|
||||
"si_name": null,
|
||||
|
||||
@@ -41,7 +41,7 @@ if t.TYPE_CHECKING:
|
||||
from searx.enginelib.traits import EngineTraits
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.result_types import EngineResults
|
||||
from searx.search.processors import OfflineParamTypes, OnlineParamTypes
|
||||
from searx.search.processors import OfflineParamTypes, OnlineParamTypes, ProcessorType
|
||||
|
||||
ENGINES_CACHE: ExpireCacheSQLite = ExpireCacheSQLite.build_cache(
|
||||
ExpireCacheCfg(
|
||||
@@ -180,7 +180,7 @@ class EngineCache:
|
||||
return ENGINES_CACHE.secret_hash(name=name)
|
||||
|
||||
|
||||
class EngineAbout(msgspec.Struct):
|
||||
class EngineAbout(msgspec.Struct, kw_only=True):
|
||||
"""Additional fields describing the engine.
|
||||
|
||||
.. code:: yaml
|
||||
@@ -207,19 +207,22 @@ class EngineAbout(msgspec.Struct):
|
||||
|
||||
use_official_api: bool = False
|
||||
"""SearXNG engine makes use of the official API or not"""
|
||||
|
||||
require_api_key: bool = False
|
||||
"""API requires a key or not."""
|
||||
|
||||
results: str = ""
|
||||
"""Data format of the source (online-engines: of the response)."""
|
||||
|
||||
language: str = ""
|
||||
"""If the engine supports only one language, this language is specified
|
||||
here (``en``, ``de``, ``"no"`` or ..); otherwise, the value remains empty.
|
||||
description: str = ""
|
||||
"""Brief description of the engine and where it gets its data from.
|
||||
|
||||
For the YAML configuration: think of the `YAML-Norway problem
|
||||
<https://ruuda.nl/2023/the-yaml-document-from-hell#the-norway-problem>`_
|
||||
"""
|
||||
This value should only be set as long as no description of the data source
|
||||
is available via a :py:obj:`EngineAbout.wikidata_id`."""
|
||||
|
||||
language: str = ""
|
||||
"""Deprecated! Migrate your setting from `engine.about.language` to
|
||||
`engine.language`"""
|
||||
|
||||
|
||||
class Engine(abc.ABC): # pylint: disable=too-few-public-methods
|
||||
@@ -227,6 +230,8 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
|
||||
|
||||
Further documentation see :ref:`general engine configuration`.
|
||||
|
||||
The defaults are taken from :py:obj:`searx.engines.ENGINE_DEFAULT_ARGS`.
|
||||
|
||||
.. hint::
|
||||
|
||||
This class is currently never initialized and only used for type hinting.
|
||||
@@ -234,49 +239,27 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
|
||||
|
||||
logger: logging.Logger
|
||||
|
||||
# Common options in the engine module
|
||||
# Common options of the engine module
|
||||
|
||||
engine_type: str
|
||||
engine_type: "ProcessorType" = "online"
|
||||
"""Type of the engine (:ref:`searx.search.processors`)"""
|
||||
|
||||
paging: bool
|
||||
paging: bool = False
|
||||
"""Engine supports multiple pages."""
|
||||
|
||||
max_page: int = 0
|
||||
"""If the engine supports paging, then this is the value for the last page
|
||||
that is still supported. ``0`` means unlimited numbers of pages."""
|
||||
|
||||
time_range_support: bool
|
||||
time_range_support: bool = False
|
||||
"""Engine supports search time range."""
|
||||
|
||||
safesearch: bool
|
||||
safesearch: bool = False
|
||||
"""Engine supports SafeSearch"""
|
||||
|
||||
language_support: bool
|
||||
language_support: bool = False
|
||||
"""Engine supports languages (locales) search."""
|
||||
|
||||
language: str
|
||||
"""For an engine, when there is ``language: ...`` in the YAML settings the engine
|
||||
does support only this one language:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: google french
|
||||
engine: google
|
||||
language: fr
|
||||
"""
|
||||
|
||||
region: str
|
||||
"""For an engine, when there is ``region: ...`` in the YAML settings the engine
|
||||
does support only this one region::
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: google belgium
|
||||
engine: google
|
||||
region: fr-BE
|
||||
"""
|
||||
|
||||
fetch_traits: "Callable[[EngineTraits, bool], None]"
|
||||
"""Function to to fetch engine's traits from origin."""
|
||||
|
||||
@@ -285,9 +268,6 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
|
||||
|
||||
# settings.yml
|
||||
|
||||
categories: list[str]
|
||||
"""Specifies to which :ref:`engine categories` the engine should be added."""
|
||||
|
||||
name: str
|
||||
"""Name that will be used across SearXNG to define this engine. In settings, on
|
||||
the result page .."""
|
||||
@@ -297,6 +277,43 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
|
||||
this search engine (file name from :origin:`searx/engines` without
|
||||
``.py``)."""
|
||||
|
||||
categories: list[str] = ["general"]
|
||||
"""Specifies to which :ref:`engine categories` the engine should be added."""
|
||||
|
||||
language: str = ""
|
||||
"""If the engine supports only one language, this language is specified here
|
||||
(``en``, ``de``, ``"no"`` or ..); otherwise, the value remains empty. For
|
||||
the YAML configuration: think of the `YAML-Norway problem
|
||||
<https://ruuda.nl/2023/the-yaml-document-from-hell#the-norway-problem>`_
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: google norway
|
||||
engine: google
|
||||
language: "no"
|
||||
|
||||
Depending on ``language_support``, this value has similar but also slightly
|
||||
different meanings.
|
||||
|
||||
- When ``language_support`` is **true**, the map of
|
||||
:py:obj:`traits.EngineTraits.languages` is reduced to the selected
|
||||
language
|
||||
|
||||
- When ``language_support`` is **false**, then the implementation of the
|
||||
engine only supports this one ``language``
|
||||
"""
|
||||
|
||||
region: str = ""
|
||||
"""For an engine, when there is ``region: ...`` in the YAML settings the engine
|
||||
does support only this one region::
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: google belgium
|
||||
engine: google
|
||||
region: fr-BE
|
||||
"""
|
||||
|
||||
enable_http: bool
|
||||
"""Enable HTTP (by default only HTTPS is enabled)."""
|
||||
|
||||
@@ -309,6 +326,31 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
|
||||
display_error_messages: bool
|
||||
"""Display error messages on the web UI."""
|
||||
|
||||
disabled: bool = False
|
||||
"""To disable by default the engine, but not deleting it. It will allow the
|
||||
user to manually activate it in the settings."""
|
||||
|
||||
inactive: bool = False
|
||||
"""Remove the engine from the settings (*disabled & removed*)."""
|
||||
|
||||
about: EngineAbout = EngineAbout()
|
||||
"""Additional fields describing the engine."""
|
||||
|
||||
using_tor_proxy: bool = False
|
||||
"""Using tor proxy (``true``) or not (``false``) for this engine."""
|
||||
|
||||
send_accept_language_header: bool = True
|
||||
"""When this option is activated (default), the language (locale) that is
|
||||
selected by the user is used to build and send a ``Accept-Language`` header
|
||||
in the request to the origin search engine."""
|
||||
|
||||
tokens: list[str] = []
|
||||
"""A list of secret tokens to make this engine *private*, more details see
|
||||
:ref:`private engines`."""
|
||||
|
||||
weight: float = 1.0
|
||||
"""Weighting of the results of this engine (:ref:`weight <settings engines>`)."""
|
||||
|
||||
proxies: dict[str, dict[str, str]]
|
||||
"""Set proxies for a specific engine (YAML):
|
||||
|
||||
@@ -319,62 +361,43 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
|
||||
https: socks5://proxy:port
|
||||
"""
|
||||
|
||||
disabled: bool
|
||||
"""To disable by default the engine, but not deleting it. It will allow the
|
||||
user to manually activate it in the settings."""
|
||||
|
||||
inactive: bool
|
||||
"""Remove the engine from the settings (*disabled & removed*)."""
|
||||
|
||||
about: EngineAbout
|
||||
"""Additional fields describing the engine."""
|
||||
|
||||
using_tor_proxy: bool
|
||||
"""Using tor proxy (``true``) or not (``false``) for this engine."""
|
||||
|
||||
send_accept_language_header: bool
|
||||
"""When this option is activated (default), the language (locale) that is
|
||||
selected by the user is used to build and send a ``Accept-Language`` header
|
||||
in the request to the origin search engine."""
|
||||
|
||||
tokens: list[str]
|
||||
"""A list of secret tokens to make this engine *private*, more details see
|
||||
:ref:`private engines`."""
|
||||
|
||||
weight: int
|
||||
"""Weighting of the results of this engine (:ref:`weight <settings engines>`)."""
|
||||
|
||||
def setup(self, engine_settings: dict[str, t.Any]) -> bool: # pylint: disable=unused-argument
|
||||
def setup(self, engine_settings: dict[str, t.Any]) -> bool | None: # pylint: disable=unused-argument
|
||||
"""Dynamic setup of the engine settings.
|
||||
|
||||
With this method, the engine's setup is carried out. For example, to
|
||||
check or dynamically adapt the values handed over in the parameter
|
||||
``engine_settings``. The return value (True/False) indicates whether
|
||||
the setup was successful and the engine can be built or rejected.
|
||||
``engine_settings``.
|
||||
|
||||
The method is optional and is called synchronously as part of the
|
||||
Whether the initialization was successful can be indicated by the return
|
||||
value ``True`` or even ``False``.
|
||||
|
||||
- If no return value (``None`` ) is given from this method , this is
|
||||
equivalent to ``True``.
|
||||
|
||||
- If an exception is thrown as part of the initialization, this is
|
||||
equivalent to ``False``.
|
||||
|
||||
The method is optional and is called **synchronously** as part of the
|
||||
initialization of the service and is therefore only suitable for simple
|
||||
(local) exams/changes at the engine setting. The :py:obj:`Engine.init`
|
||||
method must be used for longer tasks in which values of a remote must be
|
||||
determined, for example.
|
||||
(local) exams/changes at the engine setting.
|
||||
|
||||
The :py:obj:`Engine.init` method must be used for longer tasks in which
|
||||
values of a remote must be determined, for example.
|
||||
"""
|
||||
return True
|
||||
|
||||
def init(self, engine_settings: dict[str, t.Any]) -> bool | None: # pylint: disable=unused-argument
|
||||
"""Initialization of the engine.
|
||||
|
||||
The method is optional and asynchronous (in a thread). It is suitable,
|
||||
for example, for setting up a cache (for the engine) or for querying
|
||||
values (required by the engine) from a remote.
|
||||
The method is optional and called **asynchronous** (in a thread). The
|
||||
method is comparable to :py:obj:`Engine.setup`, it is suitable, for
|
||||
caching data that first needs to be requested from a remote.
|
||||
|
||||
Whether the initialization was successful can be indicated by the return
|
||||
value ``True`` or even ``False``.
|
||||
The method is optional and runs **asynchronously** (in a thread), it is
|
||||
comparable to :py:obj:`Engine.setup`. For instance, it is suitable for
|
||||
caching data that first needs to be requested from a remote source.
|
||||
|
||||
- If no return value is given from this init method (``None``), this is
|
||||
equivalent to ``True``.
|
||||
|
||||
- If an exception is thrown as part of the initialization, this is
|
||||
equivalent to ``False``.
|
||||
The evaluation of the return value is analogous to :py:obj:`Engine.setup`.
|
||||
"""
|
||||
return True
|
||||
|
||||
|
||||
@@ -142,11 +142,11 @@ class EngineTraits:
|
||||
"""
|
||||
|
||||
if self.data_type == "traits_v1":
|
||||
self._set_traits_v1(engine)
|
||||
self._set_traits_v1(engine) # pyright: ignore[reportArgumentType]
|
||||
else:
|
||||
raise TypeError("engine traits of type %s is unknown" % self.data_type)
|
||||
|
||||
def _set_traits_v1(self, engine: "Engine | types.ModuleType") -> None:
|
||||
def _set_traits_v1(self, engine: "Engine") -> None:
|
||||
# For an engine, when there is `language: ...` in the YAML settings the engine
|
||||
# does support only this one language (region)::
|
||||
#
|
||||
@@ -159,22 +159,25 @@ class EngineTraits:
|
||||
|
||||
_msg = "settings.yml - engine: '%s' / %s: '%s' not supported"
|
||||
|
||||
languages = traits.languages
|
||||
if hasattr(engine, "language"):
|
||||
if engine.language not in languages:
|
||||
raise ValueError(_msg % (engine.name, "language", engine.language))
|
||||
traits.languages = {engine.language: languages[engine.language]}
|
||||
if engine.language:
|
||||
if engine.language_support:
|
||||
if not len(traits.languages) > 1:
|
||||
raise ValueError(
|
||||
f"engine {engine.name}: activated language_support with just one or less languages"
|
||||
)
|
||||
if engine.language not in traits.languages:
|
||||
raise ValueError(_msg % (engine.name, "language", engine.language))
|
||||
traits.languages = {engine.language: traits.languages[engine.language]}
|
||||
|
||||
regions = traits.regions
|
||||
if hasattr(engine, "region"):
|
||||
if engine.region not in regions:
|
||||
if engine.region:
|
||||
if engine.region not in traits.regions:
|
||||
raise ValueError(_msg % (engine.name, "region", engine.region))
|
||||
traits.regions = {engine.region: regions[engine.region]}
|
||||
traits.regions = {engine.region: traits.regions[engine.region]}
|
||||
|
||||
engine.language_support = bool(traits.languages or traits.regions)
|
||||
|
||||
# set the copied & modified traits in engine's namespace
|
||||
engine.traits = traits # pyright: ignore[reportAttributeAccessIssue]
|
||||
engine.traits = traits
|
||||
|
||||
|
||||
class EngineTraitsMap(dict[str, EngineTraits]):
|
||||
|
||||
@@ -22,8 +22,8 @@ about = {
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "HTML",
|
||||
"language": "zh",
|
||||
}
|
||||
language = "zh"
|
||||
|
||||
# Engine Configuration
|
||||
categories = ["general"]
|
||||
|
||||
@@ -5,19 +5,19 @@ intended monkey patching of the engine modules.
|
||||
.. attention::
|
||||
|
||||
Monkey-patching modules is a practice from the past that shouldn't be
|
||||
expanded upon. In the long run, there should be an engine class that can be
|
||||
inherited. However, as long as this class doesn't exist, and as long as all
|
||||
engine modules aren't converted to an engine class, these builtin types will
|
||||
still be needed.
|
||||
expanded upon. In the long run, engines should be instances of
|
||||
:py:obj:`searx.enginelib.Engine`. However, as long as long as all engine
|
||||
modules aren't converted to this class, these builtin types will still be
|
||||
needed.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from searx.enginelib import traits as _traits
|
||||
|
||||
logger: logging.Logger
|
||||
supported_languages: str
|
||||
language_aliases: str
|
||||
language_support: bool
|
||||
language: str
|
||||
region: str
|
||||
traits: _traits.EngineTraits
|
||||
|
||||
# from searx.engines.ENGINE_DEFAULT_ARGS
|
||||
|
||||
@@ -14,40 +14,48 @@ import sys
|
||||
import copy
|
||||
import os
|
||||
from os.path import realpath, dirname
|
||||
import warnings
|
||||
|
||||
import types
|
||||
import inspect
|
||||
import msgspec
|
||||
|
||||
from searx import logger, settings
|
||||
from searx.utils import load_module
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.enginelib import Engine
|
||||
from searx.data import ENGINE_TRAITS
|
||||
from searx.enginelib import Engine, EngineAbout
|
||||
|
||||
logger = logger.getChild('engines')
|
||||
ENGINE_DIR = dirname(realpath(__file__))
|
||||
|
||||
# Defaults for the namespace of an engine module, see load_engine()
|
||||
ENGINE_DEFAULT_ARGS: dict[str, int | str | list[t.Any] | dict[str, t.Any] | bool] = {
|
||||
ENGINE_DEFAULT_ARGS: dict[str, t.Any] = {
|
||||
# Common options in the engine module
|
||||
"engine_type": "online",
|
||||
"paging": False,
|
||||
"max_page": 0,
|
||||
"time_range_support": False,
|
||||
"safesearch": False,
|
||||
"language_support": False,
|
||||
# settings.yml
|
||||
"categories": ["general"],
|
||||
"language": "",
|
||||
"region": "",
|
||||
"enable_http": False,
|
||||
"shortcut": "-",
|
||||
"timeout": settings["outgoing"]["request_timeout"],
|
||||
"display_error_messages": True,
|
||||
"disabled": False,
|
||||
"inactive": False,
|
||||
"about": {},
|
||||
"about": EngineAbout(),
|
||||
"using_tor_proxy": False,
|
||||
"send_accept_language_header": True,
|
||||
"tokens": [],
|
||||
"max_page": 0,
|
||||
"weight": 1.0,
|
||||
}
|
||||
"""Default values that are set in an engine of type *module*, please compare
|
||||
with the class :py:obj:`searx.enginelib.Engine`."""
|
||||
|
||||
# set automatically when an engine does not have any tab category
|
||||
DEFAULT_CATEGORY = 'other'
|
||||
|
||||
@@ -177,14 +185,41 @@ def set_loggers(engine: "Engine|types.ModuleType", engine_name: str):
|
||||
|
||||
|
||||
def update_engine_attributes(engine: "Engine | types.ModuleType", engine_data: dict[str, t.Any]):
|
||||
# pylint: disable=too-many-branches
|
||||
|
||||
# set engine attributes from engine_data
|
||||
kvargs: dict[str, t.Any]
|
||||
if isinstance(engine.about, EngineAbout):
|
||||
kvargs = {**msgspec.to_builtins(engine.about), **engine_data.get("about", {})}
|
||||
else:
|
||||
kvargs = {**engine.about, **engine_data.get("about", {})}
|
||||
|
||||
try:
|
||||
engine.about = EngineAbout(**kvargs)
|
||||
except TypeError as exc:
|
||||
raise TypeError(
|
||||
f"engine '{engine_data['name']}' ({engine_data['engine']}) - in the about section --> {exc}"
|
||||
) from exc
|
||||
|
||||
# warn about deprecated engine settings
|
||||
|
||||
if engine.about.language:
|
||||
if hasattr(engine, "language") and not engine.language:
|
||||
engine.language = engine.about.language
|
||||
warnings.warn(
|
||||
f"engine '{engine_data['name']}' ({engine_data['engine']})"
|
||||
f" - migrate engine.about.language to engine.language!",
|
||||
DeprecationWarning,
|
||||
2,
|
||||
)
|
||||
|
||||
for param_name, param_value in engine_data.items():
|
||||
if param_name == "about":
|
||||
continue
|
||||
if param_name == 'categories':
|
||||
if isinstance(param_value, str):
|
||||
param_value = list(map(str.strip, param_value.split(',')))
|
||||
engine.categories = param_value # type: ignore
|
||||
elif hasattr(engine, 'about') and param_name == 'about':
|
||||
engine.about = {**engine.about, **engine_data['about']} # type: ignore
|
||||
else:
|
||||
setattr(engine, param_name, param_value)
|
||||
|
||||
@@ -193,6 +228,9 @@ def update_engine_attributes(engine: "Engine | types.ModuleType", engine_data: d
|
||||
if not hasattr(engine, arg_name):
|
||||
setattr(engine, arg_name, copy.deepcopy(arg_value))
|
||||
|
||||
if ENGINE_TRAITS.get(engine.name, {}).get("languages") and not engine.language_support:
|
||||
raise ValueError(f"engine '{engine.name}' ({engine_data['engine']}) language_support should be set to True")
|
||||
|
||||
|
||||
def update_attributes_for_tor(engine: "Engine | types.ModuleType"):
|
||||
if using_tor_proxy(engine) and hasattr(engine, 'onion_url'):
|
||||
@@ -231,21 +269,27 @@ def is_engine_active(engine: "Engine | types.ModuleType"):
|
||||
|
||||
|
||||
def call_engine_setup(engine: "Engine | types.ModuleType", engine_data: dict[str, t.Any]) -> bool:
|
||||
setup_ok = False
|
||||
|
||||
setup_ok: bool | None = False
|
||||
setup_func = getattr(engine, "setup", None)
|
||||
|
||||
if setup_func is None:
|
||||
setup_ok = True
|
||||
elif not callable(setup_func):
|
||||
logger.error("engine's setup method isn't a callable (is of type: %s)", type(setup_func))
|
||||
logger.error(f"engine's setup method isn't a callable (is of type: {type(setup_func)})")
|
||||
else:
|
||||
try:
|
||||
setup_ok = engine.setup(engine_data)
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
logger.exception('exception : {0}'.format(e))
|
||||
logger.exception(f"(PID {os.getpid()}) {engine.name}: engine SETUP failed, exception: {e}")
|
||||
setup_ok = False
|
||||
|
||||
# The evaluation of the return value is analogous to Engine.init
|
||||
if setup_ok is None:
|
||||
setup_ok = True
|
||||
|
||||
if not setup_ok:
|
||||
logger.error("%s: Engine setup was not successful, engine is set to inactive.", engine.name)
|
||||
logger.error(f"(PID {os.getpid()}) {engine.name}: engine setup was not successful")
|
||||
return setup_ok
|
||||
|
||||
|
||||
@@ -273,14 +317,16 @@ def load_engines(engine_list: list[dict[str, t.Any]]):
|
||||
for engine_data in engine_list:
|
||||
if engine_data.get("inactive") is True:
|
||||
continue
|
||||
|
||||
engine = load_engine(engine_data)
|
||||
|
||||
if engine:
|
||||
register_engine(engine)
|
||||
else:
|
||||
# if an engine can't be loaded (if for example the engine is missing
|
||||
# tor or some other requirements) its set to inactive!
|
||||
logger.error(
|
||||
f"(PID {os.getpid()}) loading engine %s failed: set engine to inactive!", engine_data.get("name", "???")
|
||||
f"(PID {os.getpid()}) {engine_data.get('name', '???')}: can't register engine (loading engine failed)"
|
||||
)
|
||||
engine_data["inactive"] = True
|
||||
return engines
|
||||
|
||||
@@ -16,12 +16,12 @@ about = {
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "HTML",
|
||||
"language": "zh",
|
||||
}
|
||||
|
||||
# Engine Configuration
|
||||
categories = ["videos"]
|
||||
paging = True
|
||||
language = "zh"
|
||||
|
||||
# Base URL
|
||||
base_url = "https://www.acfun.cn"
|
||||
@@ -83,7 +83,7 @@ def extract_video_data(video_block):
|
||||
published_date = None
|
||||
if create_time:
|
||||
try:
|
||||
published_date = datetime.strptime(create_time.strip(), "%Y-%m-%d")
|
||||
published_date = datetime.fromisoformat(create_time.strip())
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
|
||||
@@ -64,6 +64,7 @@ about: dict[str, t.Any] = {
|
||||
# engine dependent config
|
||||
categories = ["files", "books"]
|
||||
paging: bool = True
|
||||
language_support = True
|
||||
|
||||
# search-url
|
||||
base_url: list[str] | str = []
|
||||
|
||||
@@ -42,8 +42,8 @@ about = {
|
||||
'use_official_api': False,
|
||||
'require_api_key': False,
|
||||
'results': 'HTML',
|
||||
'language': 'it',
|
||||
}
|
||||
language = "it"
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
||||
@@ -1,210 +0,0 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""AOL supports WEB, image, and video search. Internally, it uses the Bing
|
||||
index.
|
||||
|
||||
AOL doesn't seem to support setting the language via request parameters, instead
|
||||
the results are based on the URL. For example, there is
|
||||
|
||||
- `search.aol.com <https://search.aol.com>`_ for English results
|
||||
- `suche.aol.de <https://suche.aol.de>`_ for German results
|
||||
|
||||
However, AOL offers its services only in a few regions:
|
||||
|
||||
- en-US: search.aol.com
|
||||
- de-DE: suche.aol.de
|
||||
- fr-FR: recherche.aol.fr
|
||||
- en-GB: search.aol.co.uk
|
||||
- en-CA: search.aol.ca
|
||||
|
||||
In order to still offer sufficient support for language and region, the `search
|
||||
keywords`_ known from Bing, ``language`` and ``loc`` (region), are added to the
|
||||
search term (AOL is basically just a proxy for Bing).
|
||||
|
||||
.. _search keywords:
|
||||
https://support.microsoft.com/en-us/topic/advanced-search-keywords-ea595928-5d63-4a0b-9c6b-0b769865e78a
|
||||
|
||||
"""
|
||||
|
||||
from urllib.parse import urlencode, unquote_plus
|
||||
import typing as t
|
||||
|
||||
from lxml import html
|
||||
from dateutil import parser
|
||||
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import eval_xpath_list, eval_xpath, extract_text
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
about = {
|
||||
"website": "https://www.aol.com",
|
||||
"wikidata_id": "Q27585",
|
||||
"official_api_documentation": None,
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "HTML",
|
||||
}
|
||||
|
||||
categories = ["general"]
|
||||
search_type = "search" # supported: search, image, video
|
||||
|
||||
paging = True
|
||||
safesearch = True
|
||||
time_range_support = True
|
||||
results_per_page = 10
|
||||
|
||||
|
||||
base_url = "https://search.aol.com"
|
||||
time_range_map = {"day": "1d", "week": "1w", "month": "1m", "year": "1y"}
|
||||
safesearch_map = {0: "p", 1: "r", 2: "i"}
|
||||
|
||||
enable_http2 = False
|
||||
|
||||
|
||||
def init(_):
|
||||
if search_type not in ("search", "image", "video"):
|
||||
raise ValueError(f"unsupported search type {search_type}")
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
|
||||
language, region = (params["searxng_locale"].split("-") + [None])[:2]
|
||||
if language and language != "all":
|
||||
query = f"{query} language:{language}"
|
||||
if region:
|
||||
query = f"{query} loc:{region}"
|
||||
|
||||
args: dict[str, str | int | None] = {
|
||||
"q": query,
|
||||
"b": params["pageno"] * results_per_page + 1, # page is 1-indexed
|
||||
"pz": results_per_page,
|
||||
}
|
||||
|
||||
if params["time_range"]:
|
||||
args["fr2"] = "time"
|
||||
args["age"] = params["time_range"]
|
||||
else:
|
||||
args["fr2"] = "sb-top-search"
|
||||
|
||||
params["cookies"]["sB"] = f"vm={safesearch_map[params['safesearch']]}"
|
||||
params["url"] = f"{base_url}/aol/{search_type}?{urlencode(args)}"
|
||||
logger.debug(params)
|
||||
|
||||
|
||||
def _deobfuscate_url(obfuscated_url: str) -> str | None:
|
||||
# URL looks like "https://search.aol.com/click/_ylt=AwjFSDjd;_ylu=JfsdjDFd/RV=2/RE=1774058166/RO=10/RU=https%3a%2f%2fen.wikipedia.org%2fwiki%2fTree/RK=0/RS=BP2CqeMLjscg4n8cTmuddlEQA2I-" # pylint: disable=line-too-long
|
||||
if not obfuscated_url:
|
||||
return None
|
||||
|
||||
for part in obfuscated_url.split("/"):
|
||||
if part.startswith("RU="):
|
||||
return unquote_plus(part[3:])
|
||||
# pattern for de-obfuscating URL not found, fall back to Yahoo's tracking link
|
||||
return obfuscated_url
|
||||
|
||||
|
||||
def _general_results(doc: html.HtmlElement) -> EngineResults:
|
||||
res = EngineResults()
|
||||
|
||||
for result in eval_xpath_list(doc, "//div[@id='web']//ol/li[not(contains(@class, 'first'))]"):
|
||||
obfuscated_url = extract_text(eval_xpath(result, ".//h3/a/@href"))
|
||||
if not obfuscated_url:
|
||||
continue
|
||||
|
||||
url = _deobfuscate_url(obfuscated_url)
|
||||
if not url:
|
||||
continue
|
||||
|
||||
res.add(
|
||||
res.types.MainResult(
|
||||
url=url,
|
||||
title=extract_text(eval_xpath(result, ".//h3/a")) or "",
|
||||
content=extract_text(eval_xpath(result, ".//div[contains(@class, 'compText')]")) or "",
|
||||
thumbnail=extract_text(eval_xpath(result, ".//a[contains(@class, 'thm')]/img/@data-src")) or "",
|
||||
)
|
||||
)
|
||||
return res
|
||||
|
||||
|
||||
def _video_results(doc: html.HtmlElement) -> EngineResults:
|
||||
res = EngineResults()
|
||||
|
||||
for result in eval_xpath_list(doc, "//div[contains(@class, 'results')]//ol/li"):
|
||||
obfuscated_url = extract_text(eval_xpath(result, ".//a/@href"))
|
||||
if not obfuscated_url:
|
||||
continue
|
||||
|
||||
url = _deobfuscate_url(obfuscated_url)
|
||||
if not url:
|
||||
continue
|
||||
|
||||
published_date_raw = extract_text(eval_xpath(result, ".//div[contains(@class, 'v-age')]"))
|
||||
try:
|
||||
published_date = parser.parse(published_date_raw or "")
|
||||
except parser.ParserError:
|
||||
published_date = None
|
||||
|
||||
res.add(
|
||||
res.types.LegacyResult(
|
||||
{
|
||||
"template": "videos.html",
|
||||
"url": url,
|
||||
"title": extract_text(eval_xpath(result, ".//h3")),
|
||||
"content": extract_text(eval_xpath(result, ".//div[contains(@class, 'compText')]")),
|
||||
"thumbnail": extract_text(eval_xpath(result, ".//img[contains(@class, 'thm')]/@src")),
|
||||
"length": extract_text(eval_xpath(result, ".//span[contains(@class, 'v-time')]")),
|
||||
"publishedDate": published_date,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
def _image_results(doc: html.HtmlElement) -> EngineResults:
|
||||
res = EngineResults()
|
||||
|
||||
for result in eval_xpath_list(doc, "//section[@id='results']//ul/li"):
|
||||
obfuscated_url = extract_text(eval_xpath(result, "./a/@href"))
|
||||
if not obfuscated_url:
|
||||
continue
|
||||
|
||||
url = _deobfuscate_url(obfuscated_url)
|
||||
if not url:
|
||||
continue
|
||||
|
||||
res.add(
|
||||
res.types.LegacyResult(
|
||||
{
|
||||
"template": "images.html",
|
||||
# results don't have an extra URL, only the image source
|
||||
"url": url,
|
||||
"title": extract_text(eval_xpath(result, ".//a/@aria-label")),
|
||||
"thumbnail_src": extract_text(eval_xpath(result, ".//img/@src")),
|
||||
"img_src": url,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
doc = html.fromstring(resp.text)
|
||||
|
||||
match search_type:
|
||||
case "search":
|
||||
results = _general_results(doc)
|
||||
case "image":
|
||||
results = _image_results(doc)
|
||||
case "video":
|
||||
results = _video_results(doc)
|
||||
case _:
|
||||
raise ValueError("unsupported search type")
|
||||
|
||||
for suggestion in eval_xpath_list(doc, ".//ol[contains(@class, 'searchRightBottom')]//table//a"):
|
||||
results.add(results.types.LegacyResult({"suggestion": extract_text(suggestion)}))
|
||||
|
||||
return results
|
||||
@@ -35,6 +35,7 @@ about = {
|
||||
categories = ["it", "software wikis"]
|
||||
paging = True
|
||||
main_wiki = "wiki.archlinux.org"
|
||||
language_support = True
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
||||
@@ -109,7 +109,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
comments_elements = eval_xpath_getindex(entry, xpath_comment, 0, default=None)
|
||||
comments: str = "" if comments_elements is None else comments_elements.text
|
||||
|
||||
publishedDate = datetime.strptime(eval_xpath_getindex(entry, xpath_published, 0).text, "%Y-%m-%dT%H:%M:%SZ")
|
||||
publishedDate = datetime.fromisoformat(eval_xpath_getindex(entry, xpath_published, 0).text.rstrip("Z"))
|
||||
|
||||
res.add(
|
||||
res.types.Paper(
|
||||
|
||||
@@ -54,8 +54,8 @@ about = {
|
||||
"use_official_api": True,
|
||||
"require_api_key": True,
|
||||
"results": "JSON",
|
||||
"language": "en",
|
||||
}
|
||||
language = "en"
|
||||
|
||||
CACHE: EngineCache
|
||||
"""Persistent (SQLite) key/value cache that deletes its values after ``expire``
|
||||
|
||||
@@ -7,13 +7,17 @@
|
||||
# There exits a https://github.com/ohblue/baidu-serp-api/
|
||||
# but we don't use it here (may we can learn from).
|
||||
|
||||
import typing as t
|
||||
|
||||
from urllib.parse import urlencode
|
||||
from datetime import datetime
|
||||
from html import unescape
|
||||
import time
|
||||
import json
|
||||
|
||||
from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException
|
||||
from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException, SearxEngineAccessDeniedException
|
||||
from searx.enginelib import EngineCache
|
||||
from searx.network import get as http_get
|
||||
from searx.utils import html_to_text
|
||||
|
||||
about = {
|
||||
@@ -23,8 +27,8 @@ about = {
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
"language": "zh",
|
||||
}
|
||||
language = "zh"
|
||||
|
||||
paging = True
|
||||
categories = []
|
||||
@@ -35,6 +39,31 @@ baidu_category = 'general'
|
||||
time_range_support = True
|
||||
time_range_dict = {"day": 86400, "week": 604800, "month": 2592000, "year": 31536000}
|
||||
|
||||
image_base_url = "https://image.baidu.com/"
|
||||
|
||||
COOKIE_CACHE_KEY = "cookie"
|
||||
COOKIE_CACHE_EXPIRATION_SECONDS = 3600
|
||||
|
||||
CACHE: EngineCache
|
||||
"""Stores cookies from Baidu image search warmup."""
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool:
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache(engine_settings["name"])
|
||||
return True
|
||||
|
||||
|
||||
def get_image_cookies(headers: dict[str, str]) -> dict[str, str]:
|
||||
cookies: dict[str, str] | None = CACHE.get(COOKIE_CACHE_KEY)
|
||||
if cookies:
|
||||
return cookies
|
||||
|
||||
warmup = http_get(image_base_url, headers=headers, timeout=10)
|
||||
cookies = dict(warmup.cookies.items())
|
||||
CACHE.set(key=COOKIE_CACHE_KEY, value=cookies, expire=COOKIE_CACHE_EXPIRATION_SECONDS)
|
||||
return cookies
|
||||
|
||||
|
||||
def init(_):
|
||||
if baidu_category not in ('general', 'images', 'it'):
|
||||
@@ -88,6 +117,9 @@ def request(query, params):
|
||||
if baidu_category == 'it':
|
||||
query_params["paramList"] += f",timestamp_range={past}-{now}"
|
||||
|
||||
if baidu_category == 'images':
|
||||
params["cookies"] = get_image_cookies(params["headers"])
|
||||
|
||||
params["url"] = f"{query_url}?{urlencode(query_params)}"
|
||||
params["allow_redirects"] = False
|
||||
return params
|
||||
@@ -103,6 +135,8 @@ def response(resp):
|
||||
# baidu's JSON encoder wrongly quotes / and ' characters by \\ and \'
|
||||
text = text.replace(r"\/", "/").replace(r"\'", "'")
|
||||
data = json.loads(text, strict=False)
|
||||
if data.get("antiFlag") == 1:
|
||||
raise SearxEngineAccessDeniedException(data.get("message", "Forbid spider access"))
|
||||
parsers = {'general': parse_general, 'images': parse_images, 'it': parse_it}
|
||||
|
||||
return parsers[baidu_category](data)
|
||||
@@ -152,7 +186,7 @@ def parse_images(data):
|
||||
img_date = item.get("bdImgnewsDate")
|
||||
publishedDate = None
|
||||
if img_date:
|
||||
publishedDate = datetime.strptime(img_date, "%Y-%m-%d %H:%M")
|
||||
publishedDate = datetime.fromisoformat(img_date)
|
||||
results.append(
|
||||
{
|
||||
"template": "images.html",
|
||||
|
||||
@@ -8,6 +8,7 @@ import random
|
||||
import string
|
||||
from urllib.parse import urlencode
|
||||
from datetime import datetime, timedelta
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from searx import utils
|
||||
|
||||
@@ -39,6 +40,32 @@ cookie = {
|
||||
"home_feed_column": "4",
|
||||
}
|
||||
|
||||
_CN_TZ = ZoneInfo("Asia/Shanghai")
|
||||
|
||||
# Calendar-day time filter (Asia/Shanghai); dict values are days to subtract from today.
|
||||
time_range_support = True
|
||||
time_range_dict = {"day": 0, "week": 6, "month": 29, "year": 364}
|
||||
|
||||
|
||||
def _pubtime_range(time_range: str) -> tuple[int, int]:
|
||||
"""Return ``(pubtime_begin_s, pubtime_end_s)`` for Bilibili's search API.
|
||||
|
||||
Time ranges follow Bilibili's website semantics: they are counted in
|
||||
**calendar days** in China Standard Time (``Asia/Shanghai``), not as
|
||||
sliding 24-hour windows. For example, ``day`` means from 00:00:00 to
|
||||
23:59:59 of the current local day; ``week`` spans from 00:00:00 on the
|
||||
calendar day six days ago through the end of today, and so on.
|
||||
|
||||
The returned Unix timestamps (seconds) map to Bilibili's
|
||||
``pubtime_begin_s`` and ``pubtime_end_s`` query parameters.
|
||||
"""
|
||||
now = datetime.now(_CN_TZ)
|
||||
pubtime_end_s = int(now.replace(hour=23, minute=59, second=59, microsecond=0).timestamp())
|
||||
begin_day = now - timedelta(days=time_range_dict[time_range])
|
||||
pubtime_begin_s = int(begin_day.replace(hour=0, minute=0, second=0, microsecond=0).timestamp())
|
||||
|
||||
return pubtime_begin_s, pubtime_end_s
|
||||
|
||||
|
||||
def request(query, params):
|
||||
query_params = {
|
||||
@@ -50,6 +77,11 @@ def request(query, params):
|
||||
"search_type": "video",
|
||||
}
|
||||
|
||||
if params.get("time_range") in time_range_dict:
|
||||
pubtime_begin_s, pubtime_end_s = _pubtime_range(params["time_range"])
|
||||
query_params["pubtime_begin_s"] = pubtime_begin_s
|
||||
query_params["pubtime_end_s"] = pubtime_end_s
|
||||
|
||||
params["url"] = f"{base_url}?{urlencode(query_params)}"
|
||||
params["headers"]["Referer"] = "https://www.bilibili.com/"
|
||||
params["headers"]["Accept"] = "application/json, text/javascript, */*; q=0.01"
|
||||
|
||||
@@ -44,7 +44,7 @@ def response(resp):
|
||||
"url": 'https://www.bitchute.com/video/' + item['video_id'],
|
||||
"content": html_to_text(item['description']),
|
||||
"author": item['channel']['channel_name'],
|
||||
"publishedDate": datetime.strptime(item["date_published"], "%Y-%m-%dT%H:%M:%S.%fZ"),
|
||||
"publishedDate": datetime.fromisoformat(item["date_published"].rstrip("Z")),
|
||||
"length": item['duration'],
|
||||
"views": item['view_count'],
|
||||
"thumbnail": item['thumbnail_url'],
|
||||
|
||||
@@ -34,6 +34,7 @@ about = {
|
||||
categories = ["general", "social media"]
|
||||
paging = True
|
||||
time_range_support = True
|
||||
language_support = True
|
||||
|
||||
base_url = "https://boardreader.com"
|
||||
time_range_map = {"day": "1", "week": "7", "month": "30", "year": "365"}
|
||||
@@ -103,7 +104,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
title=_remove_keyword_marker(result["Subject"]),
|
||||
content=_remove_keyword_marker(result["Text"]),
|
||||
url=result["Url"],
|
||||
publishedDate=datetime.strptime(result["Published"], "%Y-%m-%d %H:%M:%S"),
|
||||
publishedDate=datetime.fromisoformat(result["Published"]),
|
||||
metadata=gettext.gettext("Posted by {author}").format(author=result["Author"]),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -13,8 +13,8 @@ about = {
|
||||
'use_official_api': False,
|
||||
'require_api_key': False,
|
||||
'results': 'JSON',
|
||||
'language': 'de',
|
||||
}
|
||||
language = "de"
|
||||
|
||||
paging = True
|
||||
categories = ['general']
|
||||
|
||||
@@ -31,6 +31,7 @@ from dateutil import parser
|
||||
|
||||
from searx.exceptions import SearxEngineAPIException
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import html_to_text
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
@@ -75,6 +76,7 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
"q": query,
|
||||
"count": results_per_page,
|
||||
"offset": (params["pageno"] - 1) * results_per_page,
|
||||
"text_decorations": False,
|
||||
}
|
||||
|
||||
# Apply time filter if specified
|
||||
@@ -112,14 +114,19 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
res = EngineResults()
|
||||
data = resp.json()
|
||||
|
||||
for result in data.get("web", {}).get("results", []):
|
||||
for result in (data.get("web") or {}).get("results", []):
|
||||
thumbnail_obj = result.get("thumbnail")
|
||||
thumbnail = ""
|
||||
if thumbnail_obj and not thumbnail_obj.get("logo", False):
|
||||
thumbnail = thumbnail_obj.get("src") or ""
|
||||
|
||||
res.add(
|
||||
res.types.MainResult(
|
||||
url=result["url"],
|
||||
title=result["title"],
|
||||
content=result.get("description", ""),
|
||||
title=html_to_text(result["title"]),
|
||||
content=html_to_text(result.get("description", "")),
|
||||
publishedDate=_extract_published_date(result.get("age")),
|
||||
thumbnail=result.get("thumbnail", {}).get("src"),
|
||||
thumbnail=thumbnail,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ from searx.extended_types import SXNG_Response
|
||||
from searx.network import get, post
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import html_to_text
|
||||
from searx.enginelib import EngineCache
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.search.processors import OnlineParams
|
||||
@@ -42,21 +41,7 @@ search_index = "cw22"
|
||||
<https://www.chatnoir.eu/docs/api-general>`_ for a full list."""
|
||||
|
||||
|
||||
CACHE: EngineCache
|
||||
"""Cache to store session info (i.e. api key, csrf token, session id)."""
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool:
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache(engine_settings["name"])
|
||||
return True
|
||||
|
||||
|
||||
def _obtain_api_key() -> tuple[str, str, str]:
|
||||
cached_session = CACHE.get("session")
|
||||
if cached_session:
|
||||
return tuple(cached_session.split("|"))
|
||||
|
||||
home_resp = get(base_url)
|
||||
if not home_resp.ok:
|
||||
raise SearxEngineAPIException("failed to obtain api key")
|
||||
@@ -76,10 +61,6 @@ def _obtain_api_key() -> tuple[str, str, str]:
|
||||
session_id = token_resp.cookies["sessionid"]
|
||||
scraped_api_key = token_resp.json()["token"]["token"]
|
||||
|
||||
# session keys seem to become rate-limited very fast, so only remembering
|
||||
# for 1 minute here
|
||||
CACHE.set("session", f"{csrf_token}|{session_id}|{scraped_api_key}", expire=60)
|
||||
|
||||
return csrf_token, session_id, scraped_api_key
|
||||
|
||||
|
||||
|
||||
@@ -10,8 +10,8 @@ about = {
|
||||
'use_official_api': False,
|
||||
'require_api_key': False,
|
||||
'results': 'JSON',
|
||||
'language': 'de',
|
||||
}
|
||||
language = "de"
|
||||
|
||||
paging = True
|
||||
categories = []
|
||||
@@ -43,7 +43,7 @@ def response(resp):
|
||||
|
||||
publishedDate = None
|
||||
if recipe['submissionDate']:
|
||||
publishedDate = datetime.strptime(result['recipe']['submissionDate'][:19], "%Y-%m-%dT%H:%M:%S")
|
||||
publishedDate = datetime.fromisoformat(result['recipe']['submissionDate'][:19])
|
||||
|
||||
content = [
|
||||
f"Schwierigkeitsstufe (1-3): {recipe['difficulty']}",
|
||||
|
||||
@@ -19,11 +19,9 @@ Configuration
|
||||
|
||||
The engine has the following additional settings:
|
||||
|
||||
- :py:obj:`chinaso_category` (:py:obj:`ChinasoCategoryType`)
|
||||
- :py:obj:`chinaso_news_source` (:py:obj:`ChinasoNewsSourceType`)
|
||||
|
||||
In the example below, all three ChinaSO engines are using the :ref:`network
|
||||
<engine network>` from the ``chinaso news`` engine.
|
||||
In the example below, ChinaSO is configured for news search.
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
@@ -31,23 +29,8 @@ In the example below, all three ChinaSO engines are using the :ref:`network
|
||||
engine: chinaso
|
||||
shortcut: chinaso
|
||||
categories: [news]
|
||||
chinaso_category: news
|
||||
chinaso_news_source: all
|
||||
|
||||
- name: chinaso images
|
||||
engine: chinaso
|
||||
network: chinaso news
|
||||
shortcut: chinasoi
|
||||
categories: [images]
|
||||
chinaso_category: images
|
||||
|
||||
- name: chinaso videos
|
||||
engine: chinaso
|
||||
network: chinaso news
|
||||
shortcut: chinasov
|
||||
categories: [videos]
|
||||
chinaso_category: videos
|
||||
|
||||
|
||||
Implementations
|
||||
===============
|
||||
@@ -70,26 +53,13 @@ about = {
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
"language": "zh",
|
||||
}
|
||||
|
||||
paging = True
|
||||
time_range_support = True
|
||||
results_per_page = 10
|
||||
categories = []
|
||||
|
||||
ChinasoCategoryType = t.Literal['news', 'videos', 'images']
|
||||
"""ChinaSo supports news, videos, images search.
|
||||
|
||||
- ``news``: search for news
|
||||
- ``videos``: search for videos
|
||||
- ``images``: search for images
|
||||
|
||||
In the category ``news`` you can additionally filter by option
|
||||
:py:obj:`chinaso_news_source`.
|
||||
"""
|
||||
chinaso_category = 'news'
|
||||
"""Configure ChinaSo category (:py:obj:`ChinasoCategoryType`)."""
|
||||
language = "zh"
|
||||
|
||||
ChinasoNewsSourceType = t.Literal['CENTRAL', 'LOCAL', 'BUSINESS', 'EPAPER', 'all']
|
||||
"""Filtering ChinaSo-News results by source:
|
||||
@@ -109,39 +79,24 @@ base_url = "https://www.chinaso.com"
|
||||
|
||||
|
||||
def init(_):
|
||||
if chinaso_category not in ('news', 'videos', 'images'):
|
||||
raise ValueError(f"Unsupported category: {chinaso_category}")
|
||||
if chinaso_category == 'news' and chinaso_news_source not in t.get_args(ChinasoNewsSourceType):
|
||||
if chinaso_news_source not in t.get_args(ChinasoNewsSourceType):
|
||||
raise ValueError(f"Unsupported news source: {chinaso_news_source}")
|
||||
|
||||
|
||||
def request(query, params):
|
||||
query_params = {"q": query}
|
||||
query_params = {'q': query, 'pn': params["pageno"], 'ps': results_per_page}
|
||||
|
||||
if time_range_dict.get(params['time_range']):
|
||||
query_params["stime"] = time_range_dict[params['time_range']]
|
||||
query_params["etime"] = 'now'
|
||||
|
||||
category_config = {
|
||||
'news': {'endpoint': '/v5/general/v1/web/search', 'params': {'pn': params["pageno"], 'ps': results_per_page}},
|
||||
'images': {
|
||||
'endpoint': '/v5/general/v1/search/image',
|
||||
'params': {'start_index': (params["pageno"] - 1) * results_per_page, 'rn': results_per_page},
|
||||
},
|
||||
'videos': {
|
||||
'endpoint': '/v5/general/v1/search/video',
|
||||
'params': {'start_index': (params["pageno"] - 1) * results_per_page, 'rn': results_per_page},
|
||||
},
|
||||
}
|
||||
if chinaso_news_source != 'all':
|
||||
if chinaso_news_source == 'EPAPER':
|
||||
category_config['news']['params']["type"] = 'EPAPER'
|
||||
query_params["type"] = 'EPAPER'
|
||||
else:
|
||||
category_config['news']['params']["cate"] = chinaso_news_source
|
||||
query_params["cate"] = chinaso_news_source
|
||||
|
||||
query_params.update(category_config[chinaso_category]['params'])
|
||||
|
||||
params["url"] = f"{base_url}{category_config[chinaso_category]['endpoint']}?{urlencode(query_params)}"
|
||||
params["url"] = f"{base_url}/v5/general/v1/web/search?{urlencode(query_params)}"
|
||||
cookie = {
|
||||
"uid": base64.b64encode(secrets.token_bytes(16)).decode("utf-8"),
|
||||
}
|
||||
@@ -156,12 +111,13 @@ def response(resp):
|
||||
except Exception as e:
|
||||
raise SearxEngineAPIException(f"Invalid response: {e}") from e
|
||||
|
||||
parsers = {'news': parse_news, 'images': parse_images, 'videos': parse_videos}
|
||||
# Upstream returns {'status': 0, 'msg': 'empty result', 'data': {}} when there
|
||||
# are no results; this is a valid empty result rather than an API error.
|
||||
if not isinstance(data, dict) or "data" not in data:
|
||||
raise SearxEngineAPIException("Invalid response")
|
||||
if not data["data"]:
|
||||
return []
|
||||
|
||||
return parsers[chinaso_category](data)
|
||||
|
||||
|
||||
def parse_news(data):
|
||||
results = []
|
||||
if not data.get("data", {}).get("data"):
|
||||
raise SearxEngineAPIException("Invalid response")
|
||||
@@ -183,47 +139,3 @@ def parse_news(data):
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def parse_images(data):
|
||||
results = []
|
||||
if not data.get("data", {}).get("arrRes"):
|
||||
raise SearxEngineAPIException("Invalid response")
|
||||
|
||||
for entry in data["data"]["arrRes"]:
|
||||
results.append(
|
||||
{
|
||||
'url': entry["web_url"],
|
||||
'title': html_to_text(entry["title"]),
|
||||
'content': html_to_text(entry.get("ImageInfo", "")),
|
||||
'template': 'images.html',
|
||||
'img_src': entry["url"].replace("http://", "https://"),
|
||||
'thumbnail_src': entry["largeimage"].replace("http://", "https://"),
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def parse_videos(data):
|
||||
results = []
|
||||
if not data.get("data", {}).get("arrRes"):
|
||||
raise SearxEngineAPIException("Invalid response")
|
||||
|
||||
for entry in data["data"]["arrRes"]:
|
||||
published_date = None
|
||||
if entry.get("VideoPubDate"):
|
||||
try:
|
||||
published_date = datetime.fromtimestamp(int(entry["VideoPubDate"]))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
results.append(
|
||||
{
|
||||
'url': entry["url"],
|
||||
'title': html_to_text(entry["raw_title"]),
|
||||
'template': 'videos.html',
|
||||
'publishedDate': published_date,
|
||||
'thumbnail': entry["image_src"].replace("http://", "https://"),
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
@@ -40,6 +40,7 @@ categories = ["videos"]
|
||||
paging = True
|
||||
page_size = 10
|
||||
|
||||
language_support = True
|
||||
time_range_support = True
|
||||
time_delta_dict = {
|
||||
"day": timedelta(days=1),
|
||||
|
||||
@@ -24,7 +24,7 @@ import typing as t
|
||||
import json
|
||||
|
||||
from searx.result_types import EngineResults
|
||||
from searx.enginelib import EngineCache
|
||||
from searx.enginelib import EngineCache, EngineAbout
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.search.processors import RequestParams
|
||||
@@ -35,13 +35,11 @@ categories = ["general"]
|
||||
disabled = True
|
||||
timeout = 2.0
|
||||
|
||||
about = {
|
||||
"wikidata_id": None,
|
||||
"official_api_documentation": None,
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
}
|
||||
language = "en"
|
||||
about = EngineAbout(
|
||||
results="JSON",
|
||||
description="Demo offline engine Engine with results in the English language.",
|
||||
)
|
||||
|
||||
# if there is a need for globals, use a leading underline
|
||||
_my_offline_engine: str = ""
|
||||
|
||||
@@ -25,6 +25,7 @@ import typing as t
|
||||
|
||||
from urllib.parse import urlencode
|
||||
from searx.result_types import EngineResults
|
||||
from searx.enginelib import EngineAbout
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
@@ -43,14 +44,14 @@ page_size = 20
|
||||
search_api = "https://api.artic.edu/api/v1/artworks/search"
|
||||
image_api = "https://www.artic.edu/iiif/2/"
|
||||
|
||||
about = {
|
||||
"website": "https://www.artic.edu",
|
||||
"wikidata_id": "Q239303",
|
||||
"official_api_documentation": "http://api.artic.edu/docs/",
|
||||
"use_official_api": True,
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
}
|
||||
about = EngineAbout(
|
||||
website="https://www.artic.edu",
|
||||
wikidata_id="Q239303",
|
||||
official_api_documentation="http://api.artic.edu/docs/",
|
||||
use_official_api=True,
|
||||
require_api_key=False,
|
||||
results="JSON",
|
||||
)
|
||||
|
||||
|
||||
# if there is a need for globals, use a leading underline
|
||||
|
||||
@@ -11,8 +11,8 @@ about = {
|
||||
'use_official_api': False,
|
||||
'require_api_key': False,
|
||||
'results': 'HTML',
|
||||
'language': 'de',
|
||||
}
|
||||
language = "de"
|
||||
|
||||
categories = []
|
||||
paging = True
|
||||
|
||||
@@ -203,6 +203,7 @@ about: dict[str, str | bool] = {
|
||||
categories: list[str] = ["general", "web"]
|
||||
paging: bool = True
|
||||
time_range_support: bool = True
|
||||
language_support = True
|
||||
safesearch: bool = True
|
||||
"""DDG-lite: user can't select but the results are filtered."""
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ about = {
|
||||
"require_api_key": False,
|
||||
"results": "JSON (site requires js to get images)",
|
||||
}
|
||||
language_support = True
|
||||
|
||||
# engine dependent config
|
||||
categories = []
|
||||
|
||||
@@ -26,6 +26,7 @@ about = {
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
}
|
||||
language_support = True
|
||||
|
||||
# engine dependent config
|
||||
categories = ["weather"]
|
||||
|
||||
@@ -14,8 +14,8 @@ about = {
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": 'HTML',
|
||||
"language": 'de',
|
||||
}
|
||||
language = "de"
|
||||
|
||||
categories = ['dictionaries']
|
||||
paging = True
|
||||
|
||||
@@ -55,7 +55,7 @@ about = {
|
||||
'official_api_documentation': 'https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html',
|
||||
'use_official_api': True,
|
||||
'require_api_key': False,
|
||||
'format': 'JSON',
|
||||
"results": "JSON",
|
||||
}
|
||||
|
||||
base_url = 'http://localhost:9200'
|
||||
|
||||
118
searx/engines/findfiles.py
Normal file
118
searx/engines/findfiles.py
Normal file
@@ -0,0 +1,118 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""FindFiles.net_ is a Germany-based file search engine.
|
||||
|
||||
FindFiles.net_ is a specialized file search engine designed to help you search
|
||||
files online with precision. Unlike traditional search engines that mainly index
|
||||
web pages, FindFiles focuses on finding real files on the internet - including
|
||||
PDFs, documents, archives, videos, datasets, and more.
|
||||
|
||||
.. _FindFiles.net: https://findfiles.net
|
||||
"""
|
||||
|
||||
from os.path import basename
|
||||
from urllib.parse import urlencode
|
||||
import typing as t
|
||||
|
||||
from lxml import html
|
||||
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import extract_text, eval_xpath, eval_xpath_list
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from extended_types import SXNG_Response
|
||||
from search.processors import OnlineParams
|
||||
|
||||
about = {
|
||||
"website": "https://findfiles.net",
|
||||
"wikidata_id": None,
|
||||
"official_api_documentation": None,
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "HTML",
|
||||
}
|
||||
|
||||
base_url = "https://findfiles.net"
|
||||
categories = ["files"]
|
||||
paging = True
|
||||
safeserach = True
|
||||
|
||||
safesearch_map = {
|
||||
0: "contentguard.off",
|
||||
1: "contentguard.moderate",
|
||||
2: "contentguard.strict",
|
||||
}
|
||||
|
||||
FindFilesCategory = t.Literal[
|
||||
"all",
|
||||
"document",
|
||||
"text",
|
||||
"image",
|
||||
"audio",
|
||||
"video",
|
||||
]
|
||||
FINDFILES_CATEGORIES = t.get_args(FindFilesCategory)
|
||||
|
||||
findfiles_categ: FindFilesCategory = "all"
|
||||
"""Category to search in."""
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool:
|
||||
if findfiles_categ not in FINDFILES_CATEGORIES:
|
||||
raise ValueError("invalid category: %s" % findfiles_categ)
|
||||
return True
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
args = {
|
||||
"query": query,
|
||||
"contentguard": safesearch_map[params["safesearch"]],
|
||||
"page": params["pageno"],
|
||||
}
|
||||
# the language in the path doesn't change anything about the results, it
|
||||
# only changes the UI
|
||||
params["url"] = f"{base_url}/en/serp/{findfiles_categ}/?{urlencode(args)}"
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
res = EngineResults()
|
||||
|
||||
dom = html.fromstring(resp.text)
|
||||
if findfiles_categ == "image":
|
||||
for result in eval_xpath_list(
|
||||
dom, "//div[contains(@class, 'image-mosaic')]/div[contains(@class, 'image-item')]"
|
||||
):
|
||||
res.add(
|
||||
res.types.Image(
|
||||
url=extract_text(eval_xpath(result, ".//div[contains(@class, 'caption')]/a/@href")) or "",
|
||||
title=extract_text(eval_xpath(result, ".//div[contains(@class, 'caption')]/a")) or "",
|
||||
thumbnail_src=extract_text(eval_xpath(result, ".//img/@src")) or "",
|
||||
)
|
||||
)
|
||||
elif findfiles_categ == "video":
|
||||
for result in eval_xpath_list(
|
||||
dom, "//div[contains(@class, 'video-mosaic')]/div[contains(@class, 'video-item')]"
|
||||
):
|
||||
video_src = extract_text(eval_xpath(result, ".//video/@src")) or ""
|
||||
res.add(
|
||||
res.types.LegacyResult(
|
||||
template="videos.html",
|
||||
url=video_src,
|
||||
title=extract_text(eval_xpath(result, ".//div[contains(@class, 'caption')]/span")) or "",
|
||||
iframe_src=video_src or "",
|
||||
)
|
||||
)
|
||||
else:
|
||||
for result in eval_xpath_list(dom, "//ol/li[contains(@class, 'result-item')]/article"):
|
||||
filename = basename(extract_text(eval_xpath(result, ".//h3")) or "")
|
||||
res.add(
|
||||
res.types.File(
|
||||
url=extract_text(eval_xpath(result, ".//h3/a/@href")) or "",
|
||||
title=filename,
|
||||
content=" ".join(extract_text(el) or "" for el in eval_xpath_list(result, "./div/span")),
|
||||
filename=filename,
|
||||
size=extract_text(eval_xpath(result, "(.//span[@id])[1]")) or "",
|
||||
embedded=extract_text(eval_xpath(result, ".//audio/@src")) or "",
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
@@ -37,7 +37,7 @@ def response(resp):
|
||||
for item in search_res:
|
||||
img = 'https://s3.thehackerblog.com/findthatmeme/' + item['image_path']
|
||||
thumb = 'https://s3.thehackerblog.com/findthatmeme/thumb/' + item.get('thumbnail', '')
|
||||
date = datetime.strptime(item["updated_at"].split("T")[0], "%Y-%m-%d")
|
||||
date = datetime.fromisoformat(item["updated_at"].split("T")[0])
|
||||
formatted_date = datetime.fromtimestamp(date.timestamp())
|
||||
|
||||
results.append(
|
||||
|
||||
@@ -47,7 +47,7 @@ def response(resp: "SXNG_Response"):
|
||||
title=result["title"],
|
||||
content=result["description"],
|
||||
thumbnail=result["smallImageURL"],
|
||||
publishedDate=datetime.strptime(result["status_since"], "%Y-%m-%d %H:%M:%S"),
|
||||
publishedDate=datetime.fromisoformat(result["status_since"]),
|
||||
metadata=f"Rank: {result['rank']} || {result['episode_count']} episodes",
|
||||
)
|
||||
)
|
||||
|
||||
@@ -27,8 +27,8 @@ about = {
|
||||
'official_api_documentation': None,
|
||||
'require_api_key': False,
|
||||
'results': 'HTML',
|
||||
'language': 'de',
|
||||
}
|
||||
language = "de"
|
||||
paging = True
|
||||
categories = ['shopping']
|
||||
|
||||
|
||||
127
searx/engines/giphy.py
Normal file
127
searx/engines/giphy.py
Normal file
@@ -0,0 +1,127 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Giphy (images)"""
|
||||
|
||||
import random
|
||||
from urllib.parse import urlencode
|
||||
import re
|
||||
|
||||
import typing as t
|
||||
|
||||
from lxml import html
|
||||
|
||||
from searx.enginelib import EngineCache
|
||||
from searx.exceptions import SearxEngineAPIException
|
||||
from searx.network import get
|
||||
from searx.result_types import EngineResults
|
||||
from searx.result_types.image import ImageRef
|
||||
from searx.utils import eval_xpath_list, humanize_bytes
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
|
||||
about = {
|
||||
"website": "https://giphy.com",
|
||||
"wikidata_id": "Q17054335",
|
||||
"official_api_documentation": None,
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
}
|
||||
|
||||
base_url = "https://giphy.com"
|
||||
api_url = "https://api.giphy.com"
|
||||
|
||||
categories = ["images"]
|
||||
paging = True
|
||||
page_size = 15
|
||||
|
||||
GiphyCategs = t.Literal["gifs", "stickers", "clips"]
|
||||
giphy_categ: GiphyCategs = "gifs"
|
||||
"""Giphy category to search in."""
|
||||
|
||||
CACHE: EngineCache
|
||||
"""Cache for storing the extracted api key."""
|
||||
|
||||
|
||||
_GIPHY_API_KEY_RE = re.compile(r"[Aa]piKey\s*:\s*\"(\w+)\"")
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, str]) -> bool:
|
||||
if giphy_categ not in t.get_args(GiphyCategs):
|
||||
raise ValueError("invalid category: %s" % giphy_categ)
|
||||
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache(engine_settings["name"])
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _get_api_key() -> str:
|
||||
"""
|
||||
Extract the Giphy API key from the JavaScript code. There are different API keys
|
||||
(e.g. for mobile, desktop, ...), so we just pick a random one of these.
|
||||
"""
|
||||
cached = CACHE.get("api_key")
|
||||
if cached:
|
||||
return cached
|
||||
|
||||
homepage_resp = get(base_url)
|
||||
homepage_doc = html.fromstring(homepage_resp.text)
|
||||
|
||||
for script_src in eval_xpath_list(homepage_doc, "//script[contains(@src, 'layout')]/@src"):
|
||||
script_resp = get(base_url + script_src)
|
||||
api_keys = _GIPHY_API_KEY_RE.findall(script_resp.text)
|
||||
if api_keys:
|
||||
api_key = random.choice(api_keys)
|
||||
CACHE.set("api_key", api_key, expire=60 * 60 * 6) # 6 hours
|
||||
return api_key
|
||||
|
||||
raise SearxEngineAPIException("failed to extract api keys")
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
args = {
|
||||
"q": query,
|
||||
"api_key": _get_api_key(),
|
||||
"limit": page_size,
|
||||
"offset": (params["pageno"] - 1) * page_size,
|
||||
"type": giphy_categ,
|
||||
}
|
||||
params["url"] = f"{api_url}/v1/{giphy_categ}/search?{urlencode(args)}"
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response"):
|
||||
res = EngineResults()
|
||||
|
||||
result: dict[str, t.Any]
|
||||
for result in resp.json()["data"]:
|
||||
img = result['images']['original']
|
||||
formats = [
|
||||
ImageRef(url=img["mp4"], subtype="mp4"), # type: ignore
|
||||
ImageRef(url=img["webp"], subtype="webp"), # type: ignore
|
||||
]
|
||||
thumb = (
|
||||
result["images"].get("downsized")
|
||||
or result["images"].get("downsized_medium")
|
||||
or result["images"].get("downsized_small")
|
||||
or result["images"].get("downsized_large")
|
||||
)
|
||||
res.add(
|
||||
res.types.Image(
|
||||
title=result["title"],
|
||||
content=", ".join(result.get("tags", [])),
|
||||
url=result["url"],
|
||||
thumbnail_src=thumb.get("url") or img["url"],
|
||||
img_src=img["url"],
|
||||
resolution=f"{img['width']}x{img['height']}",
|
||||
img_format="GIF",
|
||||
formats=formats,
|
||||
author=result["username"],
|
||||
filesize=humanize_bytes(int(img["size"])),
|
||||
source=result.get("source_tld") or "",
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
@@ -32,7 +32,6 @@ from searx.utils import (
|
||||
eval_xpath_getindex,
|
||||
eval_xpath_list,
|
||||
extract_text,
|
||||
gen_gsa_useragent,
|
||||
)
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
@@ -57,6 +56,7 @@ max_page = 50
|
||||
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
|
||||
"""
|
||||
time_range_support = True
|
||||
language_support = True
|
||||
safesearch = True
|
||||
|
||||
time_range_dict = {"day": "d", "week": "w", "month": "m", "year": "y"}
|
||||
@@ -196,7 +196,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
|
||||
# https://developers.google.com/custom-search/docs/xml_results_appendices#interfaceLanguages
|
||||
|
||||
# https://github.com/searxng/searxng/issues/2515#issuecomment-1607150817
|
||||
ret_val["params"]["hl"] = f"{lang_code}-{country}"
|
||||
ret_val["params"]["hl"] = f"{lang_code}"
|
||||
|
||||
# lr parameter:
|
||||
# The lr (language restrict) parameter restricts search results to
|
||||
@@ -267,7 +267,6 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
|
||||
# HTTP headers
|
||||
|
||||
ret_val["headers"]["Accept"] = "*/*"
|
||||
ret_val["headers"]["User-Agent"] = gen_gsa_useragent()
|
||||
|
||||
# Cookies
|
||||
|
||||
|
||||
190
searx/engines/google_cse.py
Normal file
190
searx/engines/google_cse.py
Normal file
@@ -0,0 +1,190 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Google Custom Search Engine"""
|
||||
|
||||
import datetime
|
||||
import typing as t
|
||||
from json import loads
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from searx.enginelib import EngineCache
|
||||
from searx.exceptions import SearxEngineAPIException, SearxEngineTooManyRequestsException
|
||||
from searx.network import get
|
||||
from searx.result_types import EngineResults, Result, MainResult, Image
|
||||
|
||||
from searx.engines.google import fetch_traits # pylint: disable=unused-import
|
||||
from searx.engines.google import filter_mapping, get_google_info
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
about = {
|
||||
"website": "https://www.google.com",
|
||||
"wikidata_id": "Q2233943",
|
||||
"official_api_documentation": "https://developers.google.com/custom-search/docs/element",
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "JSONP",
|
||||
"description": "Platform for creating custom search engines based on Google Search.",
|
||||
}
|
||||
|
||||
categories = ["general", "web"]
|
||||
paging = True
|
||||
max_page = 5
|
||||
page_size = 20
|
||||
time_range_support = True
|
||||
language_support = True
|
||||
safesearch = True
|
||||
|
||||
GoogleCategType = t.Literal["", "image"]
|
||||
google_categ: GoogleCategType = ""
|
||||
"""Google CSE category. Set to ``""`` for web search."""
|
||||
|
||||
CX = "partner-pub-8993703457585266:4862972284" # blackle.com
|
||||
|
||||
CACHE: EngineCache
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool:
|
||||
global CACHE # pylint: disable=global-statement
|
||||
|
||||
if google_categ not in t.get_args(GoogleCategType):
|
||||
raise ValueError("invalid google cse category: %s" % google_categ)
|
||||
|
||||
CACHE = EngineCache(engine_settings["name"])
|
||||
return True
|
||||
|
||||
|
||||
def _cse_token() -> dict[str, str]:
|
||||
token: dict[str, str] = CACHE.get(CX)
|
||||
if token:
|
||||
return token
|
||||
|
||||
resp = get(f"https://www.google.com/cse/cse.js?cx={CX}", timeout=10)
|
||||
if not resp.ok:
|
||||
raise SearxEngineAPIException("failed to obtain cse token")
|
||||
|
||||
end = resp.text.rfind("});")
|
||||
start = resp.text.rfind("({")
|
||||
opts: dict[str, str] = loads(resp.text[start + 1 : end + 1])
|
||||
|
||||
cse_tok = opts.get("cse_token")
|
||||
if not cse_tok:
|
||||
raise SearxEngineAPIException("failed to obtain cse token")
|
||||
|
||||
exp = opts.get("exp")
|
||||
token = {
|
||||
"cse_tok": cse_tok,
|
||||
"cselibv": opts.get("cselibVersion", ""),
|
||||
"exp": ",".join(exp) if exp else "",
|
||||
}
|
||||
CACHE.set(CX, token, expire=3600)
|
||||
return token
|
||||
|
||||
|
||||
def _get_start_and_end_date_str(time_range: str) -> tuple[str, str]:
|
||||
time_range_map = {"day": 1, "week": 7, "month": 30, "year": 365}
|
||||
|
||||
end_date = datetime.datetime.now()
|
||||
start_date = end_date - datetime.timedelta(days=time_range_map[time_range])
|
||||
|
||||
return start_date.strftime("%Y%m%d"), end_date.strftime("%Y%m%d")
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
token = _cse_token()
|
||||
|
||||
google_info = get_google_info(params, traits)
|
||||
info: dict[str, str] = google_info["params"]
|
||||
|
||||
args = {
|
||||
"rsz": "filtered_cse",
|
||||
"num": str(page_size),
|
||||
"hl": info["hl"],
|
||||
"cselibv": token["cselibv"],
|
||||
"cx": CX,
|
||||
"q": query,
|
||||
"safe": filter_mapping[params["safesearch"]],
|
||||
"cse_tok": token["cse_tok"],
|
||||
"callback": "_",
|
||||
"rurl": "",
|
||||
"searchtype": google_categ,
|
||||
}
|
||||
if params["time_range"]:
|
||||
start_date, end_date = _get_start_and_end_date_str(params["time_range"])
|
||||
args["sort"] = f"date:r:{start_date}:{end_date}"
|
||||
|
||||
if info.get("lr"):
|
||||
args["lr"] = info["lr"]
|
||||
if info.get("cr"):
|
||||
args["cr"] = info["cr"]
|
||||
if google_info["country"] not in (None, "ZZ"):
|
||||
args["gl"] = google_info["country"]
|
||||
if token["exp"]:
|
||||
args["exp"] = token["exp"]
|
||||
|
||||
start = (params["pageno"] - 1) * page_size
|
||||
if start:
|
||||
args["start"] = str(start)
|
||||
|
||||
params["url"] = "https://cse.google.com/cse/element/v1?" + urlencode(args)
|
||||
params["cookies"] = google_info["cookies"]
|
||||
params["headers"].update(google_info["headers"])
|
||||
params["headers"]["Referer"] = "https://cse.google.com/"
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
json_resp = resp.text[resp.text.find("{") : resp.text.rfind("}") + 1]
|
||||
data = loads(json_resp)
|
||||
|
||||
# not the real types, but a sufficient approximation
|
||||
item: dict[str, str]
|
||||
error: dict[str, str | int]
|
||||
|
||||
if error := data.get("error"):
|
||||
message = error.get("message", "unknown error")
|
||||
if error.get("code") == 429:
|
||||
raise SearxEngineTooManyRequestsException(message=f"google cse: {message}")
|
||||
raise SearxEngineAPIException(f"google cse: {message}")
|
||||
|
||||
results = EngineResults()
|
||||
|
||||
for item in data.get("results", []):
|
||||
|
||||
res: Result | None
|
||||
if google_categ == "":
|
||||
res = web_item(item)
|
||||
elif google_categ == "image":
|
||||
res = img_item(item)
|
||||
|
||||
if res is not None:
|
||||
results.add(res)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def web_item(item: dict[str, str]) -> MainResult | None:
|
||||
url = item.get("unescapedUrl")
|
||||
if not url:
|
||||
return None
|
||||
return MainResult(
|
||||
url=url,
|
||||
title=item.get("titleNoFormatting", ""),
|
||||
content=item.get("contentNoFormatting", ""),
|
||||
thumbnail=item.get("richSnippet", {}).get("cseThumbnail", {}).get("src", ""), # type: ignore
|
||||
)
|
||||
|
||||
|
||||
def img_item(item: dict[str, str]) -> Image | None:
|
||||
resolution = ""
|
||||
if item.get("height") and item.get("width"):
|
||||
resolution = f"{item['width']}x{item['height']}"
|
||||
return Image(
|
||||
url=item["originalContextUrl"],
|
||||
title=item.get("titleNoFormatting", ""),
|
||||
content=item.get("contentNoFormatting", ""),
|
||||
img_src=item["unescapedUrl"],
|
||||
thumbnail_src=item["tbUrl"],
|
||||
resolution=resolution,
|
||||
img_format=item["fileFormat"].split("/")[-1],
|
||||
)
|
||||
@@ -43,6 +43,7 @@ max_page = 50
|
||||
"""
|
||||
|
||||
time_range_support = True
|
||||
language_support = True
|
||||
safesearch = True
|
||||
|
||||
filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
|
||||
|
||||
@@ -66,6 +66,7 @@ about = {
|
||||
categories = ["news"]
|
||||
paging = False
|
||||
time_range_support = False
|
||||
language_support = True
|
||||
|
||||
# Google-News results are always *SafeSearch*. Option 'safesearch' is set to
|
||||
# False here.
|
||||
|
||||
@@ -14,8 +14,11 @@ from urllib.parse import urlencode
|
||||
|
||||
import typing as t
|
||||
|
||||
from searx.exceptions import SearxEngineAccessDeniedException
|
||||
from searx.enginelib import EngineCache
|
||||
from searx.network import get
|
||||
from searx.exceptions import SearxEngineAPIException, SearxEngineAccessDeniedException
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import gen_useragent
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
@@ -38,14 +41,46 @@ heexy_categ = "web"
|
||||
"""Category to search in. Can be either "web" or "image"."""
|
||||
|
||||
|
||||
base_url = "https://seapi.heexy.org"
|
||||
base_url = "https://heexy.org"
|
||||
api_url = "https://seapi.heexy.org"
|
||||
safe_search_map = {0: "off", 1: "on", 2: "on"}
|
||||
|
||||
CACHE: EngineCache
|
||||
"""Cache for storing the ``X-Data-Cacheft`` token (acts like an API key)."""
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool:
|
||||
global CACHE # pylint: disable=global-statement
|
||||
|
||||
def init(_):
|
||||
if heexy_categ not in ("web", "image"):
|
||||
raise ValueError("invalid search category: %s" % heexy_categ)
|
||||
|
||||
CACHE = EngineCache(engine_settings["name"])
|
||||
return True
|
||||
|
||||
|
||||
def _get_api_token(query: str) -> str:
|
||||
"""The API token is independent of the search query. We just need any query
|
||||
to obtain it initially, and don't hardcode it here to decrease chances of
|
||||
getting blocked. The token must be passed as ``X-Data-Cacheft`` header."""
|
||||
|
||||
cached_token: str = CACHE.get("token")
|
||||
if cached_token:
|
||||
return cached_token
|
||||
|
||||
resp = get(
|
||||
f"{base_url}/search?q={query}", headers={"User-Agent": gen_useragent(), "Accept-Language": "en-US,en:q=0.9"}
|
||||
)
|
||||
if not resp.ok:
|
||||
raise SearxEngineAPIException("failed to obtain request token: invalid response code")
|
||||
|
||||
token = resp.cookies["cacheft"]
|
||||
if not token:
|
||||
raise SearxEngineAPIException("failed to obtain request token: no token found")
|
||||
|
||||
CACHE.set("token", token, expire=3 * 60)
|
||||
return token
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
args = {
|
||||
@@ -56,8 +91,10 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
if params["searxng_locale"] != "all":
|
||||
args["lang"] = params["searxng_locale"].split("-")[0]
|
||||
|
||||
params["url"] = f"{base_url}/search/{heexy_categ}?{urlencode(args)}"
|
||||
params["headers"]["Origin"] = base_url
|
||||
params["url"] = f"{api_url}/search/{heexy_categ}?{urlencode(args)}"
|
||||
|
||||
params["headers"]["Origin"] = api_url
|
||||
params["cookies"]["cacheft"] = _get_api_token(query)
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response"):
|
||||
|
||||
@@ -91,7 +91,7 @@ def response(resp) -> EngineResults:
|
||||
|
||||
published_date = None
|
||||
try:
|
||||
published_date = datetime.strptime(entry["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
published_date = datetime.fromisoformat(entry["createdAt"].rstrip("Z"))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
|
||||
@@ -34,8 +34,8 @@ about = {
|
||||
"use_official_api": True,
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
"language": "it",
|
||||
}
|
||||
language = "it"
|
||||
|
||||
|
||||
def request(query, params):
|
||||
|
||||
@@ -16,8 +16,8 @@ about = {
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": 'HTML',
|
||||
"language": 'fr',
|
||||
}
|
||||
language = "fr"
|
||||
|
||||
# engine dependent config
|
||||
categories = ['videos']
|
||||
|
||||
@@ -14,9 +14,9 @@ about = {
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
"language": "zh",
|
||||
}
|
||||
|
||||
language = "zh"
|
||||
paging = True
|
||||
time_range_support = True
|
||||
categories = ["videos"]
|
||||
@@ -43,7 +43,7 @@ def _result(video: dict[str, typing.Any], album_info: dict[str, typing.Any]):
|
||||
release_time = album_info.get("releaseTime", {}).get("value")
|
||||
if release_time:
|
||||
try:
|
||||
published_date = datetime.strptime(release_time, "%Y-%m-%d")
|
||||
published_date = datetime.fromisoformat(release_time)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
|
||||
88
searx/engines/iseek.py
Normal file
88
searx/engines/iseek.py
Normal file
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""iseek_ is a search engine by the AI company Vantage Labs LLC,
|
||||
that focuses on medical and educational applicances.
|
||||
Although it's an AI company, it doesn't include any AI stuff in its results.
|
||||
|
||||
.. _iseek : https://www.iseek.ai/
|
||||
"""
|
||||
|
||||
import base64
|
||||
from hashlib import sha256
|
||||
import typing as t
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.search.processors import OnlineParams
|
||||
from searx.extended_types import SXNG_Response
|
||||
|
||||
|
||||
about = {
|
||||
"website": 'https://www.iseek.com',
|
||||
"wikidata_id": None,
|
||||
"official_api_documentation": None,
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
}
|
||||
categories = ["general"]
|
||||
paging = True
|
||||
|
||||
base_url = "https://api.iseek.com"
|
||||
page_size = 10
|
||||
|
||||
|
||||
def _get_new_token(query: str, pageno: int) -> str:
|
||||
"""Create a new ``qToken``. This reduced the time for fetching subsequent pages
|
||||
from 4 seconds to 200ms when testing."""
|
||||
# The website uses a random value as qToken for the first page. For our use case,
|
||||
# it's easier if the qToken can be deterministically re-calculated based on the search query,
|
||||
# so that we can the same result when calling _get_new_token for the second, third, ... page
|
||||
#
|
||||
# var qToken = Math.ceil(Math.random() * parseInt("ZZZZ", 36)).toString(36);
|
||||
# while (qToken.length < 4) qToken = '0' + qToken;
|
||||
# qToken = qToken + "_" + pageno
|
||||
query_hash = sha256(query.encode()).digest()
|
||||
hash_start = base64.b64encode(query_hash).decode()[0:4]
|
||||
return f"{hash_start}_{pageno}"
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams"):
|
||||
offset = (params["pageno"] - 1) * page_size
|
||||
|
||||
# always seems to find 20 results max
|
||||
if offset >= 20:
|
||||
params["url"] = None
|
||||
return
|
||||
|
||||
args = {
|
||||
"q": query,
|
||||
"key": "core-web",
|
||||
"num": str(page_size),
|
||||
"off": offset,
|
||||
"rSort": "__metasearch_score_d:desc",
|
||||
# it supports many more fields, but none of them are really relevant
|
||||
"names": "title_t,content_txt,url_s",
|
||||
"qNames": "title_t",
|
||||
"qToken": _get_new_token(query, params["pageno"]),
|
||||
}
|
||||
params["url"] = f"{base_url}/search?{urlencode(args)}"
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
res = EngineResults()
|
||||
|
||||
for group in resp.json()["data"]:
|
||||
group: dict[str, t.Any]
|
||||
for result in group["doclist"]["docs"]:
|
||||
result: dict[str, str]
|
||||
res.add(
|
||||
res.types.MainResult(
|
||||
url=result["url_s"],
|
||||
title=result["title_t"],
|
||||
content="".join(result["content_txt"]),
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
@@ -13,8 +13,8 @@ about = {
|
||||
"use_official_api": True,
|
||||
"require_api_key": False,
|
||||
"results": 'JSON',
|
||||
"language": 'ja',
|
||||
}
|
||||
language = "ja"
|
||||
|
||||
categories = ['dictionaries']
|
||||
paging = False
|
||||
@@ -110,8 +110,8 @@ def get_infobox(alt_forms, result_url, definitions):
|
||||
# definitions
|
||||
infobox_content.append(
|
||||
'''
|
||||
<small><a href="https://www.edrdg.org/wiki/index.php/JMdict-EDICT_Dictionary_Project">JMdict</a>
|
||||
and <a href="https://www.edrdg.org/enamdict/enamdict_doc.html">JMnedict</a>
|
||||
<small><a href="https://www.edrdg.org/wiki/index.php/JMdict-EDICT_Dictionary_Project">JMdict</a>
|
||||
and <a href="https://www.edrdg.org/enamdict/enamdict_doc.html">JMnedict</a>
|
||||
by <a href="https://www.edrdg.org/edrdg/licence.html">EDRDG</a>, CC BY-SA 3.0.</small>
|
||||
<ul>
|
||||
'''
|
||||
|
||||
@@ -79,6 +79,9 @@ from json import loads
|
||||
from urllib.parse import urlencode
|
||||
from searx.utils import to_string, html_to_text
|
||||
from searx.network import raise_for_httperror
|
||||
from searx.enginelib import EngineAbout
|
||||
|
||||
about = EngineAbout()
|
||||
|
||||
search_url = None
|
||||
"""
|
||||
|
||||
@@ -42,15 +42,14 @@ To enable Kagi, add the following to the ``engines`` seciton of
|
||||
.. _Api Portal: https://help.kagi.com/kagi/api/overview.html
|
||||
"""
|
||||
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import typing as t
|
||||
import html
|
||||
|
||||
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import parse_duration_string
|
||||
from searx.utils import html_to_text, parse_duration_string
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.search.processors import OnlineParams
|
||||
@@ -77,7 +76,12 @@ kagi_categ: t.Literal["search", "images", "news", "videos"] = "search"
|
||||
base_url = "https://kagi.com"
|
||||
|
||||
safe_search_map = {0: False, 1: True, 2: True}
|
||||
time_range_to_days_map: dict[TimeRangeType, int] = {"day": 1, "week": 7, "month": 30, "year": 365}
|
||||
time_range_to_days_map: dict[TimeRangeType, int] = {
|
||||
"day": 1,
|
||||
"week": 7,
|
||||
"month": 30,
|
||||
"year": 365,
|
||||
}
|
||||
|
||||
api_key = ""
|
||||
"""Kagi API key. Required for using this engine."""
|
||||
@@ -135,9 +139,13 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
|
||||
if kagi_categ in ("images", "videos"):
|
||||
# the JSON key is "image" for "images" and "video" for "videos"
|
||||
json_results = json_data["data"][kagi_categ[:-1]]
|
||||
json_results = json_data["data"].get(kagi_categ[:-1])
|
||||
else:
|
||||
json_results = json_data["data"][kagi_categ]
|
||||
json_results = json_data["data"].get(kagi_categ)
|
||||
|
||||
# if no results were found, the response doesn't contain the results field
|
||||
if not json_results:
|
||||
return res
|
||||
|
||||
for result in json_results:
|
||||
published_date: datetime | None = None
|
||||
@@ -148,8 +156,8 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
res.add(
|
||||
res.types.MainResult(
|
||||
url=result["url"],
|
||||
title=html.unescape(result["title"]),
|
||||
content=html.unescape(result["snippet"]),
|
||||
title=html_to_text(result.get("title", "no title available")),
|
||||
content=html_to_text(result.get("snippet", "")),
|
||||
thumbnail=result.get("image", {}).get("url") or "",
|
||||
publishedDate=published_date,
|
||||
)
|
||||
@@ -158,15 +166,15 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
res.add(
|
||||
res.types.Image(
|
||||
url=result["url"],
|
||||
title=html.unescape(result.get("title")),
|
||||
title=html_to_text(result.get("title", "no title available")),
|
||||
img_src=result.get("image", {}).get("url"),
|
||||
resolution=f"{result['image']['width']}x{result['image']['height']}",
|
||||
resolution=f"{result.get('image', {}).get('width')}x{result.get('image', {}).get('height')}",
|
||||
thumbnail_src=result.get("props", {}).get("thumbnail", {}).get("url"),
|
||||
)
|
||||
)
|
||||
elif kagi_categ == "videos":
|
||||
length: timedelta | None = None
|
||||
if result["props"].get("duration"):
|
||||
if result.get("props", {}).get("duration"):
|
||||
length = parse_duration_string(result["props"]["duration"])
|
||||
|
||||
res.add(
|
||||
@@ -174,11 +182,11 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
{
|
||||
"template": "videos.html",
|
||||
"url": result["url"],
|
||||
"title": html.unescape(result["title"]),
|
||||
"content": html.unescape(result["snippet"]),
|
||||
"title": html_to_text(result.get("title", "no title available")),
|
||||
"content": html_to_text(result.get("snippet", "")),
|
||||
"thumbnail": result.get("image", {}).get("url"),
|
||||
"publishedDate": published_date,
|
||||
"author": result["props"].get("creator_name"),
|
||||
"author": result.get("props", {}).get("creator_name"),
|
||||
"length": length,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -92,7 +92,7 @@ def _get_communities(json):
|
||||
'title': result['community']['title'],
|
||||
'content': markdown_to_text(result['community'].get('description', '')),
|
||||
'thumbnail': result['community'].get('icon', result['community'].get('banner')),
|
||||
'publishedDate': datetime.strptime(counts['published'][:19], '%Y-%m-%dT%H:%M:%S'),
|
||||
'publishedDate': datetime.fromisoformat(counts['published'][:19]),
|
||||
'metadata': metadata,
|
||||
}
|
||||
)
|
||||
@@ -141,7 +141,7 @@ def _get_posts(json):
|
||||
'title': result['post']['name'],
|
||||
'content': content,
|
||||
'thumbnail': thumbnail,
|
||||
'publishedDate': datetime.strptime(result['post']['published'][:19], '%Y-%m-%dT%H:%M:%S'),
|
||||
'publishedDate': datetime.fromisoformat(result['post']['published'][:19]),
|
||||
'metadata': metadata,
|
||||
}
|
||||
)
|
||||
@@ -170,7 +170,7 @@ def _get_comments(json):
|
||||
'url': result['comment']['ap_id'],
|
||||
'title': result['post']['name'],
|
||||
'content': markdown_to_text(result['comment']['content']),
|
||||
'publishedDate': datetime.strptime(result['comment']['published'][:19], '%Y-%m-%dT%H:%M:%S'),
|
||||
'publishedDate': datetime.fromisoformat(result['comment']['published'][:19]),
|
||||
'metadata': metadata,
|
||||
}
|
||||
)
|
||||
|
||||
62
searx/engines/magnific.py
Normal file
62
searx/engines/magnific.py
Normal file
@@ -0,0 +1,62 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Magnific_ is a database for images.
|
||||
|
||||
.. _Magnific: https://www.magnific.com
|
||||
"""
|
||||
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import typing as t
|
||||
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
|
||||
about = {
|
||||
"website": "https://www.magnific.com",
|
||||
"wikidata_id": "Q104211654",
|
||||
"official_api_documentation": None,
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
}
|
||||
|
||||
base_url = "https://www.magnific.com"
|
||||
|
||||
categories = ["images"]
|
||||
paging = True
|
||||
|
||||
free_images_only = True
|
||||
"""
|
||||
Whether to only load images that may be used for free, without a Magnific account.
|
||||
"""
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
args = {"term": query, "filters[ai-generated][excluded]": 1, "page": params["pageno"], "locale": "en"}
|
||||
if free_images_only:
|
||||
args["filters[license]"] = "free"
|
||||
|
||||
params["headers"]["Referer"] = f"{base_url}/search"
|
||||
params["url"] = f"{base_url}/api/regular/search?{urlencode(args)}"
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response"):
|
||||
res = EngineResults()
|
||||
|
||||
result: dict[str, t.Any] # TBH: dict[str, t.Any]
|
||||
for result in resp.json()["items"]:
|
||||
res.add(
|
||||
res.types.Image(
|
||||
title=result["name"],
|
||||
url=result["url"],
|
||||
thumbnail_src=result["preview"]["url"],
|
||||
img_src=result["preview"]["url"],
|
||||
resolution=f"{result['preview']['width']}x{result['preview']['height']}",
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
@@ -44,7 +44,7 @@ about = {
|
||||
|
||||
base_url = "https://api2.marginalia-search.com"
|
||||
safesearch = True
|
||||
categories = ["general"]
|
||||
categories = ["general", "blogs"]
|
||||
paging = True
|
||||
results_per_page = 20
|
||||
api_key = None
|
||||
|
||||
@@ -60,7 +60,7 @@ def response(resp):
|
||||
'title': result['username'] + f" ({result['followers_count']} followers)",
|
||||
'content': result['note'],
|
||||
'thumbnail': result.get('avatar'),
|
||||
'publishedDate': datetime.strptime(result['created_at'][:10], "%Y-%m-%d"),
|
||||
'publishedDate': datetime.fromisoformat(result['created_at'][:10]),
|
||||
}
|
||||
)
|
||||
elif mastodon_type == "hashtags":
|
||||
|
||||
@@ -11,9 +11,9 @@ about = {
|
||||
"use_official_api": True,
|
||||
"require_api_key": False,
|
||||
"results": 'JSON',
|
||||
"language": "de",
|
||||
}
|
||||
|
||||
language = "de"
|
||||
categories = ['videos']
|
||||
paging = True
|
||||
time_range_support = False
|
||||
|
||||
@@ -20,6 +20,7 @@ about = {
|
||||
}
|
||||
paging = True # paging is only supported for general search
|
||||
safesearch = True
|
||||
language_support = True
|
||||
time_range_support = True # time range search is supported for general and news
|
||||
max_page = 10
|
||||
|
||||
|
||||
@@ -35,8 +35,9 @@ about = {
|
||||
'use_official_api': False,
|
||||
'require_api_key': False,
|
||||
'results': 'JSON',
|
||||
'language': 'de',
|
||||
}
|
||||
language = "de"
|
||||
|
||||
paging = True
|
||||
categories = ["movies"]
|
||||
|
||||
|
||||
@@ -26,8 +26,8 @@ about = {
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "HTML",
|
||||
"language": "ko",
|
||||
}
|
||||
language = "ko"
|
||||
|
||||
categories = []
|
||||
paging = True
|
||||
@@ -99,23 +99,35 @@ def parse_general(data):
|
||||
|
||||
dom = html.fromstring(data)
|
||||
|
||||
for item in eval_xpath_list(dom, "//ul[contains(@class, 'lst_total')]/li[contains(@class, 'bx')]"):
|
||||
thumbnail = None
|
||||
for item in eval_xpath_list(dom, "//div[contains(@class, 'fds-web-normal-doc-root')]"):
|
||||
thumbnail = extract_text(
|
||||
eval_xpath(
|
||||
item,
|
||||
".//div[contains(@class, 'sds-comps-image') and not(contains(@class, 'sds-comps-image-circle'))]/img/@src",
|
||||
)
|
||||
)
|
||||
|
||||
title = extract_text(eval_xpath(item, ".//span[contains(@class, 'sds-comps-text-type-headline1')]"))
|
||||
|
||||
url = None
|
||||
try:
|
||||
thumbnail = eval_xpath_getindex(item, ".//div[contains(@class, 'thumb_single')]//img/@data-lazysrc", 0)
|
||||
url = eval_xpath_getindex(
|
||||
item, ".//a[starts-with(@href, 'http') and not(contains(@href, 'keep.naver.com'))]/@href", 0
|
||||
)
|
||||
except (ValueError, TypeError, SearxEngineXPathException):
|
||||
pass
|
||||
|
||||
results.add(
|
||||
MainResult(
|
||||
title=extract_text(eval_xpath(item, ".//a[contains(@class, 'link_tit')]")),
|
||||
url=eval_xpath_getindex(item, ".//a[contains(@class, 'link_tit')]/@href", 0),
|
||||
content=extract_text(
|
||||
eval_xpath(item, ".//div[contains(@class, 'total_dsc_wrap')]//a[contains(@class, 'api_txt_lines')]")
|
||||
),
|
||||
thumbnail=thumbnail,
|
||||
content = extract_text(eval_xpath(item, ".//span[contains(@class, 'sds-comps-text-type-body1')]"))
|
||||
|
||||
if title and url:
|
||||
results.add(
|
||||
MainResult(
|
||||
title=title,
|
||||
url=url,
|
||||
content=content or "",
|
||||
thumbnail=thumbnail or "",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
@@ -173,7 +185,7 @@ def parse_news(data):
|
||||
title=title,
|
||||
url=url,
|
||||
content=content,
|
||||
thumbnail=thumbnail,
|
||||
thumbnail=thumbnail or "",
|
||||
)
|
||||
)
|
||||
|
||||
@@ -196,7 +208,7 @@ def parse_videos(data):
|
||||
|
||||
length = None
|
||||
try:
|
||||
length = parse_duration_string(extract_text(eval_xpath(item, ".//span[contains(@class, 'time')]")))
|
||||
length = parse_duration_string(extract_text(eval_xpath(item, ".//span[contains(@class, 'time')]")) or "")
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
|
||||
66
searx/engines/neocities.py
Normal file
66
searx/engines/neocities.py
Normal file
@@ -0,0 +1,66 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Neocities_ is open source software for creating blogs.
|
||||
|
||||
.. _Neocities : https://github.com/neocities/neocities
|
||||
"""
|
||||
|
||||
from urllib.parse import urlencode
|
||||
import typing as t
|
||||
|
||||
from lxml import html
|
||||
|
||||
from searx.utils import eval_xpath, eval_xpath_list, extract_text
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from extended_types import SXNG_Response
|
||||
from search.processors import OnlineParams
|
||||
|
||||
|
||||
# Engine metadata
|
||||
about = {
|
||||
"website": "https://neocities.org/",
|
||||
"wikidata_id": "Q17071099",
|
||||
"official_api_documentation": None,
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "HTML",
|
||||
}
|
||||
|
||||
# Engine configuration
|
||||
categories = ["general", "blogs"]
|
||||
paging = True
|
||||
|
||||
# Search URL
|
||||
base_url = "https://neocities.org"
|
||||
|
||||
results_xpath = "//div[@class='result-item']"
|
||||
url_xpath = './/div[@class="result-url"]/a/@href'
|
||||
title_xpath = './/h3[@class="result-title"]/a/text()'
|
||||
content_xpath = './/p[@class="result-snippet"]//text()'
|
||||
screenshot_xpath = './/a[@class="result-screenshot"]/img/@src'
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
query_params: dict[str, t.Any] = {"q": query}
|
||||
if params['pageno'] > 1:
|
||||
offset = (params["pageno"] - 1) * 100
|
||||
query_params["start"] = offset
|
||||
params["url"] = f"{base_url}/search?{urlencode(query_params)}"
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
results = EngineResults()
|
||||
dom = html.fromstring(resp.text)
|
||||
|
||||
for result in eval_xpath_list(dom, results_xpath):
|
||||
results.add(
|
||||
results.types.MainResult(
|
||||
url=extract_text(eval_xpath(result, url_xpath)),
|
||||
title=extract_text(eval_xpath(result, title_xpath)) or "",
|
||||
content=extract_text(eval_xpath(result, content_xpath)) or "",
|
||||
thumbnail=base_url + (extract_text(eval_xpath(result, screenshot_xpath)) or ""),
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
92
searx/engines/neosearch.py
Normal file
92
searx/engines/neosearch.py
Normal file
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Neosearch_ aims to be a privacy-first alternative to Google.
|
||||
|
||||
.. _Neosearch: https://neosearch.org/About
|
||||
"""
|
||||
|
||||
from json import loads
|
||||
import typing as t
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.enginelib.traits import EngineTraits
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
traits: EngineTraits
|
||||
|
||||
about = {
|
||||
"website": "https://neosearch.org",
|
||||
"official_api_documentation": None,
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
}
|
||||
|
||||
base_url = "https://neosearch.org"
|
||||
categories = ["general"]
|
||||
|
||||
paging = False
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams"):
|
||||
args = {"q": query, "generate": "auto"}
|
||||
countrycode = params["searxng_locale"].split("-")[-1].upper()
|
||||
if countrycode in traits.custom["countrycodes"]:
|
||||
args["loc"] = countrycode
|
||||
params["url"] = f"{base_url}/search?{urlencode(args)}"
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
res = EngineResults()
|
||||
|
||||
# first line contains something like `{"location": "de"}`
|
||||
# second line contains the actual results
|
||||
json_resp = loads(resp.text.splitlines()[-1])
|
||||
for lens in json_resp["lenses"]:
|
||||
for category in lens["categories"]:
|
||||
for result in category["links"]:
|
||||
if not result["url"]:
|
||||
continue
|
||||
|
||||
res.add(
|
||||
res.types.MainResult(
|
||||
url=result["url"],
|
||||
title=result["title"],
|
||||
content=result["snippet"] or result["description"],
|
||||
)
|
||||
)
|
||||
|
||||
for suggestion in json_resp.get("suggestions", []):
|
||||
res.add(res.types.LegacyResult(suggestion=suggestion))
|
||||
|
||||
return res
|
||||
|
||||
|
||||
def fetch_traits(engine_traits: "EngineTraits") -> None:
|
||||
# pylint: disable=import-outside-toplevel
|
||||
from searx.network import get
|
||||
from searx.utils import extr, js_obj_str_to_python
|
||||
from babel.core import get_global
|
||||
|
||||
resp = get(base_url)
|
||||
|
||||
locations_js_raw = extr(resp.text, "const LOCATIONS = ", ";")
|
||||
if not locations_js_raw:
|
||||
raise RuntimeError("failed to find locations in neosearch HTML response")
|
||||
locations: list[dict[str, str]] = js_obj_str_to_python(locations_js_raw)
|
||||
|
||||
babel_reg_list = get_global("territory_languages").keys()
|
||||
|
||||
countrycodes: list[str] = []
|
||||
for loc in locations:
|
||||
_reg = loc["code"].upper()
|
||||
if _reg not in babel_reg_list:
|
||||
print(f"ERROR: region tag {_reg} is unknown by babel")
|
||||
continue
|
||||
countrycodes.append(_reg)
|
||||
|
||||
countrycodes.sort()
|
||||
engine_traits.custom["countrycodes"] = countrycodes
|
||||
@@ -13,8 +13,8 @@ about = {
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "HTML",
|
||||
"language": "ja",
|
||||
}
|
||||
language = "ja"
|
||||
|
||||
categories = ["videos"]
|
||||
paging = True
|
||||
|
||||
@@ -44,7 +44,7 @@ def response(resp) -> EngineResults:
|
||||
|
||||
cve_id = item["cve"]["id"]
|
||||
description = item["cve"]["descriptions"][0]["value"]
|
||||
date = datetime.strptime(item["cve"]["published"], "%Y-%m-%dT%H:%M:%S.%f")
|
||||
date = datetime.fromisoformat(item["cve"]["published"])
|
||||
|
||||
# Extract severity (Low, Medium, High, or Critical) and CVSS score, if available
|
||||
info = item["cve"].get("metrics", {}).get("cvssMetricV31", [{}])[0].get("cvssData", {})
|
||||
|
||||
@@ -26,6 +26,7 @@ about = {
|
||||
# Engine configuration
|
||||
paging = True
|
||||
time_range_support = True
|
||||
language_support = True
|
||||
results_per_page = 20
|
||||
categories = ["videos"]
|
||||
|
||||
@@ -75,7 +76,7 @@ def response(resp):
|
||||
release_time = item["release_time"]
|
||||
duration = item["duration"]
|
||||
|
||||
release_date = datetime.strptime(release_time.split("T")[0], "%Y-%m-%d")
|
||||
release_date = datetime.fromisoformat(release_time.split("T")[0])
|
||||
formatted_date = datetime.fromtimestamp(release_date.timestamp())
|
||||
|
||||
url = f"https://odysee.com/{name}:{claim_id}"
|
||||
|
||||
@@ -25,6 +25,7 @@ about = {
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
}
|
||||
language_support = True
|
||||
|
||||
# engine dependent config
|
||||
categories = ["videos"]
|
||||
|
||||
@@ -28,7 +28,7 @@ search_string = 'api/?{query}&limit={limit}'
|
||||
result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'
|
||||
|
||||
# list of supported languages
|
||||
supported_languages = ['de', 'en', 'fr', 'it']
|
||||
photon_supported_languages = ["de", "en", "fr", "it"]
|
||||
|
||||
|
||||
# do search-request
|
||||
@@ -37,7 +37,7 @@ def request(query, params):
|
||||
|
||||
if params['language'] != 'all':
|
||||
language = params['language'].split('_')[0]
|
||||
if language in supported_languages:
|
||||
if language in photon_supported_languages:
|
||||
params['url'] = params['url'] + "&lang=" + language
|
||||
|
||||
# using SearXNG User-Agent
|
||||
|
||||
63
searx/engines/picjumbo.py
Normal file
63
searx/engines/picjumbo.py
Normal file
@@ -0,0 +1,63 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Picjumbo_ provides free stock photos.
|
||||
|
||||
.. _Picjumbo: https://picjumbo.com
|
||||
"""
|
||||
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
import typing as t
|
||||
|
||||
from lxml import html
|
||||
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import eval_xpath, eval_xpath_list, extract_text
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
|
||||
about = {
|
||||
"website": "https://picjumbo.com",
|
||||
"wikidata_id": None,
|
||||
"official_api_documentation": None,
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "HTML",
|
||||
}
|
||||
|
||||
base_url = "https://picjumbo.com"
|
||||
|
||||
categories = ["images"]
|
||||
paging = True
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
params["url"] = f"{base_url}/search/{query}/page/{params['pageno']}"
|
||||
|
||||
|
||||
def _get_max_res_url(url: str) -> str:
|
||||
"""Get the maximum resolution of the image based on the thumbnail URL."""
|
||||
parsed_url = urlparse(url)
|
||||
max_res_url = parsed_url._replace(query="w=10000&quality=100")
|
||||
return urlunparse(max_res_url)
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response"):
|
||||
res = EngineResults()
|
||||
|
||||
doc = html.fromstring(resp.text)
|
||||
|
||||
for result in eval_xpath_list(doc, "//div[contains(@class, 'photo_query')]/div[contains(@class, 'photo_item')]"):
|
||||
thumbnail = extract_text(eval_xpath(result, ".//img[contains(@class, 'image')]/@src")) or ""
|
||||
res.add(
|
||||
res.types.Image(
|
||||
url=extract_text(eval_xpath(result, ".//a[contains(@class, 'image')]/@href")) or "",
|
||||
title=extract_text(eval_xpath(result, ".//h3")) or "",
|
||||
content=extract_text(eval_xpath(result, ".//meta[@itemprop='keywords']/@content")) or "",
|
||||
thumbnail_src=thumbnail,
|
||||
img_src=_get_max_res_url(thumbnail),
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
@@ -54,7 +54,7 @@ def response(resp: "SXNG_Response"):
|
||||
title=result["title"],
|
||||
content=result["description"],
|
||||
thumbnail=result["image_url"],
|
||||
publishedDate=datetime.strptime(result["created_at"], "%Y-%m-%d %H:%M:%S"),
|
||||
publishedDate=datetime.fromisoformat(result["created_at"]),
|
||||
metadata=" | ".join(metadata),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -77,7 +77,7 @@ from searx.utils import gen_useragent, html_to_text, parse_duration_string
|
||||
|
||||
about = {
|
||||
"website": "https://presearch.io",
|
||||
"wikidiata_id": "Q7240905",
|
||||
"wikidata_id": "Q7240905",
|
||||
"official_api_documentation": "https://docs.presearch.io/nodes/api",
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
|
||||
@@ -140,7 +140,7 @@ def response(resp):
|
||||
results.append(
|
||||
{
|
||||
'template': 'images.html',
|
||||
'url': _clean_url(f"{about['website']}/images/{result['objectID']}"),
|
||||
'url': _clean_url(f"{pdia_base_url}/images/{result['objectID']}"),
|
||||
'img_src': _clean_url(base_image_url),
|
||||
'thumbnail_src': _clean_url(base_image_url + THUMBNAIL_SUFFIX),
|
||||
'title': f"{result['title'].strip()} by {result['artist']} {result.get('displayYear', '')}",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user