mirror of
https://github.com/searxng/searxng.git
synced 2025-12-24 04:30:02 +00:00
[mod] typification of SearXNG: add new result type File
This PR adds a new result type: File
Python class: searx/result_types/file.py
Jinja template: searx/templates/simple/result_templates/file.html
CSS (less) client/simple/src/less/result_types/file.less
Class 'File' (singular) replaces template 'files.html' (plural). The renaming
was carried out because there is only one file (singular) in a result. Not to be
confused with the category 'files' where in multiple results can exist.
As mentioned in issue [1], the class '.category-files' was removed from the CSS
and the stylesheet was adopted in result_types/file.less (there based on the
templates and no longer based on the category).
[1] https://github.com/searxng/searxng/issues/5198
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser
committed by
Markus Heiser
Markus Heiser
parent
ee6d4f322f
commit
9371658531
@@ -1,102 +1,208 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Wikimedia Commons (images)"""
|
||||
"""`Wikimedia Commons`_ is a collection of more than 120 millions freely usable
|
||||
media files to which anyone can contribute.
|
||||
|
||||
This engine uses the `MediaWiki query API`_, with which engines can be configured
|
||||
for searching images, videos, audio, and other files in the Wikimedia.
|
||||
|
||||
.. _MediaWiki query API: https://commons.wikimedia.org/w/api.php?action=help&modules=query
|
||||
.. _Wikimedia Commons: https://commons.wikimedia.org/
|
||||
|
||||
|
||||
Configuration
|
||||
=============
|
||||
|
||||
The engine has the following additional settings:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: wikicommons.images
|
||||
engine: wikicommons
|
||||
wc_search_type: image
|
||||
|
||||
- name: wikicommons.videos
|
||||
engine: wikicommons
|
||||
wc_search_type: video
|
||||
|
||||
- name: wikicommons.audio
|
||||
engine: wikicommons
|
||||
wc_search_type: audio
|
||||
|
||||
- name: wikicommons.files
|
||||
engine: wikicommons
|
||||
wc_search_type: file
|
||||
|
||||
|
||||
Implementations
|
||||
===============
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
|
||||
import datetime
|
||||
|
||||
from urllib.parse import urlencode
|
||||
import pathlib
|
||||
from urllib.parse import urlencode, unquote
|
||||
|
||||
from searx.utils import html_to_text, humanize_bytes
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
# about
|
||||
about = {
|
||||
"website": 'https://commons.wikimedia.org/',
|
||||
"wikidata_id": 'Q565',
|
||||
"official_api_documentation": 'https://commons.wikimedia.org/w/api.php',
|
||||
"website": "https://commons.wikimedia.org/",
|
||||
"wikidata_id": "Q565",
|
||||
"official_api_documentation": "https://commons.wikimedia.org/w/api.php",
|
||||
"use_official_api": True,
|
||||
"require_api_key": False,
|
||||
"results": 'JSON',
|
||||
"results": "JSON",
|
||||
}
|
||||
categories = ['images']
|
||||
search_type = 'images'
|
||||
|
||||
base_url = "https://commons.wikimedia.org"
|
||||
search_prefix = (
|
||||
'?action=query'
|
||||
'&format=json'
|
||||
'&generator=search'
|
||||
'&gsrnamespace=6'
|
||||
'&gsrprop=snippet'
|
||||
'&prop=info|imageinfo'
|
||||
'&iiprop=url|size|mime'
|
||||
'&iiurlheight=180' # needed for the thumb url
|
||||
)
|
||||
categories: list[str] = []
|
||||
paging = True
|
||||
number_of_results = 10
|
||||
|
||||
search_types = {
|
||||
'images': 'bitmap|drawing',
|
||||
'videos': 'video',
|
||||
'audio': 'audio',
|
||||
'files': 'multimedia|office|archive|3d',
|
||||
wc_api_url = "https://commons.wikimedia.org/w/api.php"
|
||||
wc_search_type: str = ""
|
||||
|
||||
SEARCH_TYPES: dict[str, str] = {
|
||||
"image": "bitmap|drawing",
|
||||
"video": "video",
|
||||
"audio": "audio",
|
||||
"file": "multimedia|office|archive|3d",
|
||||
}
|
||||
# FileType = t.Literal["bitmap", "drawing", "video", "audio", "multimedia", "office", "archive", "3d"]
|
||||
# FILE_TYPES = list(t.get_args(FileType))
|
||||
|
||||
|
||||
def request(query, params):
|
||||
language = 'en'
|
||||
if params['language'] != 'all':
|
||||
language = params['language'].split('-')[0]
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool:
|
||||
"""Initialization of the Wikimedia engine, checks if the value configured in
|
||||
:py:obj:`wc_search_type` is valid."""
|
||||
|
||||
if search_type not in search_types:
|
||||
raise ValueError(f"Unsupported search type: {search_type}")
|
||||
if engine_settings.get("wc_search_type") not in SEARCH_TYPES:
|
||||
logger.error(
|
||||
"wc_search_type: %s isn't a valid file type (%s)",
|
||||
engine_settings.get("wc_search_type"),
|
||||
",".join(SEARCH_TYPES.keys()),
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
filetype = search_types[search_type]
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
uselang: str = "en"
|
||||
if params["searxng_locale"] != "all":
|
||||
uselang = params["searxng_locale"].split("-")[0]
|
||||
filetype = SEARCH_TYPES[wc_search_type]
|
||||
args = {
|
||||
'uselang': language,
|
||||
'gsrlimit': number_of_results,
|
||||
'gsroffset': number_of_results * (params["pageno"] - 1),
|
||||
'gsrsearch': f"filetype:{filetype} {query}",
|
||||
# https://commons.wikimedia.org/w/api.php
|
||||
"format": "json",
|
||||
"uselang": uselang,
|
||||
"action": "query",
|
||||
# https://commons.wikimedia.org/w/api.php?action=help&modules=query
|
||||
"prop": "info|imageinfo",
|
||||
# generator (gsr optins) https://commons.wikimedia.org/w/api.php?action=help&modules=query%2Bsearch
|
||||
"generator": "search",
|
||||
"gsrnamespace": "6", # https://www.mediawiki.org/wiki/Help:Namespaces#Renaming_namespaces
|
||||
"gsrprop": "snippet",
|
||||
"gsrlimit": number_of_results,
|
||||
"gsroffset": number_of_results * (params["pageno"] - 1),
|
||||
"gsrsearch": f"filetype:{filetype} {query}",
|
||||
# imageinfo: https://commons.wikimedia.org/w/api.php?action=help&modules=query%2Bimageinfo
|
||||
"iiprop": "url|size|mime",
|
||||
"iiurlheight": "180", # needed for the thumb url
|
||||
}
|
||||
|
||||
params["url"] = f"{base_url}/w/api.php{search_prefix}&{urlencode(args, safe=':|')}"
|
||||
return params
|
||||
params["url"] = f"{wc_api_url}?{urlencode(args, safe=':|')}"
|
||||
|
||||
|
||||
def response(resp):
|
||||
results = []
|
||||
json = resp.json()
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
|
||||
if not json.get("query", {}).get("pages"):
|
||||
return results
|
||||
for item in json["query"]["pages"].values():
|
||||
res = EngineResults()
|
||||
json_data = resp.json()
|
||||
pages = json_data.get("query", {}).get("pages", {}).values()
|
||||
|
||||
for item in pages:
|
||||
|
||||
if not item.get("imageinfo", []):
|
||||
continue
|
||||
imageinfo = item["imageinfo"][0]
|
||||
title = item["title"].replace("File:", "").rsplit('.', 1)[0]
|
||||
result = {
|
||||
'url': imageinfo["descriptionurl"],
|
||||
'title': title,
|
||||
'content': html_to_text(item["snippet"]),
|
||||
}
|
||||
|
||||
if search_type == "images":
|
||||
result['template'] = 'images.html'
|
||||
result['img_src'] = imageinfo["url"]
|
||||
result['thumbnail_src'] = imageinfo["thumburl"]
|
||||
result['resolution'] = f'{imageinfo["width"]} x {imageinfo["height"]}'
|
||||
else:
|
||||
result['thumbnail'] = imageinfo["thumburl"]
|
||||
title: str = item["title"].replace("File:", "").rsplit(".", 1)[0]
|
||||
content = html_to_text(item["snippet"])
|
||||
|
||||
if search_type == "videos":
|
||||
result['template'] = 'videos.html'
|
||||
if imageinfo.get('duration'):
|
||||
result['length'] = datetime.timedelta(seconds=int(imageinfo['duration']))
|
||||
result['iframe_src'] = imageinfo['url']
|
||||
elif search_type == "files":
|
||||
result['template'] = 'files.html'
|
||||
result['metadata'] = imageinfo['mime']
|
||||
result['size'] = humanize_bytes(imageinfo['size'])
|
||||
elif search_type == "audio":
|
||||
result['iframe_src'] = imageinfo['url']
|
||||
url: str = imageinfo["descriptionurl"]
|
||||
media_url: str = imageinfo["url"]
|
||||
mimetype: str = imageinfo["mime"]
|
||||
thumbnail: str = imageinfo["thumburl"]
|
||||
size = imageinfo.get("size")
|
||||
if size:
|
||||
size = humanize_bytes(size)
|
||||
|
||||
results.append(result)
|
||||
duration = None
|
||||
seconds: str = imageinfo.get("duration")
|
||||
if seconds:
|
||||
try:
|
||||
duration = datetime.timedelta(seconds=int(seconds))
|
||||
except OverflowError:
|
||||
pass
|
||||
|
||||
return results
|
||||
if wc_search_type == "file":
|
||||
res.add(
|
||||
res.types.File(
|
||||
title=title,
|
||||
url=url,
|
||||
content=content,
|
||||
size=size,
|
||||
mimetype=mimetype,
|
||||
filename=unquote(pathlib.Path(media_url).name),
|
||||
embedded=media_url,
|
||||
thumbnail=thumbnail,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
if wc_search_type == "image":
|
||||
res.add(
|
||||
res.types.LegacyResult(
|
||||
template="images.html",
|
||||
title=title,
|
||||
url=url,
|
||||
content=content,
|
||||
img_src=imageinfo["url"],
|
||||
thumbnail_src=thumbnail,
|
||||
resolution=f"{imageinfo['width']} x {imageinfo['height']}",
|
||||
img_format=imageinfo["mime"],
|
||||
filesize=size,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
if wc_search_type == "video":
|
||||
res.add(
|
||||
res.types.LegacyResult(
|
||||
template="videos.html",
|
||||
title=title,
|
||||
url=url,
|
||||
content=content,
|
||||
iframe_src=media_url,
|
||||
length=duration,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
if wc_search_type == "audio":
|
||||
res.add(
|
||||
res.types.MainResult(
|
||||
template="default.html",
|
||||
title=title,
|
||||
url=url,
|
||||
content=content,
|
||||
audio_src=media_url,
|
||||
length=duration,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
return res
|
||||
|
||||
Reference in New Issue
Block a user