From 274b63b677abe1dad8c7f8284a90a365b9f16658 Mon Sep 17 00:00:00 2001 From: Bnyro Date: Sun, 13 Sep 2026 14:01:58 +0200 Subject: [PATCH] [refactor] engines: migrate to `SXNG_Response.html()` (#6718) --- searx/engines/1337x.py | 3 +-- searx/engines/360search.py | 3 +-- searx/engines/alpinelinux.py | 3 +-- searx/engines/annas_archive.py | 5 ++--- searx/engines/ansa.py | 3 +-- searx/engines/apkmirror.py | 3 +-- searx/engines/archlinux.py | 5 ++--- searx/engines/bandcamp.py | 3 +-- searx/engines/bing.py | 5 ++--- searx/engines/bing_images.py | 4 +--- searx/engines/bing_news.py | 4 +--- searx/engines/bing_videos.py | 4 +--- searx/engines/brave.py | 7 +++---- searx/engines/btdigg.py | 2 +- searx/engines/destatis.py | 3 +-- searx/engines/deviantart.py | 3 +-- searx/engines/dictzone.py | 3 +-- searx/engines/digbt.py | 3 +-- searx/engines/duckduckgo.py | 3 +-- searx/engines/duckduckgo_web.py | 3 +-- searx/engines/duden.py | 3 +-- searx/engines/ebay.py | 3 +-- searx/engines/emojipedia.py | 3 +-- searx/engines/fdroid.py | 3 +-- searx/engines/findfiles.py | 4 +--- searx/engines/geizhals.py | 3 +-- searx/engines/giphy.py | 5 +---- searx/engines/gmx.py | 3 +-- searx/engines/goodreads.py | 3 +-- searx/engines/google_play.py | 5 ++--- searx/engines/google_scholar.py | 3 +-- searx/engines/imgur.py | 3 +-- searx/engines/ina.py | 3 +-- searx/engines/ipernity.py | 3 +-- searx/engines/kickass.py | 3 +-- searx/engines/lib_rs.py | 3 +-- searx/engines/luxxle.py | 3 +-- searx/engines/mojeek.py | 5 ++--- searx/engines/neocities.py | 4 +--- searx/engines/neosearch.py | 4 +--- searx/engines/niconico.py | 3 +-- searx/engines/nyaa.py | 3 +-- searx/engines/ollama.py | 3 +-- searx/engines/openclipart.py | 3 +-- searx/engines/pexels.py | 3 +-- searx/engines/picjumbo.py | 4 +--- searx/engines/pkg_go_dev.py | 3 +-- searx/engines/privacywall.py | 5 ++--- searx/engines/pypi.py | 3 +-- searx/engines/resulthunter.py | 4 +--- searx/engines/rottentomatoes.py | 3 +-- searx/engines/rumble.py | 3 +-- searx/engines/s1search.py | 4 +--- searx/engines/searchrockit.py | 3 +-- searx/engines/semantic_scholar.py | 3 +-- searx/engines/seznam.py | 5 ++--- searx/engines/shopify_stock.py | 4 +--- searx/engines/sogou.py | 2 +- searx/engines/sogou_wechat.py | 3 +-- searx/engines/solidtorrents.py | 4 +--- searx/engines/sourcehut.py | 3 +-- searx/engines/startpage.py | 5 ++--- searx/engines/tiger.py | 5 ++--- searx/engines/tokyotoshokan.py | 3 +-- searx/engines/tonline.py | 4 +--- searx/engines/uxwing.py | 3 +-- searx/engines/vuhuv.py | 4 +--- searx/engines/wikipedia.py | 3 +-- searx/engines/xpath.py | 3 +-- searx/engines/yahoo.py | 5 +---- searx/engines/yahoo_news.py | 3 +-- searx/engines/yandex.py | 2 +- searx/engines/yep.py | 3 +-- searx/engines/zlibrary.py | 5 ++--- tests/unit/engines/test_xpath.py | 20 ++++++++++++++++---- 75 files changed, 102 insertions(+), 178 deletions(-) diff --git a/searx/engines/1337x.py b/searx/engines/1337x.py index d6f515dc4..08a487a72 100644 --- a/searx/engines/1337x.py +++ b/searx/engines/1337x.py @@ -3,7 +3,6 @@ """1337x""" from urllib.parse import quote, urljoin -from lxml import html from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex # about @@ -31,7 +30,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath_list(dom, '//table[contains(@class, "table-list")]/tbody//tr'): href = urljoin(url, eval_xpath_getindex(result, './td[contains(@class, "name")]/a[2]/@href', 0)) diff --git a/searx/engines/360search.py b/searx/engines/360search.py index bf1ba9821..bb12a6409 100644 --- a/searx/engines/360search.py +++ b/searx/engines/360search.py @@ -5,7 +5,6 @@ import typing as t from urllib.parse import urlencode -from lxml import html from searx import logger from searx.enginelib import EngineCache @@ -88,7 +87,7 @@ def response(resp): if not resp.text or not resp.text.strip(): return [] - dom = html.fromstring(resp.text) + dom = resp.html() results = [] for item in dom.xpath('//li[contains(@class, "res-list")]'): diff --git a/searx/engines/alpinelinux.py b/searx/engines/alpinelinux.py index e5dcefed1..af26903e9 100644 --- a/searx/engines/alpinelinux.py +++ b/searx/engines/alpinelinux.py @@ -12,7 +12,6 @@ servers and for Docker images. import re from urllib.parse import urlencode -from lxml import html from dateutil import parser from searx.utils import eval_xpath, eval_xpath_list, extract_text @@ -57,7 +56,7 @@ def request(query, params): def response(resp): results = [] - doc = html.fromstring(resp.text) + doc = resp.html() for result in eval_xpath_list(doc, "//table/tbody/tr"): if len(result.xpath("./td")) < 9: diff --git a/searx/engines/annas_archive.py b/searx/engines/annas_archive.py index c4676cc31..4c547f4da 100644 --- a/searx/engines/annas_archive.py +++ b/searx/engines/annas_archive.py @@ -39,7 +39,6 @@ import random import typing as t from urllib.parse import urlencode -from lxml import html from lxml.etree import ElementBase from searx.data import ENGINE_TRAITS @@ -143,7 +142,7 @@ def request(query: str, params: "OnlineParams") -> None: def response(resp: "SXNG_Response") -> EngineResults: res = EngineResults() - dom = html.fromstring(resp.text) + dom = resp.html() # Each result is a div with class "flex" inside "js-aarecord-list-outer" # container. The "flex" filter excludes non-result div such as section @@ -255,7 +254,7 @@ def fetch_traits(engine_traits: EngineTraits) -> None: if not resp.ok: raise RuntimeError("Response from Anna's Archive is not OK.") - dom = html.fromstring(resp.text) + dom = resp.html() # supported language codes lang_map: dict[str, str] = {} diff --git a/searx/engines/ansa.py b/searx/engines/ansa.py index 21afed900..a26b9d4c4 100644 --- a/searx/engines/ansa.py +++ b/searx/engines/ansa.py @@ -14,7 +14,6 @@ list in ``settings.yml``: """ from urllib.parse import urlencode -from lxml import html from searx.result_types import EngineResults, MainResult from searx.utils import eval_xpath, eval_xpath_list, extract_text @@ -62,7 +61,7 @@ def request(query, params): def response(resp) -> EngineResults: res = EngineResults() - doc = html.fromstring(resp.text) + doc = resp.html() for result in eval_xpath_list(doc, "//div[@class='article']"): diff --git a/searx/engines/apkmirror.py b/searx/engines/apkmirror.py index 4dc5be8f0..48604f85a 100644 --- a/searx/engines/apkmirror.py +++ b/searx/engines/apkmirror.py @@ -4,7 +4,6 @@ # pylint: disable=invalid-name from urllib.parse import urlencode -from lxml import html from searx.utils import ( eval_xpath_list, @@ -43,7 +42,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() # parse results for result in eval_xpath_list(dom, "//div[@id='content']//div[@class='listWidget']/div/div[@class='appRow']"): diff --git a/searx/engines/archlinux.py b/searx/engines/archlinux.py index 04787b5aa..acf31c733 100644 --- a/searx/engines/archlinux.py +++ b/searx/engines/archlinux.py @@ -11,7 +11,6 @@ Arch Wiki blocks access to it. from urllib.parse import urlencode, urljoin, urlparse import babel -import lxml from searx.enginelib.traits import EngineTraits from searx.locales import language_tag @@ -70,7 +69,7 @@ def request(query, params): def response(resp): results = [] - dom = lxml.html.fromstring(resp.text) # type: ignore + dom = resp.html() # type: ignore # get the base URL for the language in which request was made sxng_lang = resp.search_params["searxng_locale"].split("-")[0] @@ -131,7 +130,7 @@ def fetch_traits(engine_traits: EngineTraits): if not resp.ok: raise RuntimeError("Response from Arch Linux Wiki is not OK.") - dom = lxml.html.fromstring(resp.text) # type: ignore + dom = resp.html() # type: ignore for a in eval_xpath_list(dom, "//a[@class='interlanguage-link-target']"): sxng_tag = language_tag(babel.Locale.parse(a.get("lang"), sep="-")) # zh_Hans --> zh diff --git a/searx/engines/bandcamp.py b/searx/engines/bandcamp.py index 590d206d2..d0212b404 100644 --- a/searx/engines/bandcamp.py +++ b/searx/engines/bandcamp.py @@ -10,7 +10,6 @@ from urllib.parse import urlencode, urlparse, parse_qs from dateutil.parser import parse as dateparse -from lxml import html from searx.utils import ( eval_xpath_getindex, @@ -46,7 +45,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath_list(dom, '//li[contains(@class, "searchresult")]'): diff --git a/searx/engines/bing.py b/searx/engines/bing.py index 955e5c43c..b9534b9a3 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -18,7 +18,6 @@ from urllib.parse import parse_qs, urlencode, urlparse import babel import babel.languages -from lxml import html from searx.enginelib.traits import EngineTraits from searx.locales import region_tag @@ -96,7 +95,7 @@ def response(resp: "SXNG_Response") -> list[dict[str, t.Any]]: results: list[dict[str, t.Any]] = [] - dom = html.fromstring(resp.text) + dom = resp.html() for item in eval_xpath_list(dom, '//ol[@id="b_results"]/li[contains(@class, "b_algo")]'): link = eval_xpath_getindex(item, ".//h2/a", 0, None) @@ -156,7 +155,7 @@ def fetch_traits(engine_traits: EngineTraits) -> None: if not resp.ok: raise RuntimeError("Response from Bing is not OK.") - dom = html.fromstring(resp.text) + dom = resp.html() map_market_codes: dict[str, str] = { "zh-hk": "en-hk", # not sure why, but at Microslop this is the market code for Hongkong diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py index 989adba06..c6d25bb61 100644 --- a/searx/engines/bing_images.py +++ b/searx/engines/bing_images.py @@ -5,8 +5,6 @@ import typing as t import json from urllib.parse import urlencode -from lxml import html - from searx.engines.bing import fetch_traits # pylint: disable=unused-import from searx.result_types import EngineResults @@ -75,7 +73,7 @@ def response(resp: "SXNG_Response") -> EngineResults: res = EngineResults() - dom = html.fromstring(resp.text) + dom = resp.html() for result in dom.xpath('//ul[contains(@class, "dgControl_list")]/li'): metadata = result.xpath('.//a[@class="iusc"]/@m') diff --git a/searx/engines/bing_news.py b/searx/engines/bing_news.py index 6a9df335a..112c87b74 100644 --- a/searx/engines/bing_news.py +++ b/searx/engines/bing_news.py @@ -9,8 +9,6 @@ from urllib.parse import urlencode -from lxml import html - from searx.enginelib.traits import EngineTraits from searx.engines.bing import get_locale_params from searx.utils import eval_xpath, eval_xpath_getindex, eval_xpath_list, extract_text @@ -78,7 +76,7 @@ def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for newsitem in eval_xpath_list(dom, '//div[contains(@class, "newsitem")]'): link = eval_xpath_getindex(newsitem, './/a[@class="title"]', 0, None) diff --git a/searx/engines/bing_videos.py b/searx/engines/bing_videos.py index 8b857aff3..d78b01213 100644 --- a/searx/engines/bing_videos.py +++ b/searx/engines/bing_videos.py @@ -4,8 +4,6 @@ import json from urllib.parse import urlencode -from lxml import html - from searx.engines.bing import ( # pylint: disable=unused-import fetch_traits, get_locale_params, @@ -68,7 +66,7 @@ def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for result in dom.xpath('//div[contains(@id, "mc_vtvc_video")]'): metadata = json.loads(eval_xpath_getindex(result, './/div[@class="vrhdata"]/@vrhm', index=0)) diff --git a/searx/engines/brave.py b/searx/engines/brave.py index 8d0994227..31e9d829a 100644 --- a/searx/engines/brave.py +++ b/searx/engines/brave.py @@ -125,7 +125,6 @@ from urllib.parse import ( ) from dateutil import parser -from lxml import html from searx import locales from searx.enginelib.traits import EngineTraits @@ -289,7 +288,7 @@ def response(resp: SXNG_Response) -> EngineResults: def _parse_search(resp: SXNG_Response) -> EngineResults: res = EngineResults() - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath_list(dom, "//div[contains(@class, 'snippet ')]"): url: str | None = eval_xpath_getindex(result, ".//a/@href", 0, default=None) @@ -352,7 +351,7 @@ def _parse_search(resp: SXNG_Response) -> EngineResults: def _parse_news(resp: SXNG_Response) -> EngineResults: res = EngineResults() - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath_list(dom, "//div[@data-type='news']"): url = eval_xpath_getindex(result, ".//a/@href", 0, default=None) @@ -434,7 +433,7 @@ def fetch_traits(engine_traits: EngineTraits): if not resp.ok: raise RuntimeError("Response from Brave languages is not OK.") - dom = html.fromstring(resp.text) + dom = resp.html() for option in dom.xpath("//section//option[@value='en-us']/../option"): ui_lang = option.get("value") diff --git a/searx/engines/btdigg.py b/searx/engines/btdigg.py index c5254fcf4..67bc32efc 100644 --- a/searx/engines/btdigg.py +++ b/searx/engines/btdigg.py @@ -38,7 +38,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() search_res = dom.xpath('//div[@class="one_result"]') diff --git a/searx/engines/destatis.py b/searx/engines/destatis.py index 3622cd173..05b09d31d 100644 --- a/searx/engines/destatis.py +++ b/searx/engines/destatis.py @@ -2,7 +2,6 @@ """DeStatis""" from urllib.parse import urlencode -from lxml import html from searx.utils import eval_xpath, eval_xpath_list, extract_text about = { @@ -42,7 +41,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() # filter out suggested results on further page because they're the same on each page extra_xpath = results_xpath_filter_recommended if resp.search_params['pageno'] > 1 else '' diff --git a/searx/engines/deviantart.py b/searx/engines/deviantart.py index 776a3eef9..f3b599117 100644 --- a/searx/engines/deviantart.py +++ b/searx/engines/deviantart.py @@ -4,7 +4,6 @@ import typing as t import urllib.parse -from lxml import html from searx.result_types import EngineResults from searx.utils import extract_text, eval_xpath, eval_xpath_list @@ -52,7 +51,7 @@ def request(query: str, params: "OnlineParams"): def response(resp: "SXNG_Response") -> EngineResults: res = EngineResults() - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath_list(dom, results_xpath): thumbnail_src = extract_text(eval_xpath(result, thumbnail_src_xpath)) diff --git a/searx/engines/dictzone.py b/searx/engines/dictzone.py index aa6c4806c..c9355bbf1 100644 --- a/searx/engines/dictzone.py +++ b/searx/engines/dictzone.py @@ -4,7 +4,6 @@ Dictzone """ import urllib.parse -from lxml import html from searx.utils import eval_xpath, extract_text from searx.result_types import EngineResults @@ -50,7 +49,7 @@ def response(resp) -> EngineResults: if not resp.ok: return results - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath(dom, ".//table[@id='r']//tr"): diff --git a/searx/engines/digbt.py b/searx/engines/digbt.py index cbe2e170b..4a23c4fce 100644 --- a/searx/engines/digbt.py +++ b/searx/engines/digbt.py @@ -4,7 +4,6 @@ DigBT (Videos, Music, Files) """ from urllib.parse import urljoin -from lxml import html from searx.utils import extract_text # about @@ -33,7 +32,7 @@ def request(query, params): def response(resp): - dom = html.fromstring(resp.text) + dom = resp.html() search_res = dom.xpath('.//td[@class="x-item"]') if not search_res: diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py index b32e0ed77..0533b3983 100644 --- a/searx/engines/duckduckgo.py +++ b/searx/engines/duckduckgo.py @@ -172,7 +172,6 @@ import re import typing as t import babel -import lxml.html from searx import locales from searx.enginelib import EngineCache @@ -470,7 +469,7 @@ def response(resp: "SXNG_Response") -> EngineResults: if resp.status_code == 303: return res - doc = lxml.html.fromstring(resp.text) + doc = resp.html() params = resp.search_params if is_ddg_captcha(doc): diff --git a/searx/engines/duckduckgo_web.py b/searx/engines/duckduckgo_web.py index 1ac174238..de8960adf 100644 --- a/searx/engines/duckduckgo_web.py +++ b/searx/engines/duckduckgo_web.py @@ -17,7 +17,6 @@ import typing as t import re from urllib.parse import quote_plus, urljoin -from lxml import html from searx.utils import html_to_text, extract_text, eval_xpath from searx.result_types import EngineResults @@ -81,7 +80,7 @@ def _fetch_first_page_link( if resp.status_code != 200: logger.error("vqd: got HTTP %s from duckduckgo.com", resp.status_code) - dom = html.fromstring(resp.text) + dom = resp.html() first_page_link = extract_text(eval_xpath(dom, "//link[@id='deep_preload_link']/@href")) if not first_page_link: diff --git a/searx/engines/duden.py b/searx/engines/duden.py index 77664f887..8aa67bf67 100644 --- a/searx/engines/duden.py +++ b/searx/engines/duden.py @@ -2,7 +2,6 @@ """Duden""" from urllib.parse import quote, urljoin -from lxml import html from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex from searx.network import raise_for_httperror @@ -48,7 +47,7 @@ def response(resp): raise_for_httperror(resp) - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath_list(dom, '//section[not(contains(@class, "essay"))]'): url = eval_xpath_getindex(result, './/h2/a', 0).get('href') diff --git a/searx/engines/ebay.py b/searx/engines/ebay.py index 365c5b5cc..658810cdf 100644 --- a/searx/engines/ebay.py +++ b/searx/engines/ebay.py @@ -5,7 +5,6 @@ Ebay (Videos, Music, Files) from urllib.parse import quote -from lxml import html from searx.engines.xpath import extract_text # about @@ -44,7 +43,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() results_dom = dom.xpath(results_xpath) if not results_dom: return [] diff --git a/searx/engines/emojipedia.py b/searx/engines/emojipedia.py index 9bda21234..b6d4efd5a 100644 --- a/searx/engines/emojipedia.py +++ b/searx/engines/emojipedia.py @@ -9,7 +9,6 @@ since 2021. Emojipedia is a voting member of The Unicode Consortium.[1] """ from urllib.parse import urlencode -from lxml import html from searx.utils import ( eval_xpath_list, @@ -41,7 +40,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath_list(dom, '//div[starts-with(@class, "EmojisList")]/a'): diff --git a/searx/engines/fdroid.py b/searx/engines/fdroid.py index 144a47032..0de834c75 100644 --- a/searx/engines/fdroid.py +++ b/searx/engines/fdroid.py @@ -4,7 +4,6 @@ F-Droid (a repository of FOSS applications for Android) """ from urllib.parse import urlencode -from lxml import html from searx.utils import extract_text # about @@ -37,7 +36,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for app in dom.xpath('//a[@class="package-header"]'): app_url = app.xpath('./@href')[0] diff --git a/searx/engines/findfiles.py b/searx/engines/findfiles.py index 2ce7d8e1c..a00ce635c 100644 --- a/searx/engines/findfiles.py +++ b/searx/engines/findfiles.py @@ -13,8 +13,6 @@ from os.path import basename from urllib.parse import urlencode import typing as t -from lxml import html - from searx.result_types import EngineResults from searx.utils import extract_text, eval_xpath, eval_xpath_list @@ -76,7 +74,7 @@ def request(query: str, params: "OnlineParams") -> None: def response(resp: "SXNG_Response") -> EngineResults: res = EngineResults() - dom = html.fromstring(resp.text) + dom = resp.html() if findfiles_categ == "image": for result in eval_xpath_list( dom, "//div[contains(@class, 'image-mosaic')]/div[contains(@class, 'image-item')]" diff --git a/searx/engines/geizhals.py b/searx/engines/geizhals.py index 6b8511fb9..df9289410 100644 --- a/searx/engines/geizhals.py +++ b/searx/engines/geizhals.py @@ -16,7 +16,6 @@ to the search term: import re from urllib.parse import urlencode -from lxml import html from searx.utils import eval_xpath, eval_xpath_list, extract_text @@ -67,7 +66,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath_list(dom, "//article[contains(@class, 'listview__item')]"): content = [] for spec in eval_xpath_list(result, ".//div[contains(@class, 'specs-grid__item')]"): diff --git a/searx/engines/giphy.py b/searx/engines/giphy.py index e50922c2a..1e3ec8499 100644 --- a/searx/engines/giphy.py +++ b/searx/engines/giphy.py @@ -7,8 +7,6 @@ import re import typing as t -from lxml import html - from searx.enginelib import EngineCache from searx.exceptions import SearxEngineAPIException from searx.network import get @@ -67,8 +65,7 @@ def _get_api_key() -> str: if cached: return cached - homepage_resp = get(base_url) - homepage_doc = html.fromstring(homepage_resp.text) + homepage_doc = get(base_url).html() for script_src in eval_xpath_list(homepage_doc, "//script[contains(@src, 'layout')]/@src"): script_resp = get(base_url + script_src) diff --git a/searx/engines/gmx.py b/searx/engines/gmx.py index 434d212ce..b462723b0 100644 --- a/searx/engines/gmx.py +++ b/searx/engines/gmx.py @@ -10,7 +10,6 @@ import time import typing as t from urllib.parse import urlencode -from lxml import html from searx.result_types import EngineResults from searx.exceptions import SearxEngineCaptchaException @@ -43,7 +42,7 @@ def _get_page_hash(query: str, page: int, headers: dict[str, str]) -> str: resp = get(f"{base_url}/web/result?q={query}&page={page}", headers=headers) # detect captcha (if any) - doc = html.fromstring(resp.text) + doc = resp.html() if eval_xpath(doc, "//*[@id='spam-messages']"): raise SearxEngineCaptchaException() diff --git a/searx/engines/goodreads.py b/searx/engines/goodreads.py index c1d19eece..2fc600c69 100644 --- a/searx/engines/goodreads.py +++ b/searx/engines/goodreads.py @@ -3,7 +3,6 @@ from urllib.parse import urlencode -from lxml import html from searx.utils import extract_text, eval_xpath, eval_xpath_list about = { @@ -40,7 +39,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath_list(dom, results_xpath): results.append( diff --git a/searx/engines/google_play.py b/searx/engines/google_play.py index 53712b333..f46a53f9e 100644 --- a/searx/engines/google_play.py +++ b/searx/engines/google_play.py @@ -2,7 +2,6 @@ """Google Play Apps & Google Play Movies""" from urllib.parse import urlencode -from lxml import html from searx.utils import ( eval_xpath, extract_url, @@ -52,7 +51,7 @@ def response(resp): def response_movies(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for section in eval_xpath(dom, '//c-wiz/section/header/..'): sec_name = extract_text(eval_xpath(section, './header')) @@ -79,7 +78,7 @@ def response_movies(resp): def response_apps(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() if eval_xpath(dom, '//div[@class="v6DsQb"]'): return [] diff --git a/searx/engines/google_scholar.py b/searx/engines/google_scholar.py index 3563656f4..cf314e19a 100644 --- a/searx/engines/google_scholar.py +++ b/searx/engines/google_scholar.py @@ -26,7 +26,6 @@ import typing as t from urllib.parse import urlencode from datetime import datetime -from lxml import html from curl_cffi.requests.exceptions import TooManyRedirects from searx.utils import ( @@ -106,7 +105,7 @@ def response(resp: "SXNG_Response") -> EngineResults: # pylint: disable=too-man raise TooManyRedirects(f"location {resp.headers['Location'].split('?')[0]}") res = EngineResults() - dom = html.fromstring(resp.text) + dom = resp.html() detect_google_captcha(dom) # parse results diff --git a/searx/engines/imgur.py b/searx/engines/imgur.py index ff7bdffdb..9a54b2fca 100644 --- a/searx/engines/imgur.py +++ b/searx/engines/imgur.py @@ -2,7 +2,6 @@ """Imgur (images)""" from urllib.parse import urlencode -from lxml import html from searx.utils import extract_text, eval_xpath, eval_xpath_list about = { @@ -40,7 +39,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath_list(dom, results_xpath): thumbnail_src = extract_text(eval_xpath(result, thumbnail_xpath)) diff --git a/searx/engines/ina.py b/searx/engines/ina.py index bebd3808e..8e7b1a2a8 100644 --- a/searx/engines/ina.py +++ b/searx/engines/ina.py @@ -5,7 +5,6 @@ INA (Videos) from html import unescape from urllib.parse import urlencode -from lxml import html from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex # about @@ -48,7 +47,7 @@ def response(resp): results = [] # we get html in a JSON container... - dom = html.fromstring(resp.text) + dom = resp.html() # parse results for result in eval_xpath_list(dom, results_xpath): diff --git a/searx/engines/ipernity.py b/searx/engines/ipernity.py index 6e5eb7872..430da6bc4 100644 --- a/searx/engines/ipernity.py +++ b/searx/engines/ipernity.py @@ -5,7 +5,6 @@ from datetime import datetime from json import loads, JSONDecodeError from urllib.parse import quote_plus -from lxml import html from searx.utils import extr, extract_text, eval_xpath, eval_xpath_list @@ -33,7 +32,7 @@ def request(query, params): def response(resp): results = [] - doc = html.fromstring(resp.text) + doc = resp.html() images = eval_xpath_list(doc, '//a[starts-with(@href, "/doc")]//img') diff --git a/searx/engines/kickass.py b/searx/engines/kickass.py index 311c2885b..849d61fed 100644 --- a/searx/engines/kickass.py +++ b/searx/engines/kickass.py @@ -5,7 +5,6 @@ import random from operator import itemgetter from urllib.parse import quote -from lxml import html from searx.utils import ( eval_xpath, eval_xpath_getindex, @@ -39,7 +38,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() search_res = eval_xpath_list(dom, '//table[contains(@class, "data")]//tr[descendant::a]', None) if search_res is None: diff --git a/searx/engines/lib_rs.py b/searx/engines/lib_rs.py index 79a69d19e..600dfe9d0 100644 --- a/searx/engines/lib_rs.py +++ b/searx/engines/lib_rs.py @@ -2,7 +2,6 @@ """lib.rs (packages)""" from urllib.parse import quote_plus -from lxml import html from searx.utils import eval_xpath, eval_xpath_list, extract_text about = { @@ -35,7 +34,7 @@ def request(query, params): def response(resp): results = [] - doc = html.fromstring(resp.text) + doc = resp.html() for result in eval_xpath_list(doc, results_xpath): package_name = extract_text(eval_xpath(result, title_xpath)) diff --git a/searx/engines/luxxle.py b/searx/engines/luxxle.py index 37da21d36..a496b7a2a 100644 --- a/searx/engines/luxxle.py +++ b/searx/engines/luxxle.py @@ -9,7 +9,6 @@ from json import dumps from urllib.parse import quote_plus, unquote_plus import typing as t -from lxml import html from searx.result_types import EngineResults from searx.network import get @@ -188,7 +187,7 @@ def _image_results(doc: ElementType, res: EngineResults): def response(resp: "SXNG_Response") -> EngineResults: - doc = html.fromstring(resp.text) + doc = resp.html() res = EngineResults() match luxxle_categ: diff --git a/searx/engines/mojeek.py b/searx/engines/mojeek.py index 5e0fae804..73ef2eb84 100644 --- a/searx/engines/mojeek.py +++ b/searx/engines/mojeek.py @@ -10,7 +10,6 @@ from urllib.parse import urlencode import curl_cffi from dateutil.relativedelta import relativedelta -from lxml import html from searx.exceptions import SearxEngineAPIException from searx.enginelib import EngineCache @@ -177,7 +176,7 @@ def _news_results(dom) -> EngineResults: def response(resp: "SXNG_Response") -> EngineResults: - dom = html.fromstring(resp.text) + dom = resp.html() if search_type == "": return _general_results(dom) @@ -207,7 +206,7 @@ def fetch_traits(engine_traits: EngineTraits): if not resp.ok: raise RuntimeError("Response from Mojeek is not OK.") - dom = html.fromstring(resp.text) + dom = resp.html() languages = eval_xpath_list(dom, f'//select[@name="{language_param}"]/option/@value') diff --git a/searx/engines/neocities.py b/searx/engines/neocities.py index 0e0fcb33a..8fd5bbcd7 100644 --- a/searx/engines/neocities.py +++ b/searx/engines/neocities.py @@ -7,8 +7,6 @@ from urllib.parse import urlencode import typing as t -from lxml import html - from searx.utils import eval_xpath, eval_xpath_list, extract_text from searx.result_types import EngineResults @@ -51,7 +49,7 @@ def request(query: str, params: "OnlineParams") -> None: def response(resp: "SXNG_Response") -> EngineResults: results = EngineResults() - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath_list(dom, results_xpath): results.add( diff --git a/searx/engines/neosearch.py b/searx/engines/neosearch.py index 8489bc2a0..24a75ab02 100644 --- a/searx/engines/neosearch.py +++ b/searx/engines/neosearch.py @@ -7,8 +7,6 @@ from json import loads import typing as t -from lxml import html - from searx.exceptions import SearxEngineAPIException from searx.extended_types import SXNG_Response from searx.network import get @@ -38,7 +36,7 @@ categories = ["general"] def _obtain_xsrf_token() -> str: resp = get(base_url) - doc = html.fromstring(resp.text) + doc = resp.html() xsrf_token = extract_text(eval_xpath(doc, "//meta[@name='xsrf-token']/@content")) if not xsrf_token: diff --git a/searx/engines/niconico.py b/searx/engines/niconico.py index 8a943174b..418457066 100644 --- a/searx/engines/niconico.py +++ b/searx/engines/niconico.py @@ -3,7 +3,6 @@ from urllib.parse import urlencode from datetime import datetime, timedelta -from lxml import html from searx.utils import eval_xpath_getindex, eval_xpath_list, eval_xpath, extract_text @@ -48,7 +47,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for item in eval_xpath_list(dom, results_xpath): relative_url = eval_xpath_getindex(item, url_xpath, 0) diff --git a/searx/engines/nyaa.py b/searx/engines/nyaa.py index 47c172aae..004663471 100644 --- a/searx/engines/nyaa.py +++ b/searx/engines/nyaa.py @@ -3,7 +3,6 @@ from urllib.parse import urlencode -from lxml import html from searx.utils import ( eval_xpath_getindex, extract_text, @@ -55,7 +54,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for result in dom.xpath(xpath_results): # defaults diff --git a/searx/engines/ollama.py b/searx/engines/ollama.py index 532e107dc..aafcc7a99 100644 --- a/searx/engines/ollama.py +++ b/searx/engines/ollama.py @@ -3,7 +3,6 @@ from urllib.parse import urlencode from datetime import datetime -from lxml import html from searx.utils import eval_xpath_list, eval_xpath_getindex, eval_xpath, extract_text from searx.result_types import EngineResults @@ -37,7 +36,7 @@ def request(query, params): def response(resp) -> EngineResults: res = EngineResults() - dom = html.fromstring(resp.text) + dom = resp.html() for item in eval_xpath_list(dom, results_xpath): published_date = None diff --git a/searx/engines/openclipart.py b/searx/engines/openclipart.py index 3468fcd5d..6b4df60e1 100644 --- a/searx/engines/openclipart.py +++ b/searx/engines/openclipart.py @@ -2,7 +2,6 @@ """OpenClipArt (images)""" from urllib.parse import urlencode -from lxml import html from searx.utils import extract_text, eval_xpath, eval_xpath_list about = { @@ -32,7 +31,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath_list(dom, "//div[contains(@class, 'gallery')]/div[contains(@class, 'artwork')]"): results.append( diff --git a/searx/engines/pexels.py b/searx/engines/pexels.py index d7d3a9d64..8a6ce2c52 100644 --- a/searx/engines/pexels.py +++ b/searx/engines/pexels.py @@ -5,7 +5,6 @@ import re import typing as t from urllib.parse import urlencode -from lxml import html from searx.result_types import EngineResults from searx.utils import eval_xpath_list @@ -62,7 +61,7 @@ def _get_secret_key(): if resp.status_code != 200: raise SearxEngineAPIException("failed to obtain secret key") - doc = html.fromstring(resp.text) + doc = resp.html() for script_src in eval_xpath_list(doc, "//script/@src"): script = get(script_src) if script.status_code != 200: diff --git a/searx/engines/picjumbo.py b/searx/engines/picjumbo.py index d5a95bdc6..12a8b3651 100644 --- a/searx/engines/picjumbo.py +++ b/searx/engines/picjumbo.py @@ -7,8 +7,6 @@ from urllib.parse import urlparse, urlunparse import typing as t -from lxml import html - from searx.result_types import EngineResults from searx.utils import eval_xpath, eval_xpath_list, extract_text @@ -46,7 +44,7 @@ def _get_max_res_url(url: str) -> str: def response(resp: "SXNG_Response"): res = EngineResults() - doc = html.fromstring(resp.text) + doc = resp.html() for result in eval_xpath_list(doc, "//div[contains(@class, 'photo_query')]/div[contains(@class, 'photo_item')]"): thumbnail = extract_text(eval_xpath(result, ".//img[contains(@class, 'image')]/@src")) or "" diff --git a/searx/engines/pkg_go_dev.py b/searx/engines/pkg_go_dev.py index 60d65fd4f..a8209ae42 100644 --- a/searx/engines/pkg_go_dev.py +++ b/searx/engines/pkg_go_dev.py @@ -7,7 +7,6 @@ from dateutil import parser import babel import flask_babel -from lxml import html from searx.utils import eval_xpath, eval_xpath_list, extract_text about = { @@ -51,7 +50,7 @@ def request(query, params): def response(resp): results = [] - doc = html.fromstring(resp.text) + doc = resp.html() for result in eval_xpath_list(doc, results_xpath): publishedDate = extract_text(eval_xpath(result, updated_xpath)) diff --git a/searx/engines/privacywall.py b/searx/engines/privacywall.py index e108fd519..51995cfd3 100644 --- a/searx/engines/privacywall.py +++ b/searx/engines/privacywall.py @@ -10,7 +10,6 @@ user information with Microsoft and Amazon. import typing as t from urllib.parse import urlencode, unquote_plus -from lxml import html import babel from searx.enginelib.traits import EngineTraits @@ -160,7 +159,7 @@ def _video_results(doc: "ElementBase") -> EngineResults: def response(resp: "SXNG_Response") -> EngineResults: - doc = html.fromstring(resp.text) + doc = resp.html() match privacywall_category: case "general": return _general_results(doc) @@ -187,7 +186,7 @@ def fetch_traits(engine_traits: EngineTraits) -> None: if not resp.ok: raise RuntimeError("Response from Privacywall is not OK.") - dom = html.fromstring(resp.text) + dom = resp.html() #
for onclick_listener in eval_xpath( diff --git a/searx/engines/pypi.py b/searx/engines/pypi.py index f2069126d..a97d99489 100644 --- a/searx/engines/pypi.py +++ b/searx/engines/pypi.py @@ -4,7 +4,6 @@ from urllib.parse import urlencode from dateutil import parser -from lxml import html from searx.utils import ( eval_xpath_getindex, eval_xpath_list, @@ -41,7 +40,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for entry in eval_xpath_list(dom, '/html/body/main/div/div/div/form/div/ul/li/a[@class="package-snippet"]'): url = base_url + extract_text(eval_xpath_getindex(entry, './@href', 0)) # type: ignore title = extract_text(eval_xpath_getindex(entry, './h3/span[@class="package-snippet__name"]', 0)) diff --git a/searx/engines/resulthunter.py b/searx/engines/resulthunter.py index ee0384bad..b87013aa9 100644 --- a/searx/engines/resulthunter.py +++ b/searx/engines/resulthunter.py @@ -7,8 +7,6 @@ import typing as t from urllib.parse import urlencode -from lxml import html - from searx import locales from searx.exceptions import SearxEngineResponseException from searx.result_types import EngineResults @@ -111,7 +109,7 @@ def _image_results(doc: "ElementBase") -> EngineResults: def response(resp: "SXNG_Response") -> EngineResults: - doc = html.fromstring(resp.text) + doc = resp.html() # if the request was wrong (e.g. missing params), the site doesn't contain a result container # and instead shows an "Installation required" page to download the resulthunter browser extension diff --git a/searx/engines/rottentomatoes.py b/searx/engines/rottentomatoes.py index d07504b50..2c7589e87 100644 --- a/searx/engines/rottentomatoes.py +++ b/searx/engines/rottentomatoes.py @@ -2,7 +2,6 @@ """RottenTomatoes (movies)""" from urllib.parse import quote_plus -from lxml import html from searx.utils import eval_xpath, eval_xpath_list, extract_text # about @@ -35,7 +34,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath_list(dom, results_xpath): content = [] diff --git a/searx/engines/rumble.py b/searx/engines/rumble.py index 3215c13db..f6b9d9317 100644 --- a/searx/engines/rumble.py +++ b/searx/engines/rumble.py @@ -4,7 +4,6 @@ from datetime import datetime from urllib.parse import urlencode -from lxml import html # about from searx.utils import extract_text @@ -46,7 +45,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() results_dom = dom.xpath('//li[contains(@class, "video-listing-entry")]') if not results_dom: diff --git a/searx/engines/s1search.py b/searx/engines/s1search.py index 87caeffef..221c5052f 100644 --- a/searx/engines/s1search.py +++ b/searx/engines/s1search.py @@ -11,8 +11,6 @@ Some of the engines get their results from Google, others get them from Yahoo. import typing as t from urllib.parse import urlencode, urlparse, parse_qs -from lxml import html - from searx.result_types import EngineResults from searx.enginelib import EngineCache from searx.utils import eval_xpath_list, eval_xpath, extract_text @@ -68,7 +66,7 @@ def request(query: str, params: "OnlineParams"): def response(resp: "SXNG_Response") -> EngineResults: res = EngineResults() - doc = html.fromstring(resp.text) + doc = resp.html() for suggestion in eval_xpath_list(doc, "//div[@class='aylf-yahoo-bottom' or @class='aylf-yahoo-sidebar']/div"): res.add(res.types.LegacyResult({"suggestion": extract_text(suggestion)})) diff --git a/searx/engines/searchrockit.py b/searx/engines/searchrockit.py index 90858ee4d..5ec4127da 100644 --- a/searx/engines/searchrockit.py +++ b/searx/engines/searchrockit.py @@ -4,7 +4,6 @@ but the results seem to come from Google.""" import typing as t from urllib.parse import urlencode -from lxml import html from dateutil import parser from searx.result_types import EngineResults @@ -46,7 +45,7 @@ def request(query: str, params: "OnlineParams") -> None: def response(resp: "SXNG_Response") -> EngineResults: - doc = html.fromstring(resp.text) + doc = resp.html() res = EngineResults() match searchrockit_categ: diff --git a/searx/engines/semantic_scholar.py b/searx/engines/semantic_scholar.py index 473b6d04f..42dce98b8 100644 --- a/searx/engines/semantic_scholar.py +++ b/searx/engines/semantic_scholar.py @@ -26,7 +26,6 @@ Implementations import typing as t from datetime import datetime -from lxml import html from flask_babel import gettext # pyright: ignore[reportUnknownVariableType] from searx.network import get @@ -70,7 +69,7 @@ def get_ui_version() -> str: if not resp.ok: raise RuntimeError("Can't determine Semantic Scholar UI version") - doc = html.fromstring(resp.text) + doc = resp.html() ret_val = eval_xpath_getindex(doc, "//meta[@name='s2-ui-version']/@content", 0) if not ret_val: raise RuntimeError("Can't determine Semantic Scholar UI version") diff --git a/searx/engines/seznam.py b/searx/engines/seznam.py index bc2474821..104ce3136 100644 --- a/searx/engines/seznam.py +++ b/searx/engines/seznam.py @@ -2,7 +2,6 @@ """Seznam""" from urllib.parse import urlencode -from lxml import html from searx.network import get from searx.exceptions import SearxEngineAccessDeniedException from searx.utils import ( @@ -28,7 +27,7 @@ base_url = 'https://search.seznam.cz/' def request(query, params): response_index = get(base_url, headers=params['headers'], raise_for_httperror=True, timeout=3) - dom = html.fromstring(response_index.text) + dom = response_index.html() url_params = { 'q': query, @@ -50,7 +49,7 @@ def response(resp): results = [] - dom = html.fromstring(resp.content.decode()) + dom = resp.html() for result_element in eval_xpath_list( dom, '//div[@id="searchpage-root"]//div[@class="Layout--left"]/div[@class="f2c528"]' ): diff --git a/searx/engines/shopify_stock.py b/searx/engines/shopify_stock.py index a511d92e7..e66d85860 100644 --- a/searx/engines/shopify_stock.py +++ b/searx/engines/shopify_stock.py @@ -6,8 +6,6 @@ Shopify. import typing as t from urllib.parse import urlencode -from lxml import html - from searx.result_types import EngineResults from searx.utils import eval_xpath, eval_xpath_list, extract_text @@ -45,7 +43,7 @@ def _get_download_url(url: str) -> str: def response(resp: "SXNG_Response"): res = EngineResults() - doc = html.fromstring(resp.text) + doc = resp.html() for result in eval_xpath_list(doc, "//div[contains(@class, 'js-masonry-grid')]/div"): url = base_url + (extract_text(eval_xpath(result, ".//a[contains(@class, 'photo-tile')]/@href")) or "") diff --git a/searx/engines/sogou.py b/searx/engines/sogou.py index 026f3dea7..b572b15b5 100644 --- a/searx/engines/sogou.py +++ b/searx/engines/sogou.py @@ -58,7 +58,7 @@ def response(resp): ): raise SearxEngineCaptchaException() - dom = html.fromstring(resp.text) + dom = resp.html() results = [] # pylint: disable=line-too-long diff --git a/searx/engines/sogou_wechat.py b/searx/engines/sogou_wechat.py index c7e2b877c..0de4da745 100644 --- a/searx/engines/sogou_wechat.py +++ b/searx/engines/sogou_wechat.py @@ -4,7 +4,6 @@ from urllib.parse import urlencode from datetime import datetime import re -from lxml import html from searx.utils import extract_text @@ -37,7 +36,7 @@ def request(query, params): def response(resp): - dom = html.fromstring(resp.text) + dom = resp.html() results = [] for item in dom.xpath('//li[contains(@id, "sogou_vr_")]'): diff --git a/searx/engines/solidtorrents.py b/searx/engines/solidtorrents.py index 9a3608a87..622f17281 100644 --- a/searx/engines/solidtorrents.py +++ b/searx/engines/solidtorrents.py @@ -5,8 +5,6 @@ from datetime import datetime from urllib.parse import urlencode import random -from lxml import html - from searx.utils import ( extract_text, eval_xpath, @@ -43,7 +41,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath(dom, '//li[contains(@class, "search-result")]'): torrentfile = eval_xpath_getindex(result, './/a[contains(@class, "dl-torrent")]/@href', 0, None) diff --git a/searx/engines/sourcehut.py b/searx/engines/sourcehut.py index fa3df8b23..9d22d0566 100644 --- a/searx/engines/sourcehut.py +++ b/searx/engines/sourcehut.py @@ -25,7 +25,6 @@ Implementations import typing as t from urllib.parse import urlencode -from lxml import html from searx.utils import eval_xpath, eval_xpath_list, extract_text, searxng_useragent from searx.result_types import EngineResults @@ -71,7 +70,7 @@ def request(query: str, params: "OnlineParams") -> None: def response(resp: "SXNG_Response") -> EngineResults: res = EngineResults() - doc = html.fromstring(resp.text) + doc = resp.html() for item in eval_xpath_list(doc, "(//div[@class='event-list'])[1]/div[contains(@class, 'event')]"): res.add( diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index 7c910eff4..adb2b1114 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -98,7 +98,6 @@ from unicodedata import combining, normalize import babel.localedata import dateutil.parser -import lxml.html from searx.enginelib import EngineCache from searx.enginelib.traits import EngineTraits @@ -257,7 +256,7 @@ def get_sc_code(params): message="get_sc_code: got redirected to https://www.startpage.com/sp/captcha", ) - dom = lxml.html.fromstring(resp.text) + dom = resp.html() try: sc_code = eval_xpath(dom, search_form_xpath + '//input[@name="sc"]/@value')[0] @@ -503,7 +502,7 @@ def fetch_traits(engine_traits: EngineTraits): if not resp.ok: raise RuntimeError("Response from Startpage is not OK.") - dom = lxml.html.fromstring(resp.text) + dom = resp.html() # regions diff --git a/searx/engines/tiger.py b/searx/engines/tiger.py index a7b429a89..4f0c12d69 100644 --- a/searx/engines/tiger.py +++ b/searx/engines/tiger.py @@ -11,7 +11,6 @@ from urllib.parse import urlencode import typing as t from dateutil import parser -from lxml import html from searx.exceptions import SearxEngineAPIException from searx.extended_types import SXNG_Response @@ -72,7 +71,7 @@ def _obtain_session_code() -> str: return cached_session results_page = get(f"{base_url}/checkCode.aspx") - doc = html.fromstring(results_page.text) + doc = results_page.html() extra_data: dict[str, str] = {} for extra_param in ("__VIEWSTATE", "__VIEWSTATEGENERATOR", "__EVENTVALIDATION"): @@ -130,7 +129,7 @@ def request(query: str, params: "OnlineParams"): def response(resp: "SXNG_Response") -> EngineResults: res = EngineResults() - doc = html.fromstring(resp.text) + doc = resp.html() if tiger_category == "Websuche": for result in eval_xpath_list(doc, "//div[@id='mainContainer']//table/tr"): diff --git a/searx/engines/tokyotoshokan.py b/searx/engines/tokyotoshokan.py index b53d78eb3..d43cba999 100644 --- a/searx/engines/tokyotoshokan.py +++ b/searx/engines/tokyotoshokan.py @@ -5,7 +5,6 @@ import re from datetime import datetime from urllib.parse import urlencode -from lxml import html from searx.utils import extract_text, int_or_zero # about @@ -38,7 +37,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() rows = dom.xpath('//table[@class="listing"]//tr[contains(@class, "category_0")]') # check if there are no results or page layout was changed so we cannot parse it diff --git a/searx/engines/tonline.py b/searx/engines/tonline.py index 4edf5e3ed..96e2f88f1 100644 --- a/searx/engines/tonline.py +++ b/searx/engines/tonline.py @@ -12,8 +12,6 @@ results from YouTube. import typing as t from urllib.parse import urlencode -from lxml import html - from searx.utils import eval_xpath_list, eval_xpath, extract_text, ElementType from searx.result_types import EngineResults from searx.enginelib import EngineAbout @@ -131,7 +129,7 @@ def _video_results(doc: ElementType, res: EngineResults): def response(resp: "SXNG_Response") -> EngineResults: - doc = html.fromstring(resp.text) + doc = resp.html() res = EngineResults() match tonline_categ: case "web": diff --git a/searx/engines/uxwing.py b/searx/engines/uxwing.py index fd845f980..21f62ae46 100644 --- a/searx/engines/uxwing.py +++ b/searx/engines/uxwing.py @@ -2,7 +2,6 @@ """UXwing (images)""" from urllib.parse import quote_plus -from lxml import html from searx.utils import eval_xpath, eval_xpath_list, extract_text @@ -26,7 +25,7 @@ def request(query, params): def response(resp): results = [] - doc = html.fromstring(resp.text) + doc = resp.html() for result in eval_xpath_list(doc, "//article[starts-with(@id, 'post')]"): classes = extract_text(eval_xpath(result, "./@class")).split(" ") tags = [] diff --git a/searx/engines/vuhuv.py b/searx/engines/vuhuv.py index 08a8d79a8..586edfe79 100644 --- a/searx/engines/vuhuv.py +++ b/searx/engines/vuhuv.py @@ -7,8 +7,6 @@ import typing as t from urllib.parse import urlencode -from lxml import html - from searx.result_types import EngineResults from searx.utils import eval_xpath_list, eval_xpath, extract_text @@ -101,7 +99,7 @@ def _video_results(doc: "ElementBase") -> EngineResults: def response(resp: "SXNG_Response") -> EngineResults: - doc = html.fromstring(resp.text) + doc = resp.html() match vuhuv_category: case "general": return _general_results(doc) diff --git a/searx/engines/wikipedia.py b/searx/engines/wikipedia.py index 1b41240ee..f2a645769 100644 --- a/searx/engines/wikipedia.py +++ b/searx/engines/wikipedia.py @@ -57,7 +57,6 @@ options: import urllib.parse import babel -from lxml import html from searx import locales, utils from searx import network as _network @@ -285,7 +284,7 @@ def fetch_wikimedia_traits(engine_traits: EngineTraits): if not resp.ok: raise RuntimeError("Response from Wikipedia is not OK.") - dom = html.fromstring(resp.text) + dom = resp.html() for row in dom.xpath('//table[contains(@class,"sortable")]//tbody/tr'): cols = row.xpath("./td") if not cols: diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py index 428caddbe..014a672f7 100644 --- a/searx/engines/xpath.py +++ b/searx/engines/xpath.py @@ -72,7 +72,6 @@ Implementations from urllib.parse import urlencode -from lxml import html from searx.utils import extract_text, extract_url, eval_xpath, eval_xpath_list from searx.network import raise_for_httperror from searx.result_types import EngineResults @@ -287,7 +286,7 @@ def response(resp) -> EngineResults: # pylint: disable=too-many-branches if not resp.text: return results - dom = html.fromstring(resp.text) + dom = resp.html() is_onion = 'onions' in categories if results_xpath: diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py index dffc17675..def776c8a 100644 --- a/searx/engines/yahoo.py +++ b/searx/engines/yahoo.py @@ -9,8 +9,6 @@ named `YBV`, which is cached for 24h before expiring. import typing as t from urllib.parse import unquote, urlencode, urljoin -from lxml import html - from searx.enginelib import EngineCache from searx.network import get # see https://github.com/searxng/searxng/issues/762 from searx.result_types import EngineResults @@ -184,7 +182,6 @@ def parse_url(url_string: str) -> str: def _yahoo_html(resp: "SXNG_Response") -> "SXNG_Response": cookies = dict(resp.search_params["cookies"]) params = resp.search_params - for _ in range(_YBV_HOPS): if ybv := resp.cookies.get("YBV"): cookies["YBV"] = ybv @@ -215,7 +212,7 @@ def response(resp: "SXNG_Response") -> EngineResults: results = EngineResults() if resp.status_code != 200: resp.raise_for_status() - dom = html.fromstring(resp.text) + dom = resp.html() for result in eval_xpath_list(dom, '//div[contains(@class,"algo-sr")]'): url = eval_xpath_getindex(result, './/div[contains(@class,"compTitle")]//a/@href', 0, default=None) diff --git a/searx/engines/yahoo_news.py b/searx/engines/yahoo_news.py index f9a5fee1b..074531c66 100644 --- a/searx/engines/yahoo_news.py +++ b/searx/engines/yahoo_news.py @@ -11,7 +11,6 @@ import re from urllib.parse import urlencode from datetime import datetime, timedelta from dateutil import parser -from lxml import html from searx.utils import ( eval_xpath_list, @@ -66,7 +65,7 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.text) + dom = resp.html() # parse results for result in eval_xpath_list(dom, '//ol[contains(@class,"searchCenterMiddle")]//li'): diff --git a/searx/engines/yandex.py b/searx/engines/yandex.py index 3a16d3d0e..0bc6ec911 100644 --- a/searx/engines/yandex.py +++ b/searx/engines/yandex.py @@ -135,7 +135,7 @@ def _parse_json_results(dom: html.HtmlElement) -> dict[str, t.Any]: def response(resp: "SXNG_Response") -> EngineResults: catch_bad_response(resp) results = EngineResults() - dom = html.fromstring(resp.text) + dom = resp.html() match search_type: case "web": diff --git a/searx/engines/yep.py b/searx/engines/yep.py index 81c60d627..d98920fc0 100644 --- a/searx/engines/yep.py +++ b/searx/engines/yep.py @@ -80,7 +80,6 @@ def fetch_traits(engine_traits: "EngineTraits"): # pylint: disable=import-outside-toplevel, too-many-branches - from lxml import html import babel from searx.locales import language_tag @@ -100,7 +99,7 @@ def fetch_traits(engine_traits: "EngineTraits"): if not resp.ok: raise RuntimeError("Response from Yep languages is not OK.") - doc = html.fromstring(resp.text) + doc = resp.html() url = eval_xpath_getindex(doc, "//script[contains(@src, 'PageApp')]/@src", index=0) resp = get("https:" + extract_text(url), headers=headers, timeout=5) diff --git a/searx/engines/zlibrary.py b/searx/engines/zlibrary.py index 4b1dac7a5..0649a30e1 100644 --- a/searx/engines/zlibrary.py +++ b/searx/engines/zlibrary.py @@ -38,7 +38,6 @@ from datetime import datetime from urllib.parse import quote from flask_babel import gettext # pyright: ignore[reportUnknownVariableType] -from lxml import html from searx.data import ENGINE_TRAITS from searx.enginelib.traits import EngineTraits @@ -118,7 +117,7 @@ def request(query: str, params: "OnlineParams") -> None: def response(resp: "SXNG_Response") -> EngineResults: res = EngineResults() - dom = html.fromstring(resp.text) + dom = resp.html() if domain_is_seized(dom): raise SearxException(f"zlibrary domain is seized: {base_url}") @@ -196,7 +195,7 @@ def fetch_traits(engine_traits: EngineTraits) -> None: if not resp.ok: raise RuntimeError("Response from zlibrary is not OK.") - dom = html.fromstring(resp.text) + dom = resp.html() if domain_is_seized(dom): raise RuntimeError(f"Response from zlibrary is not OK. ({base_url} seized)") diff --git a/tests/unit/engines/test_xpath.py b/tests/unit/engines/test_xpath.py index 296328102..d133bd382 100644 --- a/tests/unit/engines/test_xpath.py +++ b/tests/unit/engines/test_xpath.py @@ -2,6 +2,7 @@ # pylint: disable=missing-module-docstring,disable=missing-class-docstring,invalid-name from collections import defaultdict +import lxml.html import mock from searx.engines import xpath @@ -27,6 +28,7 @@ class TestXpathEngine(SearxTestCase): """ + empty_html = """""" def setUp(self): super().setUp() @@ -65,10 +67,14 @@ class TestXpathEngine(SearxTestCase): self.assertRaises(AttributeError, xpath.response, '') self.assertRaises(AttributeError, xpath.response, '[]') - response = mock.Mock(text='', status_code=200) + response = mock.Mock( + text=self.empty_html, status_code=200, html=mock.Mock(return_value=lxml.html.fromstring(self.empty_html)) + ) self.assertEqual(xpath.response(response), []) - response = mock.Mock(text=self.html, status_code=200) + response = mock.Mock( + text=self.html, status_code=200, html=mock.Mock(return_value=lxml.html.fromstring(self.html)) + ) results = xpath.response(response) self.assertIsInstance(results, list) self.assertEqual(len(results), 2) @@ -107,10 +113,16 @@ class TestXpathEngine(SearxTestCase): self.assertRaises(AttributeError, xpath.response, '') self.assertRaises(AttributeError, xpath.response, '[]') - response = mock.Mock(text='', status_code=200) + response = mock.Mock( + text=self.empty_html, status_code=200, html=mock.Mock(return_value=lxml.html.fromstring(self.empty_html)) + ) self.assertEqual(xpath.response(response), []) - response = mock.Mock(text=self.html, status_code=200) + response = mock.Mock( + text=self.html, + status_code=200, + html=mock.Mock(return_value=lxml.html.fromstring(self.html)), + ) results = xpath.response(response) self.assertIsInstance(results, list) self.assertEqual(len(results), 2)