[refactor] engines: migrate to SXNG_Response.html() (#6718)

searx/engines/1337x.py

@@ -3,7 +3,6 @@
 """1337x"""
 
 from urllib.parse import quote, urljoin
-from lxml import html
 from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
 
 # about
@@ -31,7 +30,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath_list(dom, '//table[contains(@class, "table-list")]/tbody//tr'):
         href = urljoin(url, eval_xpath_getindex(result, './td[contains(@class, "name")]/a[2]/@href', 0))

searx/engines/360search.py

@@ -5,7 +5,6 @@
 import typing as t
 
 from urllib.parse import urlencode
-from lxml import html
 
 from searx import logger
 from searx.enginelib import EngineCache
@@ -88,7 +87,7 @@ def response(resp):
     if not resp.text or not resp.text.strip():
         return []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
     results = []
 
     for item in dom.xpath('//li[contains(@class, "res-list")]'):

searx/engines/alpinelinux.py

@@ -12,7 +12,6 @@ servers and for Docker images.
 import re
 
 from urllib.parse import urlencode
-from lxml import html
 from dateutil import parser
 
 from searx.utils import eval_xpath, eval_xpath_list, extract_text
@@ -57,7 +56,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
     for result in eval_xpath_list(doc, "//table/tbody/tr"):
 
         if len(result.xpath("./td")) < 9:

searx/engines/annas_archive.py

@@ -39,7 +39,6 @@ import random
 import typing as t
 from urllib.parse import urlencode
 
-from lxml import html
 from lxml.etree import ElementBase
 
 from searx.data import ENGINE_TRAITS
@@ -143,7 +142,7 @@ def request(query: str, params: "OnlineParams") -> None:
 
 def response(resp: "SXNG_Response") -> EngineResults:
     res = EngineResults()
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     # Each result is a div with class "flex" inside "js-aarecord-list-outer"
     # container.  The "flex" filter excludes non-result div such as section
@@ -255,7 +254,7 @@ def fetch_traits(engine_traits: EngineTraits) -> None:
     if not resp.ok:
         raise RuntimeError("Response from Anna's Archive is not OK.")
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     # supported language codes
     lang_map: dict[str, str] = {}

searx/engines/ansa.py

@@ -14,7 +14,6 @@ list in ``settings.yml``:
 """
 
 from urllib.parse import urlencode
-from lxml import html
 from searx.result_types import EngineResults, MainResult
 from searx.utils import eval_xpath, eval_xpath_list, extract_text
 
@@ -62,7 +61,7 @@ def request(query, params):
 
 def response(resp) -> EngineResults:
     res = EngineResults()
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
 
     for result in eval_xpath_list(doc, "//div[@class='article']"):
 

searx/engines/apkmirror.py

@@ -4,7 +4,6 @@
 # pylint: disable=invalid-name
 
 from urllib.parse import urlencode
-from lxml import html
 
 from searx.utils import (
     eval_xpath_list,
@@ -43,7 +42,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     # parse results
     for result in eval_xpath_list(dom, "//div[@id='content']//div[@class='listWidget']/div/div[@class='appRow']"):

searx/engines/archlinux.py

@@ -11,7 +11,6 @@ Arch Wiki blocks access to it.
 from urllib.parse import urlencode, urljoin, urlparse
 
 import babel
-import lxml
 
 from searx.enginelib.traits import EngineTraits
 from searx.locales import language_tag
@@ -70,7 +69,7 @@ def request(query, params):
 def response(resp):
 
     results = []
-    dom = lxml.html.fromstring(resp.text)  # type: ignore
+    dom = resp.html()  # type: ignore
 
     # get the base URL for the language in which request was made
     sxng_lang = resp.search_params["searxng_locale"].split("-")[0]
@@ -131,7 +130,7 @@ def fetch_traits(engine_traits: EngineTraits):
     if not resp.ok:
         raise RuntimeError("Response from Arch Linux Wiki is not OK.")
 
-    dom = lxml.html.fromstring(resp.text)  # type: ignore
+    dom = resp.html()  # type: ignore
     for a in eval_xpath_list(dom, "//a[@class='interlanguage-link-target']"):
         sxng_tag = language_tag(babel.Locale.parse(a.get("lang"), sep="-"))
         # zh_Hans --> zh

searx/engines/bandcamp.py

@@ -10,7 +10,6 @@
 
 from urllib.parse import urlencode, urlparse, parse_qs
 from dateutil.parser import parse as dateparse
-from lxml import html
 
 from searx.utils import (
     eval_xpath_getindex,
@@ -46,7 +45,7 @@ def request(query, params):
 def response(resp):
 
     results = []
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath_list(dom, '//li[contains(@class, "searchresult")]'):
 

searx/engines/bing.py

@@ -18,7 +18,6 @@ from urllib.parse import parse_qs, urlencode, urlparse
 
 import babel
 import babel.languages
-from lxml import html
 
 from searx.enginelib.traits import EngineTraits
 from searx.locales import region_tag
@@ -96,7 +95,7 @@ def response(resp: "SXNG_Response") -> list[dict[str, t.Any]]:
 
     results: list[dict[str, t.Any]] = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for item in eval_xpath_list(dom, '//ol[@id="b_results"]/li[contains(@class, "b_algo")]'):
         link = eval_xpath_getindex(item, ".//h2/a", 0, None)
@@ -156,7 +155,7 @@ def fetch_traits(engine_traits: EngineTraits) -> None:
     if not resp.ok:
         raise RuntimeError("Response from Bing is not OK.")
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     map_market_codes: dict[str, str] = {
         "zh-hk": "en-hk",  # not sure why, but at Microslop this is the market code for Hongkong

searx/engines/bing_images.py

@@ -5,8 +5,6 @@ import typing as t
 import json
 from urllib.parse import urlencode
 
-from lxml import html
-
 from searx.engines.bing import fetch_traits  # pylint: disable=unused-import
 from searx.result_types import EngineResults
 
@@ -75,7 +73,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
 
     res = EngineResults()
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in dom.xpath('//ul[contains(@class, "dgControl_list")]/li'):
         metadata = result.xpath('.//a[@class="iusc"]/@m')

searx/engines/bing_news.py

@@ -9,8 +9,6 @@
 
 from urllib.parse import urlencode
 
-from lxml import html
-
 from searx.enginelib.traits import EngineTraits
 from searx.engines.bing import get_locale_params
 from searx.utils import eval_xpath, eval_xpath_getindex, eval_xpath_list, extract_text
@@ -78,7 +76,7 @@ def response(resp):
 
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for newsitem in eval_xpath_list(dom, '//div[contains(@class, "newsitem")]'):
         link = eval_xpath_getindex(newsitem, './/a[@class="title"]', 0, None)

searx/engines/bing_videos.py

@@ -4,8 +4,6 @@
 import json
 from urllib.parse import urlencode
 
-from lxml import html
-
 from searx.engines.bing import (  # pylint: disable=unused-import
     fetch_traits,
     get_locale_params,
@@ -68,7 +66,7 @@ def response(resp):
 
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in dom.xpath('//div[contains(@id, "mc_vtvc_video")]'):
         metadata = json.loads(eval_xpath_getindex(result, './/div[@class="vrhdata"]/@vrhm', index=0))

searx/engines/brave.py

@@ -125,7 +125,6 @@ from urllib.parse import (
 )
 
 from dateutil import parser
-from lxml import html
 
 from searx import locales
 from searx.enginelib.traits import EngineTraits
@@ -289,7 +288,7 @@ def response(resp: SXNG_Response) -> EngineResults:
 
 def _parse_search(resp: SXNG_Response) -> EngineResults:
     res = EngineResults()
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath_list(dom, "//div[contains(@class, 'snippet ')]"):
         url: str | None = eval_xpath_getindex(result, ".//a/@href", 0, default=None)
@@ -352,7 +351,7 @@ def _parse_search(resp: SXNG_Response) -> EngineResults:
 
 def _parse_news(resp: SXNG_Response) -> EngineResults:
     res = EngineResults()
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath_list(dom, "//div[@data-type='news']"):
         url = eval_xpath_getindex(result, ".//a/@href", 0, default=None)
@@ -434,7 +433,7 @@ def fetch_traits(engine_traits: EngineTraits):
     if not resp.ok:
         raise RuntimeError("Response from Brave languages is not OK.")
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for option in dom.xpath("//section//option[@value='en-us']/../option"):
         ui_lang = option.get("value")

searx/engines/btdigg.py

@@ -38,7 +38,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     search_res = dom.xpath('//div[@class="one_result"]')
 

searx/engines/destatis.py

@@ -2,7 +2,6 @@
 """DeStatis"""
 
 from urllib.parse import urlencode
-from lxml import html
 from searx.utils import eval_xpath, eval_xpath_list, extract_text
 
 about = {
@@ -42,7 +41,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     # filter out suggested results on further page because they're the same on each page
     extra_xpath = results_xpath_filter_recommended if resp.search_params['pageno'] > 1 else ''

searx/engines/deviantart.py

@@ -4,7 +4,6 @@
 import typing as t
 
 import urllib.parse
-from lxml import html
 
 from searx.result_types import EngineResults
 from searx.utils import extract_text, eval_xpath, eval_xpath_list
@@ -52,7 +51,7 @@ def request(query: str, params: "OnlineParams"):
 def response(resp: "SXNG_Response") -> EngineResults:
 
     res = EngineResults()
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath_list(dom, results_xpath):
         thumbnail_src = extract_text(eval_xpath(result, thumbnail_src_xpath))

searx/engines/dictzone.py

@@ -4,7 +4,6 @@ Dictzone
 """
 
 import urllib.parse
-from lxml import html
 
 from searx.utils import eval_xpath, extract_text
 from searx.result_types import EngineResults
@@ -50,7 +49,7 @@ def response(resp) -> EngineResults:
     if not resp.ok:
         return results
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath(dom, ".//table[@id='r']//tr"):
 

searx/engines/digbt.py

@@ -4,7 +4,6 @@ DigBT (Videos, Music, Files)
 """
 
 from urllib.parse import urljoin
-from lxml import html
 from searx.utils import extract_text
 
 # about
@@ -33,7 +32,7 @@ def request(query, params):
 
 
 def response(resp):
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
     search_res = dom.xpath('.//td[@class="x-item"]')
 
     if not search_res:

searx/engines/duckduckgo.py

@@ -172,7 +172,6 @@ import re
 import typing as t
 
 import babel
-import lxml.html
 
 from searx import locales
 from searx.enginelib import EngineCache
@@ -470,7 +469,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
     if resp.status_code == 303:
         return res
 
-    doc = lxml.html.fromstring(resp.text)
+    doc = resp.html()
     params = resp.search_params
 
     if is_ddg_captcha(doc):

searx/engines/duckduckgo_web.py

@@ -17,7 +17,6 @@ import typing as t
 import re
 
 from urllib.parse import quote_plus, urljoin
-from lxml import html
 
 from searx.utils import html_to_text, extract_text, eval_xpath
 from searx.result_types import EngineResults
@@ -81,7 +80,7 @@ def _fetch_first_page_link(
     if resp.status_code != 200:
         logger.error("vqd: got HTTP %s from duckduckgo.com", resp.status_code)
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
     first_page_link = extract_text(eval_xpath(dom, "//link[@id='deep_preload_link']/@href"))
 
     if not first_page_link:

searx/engines/duden.py

@@ -2,7 +2,6 @@
 """Duden"""
 
 from urllib.parse import quote, urljoin
-from lxml import html
 from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
 from searx.network import raise_for_httperror
 
@@ -48,7 +47,7 @@ def response(resp):
 
     raise_for_httperror(resp)
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath_list(dom, '//section[not(contains(@class, "essay"))]'):
         url = eval_xpath_getindex(result, './/h2/a', 0).get('href')

searx/engines/ebay.py

@@ -5,7 +5,6 @@ Ebay (Videos, Music, Files)
 
 from urllib.parse import quote
 
-from lxml import html
 from searx.engines.xpath import extract_text
 
 # about
@@ -44,7 +43,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
     results_dom = dom.xpath(results_xpath)
     if not results_dom:
         return []

searx/engines/emojipedia.py

@@ -9,7 +9,6 @@ since 2021. Emojipedia is a voting member of The Unicode Consortium.[1]
 """
 
 from urllib.parse import urlencode
-from lxml import html
 
 from searx.utils import (
     eval_xpath_list,
@@ -41,7 +40,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath_list(dom, '//div[starts-with(@class, "EmojisList")]/a'):
 

searx/engines/fdroid.py

@@ -4,7 +4,6 @@ F-Droid (a repository of FOSS applications for Android)
 """
 
 from urllib.parse import urlencode
-from lxml import html
 from searx.utils import extract_text
 
 # about
@@ -37,7 +36,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for app in dom.xpath('//a[@class="package-header"]'):
         app_url = app.xpath('./@href')[0]

searx/engines/findfiles.py

@@ -13,8 +13,6 @@ from os.path import basename
 from urllib.parse import urlencode
 import typing as t
 
-from lxml import html
-
 from searx.result_types import EngineResults
 from searx.utils import extract_text, eval_xpath, eval_xpath_list
 
@@ -76,7 +74,7 @@ def request(query: str, params: "OnlineParams") -> None:
 def response(resp: "SXNG_Response") -> EngineResults:
     res = EngineResults()
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
     if findfiles_categ == "image":
         for result in eval_xpath_list(
             dom, "//div[contains(@class, 'image-mosaic')]/div[contains(@class, 'image-item')]"

searx/engines/geizhals.py

@@ -16,7 +16,6 @@ to the search term:
 import re
 
 from urllib.parse import urlencode
-from lxml import html
 
 from searx.utils import eval_xpath, eval_xpath_list, extract_text
 
@@ -67,7 +66,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
     for result in eval_xpath_list(dom, "//article[contains(@class, 'listview__item')]"):
         content = []
         for spec in eval_xpath_list(result, ".//div[contains(@class, 'specs-grid__item')]"):

searx/engines/giphy.py

@@ -7,8 +7,6 @@ import re
 
 import typing as t
 
-from lxml import html
-
 from searx.enginelib import EngineCache
 from searx.exceptions import SearxEngineAPIException
 from searx.network import get
@@ -67,8 +65,7 @@ def _get_api_key() -> str:
     if cached:
         return cached
 
-    homepage_resp = get(base_url)
+    homepage_doc = get(base_url).html()
-    homepage_doc = html.fromstring(homepage_resp.text)
 
     for script_src in eval_xpath_list(homepage_doc, "//script[contains(@src, 'layout')]/@src"):
         script_resp = get(base_url + script_src)

searx/engines/gmx.py

@@ -10,7 +10,6 @@ import time
 import typing as t
 
 from urllib.parse import urlencode
-from lxml import html
 
 from searx.result_types import EngineResults
 from searx.exceptions import SearxEngineCaptchaException
@@ -43,7 +42,7 @@ def _get_page_hash(query: str, page: int, headers: dict[str, str]) -> str:
     resp = get(f"{base_url}/web/result?q={query}&page={page}", headers=headers)
 
     # detect captcha (if any)
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
     if eval_xpath(doc, "//*[@id='spam-messages']"):
         raise SearxEngineCaptchaException()
 

searx/engines/goodreads.py

@@ -3,7 +3,6 @@
 
 from urllib.parse import urlencode
 
-from lxml import html
 from searx.utils import extract_text, eval_xpath, eval_xpath_list
 
 about = {
@@ -40,7 +39,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath_list(dom, results_xpath):
         results.append(

searx/engines/google_play.py

@@ -2,7 +2,6 @@
 """Google Play Apps & Google Play Movies"""
 
 from urllib.parse import urlencode
-from lxml import html
 from searx.utils import (
     eval_xpath,
     extract_url,
@@ -52,7 +51,7 @@ def response(resp):
 def response_movies(resp):
 
     results = []
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for section in eval_xpath(dom, '//c-wiz/section/header/..'):
         sec_name = extract_text(eval_xpath(section, './header'))
@@ -79,7 +78,7 @@ def response_movies(resp):
 def response_apps(resp):
 
     results = []
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     if eval_xpath(dom, '//div[@class="v6DsQb"]'):
         return []

searx/engines/google_scholar.py

@@ -26,7 +26,6 @@ import typing as t
 
 from urllib.parse import urlencode
 from datetime import datetime
-from lxml import html
 from curl_cffi.requests.exceptions import TooManyRedirects
 
 from searx.utils import (
@@ -106,7 +105,7 @@ def response(resp: "SXNG_Response") -> EngineResults:  # pylint: disable=too-man
         raise TooManyRedirects(f"location {resp.headers['Location'].split('?')[0]}")
 
     res = EngineResults()
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
     detect_google_captcha(dom)
 
     # parse results

searx/engines/imgur.py

@@ -2,7 +2,6 @@
 """Imgur (images)"""
 
 from urllib.parse import urlencode
-from lxml import html
 from searx.utils import extract_text, eval_xpath, eval_xpath_list
 
 about = {
@@ -40,7 +39,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath_list(dom, results_xpath):
         thumbnail_src = extract_text(eval_xpath(result, thumbnail_xpath))

searx/engines/ina.py

@@ -5,7 +5,6 @@ INA (Videos)
 
 from html import unescape
 from urllib.parse import urlencode
-from lxml import html
 from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
 
 # about
@@ -48,7 +47,7 @@ def response(resp):
     results = []
 
     # we get html in a JSON container...
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     # parse results
     for result in eval_xpath_list(dom, results_xpath):

searx/engines/ipernity.py

@@ -5,7 +5,6 @@ from datetime import datetime
 from json import loads, JSONDecodeError
 
 from urllib.parse import quote_plus
-from lxml import html
 
 from searx.utils import extr, extract_text, eval_xpath, eval_xpath_list
 
@@ -33,7 +32,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
 
     images = eval_xpath_list(doc, '//a[starts-with(@href, "/doc")]//img')
 

searx/engines/kickass.py

@@ -5,7 +5,6 @@ import random
 from operator import itemgetter
 from urllib.parse import quote
 
-from lxml import html
 from searx.utils import (
     eval_xpath,
     eval_xpath_getindex,
@@ -39,7 +38,7 @@ def request(query, params):
 
 def response(resp):
     results = []
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     search_res = eval_xpath_list(dom, '//table[contains(@class, "data")]//tr[descendant::a]', None)
     if search_res is None:

searx/engines/lib_rs.py

@@ -2,7 +2,6 @@
 """lib.rs (packages)"""
 
 from urllib.parse import quote_plus
-from lxml import html
 from searx.utils import eval_xpath, eval_xpath_list, extract_text
 
 about = {
@@ -35,7 +34,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
 
     for result in eval_xpath_list(doc, results_xpath):
         package_name = extract_text(eval_xpath(result, title_xpath))

searx/engines/luxxle.py

@@ -9,7 +9,6 @@ from json import dumps
 from urllib.parse import quote_plus, unquote_plus
 
 import typing as t
-from lxml import html
 
 from searx.result_types import EngineResults
 from searx.network import get
@@ -188,7 +187,7 @@ def _image_results(doc: ElementType, res: EngineResults):
 
 
 def response(resp: "SXNG_Response") -> EngineResults:
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
     res = EngineResults()
 
     match luxxle_categ:

searx/engines/mojeek.py

@@ -10,7 +10,6 @@ from urllib.parse import urlencode
 
 import curl_cffi
 from dateutil.relativedelta import relativedelta
-from lxml import html
 
 from searx.exceptions import SearxEngineAPIException
 from searx.enginelib import EngineCache
@@ -177,7 +176,7 @@ def _news_results(dom) -> EngineResults:
 
 
 def response(resp: "SXNG_Response") -> EngineResults:
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     if search_type == "":
         return _general_results(dom)
@@ -207,7 +206,7 @@ def fetch_traits(engine_traits: EngineTraits):
     if not resp.ok:
         raise RuntimeError("Response from Mojeek is not OK.")
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     languages = eval_xpath_list(dom, f'//select[@name="{language_param}"]/option/@value')
 

searx/engines/neocities.py

@@ -7,8 +7,6 @@
 from urllib.parse import urlencode
 import typing as t
 
-from lxml import html
-
 from searx.utils import eval_xpath, eval_xpath_list, extract_text
 from searx.result_types import EngineResults
 
@@ -51,7 +49,7 @@ def request(query: str, params: "OnlineParams") -> None:
 
 def response(resp: "SXNG_Response") -> EngineResults:
     results = EngineResults()
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath_list(dom, results_xpath):
         results.add(

searx/engines/neosearch.py

@@ -7,8 +7,6 @@
 from json import loads
 import typing as t
 
-from lxml import html
-
 from searx.exceptions import SearxEngineAPIException
 from searx.extended_types import SXNG_Response
 from searx.network import get
@@ -38,7 +36,7 @@ categories = ["general"]
 
 def _obtain_xsrf_token() -> str:
     resp = get(base_url)
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
 
     xsrf_token = extract_text(eval_xpath(doc, "//meta[@name='xsrf-token']/@content"))
     if not xsrf_token:

searx/engines/niconico.py

@@ -3,7 +3,6 @@
 
 from urllib.parse import urlencode
 from datetime import datetime, timedelta
-from lxml import html
 
 from searx.utils import eval_xpath_getindex, eval_xpath_list, eval_xpath, extract_text
 
@@ -48,7 +47,7 @@ def request(query, params):
 
 def response(resp):
     results = []
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for item in eval_xpath_list(dom, results_xpath):
         relative_url = eval_xpath_getindex(item, url_xpath, 0)

searx/engines/nyaa.py

@@ -3,7 +3,6 @@
 
 from urllib.parse import urlencode
 
-from lxml import html
 from searx.utils import (
     eval_xpath_getindex,
     extract_text,
@@ -55,7 +54,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in dom.xpath(xpath_results):
         # defaults

searx/engines/ollama.py

@@ -3,7 +3,6 @@
 
 from urllib.parse import urlencode
 from datetime import datetime
-from lxml import html
 
 from searx.utils import eval_xpath_list, eval_xpath_getindex, eval_xpath, extract_text
 from searx.result_types import EngineResults
@@ -37,7 +36,7 @@ def request(query, params):
 def response(resp) -> EngineResults:
     res = EngineResults()
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for item in eval_xpath_list(dom, results_xpath):
         published_date = None

searx/engines/openclipart.py

@@ -2,7 +2,6 @@
 """OpenClipArt (images)"""
 
 from urllib.parse import urlencode
-from lxml import html
 from searx.utils import extract_text, eval_xpath, eval_xpath_list
 
 about = {
@@ -32,7 +31,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath_list(dom, "//div[contains(@class, 'gallery')]/div[contains(@class, 'artwork')]"):
         results.append(

searx/engines/pexels.py

@@ -5,7 +5,6 @@ import re
 import typing as t
 
 from urllib.parse import urlencode
-from lxml import html
 
 from searx.result_types import EngineResults
 from searx.utils import eval_xpath_list
@@ -62,7 +61,7 @@ def _get_secret_key():
     if resp.status_code != 200:
         raise SearxEngineAPIException("failed to obtain secret key")
 
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
     for script_src in eval_xpath_list(doc, "//script/@src"):
         script = get(script_src)
         if script.status_code != 200:

searx/engines/picjumbo.py

@@ -7,8 +7,6 @@
 from urllib.parse import urlparse, urlunparse
 import typing as t
 
-from lxml import html
-
 from searx.result_types import EngineResults
 from searx.utils import eval_xpath, eval_xpath_list, extract_text
 
@@ -46,7 +44,7 @@ def _get_max_res_url(url: str) -> str:
 def response(resp: "SXNG_Response"):
     res = EngineResults()
 
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
 
     for result in eval_xpath_list(doc, "//div[contains(@class, 'photo_query')]/div[contains(@class, 'photo_item')]"):
         thumbnail = extract_text(eval_xpath(result, ".//img[contains(@class, 'image')]/@src")) or ""

searx/engines/pkg_go_dev.py

@@ -7,7 +7,6 @@ from dateutil import parser
 
 import babel
 import flask_babel
-from lxml import html
 from searx.utils import eval_xpath, eval_xpath_list, extract_text
 
 about = {
@@ -51,7 +50,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
 
     for result in eval_xpath_list(doc, results_xpath):
         publishedDate = extract_text(eval_xpath(result, updated_xpath))

searx/engines/privacywall.py

@@ -10,7 +10,6 @@ user information with Microsoft and Amazon.
 import typing as t
 from urllib.parse import urlencode, unquote_plus
 
-from lxml import html
 import babel
 
 from searx.enginelib.traits import EngineTraits
@@ -160,7 +159,7 @@ def _video_results(doc: "ElementBase") -> EngineResults:
 
 
 def response(resp: "SXNG_Response") -> EngineResults:
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
     match privacywall_category:
         case "general":
             return _general_results(doc)
@@ -187,7 +186,7 @@ def fetch_traits(engine_traits: EngineTraits) -> None:
     if not resp.ok:
         raise RuntimeError("Response from Privacywall is not OK.")
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     # <div class="dropdown-option" onclick="changeMenuLanguage(&quot;CZ&quot;)"></div>
     for onclick_listener in eval_xpath(

searx/engines/pypi.py

@@ -4,7 +4,6 @@
 from urllib.parse import urlencode
 from dateutil import parser
 
-from lxml import html
 from searx.utils import (
     eval_xpath_getindex,
     eval_xpath_list,
@@ -41,7 +40,7 @@ def request(query, params):
 
 def response(resp):
     results = []
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
     for entry in eval_xpath_list(dom, '/html/body/main/div/div/div/form/div/ul/li/a[@class="package-snippet"]'):
         url = base_url + extract_text(eval_xpath_getindex(entry, './@href', 0))  # type: ignore
         title = extract_text(eval_xpath_getindex(entry, './h3/span[@class="package-snippet__name"]', 0))

searx/engines/resulthunter.py

@@ -7,8 +7,6 @@
 import typing as t
 from urllib.parse import urlencode
 
-from lxml import html
-
 from searx import locales
 from searx.exceptions import SearxEngineResponseException
 from searx.result_types import EngineResults
@@ -111,7 +109,7 @@ def _image_results(doc: "ElementBase") -> EngineResults:
 
 
 def response(resp: "SXNG_Response") -> EngineResults:
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
 
     # if the request was wrong (e.g. missing params), the site doesn't contain a result container
     # and instead shows an "Installation required" page to download the resulthunter browser extension

searx/engines/rottentomatoes.py

@@ -2,7 +2,6 @@
 """RottenTomatoes (movies)"""
 
 from urllib.parse import quote_plus
-from lxml import html
 from searx.utils import eval_xpath, eval_xpath_list, extract_text
 
 # about
@@ -35,7 +34,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath_list(dom, results_xpath):
         content = []

searx/engines/rumble.py

@@ -4,7 +4,6 @@
 from datetime import datetime
 
 from urllib.parse import urlencode
-from lxml import html
 
 # about
 from searx.utils import extract_text
@@ -46,7 +45,7 @@ def request(query, params):
 
 def response(resp):
     results = []
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
     results_dom = dom.xpath('//li[contains(@class, "video-listing-entry")]')
 
     if not results_dom:

searx/engines/s1search.py

@@ -11,8 +11,6 @@ Some of the engines get their results from Google, others get them from Yahoo.
 import typing as t
 from urllib.parse import urlencode, urlparse, parse_qs
 
-from lxml import html
-
 from searx.result_types import EngineResults
 from searx.enginelib import EngineCache
 from searx.utils import eval_xpath_list, eval_xpath, extract_text
@@ -68,7 +66,7 @@ def request(query: str, params: "OnlineParams"):
 def response(resp: "SXNG_Response") -> EngineResults:
     res = EngineResults()
 
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
 
     for suggestion in eval_xpath_list(doc, "//div[@class='aylf-yahoo-bottom' or @class='aylf-yahoo-sidebar']/div"):
         res.add(res.types.LegacyResult({"suggestion": extract_text(suggestion)}))

searx/engines/searchrockit.py

@@ -4,7 +4,6 @@ but the results seem to come from Google."""
 
 import typing as t
 from urllib.parse import urlencode
-from lxml import html
 from dateutil import parser
 
 from searx.result_types import EngineResults
@@ -46,7 +45,7 @@ def request(query: str, params: "OnlineParams") -> None:
 
 
 def response(resp: "SXNG_Response") -> EngineResults:
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
     res = EngineResults()
 
     match searchrockit_categ:

searx/engines/semantic_scholar.py

@@ -26,7 +26,6 @@ Implementations
 import typing as t
 
 from datetime import datetime
-from lxml import html
 from flask_babel import gettext  # pyright: ignore[reportUnknownVariableType]
 
 from searx.network import get
@@ -70,7 +69,7 @@ def get_ui_version() -> str:
         if not resp.ok:
             raise RuntimeError("Can't determine Semantic Scholar UI version")
 
-        doc = html.fromstring(resp.text)
+        doc = resp.html()
         ret_val = eval_xpath_getindex(doc, "//meta[@name='s2-ui-version']/@content", 0)
         if not ret_val:
             raise RuntimeError("Can't determine Semantic Scholar UI version")

searx/engines/seznam.py

@@ -2,7 +2,6 @@
 """Seznam"""
 
 from urllib.parse import urlencode
-from lxml import html
 from searx.network import get
 from searx.exceptions import SearxEngineAccessDeniedException
 from searx.utils import (
@@ -28,7 +27,7 @@ base_url = 'https://search.seznam.cz/'
 
 def request(query, params):
     response_index = get(base_url, headers=params['headers'], raise_for_httperror=True, timeout=3)
-    dom = html.fromstring(response_index.text)
+    dom = response_index.html()
 
     url_params = {
         'q': query,
@@ -50,7 +49,7 @@ def response(resp):
 
     results = []
 
-    dom = html.fromstring(resp.content.decode())
+    dom = resp.html()
     for result_element in eval_xpath_list(
         dom, '//div[@id="searchpage-root"]//div[@class="Layout--left"]/div[@class="f2c528"]'
     ):

searx/engines/shopify_stock.py

@@ -6,8 +6,6 @@ Shopify.
 import typing as t
 from urllib.parse import urlencode
 
-from lxml import html
-
 from searx.result_types import EngineResults
 from searx.utils import eval_xpath, eval_xpath_list, extract_text
 
@@ -45,7 +43,7 @@ def _get_download_url(url: str) -> str:
 def response(resp: "SXNG_Response"):
     res = EngineResults()
 
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
 
     for result in eval_xpath_list(doc, "//div[contains(@class, 'js-masonry-grid')]/div"):
         url = base_url + (extract_text(eval_xpath(result, ".//a[contains(@class, 'photo-tile')]/@href")) or "")

searx/engines/sogou.py

@@ -58,7 +58,7 @@ def response(resp):
     ):
         raise SearxEngineCaptchaException()
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
     results = []
 
     # pylint: disable=line-too-long

searx/engines/sogou_wechat.py

@@ -4,7 +4,6 @@
 from urllib.parse import urlencode
 from datetime import datetime
 import re
-from lxml import html
 
 from searx.utils import extract_text
 
@@ -37,7 +36,7 @@ def request(query, params):
 
 
 def response(resp):
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
     results = []
 
     for item in dom.xpath('//li[contains(@id, "sogou_vr_")]'):

searx/engines/solidtorrents.py

@@ -5,8 +5,6 @@ from datetime import datetime
 from urllib.parse import urlencode
 import random
 
-from lxml import html
-
 from searx.utils import (
     extract_text,
     eval_xpath,
@@ -43,7 +41,7 @@ def request(query, params):
 
 def response(resp):
     results = []
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath(dom, '//li[contains(@class, "search-result")]'):
         torrentfile = eval_xpath_getindex(result, './/a[contains(@class, "dl-torrent")]/@href', 0, None)

searx/engines/sourcehut.py

@@ -25,7 +25,6 @@ Implementations
 import typing as t
 
 from urllib.parse import urlencode
-from lxml import html
 
 from searx.utils import eval_xpath, eval_xpath_list, extract_text, searxng_useragent
 from searx.result_types import EngineResults
@@ -71,7 +70,7 @@ def request(query: str, params: "OnlineParams") -> None:
 def response(resp: "SXNG_Response") -> EngineResults:
 
     res = EngineResults()
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
 
     for item in eval_xpath_list(doc, "(//div[@class='event-list'])[1]/div[contains(@class, 'event')]"):
         res.add(

searx/engines/startpage.py

@@ -98,7 +98,6 @@ from unicodedata import combining, normalize
 
 import babel.localedata
 import dateutil.parser
-import lxml.html
 
 from searx.enginelib import EngineCache
 from searx.enginelib.traits import EngineTraits
@@ -257,7 +256,7 @@ def get_sc_code(params):
             message="get_sc_code: got redirected to https://www.startpage.com/sp/captcha",
         )
 
-    dom = lxml.html.fromstring(resp.text)
+    dom = resp.html()
 
     try:
         sc_code = eval_xpath(dom, search_form_xpath + '//input[@name="sc"]/@value')[0]
@@ -503,7 +502,7 @@ def fetch_traits(engine_traits: EngineTraits):
     if not resp.ok:
         raise RuntimeError("Response from Startpage is not OK.")
 
-    dom = lxml.html.fromstring(resp.text)
+    dom = resp.html()
 
     # regions
 

searx/engines/tiger.py

@@ -11,7 +11,6 @@ from urllib.parse import urlencode
 import typing as t
 
 from dateutil import parser
-from lxml import html
 
 from searx.exceptions import SearxEngineAPIException
 from searx.extended_types import SXNG_Response
@@ -72,7 +71,7 @@ def _obtain_session_code() -> str:
         return cached_session
 
     results_page = get(f"{base_url}/checkCode.aspx")
-    doc = html.fromstring(results_page.text)
+    doc = results_page.html()
 
     extra_data: dict[str, str] = {}
     for extra_param in ("__VIEWSTATE", "__VIEWSTATEGENERATOR", "__EVENTVALIDATION"):
@@ -130,7 +129,7 @@ def request(query: str, params: "OnlineParams"):
 
 def response(resp: "SXNG_Response") -> EngineResults:
     res = EngineResults()
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
 
     if tiger_category == "Websuche":
         for result in eval_xpath_list(doc, "//div[@id='mainContainer']//table/tr"):

searx/engines/tokyotoshokan.py

@@ -5,7 +5,6 @@ import re
 from datetime import datetime
 from urllib.parse import urlencode
 
-from lxml import html
 from searx.utils import extract_text, int_or_zero
 
 # about
@@ -38,7 +37,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
     rows = dom.xpath('//table[@class="listing"]//tr[contains(@class, "category_0")]')
 
     # check if there are no results or page layout was changed so we cannot parse it

searx/engines/tonline.py

@@ -12,8 +12,6 @@ results from YouTube.
 import typing as t
 from urllib.parse import urlencode
 
-from lxml import html
-
 from searx.utils import eval_xpath_list, eval_xpath, extract_text, ElementType
 from searx.result_types import EngineResults
 from searx.enginelib import EngineAbout
@@ -131,7 +129,7 @@ def _video_results(doc: ElementType, res: EngineResults):
 
 
 def response(resp: "SXNG_Response") -> EngineResults:
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
     res = EngineResults()
     match tonline_categ:
         case "web":

searx/engines/uxwing.py

@@ -2,7 +2,6 @@
 """UXwing (images)"""
 
 from urllib.parse import quote_plus
-from lxml import html
 
 from searx.utils import eval_xpath, eval_xpath_list, extract_text
 
@@ -26,7 +25,7 @@ def request(query, params):
 def response(resp):
     results = []
 
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
     for result in eval_xpath_list(doc, "//article[starts-with(@id, 'post')]"):
         classes = extract_text(eval_xpath(result, "./@class")).split(" ")
         tags = []

searx/engines/vuhuv.py

@@ -7,8 +7,6 @@
 import typing as t
 from urllib.parse import urlencode
 
-from lxml import html
-
 from searx.result_types import EngineResults
 from searx.utils import eval_xpath_list, eval_xpath, extract_text
 
@@ -101,7 +99,7 @@ def _video_results(doc: "ElementBase") -> EngineResults:
 
 
 def response(resp: "SXNG_Response") -> EngineResults:
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
     match vuhuv_category:
         case "general":
             return _general_results(doc)

searx/engines/wikipedia.py

@@ -57,7 +57,6 @@ options:
 import urllib.parse
 
 import babel
-from lxml import html
 
 from searx import locales, utils
 from searx import network as _network
@@ -285,7 +284,7 @@ def fetch_wikimedia_traits(engine_traits: EngineTraits):
     if not resp.ok:
         raise RuntimeError("Response from Wikipedia is not OK.")
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
     for row in dom.xpath('//table[contains(@class,"sortable")]//tbody/tr'):
         cols = row.xpath("./td")
         if not cols:

searx/engines/xpath.py

@@ -72,7 +72,6 @@ Implementations
 
 from urllib.parse import urlencode
 
-from lxml import html
 from searx.utils import extract_text, extract_url, eval_xpath, eval_xpath_list
 from searx.network import raise_for_httperror
 from searx.result_types import EngineResults
@@ -287,7 +286,7 @@ def response(resp) -> EngineResults:  # pylint: disable=too-many-branches
     if not resp.text:
         return results
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
     is_onion = 'onions' in categories
 
     if results_xpath:

searx/engines/yahoo.py

@@ -9,8 +9,6 @@ named `YBV`, which is cached for 24h before expiring.
 import typing as t
 from urllib.parse import unquote, urlencode, urljoin
 
-from lxml import html
-
 from searx.enginelib import EngineCache
 from searx.network import get  # see https://github.com/searxng/searxng/issues/762
 from searx.result_types import EngineResults
@@ -184,7 +182,6 @@ def parse_url(url_string: str) -> str:
 def _yahoo_html(resp: "SXNG_Response") -> "SXNG_Response":
     cookies = dict(resp.search_params["cookies"])
     params = resp.search_params
-
     for _ in range(_YBV_HOPS):
         if ybv := resp.cookies.get("YBV"):
             cookies["YBV"] = ybv
@@ -215,7 +212,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
     results = EngineResults()
     if resp.status_code != 200:
         resp.raise_for_status()
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     for result in eval_xpath_list(dom, '//div[contains(@class,"algo-sr")]'):
         url = eval_xpath_getindex(result, './/div[contains(@class,"compTitle")]//a/@href', 0, default=None)

searx/engines/yahoo_news.py

@@ -11,7 +11,6 @@ import re
 from urllib.parse import urlencode
 from datetime import datetime, timedelta
 from dateutil import parser
-from lxml import html
 
 from searx.utils import (
     eval_xpath_list,
@@ -66,7 +65,7 @@ def request(query, params):
 
 def response(resp):
     results = []
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     # parse results
     for result in eval_xpath_list(dom, '//ol[contains(@class,"searchCenterMiddle")]//li'):

searx/engines/yandex.py

@@ -135,7 +135,7 @@ def _parse_json_results(dom: html.HtmlElement) -> dict[str, t.Any]:
 def response(resp: "SXNG_Response") -> EngineResults:
     catch_bad_response(resp)
     results = EngineResults()
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     match search_type:
         case "web":

searx/engines/yep.py

@@ -80,7 +80,6 @@ def fetch_traits(engine_traits: "EngineTraits"):
 
     # pylint: disable=import-outside-toplevel, too-many-branches
 
-    from lxml import html
     import babel
 
     from searx.locales import language_tag
@@ -100,7 +99,7 @@ def fetch_traits(engine_traits: "EngineTraits"):
     if not resp.ok:
         raise RuntimeError("Response from Yep languages is not OK.")
 
-    doc = html.fromstring(resp.text)
+    doc = resp.html()
     url = eval_xpath_getindex(doc, "//script[contains(@src, 'PageApp')]/@src", index=0)
 
     resp = get("https:" + extract_text(url), headers=headers, timeout=5)

searx/engines/zlibrary.py

@@ -38,7 +38,6 @@ from datetime import datetime
 from urllib.parse import quote
 
 from flask_babel import gettext  # pyright: ignore[reportUnknownVariableType]
-from lxml import html
 
 from searx.data import ENGINE_TRAITS
 from searx.enginelib.traits import EngineTraits
@@ -118,7 +117,7 @@ def request(query: str, params: "OnlineParams") -> None:
 
 def response(resp: "SXNG_Response") -> EngineResults:
     res = EngineResults()
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     if domain_is_seized(dom):
         raise SearxException(f"zlibrary domain is seized: {base_url}")
@@ -196,7 +195,7 @@ def fetch_traits(engine_traits: EngineTraits) -> None:
     if not resp.ok:
         raise RuntimeError("Response from zlibrary is not OK.")
 
-    dom = html.fromstring(resp.text)
+    dom = resp.html()
 
     if domain_is_seized(dom):
         raise RuntimeError(f"Response from zlibrary is not OK. ({base_url} seized)")

tests/unit/engines/test_xpath.py

@@ -2,6 +2,7 @@
 # pylint: disable=missing-module-docstring,disable=missing-class-docstring,invalid-name
 
 from collections import defaultdict
+import lxml.html
 import mock
 
 from searx.engines import xpath
@@ -27,6 +28,7 @@ class TestXpathEngine(SearxTestCase):
         </div>
     </div>
     """
+    empty_html = """<html></html>"""
 
     def setUp(self):
         super().setUp()
@@ -65,10 +67,14 @@ class TestXpathEngine(SearxTestCase):
         self.assertRaises(AttributeError, xpath.response, '')
         self.assertRaises(AttributeError, xpath.response, '[]')
 
-        response = mock.Mock(text='<html></html>', status_code=200)
+        response = mock.Mock(
+            text=self.empty_html, status_code=200, html=mock.Mock(return_value=lxml.html.fromstring(self.empty_html))
+        )
         self.assertEqual(xpath.response(response), [])
 
-        response = mock.Mock(text=self.html, status_code=200)
+        response = mock.Mock(
+            text=self.html, status_code=200, html=mock.Mock(return_value=lxml.html.fromstring(self.html))
+        )
         results = xpath.response(response)
         self.assertIsInstance(results, list)
         self.assertEqual(len(results), 2)
@@ -107,10 +113,16 @@ class TestXpathEngine(SearxTestCase):
         self.assertRaises(AttributeError, xpath.response, '')
         self.assertRaises(AttributeError, xpath.response, '[]')
 
-        response = mock.Mock(text='<html></html>', status_code=200)
+        response = mock.Mock(
+            text=self.empty_html, status_code=200, html=mock.Mock(return_value=lxml.html.fromstring(self.empty_html))
+        )
         self.assertEqual(xpath.response(response), [])
 
-        response = mock.Mock(text=self.html, status_code=200)
+        response = mock.Mock(
+            text=self.html,
+            status_code=200,
+            html=mock.Mock(return_value=lxml.html.fromstring(self.html)),
+        )
         results = xpath.response(response)
         self.assertIsInstance(results, list)
         self.assertEqual(len(results), 2)