[fix] engines: deviantart access denied

searx/engines/deviantart.py

@@ -1,11 +1,18 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """Deviantart (Images)"""
 
+import typing as t
+
 import urllib.parse
 from lxml import html
 
+from searx.result_types import EngineResults
 from searx.utils import extract_text, eval_xpath, eval_xpath_list
 
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
+
 # about
 about = {
     "website": 'https://www.deviantart.com/',
@@ -23,63 +30,62 @@ paging = True
 # search-url
 base_url = 'https://www.deviantart.com'
 
-results_xpath = '//div[@class="V_S0t_"]/div/div/a'
+results_xpath = '//div[@data-testid="content_row"]//a[.//*[@data-testid="thumb"]]'
-url_xpath = './@href'
+img_src_xpath = './/img/@srcset'
-thumbnail_src_xpath = './div/img/@src'
+thumbnail_src_xpath = './/img/@src'
-img_src_xpath = './div/img/@srcset'
+author_xpath = './/*[@property="schema:name"]/@content'
-title_xpath = './@aria-label'
+cursor_xpath = '//a[contains(@href, "cursor=") and contains(., "Next")]/@href'
-premium_xpath = '../div/div/div/text()'
-premium_keytext = 'Watch the artist to view this deviation'
-cursor_xpath = '(//a[@class="vQ2brP"]/@href)[last()]'
 
 
-def request(query, params):
+def request(query: str, params: "OnlineParams"):
-
     # https://www.deviantart.com/search?q=foo
 
-    nextpage_url = params['engine_data'].get('nextpage')
+    args = {'q': query}
-    # don't use nextpage when user selected to jump back to page 1
+    if params['pageno'] > 1:
-    if params['pageno'] > 1 and nextpage_url is not None:
+        cursor = params['engine_data'].get('cursor')
-        params['url'] = nextpage_url
+        if cursor:
-    else:
+            args['cursor'] = cursor
-        params['url'] = f"{base_url}/search?{urllib.parse.urlencode({'q': query})}"
 
-    return params
+    params['url'] = f"{base_url}/search?{urllib.parse.urlencode(args)}"
 
 
-def response(resp):
+def response(resp: "SXNG_Response") -> EngineResults:
 
-    results = []
+    res = EngineResults()
     dom = html.fromstring(resp.text)
 
     for result in eval_xpath_list(dom, results_xpath):
-        # skip images that are blurred
+        thumbnail_src = extract_text(eval_xpath(result, thumbnail_src_xpath))
-        _text = extract_text(eval_xpath(result, premium_xpath))
-        if _text and premium_keytext in _text:
-            continue
         img_src = extract_text(eval_xpath(result, img_src_xpath))
+        # mature locked thumbs have blur transform (blur_15, blur_30 etc..)
+        if ',blur_' in f'{thumbnail_src}{img_src}':
+            continue
         if img_src:
             img_src = img_src.split(' ')[0]
             parsed_url = urllib.parse.urlparse(img_src)
             img_src = parsed_url._replace(path=parsed_url.path.split('/v1')[0]).geturl()
 
-        results.append(
+        author = extract_text(eval_xpath(result, author_xpath))
-            {
+
-                'template': 'images.html',
+        res.add(
-                'url': extract_text(eval_xpath(result, url_xpath)),
+            res.types.Image(
-                'img_src': img_src,
+                template='images.html',
-                'thumbnail_src': extract_text(eval_xpath(result, thumbnail_src_xpath)),
+                url=result.get('href'),
-                'title': extract_text(eval_xpath(result, title_xpath)),
+                img_src=img_src or "",
-            }
+                thumbnail_src=thumbnail_src or "",
+                title=result.get('aria-label'),
+                author=author or "",
+            )
         )
 
     nextpage_url = extract_text(eval_xpath(dom, cursor_xpath))
-    if nextpage_url:
+    cursor = urllib.parse.parse_qs(urllib.parse.urlparse(nextpage_url or '').query).get('cursor', [None])[0]
-        results.append(
+    if cursor:
-            {
+        res.add(
-                'engine_data': nextpage_url.replace("http://", "https://"),
+            res.types.LegacyResult(
-                'key': 'nextpage',
+                engine_data=cursor,
-            }
+                key='cursor',
+            )
         )
 
-    return results
+    return res

searx/settings.yml

@@ -761,6 +761,7 @@ engines:
     engine: deviantart
     shortcut: da
     timeout: 3.0
+    inactive: true
 
   - name: devicons
     engine: devicons