[fix] engines: deviantart access denied

searx/engines/deviantart.py

@@ -1,11 +1,18 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """Deviantart (Images)"""
 
+import typing as t
+
 import urllib.parse
 from lxml import html
 
+from searx.result_types import EngineResults
 from searx.utils import extract_text, eval_xpath, eval_xpath_list
 
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
+
 # about
 about = {
     "website": 'https://www.deviantart.com/',
@@ -23,63 +30,62 @@ paging = True
 # search-url
 base_url = 'https://www.deviantart.com'
 
-results_xpath = '//div[@class="V_S0t_"]/div/div/a'
-url_xpath = './@href'
-thumbnail_src_xpath = './div/img/@src'
-img_src_xpath = './div/img/@srcset'
-title_xpath = './@aria-label'
-premium_xpath = '../div/div/div/text()'
-premium_keytext = 'Watch the artist to view this deviation'
-cursor_xpath = '(//a[@class="vQ2brP"]/@href)[last()]'
+results_xpath = '//div[@data-testid="content_row"]//a[.//*[@data-testid="thumb"]]'
+img_src_xpath = './/img/@srcset'
+thumbnail_src_xpath = './/img/@src'
+author_xpath = './/*[@property="schema:name"]/@content'
+cursor_xpath = '//a[contains(@href, "cursor=") and contains(., "Next")]/@href'
 
 
-def request(query, params):
-
+def request(query: str, params: "OnlineParams"):
     # https://www.deviantart.com/search?q=foo
 
-    nextpage_url = params['engine_data'].get('nextpage')
-    # don't use nextpage when user selected to jump back to page 1
-    if params['pageno'] > 1 and nextpage_url is not None:
-        params['url'] = nextpage_url
-    else:
-        params['url'] = f"{base_url}/search?{urllib.parse.urlencode({'q': query})}"
+    args = {'q': query}
+    if params['pageno'] > 1:
+        cursor = params['engine_data'].get('cursor')
+        if cursor:
+            args['cursor'] = cursor
 
-    return params
+    params['url'] = f"{base_url}/search?{urllib.parse.urlencode(args)}"
 
 
-def response(resp):
+def response(resp: "SXNG_Response") -> EngineResults:
 
-    results = []
+    res = EngineResults()
     dom = html.fromstring(resp.text)
 
     for result in eval_xpath_list(dom, results_xpath):
-        # skip images that are blurred
-        _text = extract_text(eval_xpath(result, premium_xpath))
-        if _text and premium_keytext in _text:
-            continue
+        thumbnail_src = extract_text(eval_xpath(result, thumbnail_src_xpath))
         img_src = extract_text(eval_xpath(result, img_src_xpath))
+        # mature locked thumbs have blur transform (blur_15, blur_30 etc..)
+        if ',blur_' in f'{thumbnail_src}{img_src}':
+            continue
         if img_src:
             img_src = img_src.split(' ')[0]
             parsed_url = urllib.parse.urlparse(img_src)
             img_src = parsed_url._replace(path=parsed_url.path.split('/v1')[0]).geturl()
 
-        results.append(
-            {
-                'template': 'images.html',
-                'url': extract_text(eval_xpath(result, url_xpath)),
-                'img_src': img_src,
-                'thumbnail_src': extract_text(eval_xpath(result, thumbnail_src_xpath)),
-                'title': extract_text(eval_xpath(result, title_xpath)),
-            }
+        author = extract_text(eval_xpath(result, author_xpath))
+
+        res.add(
+            res.types.Image(
+                template='images.html',
+                url=result.get('href'),
+                img_src=img_src or "",
+                thumbnail_src=thumbnail_src or "",
+                title=result.get('aria-label'),
+                author=author or "",
+            )
         )
 
     nextpage_url = extract_text(eval_xpath(dom, cursor_xpath))
-    if nextpage_url:
-        results.append(
-            {
-                'engine_data': nextpage_url.replace("http://", "https://"),
-                'key': 'nextpage',
-            }
+    cursor = urllib.parse.parse_qs(urllib.parse.urlparse(nextpage_url or '').query).get('cursor', [None])[0]
+    if cursor:
+        res.add(
+            res.types.LegacyResult(
+                engine_data=cursor,
+                key='cursor',
+            )
         )
 
-    return results
+    return res

searx/settings.yml

@@ -761,6 +761,7 @@ engines:
     engine: deviantart
     shortcut: da
     timeout: 3.0
+    inactive: true
 
   - name: devicons
     engine: devicons