Merge pull request #1804 from return42/fix-core.ac.uk

core.ac.uk: use paper.html template

searx/engines/core.py

@@ -4,7 +4,6 @@
 
 """
 
-from json import loads
 from datetime import datetime
 from urllib.parse import urlencode
 
@@ -42,39 +41,74 @@ def request(query, params):
     )
     params['url'] = base_url + search_path
 
-    logger.debug("query_url --> %s", params['url'])
     return params
 
 
 def response(resp):
     results = []
-    json_data = loads(resp.text)
+    json_data = resp.json()
 
     for result in json_data['data']:
-
         source = result['_source']
+        url = None
+        if source.get('urls'):
+            url = source['urls'][0].replace('http://', 'https://', 1)
+
+        if url is None and source.get('doi'):
+            # use the DOI reference
+            url = 'https://doi.org/' + source['doi']
+
+        if url is None and source.get('downloadUrl'):
+            # use the downloadUrl
+            url = source['downloadUrl']
+
+        if url is None and source.get('identifiers'):
+            # try to find an ark id, see
+            # https://www.wikidata.org/wiki/Property:P8091
+            # and https://en.wikipedia.org/wiki/Archival_Resource_Key
+            arkids = [
+                identifier[5:]  # 5 is the length of "ark:/"
+                for identifier in source.get('identifiers')
+                if isinstance(identifier, str) and identifier.startswith('ark:/')
+            ]
+            if len(arkids) > 0:
+                url = 'https://n2t.net/' + arkids[0]
+
+        if url is None:
+            continue
+
         time = source['publishedDate'] or source['depositedDate']
         if time:
-            date = datetime.fromtimestamp(time / 1000)
-        else:
-            date = None
+            publishedDate = datetime.fromtimestamp(time / 1000)
 
-        metadata = []
-        if source['publisher'] and len(source['publisher']) > 3:
-            metadata.append(source['publisher'])
-        if source['topics']:
-            metadata.append(source['topics'][0])
-        if source['doi']:
-            metadata.append(source['doi'])
-        metadata = ' / '.join(metadata)
+        # sometimes the 'title' is None / filter None values
+        journals = [j['title'] for j in (source.get('journals') or []) if j['title']]
+
+        publisher = source['publisher']
+        if publisher:
+            publisher = source['publisher'].strip("'")
 
         results.append(
             {
-                'url': source['urls'][0].replace('http://', 'https://', 1),
+                'template': 'paper.html',
                 'title': source['title'],
-                'content': source['description'],
-                'publishedDate': date,
-                'metadata': metadata,
+                'url': url,
+                'content': source['description'] or '',
+                # 'comments': '',
+                'tags': source['topics'],
+                'publishedDate': publishedDate,
+                'type': (source['types'] or [None])[0],
+                'authors': source['authors'],
+                'editor': ', '.join(source['contributors'] or []),
+                'publisher': publisher,
+                'journal': ', '.join(journals),
+                # 'volume': '',
+                # 'pages' : '',
+                # 'number': '',
+                'doi': source['doi'],
+                'issn': source['issn'],
+                'isbn': source.get('isbn'),  # exists in the rawRecordXml
+                'pdf_url': source.get('repositoryDocument', {}).get('pdfOrigin'),
             }
         )
 

searx/templates/simple/result_templates/paper.html

@@ -1,4 +1,4 @@
-{% from 'simple/macros.html' import result_header, result_sub_header, result_sub_footer, result_footer with context %}
+{% from 'simple/macros.html' import result_header, result_sub_header, result_sub_footer, result_footer, result_link with context %}
 
 {{ result_header(result, favicons, image_proxify) -}}
 <div class="attributes">
@@ -23,7 +23,7 @@
   {%- if result.publisher %}<div class="result_publisher"><span>{{ _("Publisher") }}:</span><span>{{ result.publisher }}</span></div>{% endif -%}
   {%- if result.type %}<div class="result_type"><span>{{ _("Type") }}:</span><span>{{ result.type }}</span></div>{% endif -%}
   {%- if result.tags %}<div class="result_tags"><span>{{ _("Tags") }}:</span><span>{{ result.tags | join(", ")}}</span></div>{%- endif -%}
-  {%- if result.doi %}<div class="result_doi"><span>{{ _("DOI") }}:</span><span>{{- result.doi -}}</span></div>{% endif -%}
+  {%- if result.doi %}<div class="result_doi"><span>{{ _("DOI") }}:</span><span>{{ result_link(doi_resolver + result.doi, result.doi) }}</span></div>{% endif -%}
   {%- if result.issn %}<div class="result_issn"><span>{{ _("ISSN") }}:</span><span>{{ result.issn | join(", ") }}</span></div>{% endif -%}
   {%- if result.isbn %}<div class="result_isbn"><span>{{ _("ISBN") }}:</span><span>{{ result.isbn | join(", ") }}</span></div>{% endif -%}
 </div>

searx/webapp.py

@@ -466,6 +466,7 @@ def render(template_name: str, **kwargs):
     kwargs['proxify_results'] = settings['result_proxy']['proxify_results']
     kwargs['cache_url'] = settings['ui']['cache_url']
     kwargs['get_result_template'] = get_result_template
+    kwargs['doi_resolver'] = get_doi_resolver(request.preferences)
     kwargs['opensearch_url'] = (
         url_for('opensearch')
         + '?'