[fix] semantic scholar: method not allowed / engine doesn't work

Fixes the semantic scholar engine by extracting a ui version token.

BTW: remove html tags from the content.

Author's checklist:

- they are ratelimiting very fast, if you do approx more than 2 requests per
  minute, you have to wait some time again...

- they also have an official api at api.semanticscholar.org, but it's ratelimits
  are even harder

Closes: https://github.com/searxng/searxng/issues/4685

client/simple/package-lock.json

@@ -21,11 +21,11 @@
         "ionicons": "^7.4.0",
         "leaflet": "^1.9.4",
         "less": "^4.3.0",
-        "less-loader": "^12.2.0",
+        "less-loader": "^12.3.0",
         "normalize.css": "^8.0.1",
         "sharp": "^0.34.1",
         "style-loader": "^4.0.0",
-        "stylelint": "^16.19.0",
+        "stylelint": "^16.19.1",
         "stylelint-config-standard": "^38.0.0",
         "stylelint-config-standard-less": "^3.0.1",
         "stylelint-prettier": "^5.0.3",
@@ -34,7 +34,7 @@
         "vite": "^6.3.4",
         "vite-plugin-static-copy": "^2.3.1",
         "vite-plugin-stylelint": "^6.0.0",
-        "webpack": "^5.99.6",
+        "webpack": "^5.99.7",
         "webpack-cli": "^6.0.1"
       }
     },
@@ -4573,9 +4573,9 @@
       }
     },
     "node_modules/less-loader": {
-      "version": "12.2.0",
-      "resolved": "https://registry.npmjs.org/less-loader/-/less-loader-12.2.0.tgz",
-      "integrity": "sha512-MYUxjSQSBUQmowc0l5nPieOYwMzGPUaTzB6inNW/bdPEG9zOL3eAAD1Qw5ZxSPk7we5dMojHwNODYMV1hq4EVg==",
+      "version": "12.3.0",
+      "resolved": "https://registry.npmjs.org/less-loader/-/less-loader-12.3.0.tgz",
+      "integrity": "sha512-0M6+uYulvYIWs52y0LqN4+QM9TqWAohYSNTo4htE8Z7Cn3G/qQMEmktfHmyJT23k+20kU9zHH2wrfFXkxNLtVw==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -5748,9 +5748,9 @@
       "optional": true
     },
     "node_modules/schema-utils": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.0.tgz",
-      "integrity": "sha512-Gf9qqc58SpCA/xdziiHz35F4GNIWYWZrEshUc/G/r5BnLph6xpKuLeoJoQuj5WfBIx/eQLf+hmVPYHaxJu7V2g==",
+      "version": "4.3.2",
+      "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.2.tgz",
+      "integrity": "sha512-Gn/JaSk/Mt9gYubxTtSn/QCV4em9mpAPiR1rqy/Ocu19u/G9J5WWdNoUT4SiV6mFC3y6cxyFcFwdzPM3FgxGAQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -6098,9 +6098,9 @@
       }
     },
     "node_modules/stylelint": {
-      "version": "16.19.0",
-      "resolved": "https://registry.npmjs.org/stylelint/-/stylelint-16.19.0.tgz",
-      "integrity": "sha512-BJzc5mo/ez0H/ZSl3UbxGdkK/s0kFGsF5/k6IGu4z8wJ1qp49WrOS9RxswvcN6HMirt0g/iiJqOwLHTbWv49IQ==",
+      "version": "16.19.1",
+      "resolved": "https://registry.npmjs.org/stylelint/-/stylelint-16.19.1.tgz",
+      "integrity": "sha512-C1SlPZNMKl+d/C867ZdCRthrS+6KuZ3AoGW113RZCOL0M8xOGpgx7G70wq7lFvqvm4dcfdGFVLB/mNaLFChRKw==",
       "dev": true,
       "funding": [
         {
@@ -7004,14 +7004,15 @@
       }
     },
     "node_modules/webpack": {
-      "version": "5.99.6",
-      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.99.6.tgz",
-      "integrity": "sha512-TJOLrJ6oeccsGWPl7ujCYuc0pIq2cNsuD6GZDma8i5o5Npvcco/z+NKvZSFsP0/x6SShVb0+X2JK/JHUjKY9dQ==",
+      "version": "5.99.7",
+      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.99.7.tgz",
+      "integrity": "sha512-CNqKBRMQjwcmKR0idID5va1qlhrqVUKpovi+Ec79ksW8ux7iS1+A6VqzfZXgVYCFRKl7XL5ap3ZoMpwBJxcg0w==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "@types/eslint-scope": "^3.7.7",
         "@types/estree": "^1.0.6",
+        "@types/json-schema": "^7.0.15",
         "@webassemblyjs/ast": "^1.14.1",
         "@webassemblyjs/wasm-edit": "^1.14.1",
         "@webassemblyjs/wasm-parser": "^1.14.1",
@@ -7028,7 +7029,7 @@
         "loader-runner": "^4.2.0",
         "mime-types": "^2.1.27",
         "neo-async": "^2.6.2",
-        "schema-utils": "^4.3.0",
+        "schema-utils": "^4.3.2",
         "tapable": "^2.1.1",
         "terser-webpack-plugin": "^5.3.11",
         "watchpack": "^2.4.1",

client/simple/package.json

@@ -19,11 +19,11 @@
     "ionicons": "^7.4.0",
     "leaflet": "^1.9.4",
     "less": "^4.3.0",
-    "less-loader": "^12.2.0",
+    "less-loader": "^12.3.0",
     "normalize.css": "^8.0.1",
     "sharp": "^0.34.1",
     "style-loader": "^4.0.0",
-    "stylelint": "^16.19.0",
+    "stylelint": "^16.19.1",
     "stylelint-config-standard": "^38.0.0",
     "stylelint-config-standard-less": "^3.0.1",
     "stylelint-prettier": "^5.0.3",
@@ -32,7 +32,7 @@
     "vite": "^6.3.4",
     "vite-plugin-static-copy": "^2.3.1",
     "vite-plugin-stylelint": "^6.0.0",
-    "webpack": "^5.99.6",
+    "webpack": "^5.99.7",
     "webpack-cli": "^6.0.1"
   },
   "dependencies": {

docs/admin/settings/settings_engines.rst

@@ -148,6 +148,8 @@ engine is shown.  Most of the options have a default value or even are optional.
 ``display_error_messages`` : default ``true``
   When an engine returns an error, the message is displayed on the user interface.
 
+.. _engine network:
+
 ``network`` : optional
   Use the network configuration from another engine.
   In addition, there are two default networks:
@@ -257,4 +259,3 @@ Example configuration in settings.yml for a German and English speaker:
 
 When searching, the default google engine will return German results and
 "google english" will return English results.
-

docs/admin/settings/settings_search.rst

@@ -57,11 +57,11 @@
 
 ``default_lang``:
   Default search language - leave blank to detect from browser information or
-  use codes from :origin:`searx/languages.py`.
+  use codes from :origin:`searx/sxng_locales.py`.
 
 ``languages``:
   List of available languages - leave unset to use all codes from
-  :origin:`searx/languages.py`.  Otherwise list codes of available languages.
+  :origin:`searx/sxng_locales.py`.  Otherwise list codes of available languages.
   The ``all`` value is shown as the ``Default language`` in the user interface
   (in most cases, it is meant to send the query without a language parameter ;
   in some cases, it means the English language) Example:

docs/dev/engines/online/chinaso.rst

@@ -0,0 +1,8 @@
+.. _chinaso engine:
+
+=======
+ChinaSo
+=======
+
+.. automodule:: searx.engines.chinaso
+   :members:

requirements.txt

@@ -1,4 +1,4 @@
-certifi==2025.1.31
+certifi==2025.4.26
 babel==2.17.0
 flask-babel==4.0.0
 flask==3.1.0
@@ -11,11 +11,11 @@ httpx[http2]==0.24.1
 Brotli==1.1.0
 uvloop==0.21.0
 httpx-socks[asyncio]==0.7.7
-setproctitle==1.3.5
+setproctitle==1.3.6
 redis==5.0.8
 markdown-it-py==3.0.0
 fasttext-predict==0.9.2.4
 tomli==2.2.1; python_version < '3.11'
 msgspec==0.19.0
-typer-slim==0.15.2
+typer-slim==0.15.3
 isodate==0.7.2

searx/engines/chinaso.py

@@ -1,5 +1,60 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""ChinaSo: A search engine from ChinaSo."""
+"""ChinaSo_, a search engine for the chinese language area.
+
+.. attention::
+
+   ChinaSo engine does not return real URL, the links from these search
+   engines violate the privacy of the users!!
+
+   We try to find a solution for this problem, please follow `issue #4694`_.
+
+   As long as the problem has not been resolved, these engines are
+   not active in a standard setup (``inactive: true``).
+
+.. _ChinaSo: https://www.chinaso.com/
+.. _issue #4694: https://github.com/searxng/searxng/issues/4694
+
+Configuration
+=============
+
+The engine has the following additional settings:
+
+- :py:obj:`chinaso_category` (:py:obj:`ChinasoCategoryType`)
+- :py:obj:`chinaso_news_source` (:py:obj:`ChinasoNewsSourceType`)
+
+In the example below, all three ChinaSO engines are using the :ref:`network
+<engine network>` from the ``chinaso news`` engine.
+
+.. code:: yaml
+
+   - name: chinaso news
+     engine: chinaso
+     shortcut: chinaso
+     categories: [news]
+     chinaso_category: news
+     chinaso_news_source: all
+
+   - name: chinaso images
+     engine: chinaso
+     network: chinaso news
+     shortcut: chinasoi
+     categories: [images]
+     chinaso_category: images
+
+   - name: chinaso videos
+     engine: chinaso
+     network: chinaso news
+     shortcut: chinasov
+     categories: [videos]
+     chinaso_category: videos
+
+
+Implementations
+===============
+
+"""
+
+import typing
 
 from urllib.parse import urlencode
 from datetime import datetime
@@ -20,13 +75,31 @@ paging = True
 time_range_support = True
 results_per_page = 10
 categories = []
-chinaso_category = 'news'
+
+ChinasoCategoryType = typing.Literal['news', 'videos', 'images']
 """ChinaSo supports news, videos, images search.
 
 - ``news``: search for news
 - ``videos``: search for videos
 - ``images``: search for images
+
+In the category ``news`` you can additionally filter by option
+:py:obj:`chinaso_news_source`.
 """
+chinaso_category = 'news'
+"""Configure ChinaSo category (:py:obj:`ChinasoCategoryType`)."""
+
+ChinasoNewsSourceType = typing.Literal['CENTRAL', 'LOCAL', 'BUSINESS', 'EPAPER', 'all']
+"""Filtering ChinaSo-News results by source:
+
+- ``CENTRAL``: central publication
+- ``LOCAL``: local publication
+- ``BUSINESS``: business publication
+- ``EPAPER``: E-Paper
+- ``all``: all sources
+"""
+chinaso_news_source: ChinasoNewsSourceType = 'all'
+"""Configure ChinaSo-News type (:py:obj:`ChinasoNewsSourceType`)."""
 
 time_range_dict = {'day': '24h', 'week': '1w', 'month': '1m', 'year': '1y'}
 
@@ -35,7 +108,9 @@ base_url = "https://www.chinaso.com"
 
 def init(_):
     if chinaso_category not in ('news', 'videos', 'images'):
-        raise SearxEngineAPIException(f"Unsupported category: {chinaso_category}")
+        raise ValueError(f"Unsupported category: {chinaso_category}")
+    if chinaso_category == 'news' and chinaso_news_source not in typing.get_args(ChinasoNewsSourceType):
+        raise ValueError(f"Unsupported news source: {chinaso_news_source}")
 
 
 def request(query, params):
@@ -56,6 +131,11 @@ def request(query, params):
             'params': {'start_index': (params["pageno"] - 1) * results_per_page, 'rn': results_per_page},
         },
     }
+    if chinaso_news_source != 'all':
+        if chinaso_news_source == 'EPAPER':
+            category_config['news']['params']["type"] = 'EPAPER'
+        else:
+            category_config['news']['params']["cate"] = chinaso_news_source
 
     query_params.update(category_config[chinaso_category]['params'])
 

searx/engines/semantic_scholar.py

@@ -1,11 +1,14 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""Semantic Scholar (Science)
-"""
+"""Semantic Scholar (Science)"""
 
-from json import dumps, loads
+from json import dumps
 from datetime import datetime
+from lxml import html
 
 from flask_babel import gettext
+from searx.network import get
+from searx.utils import eval_xpath_getindex, gen_useragent, html_to_text
+
 
 about = {
     "website": 'https://www.semanticscholar.org/',
@@ -19,13 +22,31 @@ about = {
 categories = ['science', 'scientific publications']
 paging = True
 search_url = 'https://www.semanticscholar.org/api/1/search'
-paper_url = 'https://www.semanticscholar.org/paper'
+base_url = 'https://www.semanticscholar.org'
+
+
+def _get_ui_version():
+    resp = get(base_url)
+    if not resp.ok:
+        raise RuntimeError("Can't determine Semantic Scholar UI version")
+
+    doc = html.fromstring(resp.text)
+    ui_version = eval_xpath_getindex(doc, "//meta[@name='s2-ui-version']/@content", 0)
+    if not ui_version:
+        raise RuntimeError("Can't determine Semantic Scholar UI version")
+
+    return ui_version
 
 
 def request(query, params):
     params['url'] = search_url
     params['method'] = 'POST'
-    params['headers']['content-type'] = 'application/json'
+    params['headers'] = {
+        'Content-Type': 'application/json',
+        'X-S2-UI-Version': _get_ui_version(),
+        'X-S2-Client': "webapp-browser",
+        'User-Agent': gen_useragent(),
+    }
     params['data'] = dumps(
         {
             "queryString": query,
@@ -43,7 +64,8 @@ def request(query, params):
 
 
 def response(resp):
-    res = loads(resp.text)
+    res = resp.json()
+
     results = []
     for result in res['results']:
         url = result.get('primaryPaperLink', {}).get('url')
@@ -54,7 +76,7 @@ def response(resp):
             if alternatePaperLinks:
                 url = alternatePaperLinks[0].get('url')
         if not url:
-            url = paper_url + '/%s' % result['id']
+            url = base_url + '/paper/%s' % result['id']
 
         # publishedDate
         if 'pubDate' in result:
@@ -88,7 +110,7 @@ def response(resp):
                 'template': 'paper.html',
                 'url': url,
                 'title': result['title']['text'],
-                'content': result['paperAbstract']['text'],
+                'content': html_to_text(result['paperAbstract']['text']),
                 'journal': result.get('venue', {}).get('text') or result.get('journal', {}).get('name'),
                 'doi': result.get('doiInfo', {}).get('doi'),
                 'tags': result.get('fieldsOfStudy'),

searx/settings.yml

@@ -619,23 +619,37 @@ engines:
     # to show premium or plus results too:
     # skip_premium: false
 
+  # WARNING: links from chinaso.com voilate users privacy
+  # Before activate these engines its mandatory to read
+  # - https://github.com/searxng/searxng/issues/4694
+  # - https://docs.searxng.org/dev/engines/online/chinaso.html
+
   - name: chinaso news
-    chinaso_category: news
     engine: chinaso
     shortcut: chinaso
+    categories: [news]
+    chinaso_category: news
+    chinaso_news_source: all
     disabled: true
+    inactive: true
 
   - name: chinaso images
-    chinaso_category: images
     engine: chinaso
+    network: chinaso news
     shortcut: chinasoi
+    categories: [images]
+    chinaso_category: images
     disabled: true
+    inactive: true
 
   - name: chinaso videos
-    chinaso_category: videos
     engine: chinaso
+    network: chinaso news
     shortcut: chinasov
+    categories: [videos]
+    chinaso_category: videos
     disabled: true
+    inactive: true
 
   - name: cloudflareai
     engine: cloudflareai

searx/translations/ro/LC_MESSAGES/messages.po

@@ -30,7 +30,7 @@ msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2025-03-29 09:21+0000\n"
-"PO-Revision-Date: 2025-03-31 18:08+0000\n"
+"PO-Revision-Date: 2025-04-30 15:18+0000\n"
 "Last-Translator: return42 <return42@noreply.codeberg.org>\n"
 "Language-Team: Romanian <https://translate.codeberg.org/projects/searxng/"
 "searxng/ro/>\n"
@@ -544,7 +544,7 @@ msgstr ""
 
 #: searx/plugins/calculator.py:38
 msgid "Basic Calculator"
-msgstr ""
+msgstr "Calculator de bază"
 
 #: searx/plugins/calculator.py:39
 msgid "Calculate mathematical expressions via the search bar"