From 44decaa524279e2fc10ce42f044e1ee69bcc2aca Mon Sep 17 00:00:00 2001
From: Bnyro <bnyro@tutanota.com>
Date: Wed, 6 Mar 2024 11:04:55 +0100
Subject: [PATCH] [feat] pkg.go.dev: use packages.html template

---
 searx/engines/pkg_go_dev.py | 85 +++++++++++++++++++++++++++++++++++++
 searx/settings.yml          | 14 +-----
 2 files changed, 86 insertions(+), 13 deletions(-)
 create mode 100644 searx/engines/pkg_go_dev.py

diff --git a/searx/engines/pkg_go_dev.py b/searx/engines/pkg_go_dev.py
new file mode 100644
index 000000000..dcbd88451
--- /dev/null
+++ b/searx/engines/pkg_go_dev.py
@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""pkg.go.dev (packages)"""
+
+import re
+from urllib.parse import urlencode
+from dateutil import parser
+
+import babel
+import flask_babel
+from lxml import html
+from searx.utils import eval_xpath, eval_xpath_list, extract_text
+
+about = {
+    'website': 'https://pkg.go.dev/',
+    'use_official_api': False,
+    'official_api_documentation': None,
+    'require_api_key': False,
+    'results': 'HTML',
+}
+
+categories = ["packages", "it"]
+
+base_url = "https://pkg.go.dev"
+max_result_count = 50
+
+results_xpath = '/html/body/main/div[contains(@class,"SearchResults")]/div[not(@class)]/div[@class="SearchSnippet"]'
+url_xpath = './div[@class="SearchSnippet-headerContainer"]/h2/a/@href'
+title_xpath = './div[@class="SearchSnippet-headerContainer"]/h2/a/text()'
+package_name_xpath = './div[@class="SearchSnippet-headerContainer"]/h2/a/span/text()'
+version_xpath = './div[contains(@class, "SearchSnippet-infoLabel")]/span/strong[1]/text()'
+updated_xpath = (
+    './div[contains(@class, "SearchSnippet-infoLabel")]/span/span[@data-test-id="snippet-published"]/strong/text()'
+)
+content_xpath = './p[@class="SearchSnippet-synopsis"]'
+popularity_xpath = './div[contains(@class, "SearchSnippet-infoLabel")]/a/strong/text()'
+license_name_xpath = './div[contains(@class, "SearchSnippet-infoLabel")]/span[@data-test-id="snippet-license"]/a/text()'
+license_url_xpath = './div[contains(@class, "SearchSnippet-infoLabel")]/span[@data-test-id="snippet-license"]/a/@href'
+
+
+def request(query, params):
+    args = {
+        'q': query,
+        'm': 'package',
+        'limit': max_result_count,
+    }
+    params['url'] = f"{base_url}/search?{urlencode(args)}"
+
+    return params
+
+
+def response(resp):
+    results = []
+
+    doc = html.fromstring(resp.text)
+
+    for result in eval_xpath_list(doc, results_xpath):
+        publishedDate = extract_text(eval_xpath(result, updated_xpath))
+        try:
+            publishedDate = parser.parse(publishedDate)
+        except parser.ParserError:
+            publishedDate = None
+
+        # 110n 15,000.00 (EN) --> 15.000,00 (DE)
+        popularity = extract_text(eval_xpath(result, popularity_xpath)).strip()
+        popularity = babel.numbers.parse_decimal(popularity, locale='en_US')
+        # popularity is of type str ..
+        popularity = flask_babel.format_decimal(popularity)
+
+        results.append(
+            {
+                'template': 'packages.html',
+                'url': base_url + extract_text(eval_xpath(result, url_xpath)),
+                'title': extract_text(eval_xpath(result, title_xpath)),
+                'content': extract_text(eval_xpath(result, content_xpath)),
+                'package_name': re.sub(r"\(|\)", "", extract_text(eval_xpath(result, package_name_xpath))),
+                'version': extract_text(eval_xpath(result, version_xpath)),
+                'popularity': popularity,
+                'license_name': extract_text(eval_xpath(result, license_name_xpath)),
+                'license_url': base_url + extract_text(eval_xpath(result, license_url_xpath)),
+                'publishedDate': publishedDate,
+            }
+        )
+
+    return results
diff --git a/searx/settings.yml b/searx/settings.yml
index f292de18a..865cb6b77 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -2314,21 +2314,9 @@ engines:
     shortcut: bt4g
 
   - name: pkg.go.dev
-    engine: xpath
+    engine: pkg_go_dev
     shortcut: pgo
-    search_url: https://pkg.go.dev/search?limit=100&m=package&q={query}
-    results_xpath: /html/body/main/div[contains(@class,"SearchResults")]/div[not(@class)]/div[@class="SearchSnippet"]
-    url_xpath: ./div[@class="SearchSnippet-headerContainer"]/h2/a/@href
-    title_xpath: ./div[@class="SearchSnippet-headerContainer"]/h2/a
-    content_xpath: ./p[@class="SearchSnippet-synopsis"]
-    categories: [packages, it]
-    timeout: 3.0
     disabled: true
-    about:
-      website: https://pkg.go.dev/
-      use_official_api: false
-      require_api_key: false
-      results: HTML
 
 # Doku engine lets you access to any Doku wiki instance:
 # A public one or a privete/corporate one.