searxng/searx/engines/swisscows.py

"""
 Swisscows (Web, Images)

 @website     https://swisscows.ch
 @provide-api no

 @using-api   no
 @results     HTML (using search portal)
 @stable      no (HTML can change)
 @parse       url, title, content
"""

from json import loads
from urllib import urlencode, unquote
import re

# engine dependent config
categories = ['general', 'images']
paging = True
language_support = True
supported_languages = ["ar-SA", "es-AR", "en-AU", "de-AT", "fr-BE", "nl-BE", "pt-BR", "bg-BG", "en-CA", "fr-CA",
                       "es-CL", "zh-CN", "hr-HR", "cs-CZ", "da-DK", "et-EE", "fi-FI", "fr-FR", "de-DE", "el-GR",
                       "zh-HK", "hu-HU", "en-IN", "en-IE", "he-IL", "it-IT", "ja-JP", "ko-KR", "lv-LV", "lt-LT",
                       "en-MY", "es-MX", "nl-NL", "en-NZ", "nb-NO", "en-PH", "pl-PL", "pt-PT", "ro-RO", "ru-RU",
                       "en-SG", "sk-SK", "sl-SI", "en-ZA", "es-ES", "sv-SE", "de-CH", "fr-CH", "zh-TW", "th-TH",
                       "tr-TR", "uk-UA", "en-GB", "en-US", "es-US"]

# search-url
base_url = 'https://swisscows.ch/'
search_string = '?{query}&page={page}'

# regex
regex_json = re.compile(r'initialData: {"Request":(.|\n)*},\s*environment')
regex_json_remove_start = re.compile(r'^initialData:\s*')
regex_json_remove_end = re.compile(r',\s*environment$')
regex_img_url_remove_start = re.compile(r'^https?://i\.swisscows\.ch/\?link=')


# do search-request
def request(query, params):
    if params['language'] == 'all':
        ui_language = 'browser'
        region = 'browser'
    elif params['language'].split('-')[0] == 'no':
        region = 'nb-NO'
    else:
        region = params['language']
        ui_language = params['language'].split('-')[0]

    search_path = search_string.format(
        query=urlencode({'query': query,
                         'uiLanguage': ui_language,
                         'region': region}),
        page=params['pageno'])

    # image search query is something like 'image?{query}&page={page}'
    if params['category'] == 'images':
        search_path = 'image' + search_path

    params['url'] = base_url + search_path

    return params


# get response from search-request
def response(resp):
    results = []

    json_regex = regex_json.search(resp.content)

    # check if results are returned
    if not json_regex:
        return []

    json_raw = regex_json_remove_end.sub('', regex_json_remove_start.sub('', json_regex.group()))
    json = loads(json_raw)

    # parse results
    for result in json['Results'].get('items', []):
        result_title = result['Title'].replace(u'\uE000', '').replace(u'\uE001', '')

        # parse image results
        if result.get('ContentType', '').startswith('image'):
            img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))

            # append result
            results.append({'url': result['SourceUrl'],
                            'title': result['Title'],
                            'content': '',
                            'img_src': img_url,
                            'template': 'images.html'})

        # parse general results
        else:
            result_url = result['Url'].replace(u'\uE000', '').replace(u'\uE001', '')
            result_content = result['Description'].replace(u'\uE000', '').replace(u'\uE001', '')

            # append result
            results.append({'url': result_url,
                            'title': result_title,
                            'content': result_content})

    # parse images
    for result in json.get('Images', []):
        # decode image url
        img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))

        # append result
        results.append({'url': result['SourceUrl'],
                        'title': result['Title'],
                        'content': '',
                        'img_src': img_url,
                        'template': 'images.html'})

    # return results
    return results