From fe5bdd1bf2c4d6223cb56cd5d3f07106a0a408fe Mon Sep 17 00:00:00 2001 From: Bnyro Date: Fri, 18 Sep 2026 13:06:34 +0200 Subject: [PATCH] [feat] magnific: solve captcha to bypass botblocking Magnific now employs a very simple to solve CAPTCHA before all API requests (essentially summing up some numbers). Python script for testing: ```python import re from lxml import html import requests headers = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:156.0) Gecko/20100101 Firefox/156.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', 'Sec-GPC': '1', 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'same-origin', 'Priority': 'u=0, i', 'Pragma': 'no-cache', 'Cache-Control': 'no-cache', } resp = requests.get( "https://www.magnific.com/api/regular/search?term=tree&filters%5Bai-generated%5D%5Bexcluded%5D=1&page=1&locale=en&filters%5Blicense%5D=free", headers=headers, ) cookies = resp.cookies print(resp.text) _NUMBER_RE = re.compile("\d+") _BM_VERIFY = re.compile(r"\"bm-verify\":\s*\"(.*?)\"") doc = html.fromstring(resp.text) script = doc.xpath("//script")[0].text numbers = [int(m) for m in _NUMBER_RE.findall(script)] bm_verify = _BM_VERIFY.search(resp.text) if not bm_verify: exit(1) bm_verify = bm_verify.group(1) print("bm-verify: " + bm_verify) solution = sum(numbers) print(f"sum({numbers}) = {solution}") resp = requests.post( "https://www.magnific.com/_sec/verify?provider=interstitial", json={"bm-verify": bm_verify, "pow": solution}, headers=headers, cookies=cookies, ) resp = requests.get( "https://www.magnific.com/api/regular/search?term=bird&filters%5Bai-generated%5D%5Bexcluded%5D=1&page=1&locale=en&filters%5Blicense%5D=free", headers=headers, cookies=resp.cookies, ) print(resp.text) ``` --- searx/engines/magnific.py | 56 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/searx/engines/magnific.py b/searx/engines/magnific.py index ddec8dfa4..3460725fe 100644 --- a/searx/engines/magnific.py +++ b/searx/engines/magnific.py @@ -4,11 +4,16 @@ .. _Magnific: https://www.magnific.com """ +import re from urllib.parse import urlencode import typing as t +from searx.enginelib import EngineCache +from searx.exceptions import SearxEngineAPIException +from searx.network import get, post from searx.result_types import EngineResults +from searx.utils import eval_xpath_getindex, extract_text if t.TYPE_CHECKING: from searx.extended_types import SXNG_Response @@ -26,7 +31,7 @@ about = { base_url = "https://www.magnific.com" -categories = ["images"] +categories = ["stock_images"] paging = True free_images_only = True @@ -34,6 +39,47 @@ free_images_only = True Whether to only load images that may be used for free, without a Magnific account. """ +# regular expressions for solving the captcha +_NUMBER_RE = re.compile(r"\d+") +_BM_VERIFY_RE = re.compile(r"\"bm-verify\":\s*\"(.*?)\"") +_CAPTCHA_COOKIE_NAME = "ak_bmsc" + +CACHE: EngineCache +"""Cache for storing the cookie to bypass botblocking, obtained by solving a CAPTCHA.""" + + +def setup(engine_settings: dict[str, t.Any]): + global CACHE # pylint: disable=global-statement + CACHE = EngineCache(engine_settings["name"]) + + +def _solve_captcha(original_resp: "SXNG_Response") -> "SXNG_Response": + """Solves the Magnific CAPTCHA by summing all numbers in the response. + E.g., the relevant part of the script that contains the numbers could look like + ``var i = 1789729970; var j = i + Number("1947" + "43160");``.""" + doc = original_resp.html() + script = extract_text(eval_xpath_getindex(doc, "//script", 0)) or "" + + bm_verify = _BM_VERIFY_RE.search(original_resp.text) + if not bm_verify: + raise SearxEngineAPIException("failed to extract bm verify token") + bm_verify = bm_verify.group(1) + + # challenge is just adding a bunch of number together (some are strings, some are ints) + numbers = [int(m) for m in _NUMBER_RE.findall(script)] + solution = sum(numbers) + + challenge_resp = post( + f"{base_url}/_sec/verify?provider=interstitial", + json={"bm-verify": bm_verify, "pow": solution}, + cookies=original_resp.cookies, + ) + + resp = get(original_resp.url, headers=original_resp.search_params["headers"], cookies=challenge_resp.cookies) + # cookie is valid for 2 hours + CACHE.set(_CAPTCHA_COOKIE_NAME, challenge_resp.cookies[_CAPTCHA_COOKIE_NAME], expire=7200) + return resp + def request(query: str, params: "OnlineParams") -> None: args = {"term": query, "filters[ai-generated][excluded]": 1, "page": params["pageno"], "locale": "en"} @@ -41,10 +87,18 @@ def request(query: str, params: "OnlineParams") -> None: args["filters[license]"] = "free" params["headers"]["Referer"] = f"{base_url}/search" + + # cookie that is obtained after solving the CAPTCHA + if cookie := CACHE.get(_CAPTCHA_COOKIE_NAME): + params["cookies"][_CAPTCHA_COOKIE_NAME] = cookie + params["url"] = f"{base_url}/api/regular/search?{urlencode(args)}" def response(resp: "SXNG_Response"): + if resp.headers["content-type"] == "text/html": + resp = _solve_captcha(resp) + res = EngineResults() result: dict[str, t.Any] # TBH: dict[str, t.Any]