[feat] magnific: solve captcha to bypass botblocking

Magnific now employs a very simple to solve CAPTCHA before
all API requests (essentially summing up some numbers).

Python script for testing:
```python
import re
from lxml import html
import requests

headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:156.0) Gecko/20100101 Firefox/156.0',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.9',
    'Sec-GPC': '1',
    'Connection': 'keep-alive',
    'Upgrade-Insecure-Requests': '1',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'same-origin',
    'Priority': 'u=0, i',
    'Pragma': 'no-cache',
    'Cache-Control': 'no-cache',
}

resp = requests.get(
    "https://www.magnific.com/api/regular/search?term=tree&filters%5Bai-generated%5D%5Bexcluded%5D=1&page=1&locale=en&filters%5Blicense%5D=free",
    headers=headers,
)
cookies = resp.cookies
print(resp.text)

_NUMBER_RE = re.compile("\d+")
_BM_VERIFY = re.compile(r"\"bm-verify\":\s*\"(.*?)\"")

doc = html.fromstring(resp.text)
script = doc.xpath("//script")[0].text
numbers = [int(m) for m in _NUMBER_RE.findall(script)]
bm_verify = _BM_VERIFY.search(resp.text)
if not bm_verify:
    exit(1)
bm_verify = bm_verify.group(1)
print("bm-verify: " + bm_verify)

solution = sum(numbers)
print(f"sum({numbers}) = {solution}")

resp = requests.post(
    "https://www.magnific.com/_sec/verify?provider=interstitial",
    json={"bm-verify": bm_verify, "pow": solution},
    headers=headers,
    cookies=cookies,
)

resp = requests.get(
    "https://www.magnific.com/api/regular/search?term=bird&filters%5Bai-generated%5D%5Bexcluded%5D=1&page=1&locale=en&filters%5Blicense%5D=free",
    headers=headers,
    cookies=resp.cookies,
)
print(resp.text)
```
This commit is contained in:
Bnyro
2026-09-18 13:06:34 +02:00
parent 367fb6537c
commit fe5bdd1bf2

View File

@@ -4,11 +4,16 @@
.. _Magnific: https://www.magnific.com
"""
import re
from urllib.parse import urlencode
import typing as t
from searx.enginelib import EngineCache
from searx.exceptions import SearxEngineAPIException
from searx.network import get, post
from searx.result_types import EngineResults
from searx.utils import eval_xpath_getindex, extract_text
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
@@ -26,7 +31,7 @@ about = {
base_url = "https://www.magnific.com"
categories = ["images"]
categories = ["stock_images"]
paging = True
free_images_only = True
@@ -34,6 +39,47 @@ free_images_only = True
Whether to only load images that may be used for free, without a Magnific account.
"""
# regular expressions for solving the captcha
_NUMBER_RE = re.compile(r"\d+")
_BM_VERIFY_RE = re.compile(r"\"bm-verify\":\s*\"(.*?)\"")
_CAPTCHA_COOKIE_NAME = "ak_bmsc"
CACHE: EngineCache
"""Cache for storing the cookie to bypass botblocking, obtained by solving a CAPTCHA."""
def setup(engine_settings: dict[str, t.Any]):
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"])
def _solve_captcha(original_resp: "SXNG_Response") -> "SXNG_Response":
"""Solves the Magnific CAPTCHA by summing all numbers in the response.
E.g., the relevant part of the script that contains the numbers could look like
``var i = 1789729970; var j = i + Number("1947" + "43160");``."""
doc = original_resp.html()
script = extract_text(eval_xpath_getindex(doc, "//script", 0)) or ""
bm_verify = _BM_VERIFY_RE.search(original_resp.text)
if not bm_verify:
raise SearxEngineAPIException("failed to extract bm verify token")
bm_verify = bm_verify.group(1)
# challenge is just adding a bunch of number together (some are strings, some are ints)
numbers = [int(m) for m in _NUMBER_RE.findall(script)]
solution = sum(numbers)
challenge_resp = post(
f"{base_url}/_sec/verify?provider=interstitial",
json={"bm-verify": bm_verify, "pow": solution},
cookies=original_resp.cookies,
)
resp = get(original_resp.url, headers=original_resp.search_params["headers"], cookies=challenge_resp.cookies)
# cookie is valid for 2 hours
CACHE.set(_CAPTCHA_COOKIE_NAME, challenge_resp.cookies[_CAPTCHA_COOKIE_NAME], expire=7200)
return resp
def request(query: str, params: "OnlineParams") -> None:
args = {"term": query, "filters[ai-generated][excluded]": 1, "page": params["pageno"], "locale": "en"}
@@ -41,10 +87,18 @@ def request(query: str, params: "OnlineParams") -> None:
args["filters[license]"] = "free"
params["headers"]["Referer"] = f"{base_url}/search"
# cookie that is obtained after solving the CAPTCHA
if cookie := CACHE.get(_CAPTCHA_COOKIE_NAME):
params["cookies"][_CAPTCHA_COOKIE_NAME] = cookie
params["url"] = f"{base_url}/api/regular/search?{urlencode(args)}"
def response(resp: "SXNG_Response"):
if resp.headers["content-type"] == "text/html":
resp = _solve_captcha(resp)
res = EngineResults()
result: dict[str, t.Any] # TBH: dict[str, t.Any]