jozamudi
ee2a888aca
The slow-segment logs show per-connection CDN throttling (15-40 KB/s dribbles). yt-dlp's standard fix is -N 16 parallel fragment lanes; this is the 4-lane version for the proxy: HEAD for content-length, then 4 concurrent Range GETs reassembled byte-exact (verified sha256-identical to a direct CDN fetch). Small files (<512KB) still use a single GET. ANY failure (no content-length, 416, lane error, timeout) falls back to the existing single-stream path, so this is never worse than before. Caveat (measured live): when the CDN caps SHARED egress rather than per-connection, lanes just split the same pie — in that window both 1-lane and 4-lane time out. A different source is the only real fix for those windows. 18/18 tests (new test pins lane stitch + fallback; also fixed a pre-existing _get stub leak in the Jikan-shape test).
240 lines
9.5 KiB
Python
240 lines
9.5 KiB
Python
"""upstream.py — the only module that talks to anidb.app (and keyless Jikan
|
|
for intro timings).
|
|
|
|
anidb.app sits behind Cloudflare: plain curl gets 403. All requests use
|
|
curl_cffi Chrome TLS impersonation (verified to pass the challenge).
|
|
Parsers are regex against live markup — keep them here so the layout
|
|
breaks in one place, not across the app.
|
|
"""
|
|
import re
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
from curl_cffi import requests
|
|
|
|
BASE = "https://anidb.app"
|
|
H = dict(impersonate="chrome", timeout=20)
|
|
|
|
# ponytail: thread-local sessions (uvicorn threadpool); curl_cffi Sessions
|
|
# are not thread-safe, but per-thread keep-alive still reuses TCP/TLS across
|
|
# the many requests a worker handles. Saves a handshake on every call.
|
|
_tls = threading.local()
|
|
|
|
def _session():
|
|
if not hasattr(_tls, "s"):
|
|
_tls.s = requests.Session()
|
|
return _tls.s
|
|
|
|
# ponytail: in-memory TTL cache for upstream data that never changes
|
|
# (details/episodes/langs/playlist URLs). No lock: worst case a duplicate
|
|
# fetch. A dead/rotated playlist URL is recovered by the one-shot
|
|
# clear_cache()+retry in api_stream.
|
|
_CACHE = {} # key -> (expires, value)
|
|
|
|
def _cached(key, ttl, fn, *a, **kw):
|
|
v = _CACHE.get(key)
|
|
if v and v[0] > time.time():
|
|
return v[1]
|
|
val = fn(*a, **kw)
|
|
_CACHE[key] = (time.time() + ttl, val)
|
|
if len(_CACHE) > 256:
|
|
_CACHE.clear()
|
|
return val
|
|
|
|
def clear_cache():
|
|
_CACHE.clear()
|
|
|
|
_ANIME_LINK = re.compile(r'anime/([a-z0-9-]+)-(\d+)"[^>]*title="([^"]*)"')
|
|
|
|
|
|
def _get(url, **kw):
|
|
# ponytail: one retry (300ms) on transient upstream errors (Cloudflare
|
|
# 403/503, 429, 5xx); 404 and other 4xx are final. Failing fast matters:
|
|
# hls.js's fragLoadingTimeOut (20s) races curl's timeout (20s), so a
|
|
# quick 502 lets hls.js's own frag retries (6x, backoff) do the waiting
|
|
for attempt in range(2):
|
|
r = _session().get(url, **{**H, **kw}) # kw overrides H (e.g. timeout)
|
|
if not (r.status_code in (403, 429, 503) or r.status_code >= 500
|
|
or "Just a moment" in r.text[:500]):
|
|
r.raise_for_status()
|
|
return r
|
|
if attempt:
|
|
raise RuntimeError(f"upstream {r.status_code}: {url}")
|
|
time.sleep(0.3)
|
|
|
|
|
|
def get_stream(url, timeout=12):
|
|
"""Segments: open the upstream response WITHOUT reading the body, so the
|
|
proxy can forward bytes as the CDN emits them. (A buffered fetch made the
|
|
browser's time-to-first-byte = the CDN's whole upload time, which tripped
|
|
hls.js's fragLoadTimeOut on slow CDN moments.)
|
|
ponytail: skips the 'Just a moment' body check — challenge pages arrive
|
|
as 403/503 (handled below), and reading the body would defeat streaming."""
|
|
for attempt in range(2):
|
|
r = _session().get(url, **{**H, "timeout": timeout, "stream": True})
|
|
if r.status_code in (403, 429, 503) or r.status_code >= 500:
|
|
r.close()
|
|
if attempt:
|
|
raise RuntimeError(f"upstream {r.status_code}: {url}")
|
|
time.sleep(0.3)
|
|
continue
|
|
r.raise_for_status()
|
|
return r
|
|
|
|
|
|
def fetch_parallel(url, lanes=4, lane=256 * 1024, timeout=12):
|
|
"""One upstream file across N parallel byte-range lanes.
|
|
|
|
Per-connection CDN throttling is the common slow-segment cause; N lanes
|
|
turn one 60s crawl into N concurrent crawls (I/O-bound, GIL irrelevant).
|
|
Returns (body, content-type); raises on ANY failure so the caller falls
|
|
back to the single-request path — never worse than before.
|
|
Verified 2026-08 against hls.anidb.app: HEAD -> content-length,
|
|
ranges -> 206, 4 lanes -> byte-identical stitch (sha256)."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
head = _session().head(url, **{**H, "timeout": timeout})
|
|
head.raise_for_status()
|
|
total = int(head.headers.get("content-length") or 0)
|
|
ct = head.headers.get("content-type", "application/octet-stream")
|
|
if total == 0:
|
|
raise RuntimeError("no content-length")
|
|
if total <= 2 * lane: # too small to be worth N connections
|
|
r = _get(url, timeout=timeout)
|
|
return r.content, ct
|
|
n = min(lanes, total // lane)
|
|
size = (total + n - 1) // n
|
|
parts = [None] * n
|
|
|
|
def grab(i):
|
|
a = i * size
|
|
b = total - 1 if i == n - 1 else (i + 1) * size - 1
|
|
r = _get(url, headers={"Range": f"bytes={a}-{b}"}, timeout=timeout)
|
|
if r.status_code not in (200, 206) or len(r.content) != b - a + 1:
|
|
raise RuntimeError(f"lane {i}: http {r.status_code} "
|
|
f"len={len(r.content)} want={b - a + 1}")
|
|
parts[i] = r.content
|
|
|
|
with ThreadPoolExecutor(max_workers=n) as ex:
|
|
list(ex.map(grab, range(n)))
|
|
body = b"".join(parts)
|
|
if len(body) != total:
|
|
raise RuntimeError(f"stitch mismatch {len(body)} != {total}")
|
|
return body, ct
|
|
|
|
|
|
def _search(q):
|
|
"""Step 1: browse page -> [{id, slug_id, name, poster}]"""
|
|
html = _get(f"{BASE}/browse", params={"q": q}).text
|
|
out, seen = [], set()
|
|
for m in re.finditer(
|
|
r'<a href="https://anidb\.app/anime/([a-z0-9-]+-\d+)"[^>]*title="([^"]*)"[^>]*>(.*?)</a>',
|
|
html, re.S):
|
|
slug_id, name, block = m.group(1), m.group(2), m.group(3)
|
|
if slug_id in seen:
|
|
continue
|
|
seen.add(slug_id)
|
|
img = re.search(r'<img src="([^"]+)"', block)
|
|
out.append({"id": int(slug_id.rsplit("-", 1)[1]), "slug_id": slug_id,
|
|
"name": name, "poster": img.group(1) if img else None})
|
|
return out
|
|
|
|
|
|
def _details(slug_id):
|
|
"""Step 2: details page -> {name, synopsis, mal_id, poster, seasons}"""
|
|
p = _get(f"{BASE}/anime/{slug_id}").text
|
|
mal = re.search(r'myanimelist\.net/anime/(\d+)', p)
|
|
syn = re.search(r'>Synopsis</h2>.*?<p[^>]*>(.*?)</p>', p, re.S)
|
|
h1 = re.search(r'<h1[^>]*>(.*?)</h1>', p, re.S)
|
|
post = re.search(r'src="(https://cdn\.xlsbox\.com/poster/[^"\']+)"', p)
|
|
seasons, seen = [], set()
|
|
i = p.find(">Seasons<")
|
|
if i > 0:
|
|
cnt = re.search(r">(\d+) entries in this series<", p[i:i + 400])
|
|
n = int(cnt.group(1)) if cnt else 10 # fall back to cap if count markup changes
|
|
for m in _ANIME_LINK.finditer(p[i:i + 20000]):
|
|
sid = f"{m.group(1)}-{m.group(2)}"
|
|
if sid in seen:
|
|
continue
|
|
seen.add(sid)
|
|
seasons.append({"slug_id": sid, "name": m.group(3), "id": int(m.group(2))})
|
|
if len(seasons) == n:
|
|
break
|
|
return {"id": int(slug_id.rsplit("-", 1)[1]),
|
|
"name": h1.group(1).strip() if h1 else None,
|
|
"synopsis": re.sub(r"<[^>]+>", "", syn.group(1)).strip() if syn else None,
|
|
"mal_id": int(mal.group(1)) if mal else None,
|
|
"poster": post.group(1) if post else None,
|
|
"seasons": seasons}
|
|
|
|
|
|
def _episodes(anime_id):
|
|
"""Step 3: JSON API -> [{id, number, filler}]"""
|
|
d = _get(f"{BASE}/api/frontend/anime/{anime_id}/episodes").json()
|
|
return [{"id": e["id"], "number": e["number"], "filler": bool(e.get("filler"))}
|
|
for e in d["episodes"]]
|
|
|
|
|
|
def _languages(ep_id):
|
|
"""Step 4: JSON API -> [{code, name, embed_url}] (jpn=sub, eng=dub)"""
|
|
return _get(f"{BASE}/api/frontend/episode/{ep_id}/languages").json()["languages"]
|
|
|
|
|
|
def _parse_intros(eps):
|
|
"""Jikan episode list -> one entry per episode: {title?, start?, length?}.
|
|
title from MAL (episode name); start/length only when MAL has the timing."""
|
|
out = []
|
|
for e in eps:
|
|
i = e.get("intro") or {}
|
|
d = {"title": e.get("title") or None}
|
|
if i.get("start") is not None:
|
|
d.update(start=int(i["start"]), length=int(i["length"]))
|
|
out.append(d)
|
|
return out
|
|
|
|
|
|
def _intros(mal_id):
|
|
"""Jikan (keyless MAL mirror): per-episode intro start/length, seconds."""
|
|
# Jikan v4 returns the list directly under 'data'; tolerate a wrapped
|
|
# shape too, so a format change doesn't silently kill the button
|
|
r = _get("https://api.jikan.moe/v4/anime/{}".format(mal_id) + "/episodes")
|
|
d = r.json().get("data")
|
|
eps = d if isinstance(d, list) else (d or {}).get("episodes", [])
|
|
return _parse_intros(eps)
|
|
|
|
|
|
def _master_m3u8(ep_id, lang="eng"):
|
|
"""Steps 5+6: embed page -> 'file: <master m3u8>' URL (None if unavailable)"""
|
|
for l in languages(ep_id):
|
|
if l["code"] == lang:
|
|
m = re.search(r"file: '([^']*)'", _get(l["embed_url"]).text)
|
|
if m:
|
|
return m.group(1)
|
|
return None
|
|
|
|
|
|
# cached public API (TTLs in seconds)
|
|
search = lambda q: _cached(f"search:{q.lower()}", 300, _search, q)
|
|
details = lambda s: _cached(f"details:{s}", 3600, _details, s)
|
|
episodes = lambda i: _cached(f"episodes:{i}", 300, _episodes, i) # 5m: airing shows gain eps
|
|
languages = lambda e: _cached(f"languages:{e}", 3600, _languages, e)
|
|
master_m3u8 = lambda e, l="eng": _cached(f"master:{e}:{l}", 3600, _master_m3u8, e, l)
|
|
intros = lambda m: _cached(f"intros:{m}", 30 * 86400, _intros, m) # static data
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# ponytail: assert self-check instead of a test framework
|
|
results = search("chainsaw man")
|
|
assert results and any(x["name"] == "Chainsaw Man" for x in results), results
|
|
top = next(x for x in results if x["name"] == "Chainsaw Man")
|
|
d = details(top["slug_id"])
|
|
assert d["mal_id"] == 44511 and d["synopsis"], d
|
|
assert [s["id"] for s in d["seasons"]] == [922, 923], d["seasons"]
|
|
eps = episodes(top["id"])
|
|
assert eps and eps[0]["number"] == 1, eps
|
|
m = master_m3u8(eps[0]["id"], "eng")
|
|
assert m and "EXT-X-STREAM" in _get(m).text, m
|
|
print(f"OK: {top['name']} (MAL {d['mal_id']}, {len(d['seasons'])} seasons, "
|
|
f"{len(eps)} eps) -> ep1 dub: {m[:70]}...")
|
|
sys.exit(0)
|