Provider layer + hianime source (primary)
- providers/{anidb,anikoto,hianime}.py: per-source modules behind the
ANI_SOURCE fallback dispatch (upstream.py _call/_providers)
- hianime.py: pystardust/ani-cli#1897 flow — search/episode-list/
servers API, HD-1 hash -> base64 embed url, window.__P blob
(base64 + XOR 'otaku-embed-v1') -> master m3u8; registers CDN hosts
and the required Referer via stream_meta()
- upstream.py: get_stream/fetch_parallel accept extra headers;
default source order hianime,anidb,anikoto (anidb.app is 503)
- app.py: /stream allowlist accepts provider-registered hosts and
sends their Referer; status probe follows the primary provider
(hianime variant URIs are relative to the variant path)
- app.js: error-time position save floors at furthest point reached
(hls.js resets currentTime to 0 before playerFail on fatal errors)
- tests: default-order + hianime import checks; live browser suite green
This commit is contained in:
jozamudi committed
1 parent
ff8311c1a8
commit
834ca80a8c
8 files changed
+553
-123
No files matched your search
+80
-96
@@ -1,19 +1,30 @@
|
||||
"""upstream.py — the only module that talks to anidb.app (and keyless Jikan
|
||||
for intro timings).
|
||||
"""upstream.py — provider-agnostic source layer.
|
||||
|
||||
anidb.app sits behind Cloudflare: plain curl gets 403. All requests use
|
||||
curl_cffi Chrome TLS impersonation (verified to pass the challenge).
|
||||
Parsers are regex against live markup — keep them here so the layout
|
||||
breaks in one place, not across the app.
|
||||
A *provider* is a module in providers/ implementing the same interface:
|
||||
|
||||
search(q) -> [{id, slug_id, name, poster}]
|
||||
details(slug_id) -> {id, name, synopsis, mal_id, poster, seasons}
|
||||
episodes(anime_id) -> [{id, number, filler}]
|
||||
languages(ep_id) -> [{code, name, embed_url}]
|
||||
master_m3u8(ep_id, lang) -> master m3u8 url | None
|
||||
|
||||
Switching sources = add providers/<name>.py, then:
|
||||
ANI_SOURCE=other,anidb # ordered fallback: first to succeed wins
|
||||
(default ANI_SOURCE=hianime,anidb,anikoto). Exceptions trigger the fallback; a legitimate
|
||||
None from master_m3u8 ("no dub") is final, not an error.
|
||||
|
||||
Shared HTTP helpers (_get/get_stream/fetch_parallel, Chrome TLS
|
||||
impersonation, TTL cache) live here so every provider gets them for free;
|
||||
they work on any URL, not just anidb's.
|
||||
"""
|
||||
import importlib
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from curl_cffi import requests
|
||||
|
||||
BASE = "https://anidb.app"
|
||||
H = dict(impersonate="chrome", timeout=20)
|
||||
|
||||
# ponytail: thread-local sessions (uvicorn threadpool); curl_cffi Sessions
|
||||
@@ -26,10 +37,9 @@ def _session():
|
||||
_tls.s = requests.Session()
|
||||
return _tls.s
|
||||
|
||||
# ponytail: in-memory TTL cache for upstream data that never changes
|
||||
# (details/episodes/langs/playlist URLs). No lock: worst case a duplicate
|
||||
# fetch. A dead/rotated playlist URL is recovered by the one-shot
|
||||
# clear_cache()+retry in api_stream.
|
||||
# ponytail: in-memory TTL cache shared by all providers (key namespace is
|
||||
# the provider's job). No lock: worst case a duplicate fetch. A dead/rotated
|
||||
# playlist URL is recovered by the one-shot clear_cache()+retry in api_stream.
|
||||
_CACHE = {} # key -> (expires, value)
|
||||
|
||||
def _cached(key, ttl, fn, *a, **kw):
|
||||
@@ -45,8 +55,6 @@ def _cached(key, ttl, fn, *a, **kw):
|
||||
def clear_cache():
|
||||
_CACHE.clear()
|
||||
|
||||
_ANIME_LINK = re.compile(r'anime/([a-z0-9-]+)-(\d+)"[^>]*title="([^"]*)"')
|
||||
|
||||
|
||||
def _get(url, **kw):
|
||||
# ponytail: one retry (300ms) on transient upstream errors (Cloudflare
|
||||
@@ -64,7 +72,7 @@ def _get(url, **kw):
|
||||
time.sleep(0.3)
|
||||
|
||||
|
||||
def get_stream(url, timeout=12):
|
||||
def get_stream(url, timeout=12, headers=None):
|
||||
"""Segments: open the upstream response WITHOUT reading the body, so the
|
||||
proxy can forward bytes as the CDN emits them. (A buffered fetch made the
|
||||
browser's time-to-first-byte = the CDN's whole upload time, which tripped
|
||||
@@ -72,7 +80,8 @@ def get_stream(url, timeout=12):
|
||||
ponytail: skips the 'Just a moment' body check — challenge pages arrive
|
||||
as 403/503 (handled below), and reading the body would defeat streaming."""
|
||||
for attempt in range(2):
|
||||
r = _session().get(url, **{**H, "timeout": timeout, "stream": True})
|
||||
r = _session().get(url, **{**H, "timeout": timeout, "stream": True,
|
||||
"headers": headers or {}})
|
||||
if r.status_code in (403, 429, 503) or r.status_code >= 500:
|
||||
r.close()
|
||||
if attempt:
|
||||
@@ -83,18 +92,17 @@ def get_stream(url, timeout=12):
|
||||
return r
|
||||
|
||||
|
||||
def fetch_parallel(url, lanes=16, lane=32 * 1024, timeout=12):
|
||||
def fetch_parallel(url, lanes=16, lane=32 * 1024, timeout=12, headers=None):
|
||||
"""One upstream file across N parallel byte-range lanes (16, like yt-dlp's
|
||||
-N 16 / ani-cli's download path).
|
||||
|
||||
Per-connection CDN throttling is the common slow-segment cause; N lanes
|
||||
turn one 60s crawl into N concurrent crawls (I/O-bound, GIL irrelevant).
|
||||
Returns (body, content-type); raises on ANY failure so the caller falls
|
||||
back to the single-request path — never worse than before.
|
||||
Verified 2026-08 against hls.anidb.app: HEAD -> content-length,
|
||||
ranges -> 206, multi-lane -> byte-identical stitch (sha256)."""
|
||||
back to the single-request path — never worse than before."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
head = _session().head(url, **{**H, "timeout": timeout})
|
||||
head = _session().head(url, **{**H, "timeout": timeout,
|
||||
"headers": headers or {}})
|
||||
head.raise_for_status()
|
||||
total = int(head.headers.get("content-length") or 0)
|
||||
ct = head.headers.get("content-type", "application/octet-stream")
|
||||
@@ -110,7 +118,8 @@ def fetch_parallel(url, lanes=16, lane=32 * 1024, timeout=12):
|
||||
def grab(i):
|
||||
a = i * size
|
||||
b = total - 1 if i == n - 1 else (i + 1) * size - 1
|
||||
r = _get(url, headers={"Range": f"bytes={a}-{b}"}, timeout=timeout)
|
||||
r = _get(url, headers={**(headers or {}), "Range": f"bytes={a}-{b}"},
|
||||
timeout=timeout)
|
||||
if r.status_code not in (200, 206) or len(r.content) != b - a + 1:
|
||||
raise RuntimeError(f"lane {i}: http {r.status_code} "
|
||||
f"len={len(r.content)} want={b - a + 1}")
|
||||
@@ -120,66 +129,11 @@ def fetch_parallel(url, lanes=16, lane=32 * 1024, timeout=12):
|
||||
list(ex.map(grab, range(n)))
|
||||
body = b"".join(parts)
|
||||
if len(body) != total:
|
||||
raise RuntimeError(f"stitch mismatch {len(body)} != {total}")
|
||||
raise RuntimeError(f"stitch mismatch {len(body)} != total")
|
||||
return body, ct
|
||||
|
||||
|
||||
def _search(q):
|
||||
"""Step 1: browse page -> [{id, slug_id, name, poster}]"""
|
||||
html = _get(f"{BASE}/browse", params={"q": q}).text
|
||||
out, seen = [], set()
|
||||
for m in re.finditer(
|
||||
r'<a href="https://anidb\.app/anime/([a-z0-9-]+-\d+)"[^>]*title="([^"]*)"[^>]*>(.*?)</a>',
|
||||
html, re.S):
|
||||
slug_id, name, block = m.group(1), m.group(2), m.group(3)
|
||||
if slug_id in seen:
|
||||
continue
|
||||
seen.add(slug_id)
|
||||
img = re.search(r'<img src="([^"]+)"', block)
|
||||
out.append({"id": int(slug_id.rsplit("-", 1)[1]), "slug_id": slug_id,
|
||||
"name": name, "poster": img.group(1) if img else None})
|
||||
return out
|
||||
|
||||
|
||||
def _details(slug_id):
|
||||
"""Step 2: details page -> {name, synopsis, mal_id, poster, seasons}"""
|
||||
p = _get(f"{BASE}/anime/{slug_id}").text
|
||||
mal = re.search(r'myanimelist\.net/anime/(\d+)', p)
|
||||
syn = re.search(r'>Synopsis</h2>.*?<p[^>]*>(.*?)</p>', p, re.S)
|
||||
h1 = re.search(r'<h1[^>]*>(.*?)</h1>', p, re.S)
|
||||
post = re.search(r'src="(https://cdn\.xlsbox\.com/poster/[^"\']+)"', p)
|
||||
seasons, seen = [], set()
|
||||
i = p.find(">Seasons<")
|
||||
if i > 0:
|
||||
cnt = re.search(r">(\d+) entries in this series<", p[i:i + 400])
|
||||
n = int(cnt.group(1)) if cnt else 10 # fall back to cap if count markup changes
|
||||
for m in _ANIME_LINK.finditer(p[i:i + 20000]):
|
||||
sid = f"{m.group(1)}-{m.group(2)}"
|
||||
if sid in seen:
|
||||
continue
|
||||
seen.add(sid)
|
||||
seasons.append({"slug_id": sid, "name": m.group(3), "id": int(m.group(2))})
|
||||
if len(seasons) == n:
|
||||
break
|
||||
return {"id": int(slug_id.rsplit("-", 1)[1]),
|
||||
"name": h1.group(1).strip() if h1 else None,
|
||||
"synopsis": re.sub(r"<[^>]+>", "", syn.group(1)).strip() if syn else None,
|
||||
"mal_id": int(mal.group(1)) if mal else None,
|
||||
"poster": post.group(1) if post else None,
|
||||
"seasons": seasons}
|
||||
|
||||
|
||||
def _episodes(anime_id):
|
||||
"""Step 3: JSON API -> [{id, number, filler}]"""
|
||||
d = _get(f"{BASE}/api/frontend/anime/{anime_id}/episodes").json()
|
||||
return [{"id": e["id"], "number": e["number"], "filler": bool(e.get("filler"))}
|
||||
for e in d["episodes"]]
|
||||
|
||||
|
||||
def _languages(ep_id):
|
||||
"""Step 4: JSON API -> [{code, name, embed_url}] (jpn=sub, eng=dub)"""
|
||||
return _get(f"{BASE}/api/frontend/episode/{ep_id}/languages").json()["languages"]
|
||||
|
||||
# ---------- Jikan intros: provider-independent (keyless MAL mirror) ----------
|
||||
|
||||
def _parse_intros(eps):
|
||||
"""Jikan episode list -> one entry per episode: {title?, start?, length?}.
|
||||
@@ -204,27 +158,56 @@ def _intros(mal_id):
|
||||
return _parse_intros(eps)
|
||||
|
||||
|
||||
def _master_m3u8(ep_id, lang="eng"):
|
||||
"""Steps 5+6: embed page -> 'file: <master m3u8>' URL (None if unavailable)"""
|
||||
for l in languages(ep_id):
|
||||
if l["code"] == lang:
|
||||
m = re.search(r"file: '([^']*)'", _get(l["embed_url"]).text)
|
||||
if m:
|
||||
return m.group(1)
|
||||
intros = lambda m: _cached(f"jikan:intros:{m}", 30 * 86400, _intros, m) # static data
|
||||
|
||||
|
||||
# ---------- source dispatch ----------
|
||||
|
||||
def _providers():
|
||||
# ponytail: env read per call, so tests and runtime can re-point without
|
||||
# re-importing; comma list = ordered fallback
|
||||
return [p.strip() for p in os.environ.get("ANI_SOURCE", "hianime,anidb,anikoto").split(",")
|
||||
if p.strip()]
|
||||
|
||||
|
||||
def _prov(name):
|
||||
try:
|
||||
return importlib.import_module(f".providers.{name}", __package__)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _call(name, *a, **kw):
|
||||
errs = []
|
||||
for p in _providers():
|
||||
try:
|
||||
return getattr(_prov(p), name)(*a, **kw)
|
||||
except Exception as e:
|
||||
errs.append(f"{p}: {e}")
|
||||
raise RuntimeError("all sources failed -> " + " | ".join(errs))
|
||||
|
||||
|
||||
search = lambda q: _call("search", q)
|
||||
details = lambda s: _call("details", s)
|
||||
episodes = lambda i: _call("episodes", i)
|
||||
languages = lambda e: _call("languages", e)
|
||||
master_m3u8 = lambda e, l="eng": _call("master_m3u8", e, l)
|
||||
|
||||
|
||||
def stream_meta(url):
|
||||
"""Provider hook: extra upstream headers for a /stream URL (hianime's
|
||||
CDN demands the embed origin as Referer), or None if no enabled
|
||||
provider claims the host — that doubles as the allowlist."""
|
||||
for p in _providers():
|
||||
fn = getattr(_prov(p), "stream_meta", None)
|
||||
if fn and fn(url):
|
||||
return fn(url)
|
||||
return None
|
||||
|
||||
|
||||
# cached public API (TTLs in seconds)
|
||||
search = lambda q: _cached(f"search:{q.lower()}", 300, _search, q)
|
||||
details = lambda s: _cached(f"details:{s}", 3600, _details, s)
|
||||
episodes = lambda i: _cached(f"episodes:{i}", 300, _episodes, i) # 5m: airing shows gain eps
|
||||
languages = lambda e: _cached(f"languages:{e}", 3600, _languages, e)
|
||||
master_m3u8 = lambda e, l="eng": _cached(f"master:{e}:{l}", 3600, _master_m3u8, e, l)
|
||||
intros = lambda m: _cached(f"intros:{m}", 30 * 86400, _intros, m) # static data
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# ponytail: assert self-check instead of a test framework
|
||||
# ponytail: assert self-check instead of a test framework.
|
||||
# Needs a live source: ANI_SOURCE=<provider> python -m server.upstream
|
||||
results = search("chainsaw man")
|
||||
assert results and any(x["name"] == "Chainsaw Man" for x in results), results
|
||||
top = next(x for x in results if x["name"] == "Chainsaw Man")
|
||||
@@ -237,4 +220,5 @@ if __name__ == "__main__":
|
||||
assert m and "EXT-X-STREAM" in _get(m).text, m
|
||||
print(f"OK: {top['name']} (MAL {d['mal_id']}, {len(d['seasons'])} seasons, "
|
||||
f"{len(eps)} eps) -> ep1 dub: {m[:70]}...")
|
||||
import sys
|
||||
sys.exit(0)
|
||||
Reference in new issue
Block a user