From 0fd06f036a1def9f4f62d10ad5a9b93b6aa102fe Mon Sep 17 00:00:00 2001 From: Josue Zamudio Date: Sun, 30 Aug 2026 17:23:57 +0000 Subject: [PATCH] Playback: retry flaky upstream, faster proxy fail, cache /stream in browser - upstream._get: one 300ms retry on transient errors (CF 403/503, 429, 5xx); 404 stays final. Fixes the 502 cluster that hls.js couldn't recover from (fragLoadTimeOut x3 in error log) - /stream: upstream timeout 12s < hls.js fragLoadingTimeOut (20s), so a hung segment 502s first and hls.js retries instead of fatal-timing-out - /stream responses: Cache-Control public max-age=86400 (VOD + token is the exact upstream URL => immutable); re-opens/re-seeks skip the proxy - /api/stream: drop unused qualities (frontend never read it; saved a ~0.2s cold upstream round-trip per episode) - app.js/style.css: max-age=300 instead of no-store - test_history_page_and_delete: stale selector (home redesign removed the 'view all' link; use 'history ->') --- SPEC.md | 7 +++++-- server/app.py | 30 ++++++++++++++++++++---------- server/upstream.py | 25 +++++++++++++------------ tests/test_browser.py | 10 +++++----- 4 files changed, 43 insertions(+), 29 deletions(-) diff --git a/SPEC.md b/SPEC.md index 5e2b7b7..6adefb2 100644 --- a/SPEC.md +++ b/SPEC.md @@ -49,7 +49,7 @@ Two moving parts, no database, no framework on the frontend. - `GET /api/search?q=` → `[{id, slug_id, name, poster}]` (parse step 1) - `GET /api/anime/{slug_id}` → `{id, name, synopsis, mal_id, seasons:[{slug_id,name,id}]}` - `GET /api/anime/{id}/episodes` → `[{id, number, filler}]` (step 3, passthrough) -- `GET /api/episode/{ep_id}/stream?lang=eng|jpn` → `{token, qualities}` +- `GET /api/episode/{ep_id}/stream?lang=eng|jpn` → `{token}` (steps 5+6; token is base64url of the master m3u8 URL) - `GET /api/history` / `POST /api/history` → **per-user** watch history with position, persisted to @@ -58,7 +58,10 @@ Two moving parts, no database, no framework on the frontend. - `GET /stream/{token}` → proxied HLS. Token = base64url of the upstream URL. Rewrites every non-comment line of m3u8 bodies to point back at `/stream/`; proxies master, variants, and segments - (bytes, no transcoding). + (bytes, no transcoding). Responses are `Cache-Control: public, + max-age=86400` — VOD playlists/segments are immutable and the token + is the exact upstream URL, so the browser disk cache serves re-opens + and re-seeks without touching the proxy. - `GET /login` / `POST /login` / `POST /logout` → multi-user auth: PBKDF2-SHA256 passwords (`server/users.py`; `users.json` 0600), signed 30-day cookie carrying the username, 5-fail/15-min diff --git a/server/app.py b/server/app.py index 0b1f757..434bf2b 100644 --- a/server/app.py +++ b/server/app.py @@ -393,13 +393,16 @@ def _upstream(fn, *a, **kw): @app.middleware("http") async def no_cache(request, call_next): - # ponytail: no-store on everything except /stream (immutable HLS - # data) and the vendored hls.min.js (immutable, 500KB — re-downloading - # it on every page load hurt on mobile) + # ponytail: no-store on everything except /stream (immutable HLS data, + # cached per upstream URL) and the vendored/static JS+CSS (re-downloaded + # on every page load hurt on mobile) r = await call_next(request) - if request.url.path == "/hls.min.js": + p = request.url.path + if p == "/hls.min.js": r.headers["Cache-Control"] = "public, max-age=2592000" - elif not request.url.path.startswith("/stream"): + elif p in ("/app.js", "/style.css"): + r.headers["Cache-Control"] = "public, max-age=300" + elif not p.startswith("/stream"): r.headers["Cache-Control"] = "no-store" return r @@ -425,8 +428,7 @@ def api_stream(ep_id: int, lang: str = "jpn", request: Request = None): master = _upstream(upstream.master_m3u8, ep_id, lang) if not master: raise HTTPException(404, f"no {lang} stream for episode {ep_id}") - return {"token": _b64(master), - "qualities": _upstream(upstream.master_variants, master)} + return {"token": _b64(master)} try: out = go() @@ -461,6 +463,10 @@ def _rewrite_m3u8(text: str, base: str) -> str: _STREAM_CACHE = OrderedDict() # ponytail: LRU of proxied HLS data (immutable); _STREAM_MAX = 24 # segments ~1-2MB -> <= ~48MB +# ponytail: HLS VOD content is immutable and the token IS the exact upstream +# URL, so per-URL browser caching can't go stale — re-opens, re-seeks and +# auto-next come from the disk cache instead of proxy + CDN +_STREAM_CACHE_HDRS = {"Cache-Control": "public, max-age=86400"} @app.get("/stream/{token}") @@ -477,15 +483,19 @@ def stream(token: str): hit = _STREAM_CACHE.get(url) if hit: _STREAM_CACHE.move_to_end(url) - return Response(hit[1], media_type=hit[0]) - r = _upstream(upstream._get, url) + return Response(hit[1], media_type=hit[0], headers=_STREAM_CACHE_HDRS) + # ponytail: 12s < hls.js's fragLoadingTimeOut (20s) — on a hang the proxy + # 502s first, so hls.js retries the frag (backoff) instead of freezing + # on a fatal timeout race + r = _upstream(upstream._get, url, timeout=12) body = r.content if url.endswith(".m3u8") or "mpegurl" in r.headers.get("content-type", ""): body = _rewrite_m3u8(body.decode(), url).encode() _STREAM_CACHE[url] = (r.headers.get("content-type", "application/octet-stream"), body) while len(_STREAM_CACHE) > _STREAM_MAX: _STREAM_CACHE.popitem(last=False) - return Response(body, media_type=r.headers.get("content-type", "application/octet-stream")) + return Response(body, media_type=r.headers.get("content-type", "application/octet-stream"), + headers=_STREAM_CACHE_HDRS) app.mount("/", StaticFiles(directory=ROOT / "static", html=True), name="static") diff --git a/server/upstream.py b/server/upstream.py index 78f3ff5..16b1b2c 100644 --- a/server/upstream.py +++ b/server/upstream.py @@ -48,11 +48,19 @@ _ANIME_LINK = re.compile(r'anime/([a-z0-9-]+)-(\d+)"[^>]*title="([^"]*)"') def _get(url, **kw): - r = _session().get(url, **H, **kw) - if r.status_code in (403, 503) or "Just a moment" in r.text[:500]: - raise RuntimeError(f"blocked by cloudflare: {url}") - r.raise_for_status() - return r + # ponytail: one retry (300ms) on transient upstream errors (Cloudflare + # 403/503, 429, 5xx); 404 and other 4xx are final. Failing fast matters: + # hls.js's fragLoadingTimeOut (20s) races curl's timeout (20s), so a + # quick 502 lets hls.js's own frag retries (6x, backoff) do the waiting + for attempt in range(2): + r = _session().get(url, **{**H, **kw}) # kw overrides H (e.g. timeout) + if not (r.status_code in (403, 429, 503) or r.status_code >= 500 + or "Just a moment" in r.text[:500]): + r.raise_for_status() + return r + if attempt: + raise RuntimeError(f"upstream {r.status_code}: {url}") + time.sleep(0.3) def _search(q): @@ -122,19 +130,12 @@ def _master_m3u8(ep_id, lang="eng"): return None -def _master_variants(master_url): - """Step 7: parse #EXT-X-STREAM-INF lines -> ['1920x1080', ...] (for display)""" - text = _get(master_url).text - return re.findall(r"RESOLUTION=(\d+x\d+)", text) - - # cached public API (TTLs in seconds) search = lambda q: _cached(f"search:{q.lower()}", 300, _search, q) details = lambda s: _cached(f"details:{s}", 3600, _details, s) episodes = lambda i: _cached(f"episodes:{i}", 300, _episodes, i) # 5m: airing shows gain eps languages = lambda e: _cached(f"languages:{e}", 3600, _languages, e) master_m3u8 = lambda e, l="eng": _cached(f"master:{e}:{l}", 3600, _master_m3u8, e, l) -master_variants = lambda u: _cached(f"variants:{u}", 3600, _master_variants, u) if __name__ == "__main__": diff --git a/tests/test_browser.py b/tests/test_browser.py index 0879fee..327cceb 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -115,7 +115,7 @@ async def _history_page_and_delete(): op.open(urllib.request.Request(BASE + "/login", data=json.dumps({"username": "erin", "password": "erinpw12345"}).encode(), headers={"Content-Type": "application/json"}, method="POST")) - for n in range(1, 6): # 5 entries: makes the home "view all" link render + for n in range(1, 6): # 5 entries: makes the home "history →" link render e = {"title": f"Hist {n}", "slug_id": f"hist-{n}", "season_id": n, "ep_id": n, "lang": "eng", "ep": n, "pos": 5, "dur": 600} op.open(urllib.request.Request(BASE + "/api/history", @@ -129,11 +129,11 @@ async def _history_page_and_delete(): page.on("pageerror", lambda e: errors.append(str(e))) await _login(page, "erin", "erinpw12345") - # regression: "view all" must go to the ?history query, not the /history path + # regression: the home history link must go to the ?history query, not the /history path await page.goto(BASE + "/") - await page.wait_for_selector("a:has-text('view all')", timeout=15000) - assert (await page.get_attribute("a:has-text('view all')", "href")).endswith("/?history") - await page.click("a:has-text('view all')", force=True) + await page.wait_for_selector("a:has-text('history →')", timeout=15000) + assert (await page.get_attribute("a:has-text('history →')", "href")).endswith("/?history") + await page.click("a:has-text('history →')", force=True) await page.wait_for_selector(".card.hist", timeout=15000) await page.wait_for_function( "document.querySelectorAll('.card.hist').length == 5", timeout=15000, polling=POLL)