Proxy: fetch segments across 4 parallel byte-range lanes
The slow-segment logs show per-connection CDN throttling (15-40 KB/s dribbles). yt-dlp's standard fix is -N 16 parallel fragment lanes; this is the 4-lane version for the proxy: HEAD for content-length, then 4 concurrent Range GETs reassembled byte-exact (verified sha256-identical to a direct CDN fetch). Small files (<512KB) still use a single GET. ANY failure (no content-length, 416, lane error, timeout) falls back to the existing single-stream path, so this is never worse than before. Caveat (measured live): when the CDN caps SHARED egress rather than per-connection, lanes just split the same pie — in that window both 1-lane and 4-lane time out. A different source is the only real fix for those windows. 18/18 tests (new test pins lane stitch + fallback; also fixed a pre-existing _get stub leak in the Jikan-shape test).
This commit is contained in:
1 parent
2576546b43
commit
ee2a888aca
3 files changed
+120
-6
No files matched your search
@@ -330,6 +330,7 @@ def test_intros_parses_jikan_shapes():
|
||||
]}
|
||||
|
||||
u._CACHE.clear()
|
||||
real_get = u._get
|
||||
u._get = lambda url: R() # stub the network
|
||||
assert u._intros(92573) == [
|
||||
{"title": "Episode 1", "start": 111, "length": 77},
|
||||
@@ -343,4 +344,62 @@ def test_intros_parses_jikan_shapes():
|
||||
]}}
|
||||
u._get = lambda url: W()
|
||||
assert u._intros(92573)[0]["start"] == 90
|
||||
u._get = real_get # restore: later tests need the real _get
|
||||
u._CACHE.clear()
|
||||
|
||||
|
||||
def test_fetch_parallel_stitches_range_lanes():
|
||||
# fetch_parallel across N byte-range lanes must reassemble byte-identical,
|
||||
# and must raise (so the app falls back to the single-request path) when
|
||||
# the CDN refuses ranges or HEAD fails.
|
||||
import server.upstream as u
|
||||
|
||||
body = bytes((i * 7 + 3) % 256 for i in range(1024 * 1024)) # 1MiB > 2*lane, forces the lane path
|
||||
|
||||
class Resp:
|
||||
def __init__(self, content, status, clen=None):
|
||||
self.content, self.status_code = content, status
|
||||
self.headers = {"content-type": "video/mp2t"}
|
||||
if clen is not None:
|
||||
self.headers["content-length"] = str(clen)
|
||||
def raise_for_status(self):
|
||||
if self.status_code >= 400:
|
||||
raise RuntimeError(f"http {self.status_code}")
|
||||
@property
|
||||
def text(self):
|
||||
return ""
|
||||
|
||||
class Ranges:
|
||||
def head(self, url, **kw):
|
||||
return Resp(b"", 200, len(body))
|
||||
def get(self, url, **kw):
|
||||
rng = (kw.get("headers") or {}).get("Range")
|
||||
if rng:
|
||||
a, b = rng[len("bytes="):].split("-")
|
||||
return Resp(body[int(a):int(b) + 1], 206)
|
||||
return Resp(body, 200)
|
||||
|
||||
class NoRanges: # 416 on every range -> must raise, app falls back
|
||||
def head(self, url, **kw):
|
||||
return Resp(b"", 200, len(body))
|
||||
def get(self, url, **kw):
|
||||
if (kw.get("headers") or {}).get("Range"):
|
||||
return Resp(b"", 416)
|
||||
return Resp(body, 200)
|
||||
|
||||
real = u._session
|
||||
real_get = u._get
|
||||
try:
|
||||
u._session = lambda: Ranges()
|
||||
out, ct = u.fetch_parallel("http://cdn/x.xls", lanes=4)
|
||||
assert out == body and ct == "video/mp2t"
|
||||
|
||||
u._session = lambda: NoRanges()
|
||||
try:
|
||||
u.fetch_parallel("http://cdn/x.xls", lanes=4)
|
||||
raise AssertionError("should have raised")
|
||||
except RuntimeError:
|
||||
pass
|
||||
finally:
|
||||
u._session = real
|
||||
u._get = real_get
|
||||
Reference in new issue
Block a user