File size: 4,518 Bytes
a8ec9e4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | """C simultaneous streaming requests to a running zonos3-server; per stream: time to first audio and whether
playback (starting at the first chunk) would ever run dry. Stdlib only.
python3 stream_bench.py --port 1919 --conc 1,4,8,16 --tag vulkan-b8
"""
import argparse, array, base64, http.client, json, os, statistics as st, threading, time, wave
ap = argparse.ArgumentParser()
ap.add_argument("--port", type=int, default=1919)
ap.add_argument("--conc", default="1,2,4,8")
ap.add_argument("--rounds", type=int, default=2)
ap.add_argument("--tag", required=True)
ap.add_argument("--prompts", default="~/omnispace-b70/bench/prompts.jsonl")
ap.add_argument("--min-len", type=int, default=60)
ap.add_argument("--save", action="store_true", help="write each stream to bench/<tag>/wav and a timings.json for score.py")
a = ap.parse_args()
BPS = 44100 * 4 # float32 mono PCM
BUFFERS = (0.0, 0.1, 0.2)
emb = base64.b64encode(open(os.path.expanduser(
"~/omnispace-b70/models/voices/Savannah_en-American_F_1.embedding.npy"), "rb").read()).decode()
texts = [json.loads(l)["text"] for l in open(os.path.expanduser(a.prompts))]
texts = [t for t in texts if len(t) > a.min_len]
def one(i, seed, out):
body = json.dumps({"text": texts[i % len(texts)], "stream": True, "seed": seed,
"speaker_embedding_base64": emb, "speaker_mode": "embedding"})
c = http.client.HTTPConnection("127.0.0.1", a.port, timeout=600)
t0 = time.perf_counter()
c.request("POST", "/v2/generate", body, {"Content-Type": "application/json"})
r = c.getresponse()
if r.status != 200:
out[i] = {"error": f"{r.status} {r.read()[:200]!r}"}
return
got, first, arrivals, pcm = 0, None, [], bytearray() # (arrival time, audio seconds received before this chunk)
while True:
chunk = r.read1(65536)
if not chunk:
break
now = time.perf_counter()
if first is None:
first = now
arrivals.append((now, got / BPS))
got += len(chunk)
if a.save:
pcm += chunk
end = time.perf_counter()
res = {"ttfa_ms": round(1000 * (first - t0), 1), "audio_s": round(got / BPS, 2), "wall_s": round(end - t0, 2)}
for buf in BUFFERS: # playback starts buf seconds after the first chunk; a stall is the player running dry
short = [(now - first - buf) - before for now, before in arrivals[1:]]
res[f"stall_ms_b{int(buf * 1000)}"] = round(1000 * max([0.0] + short), 1)
if a.save:
d = os.path.expanduser(f"~/omnispace-b70/bench/{a.tag}/wav"); os.makedirs(d, exist_ok=True)
x = array.array("f"); x.frombytes(bytes(pcm[: len(pcm) // 4 * 4]))
res["wav"] = f"{d}/c{len(out)}_s{seed}.wav"
with wave.open(res["wav"], "wb") as w:
w.setnchannels(1); w.setsampwidth(2); w.setframerate(44100)
w.writeframes(array.array("h", (max(-32768, min(32767, int(v * 32767))) for v in x)).tobytes())
res["text"] = texts[i % len(texts)]; res["id"] = f"c{len(out)}_s{seed}"
out[i] = res
rows, SAVED = [], []
for C in [int(x) for x in a.conc.split(",")]:
for rnd in range(a.rounds):
out = [None] * C
th = [threading.Thread(target=one, args=(i, 1000 * rnd + i, out)) for i in range(C)]
t = time.perf_counter()
[x.start() for x in th]
[x.join() for x in th]
wall = time.perf_counter() - t
ok = [o for o in out if o and "error" not in o]
res = {"tag": a.tag, "conc": C, "round": rnd, "ok": len(ok), "errors": [o["error"] for o in out if o and "error" in o][:2],
"ttfa_ms_median": st.median(o["ttfa_ms"] for o in ok) if ok else None,
"ttfa_ms_max": max(o["ttfa_ms"] for o in ok) if ok else None,
**{f"stalled_b{int(b * 1000)}": sum(o[f"stall_ms_b{int(b * 1000)}"] > 0 for o in ok) for b in BUFFERS},
**{f"worst_ms_b{int(b * 1000)}": max((o[f"stall_ms_b{int(b * 1000)}"] for o in ok), default=None) for b in BUFFERS},
"audio_s_per_wall_s": round(sum(o["audio_s"] for o in ok) / wall, 1)}
rows.append(res)
SAVED.extend(o for o in ok if "wav" in o)
print(json.dumps(res), flush=True)
os.makedirs(os.path.expanduser("~/omnispace-b70/bench/stream"), exist_ok=True)
if a.save:
json.dump(SAVED, open(os.path.expanduser(f"~/omnispace-b70/bench/{a.tag}/timings.json"), "w"), indent=1)
json.dump(rows, open(os.path.expanduser(f"~/omnispace-b70/bench/stream/{a.tag}.json"), "w"), indent=1)
|