File size: 4,518 Bytes
a8ec9e4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
"""C simultaneous streaming requests to a running zonos3-server; per stream: time to first audio and whether
playback (starting at the first chunk) would ever run dry. Stdlib only.

  python3 stream_bench.py --port 1919 --conc 1,4,8,16 --tag vulkan-b8
"""
import argparse, array, base64, http.client, json, os, statistics as st, threading, time, wave

ap = argparse.ArgumentParser()
ap.add_argument("--port", type=int, default=1919)
ap.add_argument("--conc", default="1,2,4,8")
ap.add_argument("--rounds", type=int, default=2)
ap.add_argument("--tag", required=True)
ap.add_argument("--prompts", default="~/omnispace-b70/bench/prompts.jsonl")
ap.add_argument("--min-len", type=int, default=60)
ap.add_argument("--save", action="store_true", help="write each stream to bench/<tag>/wav and a timings.json for score.py")
a = ap.parse_args()

BPS = 44100 * 4  # float32 mono PCM
BUFFERS = (0.0, 0.1, 0.2)
emb = base64.b64encode(open(os.path.expanduser(
    "~/omnispace-b70/models/voices/Savannah_en-American_F_1.embedding.npy"), "rb").read()).decode()
texts = [json.loads(l)["text"] for l in open(os.path.expanduser(a.prompts))]
texts = [t for t in texts if len(t) > a.min_len]


def one(i, seed, out):
    body = json.dumps({"text": texts[i % len(texts)], "stream": True, "seed": seed,
                       "speaker_embedding_base64": emb, "speaker_mode": "embedding"})
    c = http.client.HTTPConnection("127.0.0.1", a.port, timeout=600)
    t0 = time.perf_counter()
    c.request("POST", "/v2/generate", body, {"Content-Type": "application/json"})
    r = c.getresponse()
    if r.status != 200:
        out[i] = {"error": f"{r.status} {r.read()[:200]!r}"}
        return
    got, first, arrivals, pcm = 0, None, [], bytearray()  # (arrival time, audio seconds received before this chunk)
    while True:
        chunk = r.read1(65536)
        if not chunk:
            break
        now = time.perf_counter()
        if first is None:
            first = now
        arrivals.append((now, got / BPS))
        got += len(chunk)
        if a.save:
            pcm += chunk
    end = time.perf_counter()
    res = {"ttfa_ms": round(1000 * (first - t0), 1), "audio_s": round(got / BPS, 2), "wall_s": round(end - t0, 2)}
    for buf in BUFFERS:  # playback starts buf seconds after the first chunk; a stall is the player running dry
        short = [(now - first - buf) - before for now, before in arrivals[1:]]
        res[f"stall_ms_b{int(buf * 1000)}"] = round(1000 * max([0.0] + short), 1)
    if a.save:
        d = os.path.expanduser(f"~/omnispace-b70/bench/{a.tag}/wav"); os.makedirs(d, exist_ok=True)
        x = array.array("f"); x.frombytes(bytes(pcm[: len(pcm) // 4 * 4]))
        res["wav"] = f"{d}/c{len(out)}_s{seed}.wav"
        with wave.open(res["wav"], "wb") as w:
            w.setnchannels(1); w.setsampwidth(2); w.setframerate(44100)
            w.writeframes(array.array("h", (max(-32768, min(32767, int(v * 32767))) for v in x)).tobytes())
        res["text"] = texts[i % len(texts)]; res["id"] = f"c{len(out)}_s{seed}"
    out[i] = res


rows, SAVED = [], []
for C in [int(x) for x in a.conc.split(",")]:
    for rnd in range(a.rounds):
        out = [None] * C
        th = [threading.Thread(target=one, args=(i, 1000 * rnd + i, out)) for i in range(C)]
        t = time.perf_counter()
        [x.start() for x in th]
        [x.join() for x in th]
        wall = time.perf_counter() - t
        ok = [o for o in out if o and "error" not in o]
        res = {"tag": a.tag, "conc": C, "round": rnd, "ok": len(ok), "errors": [o["error"] for o in out if o and "error" in o][:2],
               "ttfa_ms_median": st.median(o["ttfa_ms"] for o in ok) if ok else None,
               "ttfa_ms_max": max(o["ttfa_ms"] for o in ok) if ok else None,
               **{f"stalled_b{int(b * 1000)}": sum(o[f"stall_ms_b{int(b * 1000)}"] > 0 for o in ok) for b in BUFFERS},
               **{f"worst_ms_b{int(b * 1000)}": max((o[f"stall_ms_b{int(b * 1000)}"] for o in ok), default=None) for b in BUFFERS},
               "audio_s_per_wall_s": round(sum(o["audio_s"] for o in ok) / wall, 1)}
        rows.append(res)
        SAVED.extend(o for o in ok if "wav" in o)
        print(json.dumps(res), flush=True)
os.makedirs(os.path.expanduser("~/omnispace-b70/bench/stream"), exist_ok=True)
if a.save:
    json.dump(SAVED, open(os.path.expanduser(f"~/omnispace-b70/bench/{a.tag}/timings.json"), "w"), indent=1)
json.dump(rows, open(os.path.expanduser(f"~/omnispace-b70/bench/stream/{a.tag}.json"), "w"), indent=1)