Frosty40's picture
ZONOS3-Mini B70 Turbo v1: two runtime patches and clean held-out results
a8ec9e4 verified
Raw History Blame Contribute Delete
1.9 kB
"""WER of each take in <tag>/timings.json with faster-whisper large-v3 (CPU int8), the same ASR the Drama-3 B70 card used."""
import json, os, re, sys
from faster_whisper import WhisperModel
import jiwer
import wave
import numpy as np
from scipy.signal import resample_poly
def load_16k(path):
with wave.open(path) as w:
sr, ch, n = w.getframerate(), w.getnchannels(), w.getnframes()
x = np.frombuffer(w.readframes(n), dtype=np.int16).astype(np.float32) / 32768.0
if ch > 1:
x = x.reshape(-1, ch).mean(1)
g = np.gcd(16000, sr)
return resample_poly(x, 16000 // g, sr // g).astype(np.float32)
tag = sys.argv[1]
d = os.path.expanduser(f"~/omnispace-b70/bench/{tag}")
rows = json.load(open(f"{d}/timings.json"))
m = WhisperModel("large-v3", device="cpu", compute_type="int8", download_root="/mnt/data2tb/fish-asr/hf")
def norm(s):
s = re.sub(r"[^a-z0-9' ]", " ", s.lower().replace("-", " "))
return re.sub(r"\s+", " ", s).strip()
for r in rows:
if "error" in r:
continue
seg, _ = m.transcribe(load_16k(r["wav"]), language="en", beam_size=5, vad_filter=True, condition_on_previous_text=False)
r["hyp"] = " ".join(s.text.strip() for s in seg).strip()
r["wer"] = round(min(1.0, jiwer.wer(norm(r["text"]), norm(r["hyp"]) or "_")), 4)
print(r["id"], r["wer"], r["hyp"][:90], flush=True)
json.dump(rows, open(f"{d}/scored.json", "w"), indent=1)
ok = [r for r in rows if "wer" in r]
ref_words = sum(len(norm(r["text"]).split()) for r in ok)
errs = sum(jiwer.process_words(norm(r["text"]), norm(r["hyp"]) or "_").wer * len(norm(r["text"]).split()) for r in ok)
print("SCORE", tag, json.dumps(dict(
n=len(ok), corpus_wer=round(errs / ref_words, 4), mean_wer=round(sum(r["wer"] for r in ok) / len(ok), 4),
catastrophic=sum(r["wer"] > 0.5 for r in ok),
no_eos=sum(r.get("eos") is not None and r["eos"] < 0 for r in ok))))