"""WER of each take in /timings.json with faster-whisper large-v3 (CPU int8), the same ASR the Drama-3 B70 card used.""" import json, os, re, sys from faster_whisper import WhisperModel import jiwer import wave import numpy as np from scipy.signal import resample_poly def load_16k(path): with wave.open(path) as w: sr, ch, n = w.getframerate(), w.getnchannels(), w.getnframes() x = np.frombuffer(w.readframes(n), dtype=np.int16).astype(np.float32) / 32768.0 if ch > 1: x = x.reshape(-1, ch).mean(1) g = np.gcd(16000, sr) return resample_poly(x, 16000 // g, sr // g).astype(np.float32) tag = sys.argv[1] d = os.path.expanduser(f"~/omnispace-b70/bench/{tag}") rows = json.load(open(f"{d}/timings.json")) m = WhisperModel("large-v3", device="cpu", compute_type="int8", download_root="/mnt/data2tb/fish-asr/hf") def norm(s): s = re.sub(r"[^a-z0-9' ]", " ", s.lower().replace("-", " ")) return re.sub(r"\s+", " ", s).strip() for r in rows: if "error" in r: continue seg, _ = m.transcribe(load_16k(r["wav"]), language="en", beam_size=5, vad_filter=True, condition_on_previous_text=False) r["hyp"] = " ".join(s.text.strip() for s in seg).strip() r["wer"] = round(min(1.0, jiwer.wer(norm(r["text"]), norm(r["hyp"]) or "_")), 4) print(r["id"], r["wer"], r["hyp"][:90], flush=True) json.dump(rows, open(f"{d}/scored.json", "w"), indent=1) ok = [r for r in rows if "wer" in r] ref_words = sum(len(norm(r["text"]).split()) for r in ok) errs = sum(jiwer.process_words(norm(r["text"]), norm(r["hyp"]) or "_").wer * len(norm(r["text"]).split()) for r in ok) print("SCORE", tag, json.dumps(dict( n=len(ok), corpus_wer=round(errs / ref_words, 4), mean_wer=round(sum(r["wer"] for r in ok) / len(ok), 4), catastrophic=sum(r["wer"] > 0.5 for r in ok), no_eos=sum(r.get("eos") is not None and r["eos"] < 0 for r in ok))))