Download scripts/score.py from Frosty40/ZONOS3-Mini-B70-Turbo: direct link, hf CLI and curl.
- Browser
- Download file 1.9 kB
-
https://huggingface.co/Frosty40/ZONOS3-Mini-B70-Turbo/resolve/main/scripts/score.py
- Command line
-
hf download hf://Frosty40/ZONOS3-Mini-B70-Turbo/scripts/score.py
-
curl -L -o score.py https://huggingface.co/Frosty40/ZONOS3-Mini-B70-Turbo/resolve/main/scripts/score.py
1.9 kB
| """WER of each take in <tag>/timings.json with faster-whisper large-v3 (CPU int8), the same ASR the Drama-3 B70 card used.""" | |
| import json, os, re, sys | |
| from faster_whisper import WhisperModel | |
| import jiwer | |
| import wave | |
| import numpy as np | |
| from scipy.signal import resample_poly | |
| def load_16k(path): | |
| with wave.open(path) as w: | |
| sr, ch, n = w.getframerate(), w.getnchannels(), w.getnframes() | |
| x = np.frombuffer(w.readframes(n), dtype=np.int16).astype(np.float32) / 32768.0 | |
| if ch > 1: | |
| x = x.reshape(-1, ch).mean(1) | |
| g = np.gcd(16000, sr) | |
| return resample_poly(x, 16000 // g, sr // g).astype(np.float32) | |
| tag = sys.argv[1] | |
| d = os.path.expanduser(f"~/omnispace-b70/bench/{tag}") | |
| rows = json.load(open(f"{d}/timings.json")) | |
| m = WhisperModel("large-v3", device="cpu", compute_type="int8", download_root="/mnt/data2tb/fish-asr/hf") | |
| def norm(s): | |
| s = re.sub(r"[^a-z0-9' ]", " ", s.lower().replace("-", " ")) | |
| return re.sub(r"\s+", " ", s).strip() | |
| for r in rows: | |
| if "error" in r: | |
| continue | |
| seg, _ = m.transcribe(load_16k(r["wav"]), language="en", beam_size=5, vad_filter=True, condition_on_previous_text=False) | |
| r["hyp"] = " ".join(s.text.strip() for s in seg).strip() | |
| r["wer"] = round(min(1.0, jiwer.wer(norm(r["text"]), norm(r["hyp"]) or "_")), 4) | |
| print(r["id"], r["wer"], r["hyp"][:90], flush=True) | |
| json.dump(rows, open(f"{d}/scored.json", "w"), indent=1) | |
| ok = [r for r in rows if "wer" in r] | |
| ref_words = sum(len(norm(r["text"]).split()) for r in ok) | |
| errs = sum(jiwer.process_words(norm(r["text"]), norm(r["hyp"]) or "_").wer * len(norm(r["text"]).split()) for r in ok) | |
| print("SCORE", tag, json.dumps(dict( | |
| n=len(ok), corpus_wer=round(errs / ref_words, 4), mean_wer=round(sum(r["wer"] for r in ok) / len(ok), 4), | |
| catastrophic=sum(r["wer"] > 0.5 for r in ok), | |
| no_eos=sum(r.get("eos") is not None and r["eos"] < 0 for r in ok)))) | |