"""Rescore /scored.json with Whisper's English normalizer on both sides (numbers, spelling), the usual ASR WER convention.""" import json, os, sys import jiwer from whisper_normalizer.english import EnglishTextNormalizer N = EnglishTextNormalizer() for tag in sys.argv[1:]: rows = [r for r in json.load(open(os.path.expanduser(f"~/omnispace-b70/bench/{tag}/scored.json"))) if "hyp" in r] refs = [N(r["text"]) for r in rows] hyps = [N(r["hyp"]) or "_" for r in rows] per = [jiwer.wer(a, b) for a, b in zip(refs, hyps)] print("RESCORE", tag, json.dumps(dict( n=len(rows), corpus_wer=round(jiwer.wer(refs, hyps), 4), takes_with_errors=sum(p > 0 for p in per), catastrophic=sum(p > 0.5 for p in per), no_eos=sum(r.get("eos") is not None and r["eos"] < 0 for r in rows)))) for r, p, a, b in zip(rows, per, refs, hyps): if p > 0: print(" ", r["id"], round(p, 3), "|", b[:120])