Spaces:
Sleeping
Sleeping
File size: 2,544 Bytes
f1ef7e2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | """
WER evaluation for a single call's transcription.
Reports TWO metrics for honesty:
- Raw WER : lowercase + strip punctuation only (conservative)
- Normalised WER: Whisper EnglishTextNormalizer (semantic; numbers/contractions
canonicalised). This is the standard ASR-benchmark metric.
Usage:
python evaluate_wer.py [--call call_1] [--hypo path/to/transcript_data.json]
"""
import json
import argparse
import jiwer
from eval_common import normalise, normalise_raw
METADATA = r"d:\Desktop\ai-ml-capstone\data\apptek\metadata.json"
def wer_for(ref, hyp):
out = jiwer.process_words(ref, hyp)
return out.wer, out.substitutions, out.deletions, out.insertions, out.hits
def report(ref_text, hyp_text, label, norm_fn):
ref, hyp = norm_fn(ref_text), norm_fn(hyp_text)
wer, S, D, I, H = wer_for(ref, hyp)
n = len(ref.split())
print(f" {label:<10} WER {wer*100:5.1f}% (acc {(1-wer)*100:4.1f}%) "
f"[S={S} D={D} I={I} ref_words={n}]")
return wer, n
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--call", default="call_1")
ap.add_argument("--hypo", default=r"d:\Desktop\ai-ml-capstone\frontend\transcript_data.json")
args = ap.parse_args()
with open(METADATA, encoding="utf-8") as f:
meta = {c["call_id"]: c for c in json.load(f)}
with open(args.hypo, encoding="utf-8") as f:
hypo = json.load(f)
call = meta[args.call]
hyp_agent = " ".join(w["word"] for w in hypo["agent"])
hyp_customer = " ".join(w["word"] for w in hypo["customer"])
print(f"Model : {hypo.get('model','?')}")
print(f"Call : {args.call}")
print("=" * 64)
results = {}
for norm_name, norm_fn in [("RAW", normalise_raw), ("NORMALISED", normalise)]:
print(f"\n --- {norm_name} ---")
wa, na = report(call["agent_transcript"], hyp_agent, "Agent", norm_fn)
wc, nc = report(call["customer_transcript"], hyp_customer, "Customer", norm_fn)
overall = (wa * na + wc * nc) / (na + nc)
print(f" {'Overall':<10} WER {overall*100:5.1f}% (acc {(1-overall)*100:4.1f}%)")
results[norm_name] = 1 - overall
print("\n" + "=" * 64)
print(f" Raw accuracy : {results['RAW']*100:.1f}%")
print(f" Normalised accuracy : {results['NORMALISED']*100:.1f}% (target >= 90%)")
status = "PASS" if results["NORMALISED"] >= 0.90 else "NEEDS WORK"
print(f" Status : {status}")
print("=" * 64)
if __name__ == "__main__":
main()
|