ml-services/scripts/run_vad_exp.py

"""
VAD-recall experiment: does a more sensitive VAD (or no VAD) recover the dropped
speech segments we saw in error analysis, WITHOUT adding hallucination?

Runs a named decode config on a small mini-set with small.en (fast), writes to
results_vad_<config>/, and prints per-call accuracy vs the frozen small.en baseline
(results_channels). Includes a good-banking call as a REGRESSION GUARD -- a config
that helps hard calls but hurts easy ones is no good.

Usage:
  python run_vad_exp.py --config sensitive
  python run_vad_exp.py --config novad
  python run_vad_exp.py --config midvad --full   # all 12 probe calls
"""
import os, json, time, argparse, jiwer
from faster_whisper import WhisperModel
import transcribe_channels
from eval_common import normalise

DATA      = r"d:\Desktop\ai-ml-capstone\data\na_testset"
MANIFEST  = os.path.join(DATA, "manifest.json")
PROBE_SET = os.path.join(DATA, "probe_set.json")
BASELINE  = "results_channels"   # small.en Phase-1 baseline

# decode configs (merged onto the Phase-1 defaults inside transcribe_channels)
CONFIGS = {
    "baseline":  {},  # sanity: should reproduce results_channels
    "sensitive": {"vad_parameters": {"threshold": 0.3, "min_silence_duration_ms": 200,
                                     "speech_pad_ms": 600}},
    "midvad":    {"vad_parameters": {"threshold": 0.4, "min_silence_duration_ms": 300,
                                     "speech_pad_ms": 500}},
    "novad":     {"vad_filter": False},
    # no-VAD but lean on Whisper's own silence suppression to limit hallucination
    "novad_strict": {"vad_filter": False, "no_speech_threshold": 0.5,
                     "condition_on_previous_text": False},
}

# mini-set: 2 filler-heavy catastrophic + 2 bad-banking + 1 good-banking (guard)
MINISET = [
    "en_US_General_Health_1587175",   # catastrophic, lots of dropped filler/segments
    "en_CA_Aviation_1588678",         # catastrophic, API regressed here
    "en_CA_Banking_1588683",          # bad_banking
    "en_US_General_Banking_1584540",  # bad_banking, big API gain
    "en_CA_Banking_1592237",          # good_banking -- REGRESSION GUARD (94.8%)
]


def acc(ref, hyp):
    r, h = normalise(ref), normalise(hyp)
    n = len(r.split())
    return (1 - jiwer.wer(r, h)) * 100 if n else 100.0, n


def overall(m, ag, cu):
    aa, na = acc(m["agent_transcript"], ag)
    ac, nc = acc(m["customer_transcript"], cu)
    return (aa*na + ac*nc) / (na+nc), na+nc


def baseline_acc(m, accent, cid):
    with open(os.path.join(DATA, BASELINE, accent, cid + ".json"), encoding="utf-8") as f:
        b = json.load(f)
    return overall(m, " ".join(w["word"] for w in b["agent"]),
                      " ".join(w["word"] for w in b["customer"]))[0]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--config", required=True, choices=list(CONFIGS))
    ap.add_argument("--model", default="small.en")
    ap.add_argument("--full", action="store_true", help="all 12 probe calls, not mini-set")
    args = ap.parse_args()

    with open(MANIFEST, encoding="utf-8") as f:
        manifest = {m["call_id"]: m for m in json.load(f)}
    if args.full:
        with open(PROBE_SET, encoding="utf-8") as f:
            ids = [c["call_id"] for c in json.load(f)["calls"]]
    else:
        ids = MINISET

    decode = CONFIGS[args.config]
    model_tag = args.model.replace(".", "")
    # keep small.en config-only dirs (backward compat with the first mini-set runs);
    # tag non-small models so medium/large results never collide with small ones.
    out_dir = (f"results_vad_{args.config}" if args.model == "small.en"
               else f"results_vad_{args.config}_{model_tag}")
    print(f"VAD experiment | config={args.config} | model={args.model} | {len(ids)} calls")
    print(f"  decode overrides: {decode}\n")

    model = WhisperModel(args.model, device="cpu", compute_type="int8")

    print(f"  {'call_id':<32} {'base':>6} {'new':>6} {'delta':>6} {'A_wd':>5} {'C_wd':>5}")
    print("  " + "-" * 68)

    deltas, ws, news = [], [], []
    t_start = time.time()
    for cid in ids:
        m = manifest[cid]
        accent = m["accent"]
        a_wav = os.path.join(DATA, m["agent_wav"])
        c_wav = os.path.join(DATA, m["customer_wav"])
        out = os.path.join(DATA, out_dir, accent, cid + ".json")
        os.makedirs(os.path.dirname(out), exist_ok=True)

        if os.path.exists(out) and os.path.getsize(out) > 0:
            with open(out, encoding="utf-8") as f:
                r = json.load(f)
            aw, cw = r["agent"], r["customer"]
        else:
            aw, cw, segs, dur = transcribe_channels.transcribe_call(
                model, a_wav, c_wav, decode=decode)
            with open(out, "w", encoding="utf-8") as f:
                json.dump({"call_id": cid, "accent": accent, "domain": m["domain"],
                           "config": args.config, "agent": aw, "customer": cw}, f, indent=2)

        new_o, w = overall(m, " ".join(x["word"] for x in aw),
                              " ".join(x["word"] for x in cw))
        base_o = baseline_acc(m, accent, cid)
        d = new_o - base_o
        deltas.append(d * w); ws.append(w); news.append(new_o * w)
        print(f"  {cid:<32} {base_o:>5.1f}% {new_o:>5.1f}% {d:>+5.1f} {len(aw):>5} {len(cw):>5}")

    print("  " + "-" * 68)
    wavg     = sum(deltas) / sum(ws)
    overall_abs = sum(news) / sum(ws)
    print(f"  OVERALL absolute accuracy: {overall_abs:.2f}%   "
          f"(word-weighted mean delta vs small.en baseline: {wavg:+.2f})")
    print(f"  {'>> HELPS' if wavg > 0.1 else '>> NEUTRAL' if abs(wavg) <= 0.1 else '>> HURTS'}")
    if args.full:
        tag = ("CLEARS +cushion" if overall_abs >= 93.5 else
               "clears (thin)" if overall_abs >= 93.0 else "short of 93%")
        print(f"  GOAL (>=93% over 12 calls): {overall_abs:.2f}%  ->  {tag}")
    print(f"  ({(time.time()-t_start)/60:.1f} min)")


if __name__ == "__main__":
    main()