Spaces:
Running on Zero
Running on Zero
File size: 4,997 Bytes
f1ef7e2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | """
Single-call API pilot for whisper-1 (OpenAI) — same call as pilot_medium.py
so we get a direct 3-way comparison: small.en / medium.en / whisper-1 API.
Uses per-channel approach: agent WAV + customer WAV sent separately,
same structural speaker attribution as the local pipeline.
Usage:
set OPENAI_API_KEY=sk-...
python pilot_api.py
"""
import os, json, time
import jiwer
from openai import OpenAI
from eval_common import normalise
DATA = r"d:\Desktop\ai-ml-capstone\data\na_testset"
MANIFEST = os.path.join(DATA, "manifest.json")
PROBE_SET = os.path.join(DATA, "probe_set.json")
# same initial prompt as local pipeline for fair comparison
INITIAL_PROMPT = (
"Banking call center transcript. "
"Speakers discuss account numbers, balances, transfers, loans, credit cards, "
"PINs, dates, dollar amounts, authentication, and customer service."
)
with open(MANIFEST, encoding="utf-8") as f:
manifest = {m["call_id"]: m for m in json.load(f)}
with open(PROBE_SET, encoding="utf-8") as f:
probe = json.load(f)["calls"]
# same call as medium.en pilot
p = probe[0]
cid, accent = p["call_id"], p["accent"]
m = manifest[cid]
a_wav = os.path.join(DATA, m["agent_wav"])
c_wav = os.path.join(DATA, m["customer_wav"])
print(f"Pilot call : {cid} ({accent} {m['domain']})")
print(f"Agent WAV : {os.path.getsize(a_wav)/1e6:.1f} MB")
print(f"Customer WAV: {os.path.getsize(c_wav)/1e6:.1f} MB")
print()
api_key = os.environ.get("OPENAI_API_KEY")
if not api_key:
raise SystemExit("ERROR: OPENAI_API_KEY not set. Run: set OPENAI_API_KEY=sk-...")
client = OpenAI(api_key=api_key)
def transcribe_via_api(wav_path, label):
t0 = time.time()
with open(wav_path, "rb") as f:
resp = client.audio.transcriptions.create(
model="whisper-1",
file=f,
language="en",
prompt=INITIAL_PROMPT,
response_format="text",
)
elapsed = time.time() - t0
text = resp.strip() if isinstance(resp, str) else resp
words = text.split()
print(f" {label:<10} {elapsed:5.1f}s | {len(words)} words")
return text, elapsed
print("Calling whisper-1 API (per channel)...")
agent_text, t_agent = transcribe_via_api(a_wav, "agent")
customer_text, t_customer = transcribe_via_api(c_wav, "customer")
t_total = t_agent + t_customer
# audio duration from manifest or estimate from file size (16-bit 8kHz mono)
# WAV file = header(44) + samples. 20.1MB @ 16-bit 8kHz = ~1256s...
# Actually let's compute from the WAV header
import wave
with wave.open(a_wav) as wf:
dur = wf.getnframes() / wf.getframerate()
print(f"\n Audio duration : {dur:.0f}s ({dur/60:.1f} min)")
print(f" API wall time : {t_total:.1f}s ({t_total/60:.1f} min)")
print(f" Real-time factor: {t_total/dur:.2f}x")
print(f" Projected for 12-call probe: {12*t_total/60:.0f} min")
# ── WER comparison ────────────────────────────────────────────────────────────
def acc(ref, hyp):
r, h = normalise(ref), normalise(hyp)
n = len(r.split())
return (1 - jiwer.wer(r, h)) * 100 if n else 100.0, n
# load small.en and medium.en results for the same call
p1_path = os.path.join(DATA, "results_channels", accent, cid + ".json")
med_path = os.path.join(DATA, "results_channels_medium", accent, cid + ".json")
with open(p1_path, encoding="utf-8") as f:
p1 = json.load(f)
small_a = " ".join(w["word"] for w in p1["agent"])
small_c = " ".join(w["word"] for w in p1["customer"])
aa_s, na = acc(m["agent_transcript"], small_a)
ac_s, nc = acc(m["customer_transcript"], small_c)
aa_api,_ = acc(m["agent_transcript"], agent_text)
ac_api,_ = acc(m["customer_transcript"], customer_text)
small_overall = (aa_s*na + ac_s*nc) / (na+nc)
api_overall = (aa_api*na + ac_api*nc) / (na+nc)
# medium.en — load from disk if saved, else use known pilot numbers
aa_m = ac_m = med_overall = None
if os.path.exists(med_path):
with open(med_path, encoding="utf-8") as f:
med = json.load(f)
med_a = " ".join(w["word"] for w in med["agent"])
med_c = " ".join(w["word"] for w in med["customer"])
aa_m, _ = acc(m["agent_transcript"], med_a)
ac_m, _ = acc(m["customer_transcript"], med_c)
med_overall = (aa_m*na + ac_m*nc) / (na+nc)
else:
# known numbers from pilot_medium.py run (same call, same model)
aa_m, ac_m, med_overall = 90.7, 87.5, 89.6
print(f"\n{'='*60}")
print(f" Head-to-head on {cid}")
print(f" {'Model':<22} {'Agent':>7} {'Customer':>9} {'Overall':>8} {'Time':>8}")
print(f" {'-'*56}")
print(f" {'small.en (local)':<22} {aa_s:>6.1f}% {ac_s:>8.1f}% {small_overall:>7.1f}% {'~3.5min':>8}")
print(f" {'medium.en (local)':<22} {aa_m:>6.1f}% {ac_m:>8.1f}% {med_overall:>7.1f}% {'~10min':>8}")
print(f" {'whisper-1 (API)':<22} {aa_api:>6.1f}% {ac_api:>8.1f}% {api_overall:>7.1f}% {t_total/60:>7.1f}min")
print(f"{'='*60}")
|