#!/usr/bin/env python3 """Tiny eval-comparison server: http://:8080 Auto-discovers /opt/work/eval// dirs (timing.jsonl present), shows each eval sentence with side-by-side audio players + Soniox WER badges. Refresh to pick up newly evaluated checkpoints. """ import json import re from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path EVAL = Path("/opt/work/eval") SENTS = Path("/opt/work/scripts/eval_sentences.jsonl") def discover(): systems = [] for d in sorted(EVAL.iterdir()): if d.is_dir() and any(d.glob("*.wav")) and d.name != "baseline": systems.append(d.name) # baseline first, then stepN numerically def key(n): if n.startswith("baseline"): return (0, 0) m = re.search(r"(\d+)", n) return (1, int(m.group(1)) if m else 9e9) return sorted(systems, key=key) def reports(systems): out = {} for s in systems: for fn in ("asr_report_soniox.json", "asr_report.json"): p = EVAL / s / fn if p.exists(): try: rep = json.loads(p.read_text(encoding="utf-8")) out[s] = { "rows": {r["id"]: r for r in rep.get("rows", []) if "id" in r}, "summary": rep.get("summary", {}), "engine": rep.get("summary", {}).get("engine", "whisper"), } except Exception: pass break return out def render(): systems = discover() reps = reports(systems) sents = [json.loads(l) for l in SENTS.read_text(encoding="utf-8").splitlines() if l.strip()] # include ids that only exist in system timing files (e.g. mega paragraph) known = {s["id"] for s in sents} for s in systems: tj = EVAL / s / "timing.jsonl" if tj.exists(): for line in tj.read_text(encoding="utf-8").splitlines(): if line.strip(): r = json.loads(line) if r.get("id") and r["id"] not in known and r.get("text"): sents.insert(0, {"id": r["id"], "text": r["text"]}) known.add(r["id"]) head = "".join( f"{s}
{('WER %.0f%%' % (100*reps[s]['summary'].get('mean_wer',0))) if s in reps else 'scoring...'}" for s in systems) rows_html = [] for sent in sents: sid = sent["id"] cells = [] for s in systems: wav = EVAL / s / f"{sid}.wav" if wav.exists(): badge = "" r = reps.get(s, {}).get("rows", {}).get(sid) if r and "wer" in r: c = "#2a4" if r["wer"] < 0.2 else ("#a82" if r["wer"] < 0.5 else "#a33") badge = f'WER {r["wer"]:.2f}' cells.append( f'

{badge}') else: cells.append("—") rows_html.append( f'{sid}

{sent["text"]}

' + "".join(cells) + "") return f""" s2-pro Egyptian eval

s2-pro Egyptian fine-tune — A/B eval (refresh for new checkpoints)

{head} {''.join(rows_html)}

sentence

""" class H(SimpleHTTPRequestHandler): def __init__(self, *a, **kw): super().__init__(*a, directory=str(EVAL), **kw) def do_GET(self): if self.path in ("/", "/index.html"): body = render().encode("utf-8") self.send_response(200) self.send_header("Content-Type", "text/html; charset=utf-8") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) else: super().do_GET() def log_message(self, *a): pass if __name__ == "__main__": ThreadingHTTPServer(("0.0.0.0", 8080), H).serve_forever()