File size: 3,687 Bytes
0af696f 0cfc3a7 0af696f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | #!/usr/bin/env python3
"""Rescore the flagged recordings with the current scorer and list the biggest changes against the stored score.
Usage: python scripts/rescore.py # every flag that kept its recording
python scripts/rescore.py --status pending,fixed # only these review states
python scripts/rescore.py --limit 50 --top 20
python scripts/rescore.py --json out.json # every row, with the old and new sound-level results
Reads SUPABASE_URL, SUPABASE_PUBLISHABLE_KEY and SUPABASE_SECRET_KEY (as the server does) and MODEL_ID.
A flag stores the phrase it was said for, its old result and the recording, so the comparison needs no
other data. Run it before shipping a scoring change: the rows that moved most are the ones to listen to.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from dotenv import load_dotenv # noqa: E402
from app.auth import load_auth # noqa: E402
from app.lab import flag_phrase as phrase_for, flagged_sounds as sounds # noqa: E402
from app.scorer import decode_audio, load_scorer # noqa: E402
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument("--status", default="pending,confirmed,rejected,fixed")
ap.add_argument("--limit", type=int, default=0, help="at most this many flags (0: all)")
ap.add_argument("--top", type=int, default=30, help="rows to print, biggest change first")
ap.add_argument("--json", type=Path, help="write every row here")
args = ap.parse_args()
load_dotenv()
auth = load_auth()
if auth is None or not auth.secret:
print("needs SUPABASE_URL, SUPABASE_PUBLISHABLE_KEY and SUPABASE_SECRET_KEY", file=sys.stderr)
return 2
scorer = load_scorer(os.environ.get("MODEL_ID", "facebook/wav2vec2-xlsr-53-espeak-cv-ft"))
flags = [f for s in args.status.split(",") for f in auth.list_flags(s.strip()) if f.get("audio")]
if args.limit:
flags = flags[:args.limit]
rows = []
for f in flags:
audio = auth.flag_audio(f["id"])
if audio is None:
continue
try:
new = scorer.score(decode_audio(audio[0]), phrase_for(f))
except (ValueError, KeyError) as e:
print(f"flag {f['id']}: skipped ({e})", file=sys.stderr)
continue
if new.get("error"):
continue
old = f.get("result") or {}
rows.append({"id": f["id"], "status": f.get("status"), "phrase": f.get("he") or f.get("phrase"),
"old": f.get("score"), "new": new["score"], "delta": new["score"] - (f.get("score") or 0),
"old_flagged": sounds(old), "new_flagged": sounds(new),
"dialect": "mixed" if new.get("mixed_dialect") else new.get("dialect", ""),
"old_result": old, "new_result": new})
rows.sort(key=lambda r: -abs(r["delta"]))
print(f"{len(rows)} flags rescored; mean change {sum(r['delta'] for r in rows) / max(1, len(rows)):+.1f}")
for r in rows[:args.top]:
print(f"{r['id']:>6} {r['status'] or '':<9} {r['old']!s:>4} -> {r['new']:>3} ({r['delta']:+4}) "
f"{r['dialect']:<6} {r['phrase']}")
if r["old_flagged"] != r["new_flagged"]:
print(f" flagged: {' '.join(r['old_flagged']) or '-'} -> {' '.join(r['new_flagged']) or '-'}")
if args.json:
args.json.write_text(json.dumps(rows, ensure_ascii=False, indent=1), encoding="utf-8")
return 0
if __name__ == "__main__":
sys.exit(main())
|