Spaces:
Sleeping
Sleeping
File size: 2,890 Bytes
ea5dc26 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | """Score the phrasebook short-circuit against configs/eval/phrasebook_eval.json.
For every (input, lang) pair the outcome is one of:
correct — matched the expected row
wrong — matched a DIFFERENT row (worst case: user hears a wrong phrase)
false_hit — expected the LLM to handle it, but the phrasebook fired
miss — expected a row, nothing matched (falls through to LLM — safe)
correct_miss — open-ended input correctly left to the LLM
python scripts/eval_phrasebook.py # summary
python scripts/eval_phrasebook.py -v # list every non-correct item
python scripts/eval_phrasebook.py --threshold 0.85
"""
from __future__ import annotations
import argparse
import json
import sys
from collections import Counter
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from src.llm.phrasebook import DEFAULT_THRESHOLD, lookup # noqa: E402
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--file", default=str(ROOT / "configs" / "eval" / "phrasebook_eval.json"))
ap.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD)
ap.add_argument("-v", "--verbose", action="store_true")
args = ap.parse_args()
data = json.loads(Path(args.file).read_text(encoding="utf-8"))
groups = data.get("groups", {})
per_lang: dict[str, Counter] = {"bam": Counter(), "ful": Counter()}
problems: list[str] = []
for it in data["items"]:
langs = ["bam", "ful"] if it["lang"] == "both" else [it["lang"]]
exp = it["expect"]
ok_sources = set(groups.get(exp, [exp])) if exp else set()
for lang in langs:
hit = lookup(it["q"], lang, threshold=args.threshold)
got = hit["source"] if hit else None
if exp is None:
outcome = "false_hit" if hit else "correct_miss"
elif hit is None:
outcome = "miss"
else:
outcome = "correct" if got in ok_sources else "wrong"
per_lang[lang][outcome] += 1
if outcome not in ("correct", "correct_miss"):
problems.append(f" [{lang}] {outcome:9} {it['q']!r:42} expected={exp!r} got={got!r}"
+ (f" ({hit['score']})" if hit else ""))
print(f"threshold={args.threshold}")
for lang, c in per_lang.items():
n = sum(c.values())
good = c["correct"] + c["correct_miss"]
print(f"{lang}: {good}/{n} = {good / n:.0%} right | correct={c['correct']} "
f"correct_miss={c['correct_miss']} miss={c['miss']} "
f"wrong={c['wrong']} false_hit={c['false_hit']}")
if args.verbose and problems:
print("\nNon-correct items:")
print("\n".join(problems))
return 0
if __name__ == "__main__":
raise SystemExit(main())
|