eval.py

"""Fairness benchmark for the troll.

Runs the model against data/eval_seeds.jsonl and reports:
  * tactic accuracy        — does Gorm correctly name flattery/threat/genuine/...?
  * persuasiveness bands   — do weak/mid/strong genuine lines land in sane ranges?
  * the "unfairness" cases — where he folds to a bad tactic or ignores a strong appeal.

This is your calibration loop. Run it before AND after fine-tuning to prove the
fine-tune made the troll *fairer*, not just different — that delta is your
Field Notes blog post.

    BRIDGE_TROLL_MOCK=1 python eval.py     # smoke-test the harness
    python eval.py                         # real model
"""

from __future__ import annotations

import json
from collections import Counter
from pathlib import Path

from troll_engine import GameState, Tactic, build_messages, parse_judgment
from models import get_backend

SEEDS = Path(__file__).parent / "data" / "eval_seeds.jsonl"

# Expected persuasiveness range per band for "genuine" lines.
BANDS = {"weak": (0, 2), "mid": (2, 4), "strong": (4, 5), "none": (0, 0)}


def load_seeds() -> list[dict]:
    return [json.loads(l) for l in SEEDS.read_text().splitlines() if l.strip()]


def main() -> None:
    backend = get_backend()
    seeds = load_seeds()

    tactic_hits = 0
    band_hits = 0
    band_total = 0
    confusion: Counter = Counter()
    unfair: list[str] = []

    for s in seeds:
        # fresh state each time — judging a single opening line in isolation
        state = GameState()
        raw = backend.generate(build_messages(state, s["text"]))
        j = parse_judgment(raw)

        exp_tactic = s["tactic"]
        ok_tactic = j.tactic.value == exp_tactic
        tactic_hits += ok_tactic
        if not ok_tactic:
            confusion[f"{exp_tactic}->{j.tactic.value}"] += 1

        # band check (only meaningful where we predicted genuine correctly)
        lo, hi = BANDS[s["band"]]
        if s["tactic"] == "genuine":
            band_total += 1
            if ok_tactic and lo <= j.persuasiveness <= hi:
                band_hits += 1

        # unfairness: troll progresses on a non-genuine tactic, or stonewalls a strong appeal
        delta = j.resolve_delta()
        if exp_tactic != "genuine" and delta < 0:
            unfair.append(f"FOLDED to {exp_tactic}: {s['text'][:60]}")
        if s["band"] == "strong" and delta > -12:
            unfair.append(f"IGNORED strong appeal (d={delta:+d}): {s['text'][:60]}")

        print(f"[{exp_tactic:12}->{j.tactic.value:12}] p={j.persuasiveness} d={delta:+3d} "
              f"{'ok' if ok_tactic else 'MISS'}")

    n = len(seeds)
    print("\n--- SUMMARY ---")
    print(f"tactic accuracy:        {tactic_hits}/{n} = {tactic_hits / n:.0%}")
    if band_total:
        print(f"persuasiveness in-band: {band_hits}/{band_total} = {band_hits / band_total:.0%}")
    if confusion:
        print("confusions:", dict(confusion))
    if unfair:
        print("\nUNFAIR CASES (fix these via tuning):")
        for u in unfair:
            print("  -", u)
    else:
        print("\nNo unfair cases. Gorm is calibrated.")


if __name__ == "__main__":
    main()