"""ISSUE-002 probe report: judge pass rate per second-turn wording and turn, one column per eval run. Usage: python scripts/report_probe.py results/probe/target_change/base results/probe/target_change/tuned """ import json import sys from pathlib import Path ORDER = ["tgt_training_phrase", "tgt_explicit", "tgt_instead", "tgt_referent", "tgt_referent_bare", "src_clarify_control"] def main(): runs = {} for folder in sys.argv[1:]: f = next(Path(folder, "judged").glob("*.jsonl")) runs[Path(folder).name] = [json.loads(l) for l in open(f, encoding="utf-8")] print("| Wording | Turn | " + " | ".join(runs) + " |") print("|---|---|" + "---|" * len(runs)) for s in ORDER: for t in (0, 1, 2): cells = [] for rows in runs.values(): rs = [r for r in rows if r["slice"] == s and r["turn"] == t] cells.append(f"{sum(r['judge_spec_adherence'] == 1 for r in rs)}/{len(rs)}" if rs else "") if cells[0]: print(f"| {s} | {t + 1} | " + " | ".join(cells) + " |") if __name__ == "__main__": main()