File size: 1,160 Bytes
dc23a98 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | """ISSUE-002 probe report: judge pass rate per second-turn wording and turn, one column per eval run.
Usage: python scripts/report_probe.py results/probe/target_change/base results/probe/target_change/tuned
"""
import json
import sys
from pathlib import Path
ORDER = ["tgt_training_phrase", "tgt_explicit", "tgt_instead", "tgt_referent", "tgt_referent_bare",
"src_clarify_control"]
def main():
runs = {}
for folder in sys.argv[1:]:
f = next(Path(folder, "judged").glob("*.jsonl"))
runs[Path(folder).name] = [json.loads(l) for l in open(f, encoding="utf-8")]
print("| Wording | Turn | " + " | ".join(runs) + " |")
print("|---|---|" + "---|" * len(runs))
for s in ORDER:
for t in (0, 1, 2):
cells = []
for rows in runs.values():
rs = [r for r in rows if r["slice"] == s and r["turn"] == t]
cells.append(f"{sum(r['judge_spec_adherence'] == 1 for r in rs)}/{len(rs)}" if rs else "")
if cells[0]:
print(f"| {s} | {t + 1} | " + " | ".join(cells) + " |")
if __name__ == "__main__":
main()
|