File size: 1,160 Bytes
dc23a98
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
"""ISSUE-002 probe report: judge pass rate per second-turn wording and turn, one column per eval run.



Usage: python scripts/report_probe.py results/probe/target_change/base results/probe/target_change/tuned

"""
import json
import sys
from pathlib import Path

ORDER = ["tgt_training_phrase", "tgt_explicit", "tgt_instead", "tgt_referent", "tgt_referent_bare",
         "src_clarify_control"]


def main():
    runs = {}
    for folder in sys.argv[1:]:
        f = next(Path(folder, "judged").glob("*.jsonl"))
        runs[Path(folder).name] = [json.loads(l) for l in open(f, encoding="utf-8")]
    print("| Wording | Turn | " + " | ".join(runs) + " |")
    print("|---|---|" + "---|" * len(runs))
    for s in ORDER:
        for t in (0, 1, 2):
            cells = []
            for rows in runs.values():
                rs = [r for r in rows if r["slice"] == s and r["turn"] == t]
                cells.append(f"{sum(r['judge_spec_adherence'] == 1 for r in rs)}/{len(rs)}" if rs else "")
            if cells[0]:
                print(f"| {s} | {t + 1} | " + " | ".join(cells) + " |")


if __name__ == "__main__":
    main()