slm-training / scripts /report_probe.py
debudash's picture
Mirror slm-training @ 507b432 (course PDF excluded)
dc23a98 verified
Raw History Blame Contribute Delete
1.16 kB
"""ISSUE-002 probe report: judge pass rate per second-turn wording and turn, one column per eval run.
Usage: python scripts/report_probe.py results/probe/target_change/base results/probe/target_change/tuned
"""
import json
import sys
from pathlib import Path
ORDER = ["tgt_training_phrase", "tgt_explicit", "tgt_instead", "tgt_referent", "tgt_referent_bare",
"src_clarify_control"]
def main():
runs = {}
for folder in sys.argv[1:]:
f = next(Path(folder, "judged").glob("*.jsonl"))
runs[Path(folder).name] = [json.loads(l) for l in open(f, encoding="utf-8")]
print("| Wording | Turn | " + " | ".join(runs) + " |")
print("|---|---|" + "---|" * len(runs))
for s in ORDER:
for t in (0, 1, 2):
cells = []
for rows in runs.values():
rs = [r for r in rows if r["slice"] == s and r["turn"] == t]
cells.append(f"{sum(r['judge_spec_adherence'] == 1 for r in rs)}/{len(rs)}" if rs else "")
if cells[0]:
print(f"| {s} | {t + 1} | " + " | ".join(cells) + " |")
if __name__ == "__main__":
main()