Download scripts/report_probe.py from debudash/slm-training: direct link, hf CLI and curl.
- Browser
- Download file 1.16 kB
-
https://huggingface.co/debudash/slm-training/resolve/main/scripts/report_probe.py
- Command line
-
hf download hf://debudash/slm-training/scripts/report_probe.py
-
curl -L -o report_probe.py https://huggingface.co/debudash/slm-training/resolve/main/scripts/report_probe.py
1.16 kB
| """ISSUE-002 probe report: judge pass rate per second-turn wording and turn, one column per eval run. | |
| Usage: python scripts/report_probe.py results/probe/target_change/base results/probe/target_change/tuned | |
| """ | |
| import json | |
| import sys | |
| from pathlib import Path | |
| ORDER = ["tgt_training_phrase", "tgt_explicit", "tgt_instead", "tgt_referent", "tgt_referent_bare", | |
| "src_clarify_control"] | |
| def main(): | |
| runs = {} | |
| for folder in sys.argv[1:]: | |
| f = next(Path(folder, "judged").glob("*.jsonl")) | |
| runs[Path(folder).name] = [json.loads(l) for l in open(f, encoding="utf-8")] | |
| print("| Wording | Turn | " + " | ".join(runs) + " |") | |
| print("|---|---|" + "---|" * len(runs)) | |
| for s in ORDER: | |
| for t in (0, 1, 2): | |
| cells = [] | |
| for rows in runs.values(): | |
| rs = [r for r in rows if r["slice"] == s and r["turn"] == t] | |
| cells.append(f"{sum(r['judge_spec_adherence'] == 1 for r in rs)}/{len(rs)}" if rs else "") | |
| if cells[0]: | |
| print(f"| {s} | {t + 1} | " + " | ".join(cells) + " |") | |
| if __name__ == "__main__": | |
| main() | |