Download report_ceiling.py from debudash/slm-training: direct link, hf CLI and curl.
- Browser
- Download file 4.59 kB
-
https://huggingface.co/debudash/slm-training/resolve/main/report_ceiling.py
- Command line
-
hf download hf://debudash/slm-training/report_ceiling.py
-
curl -L -o report_ceiling.py https://huggingface.co/debudash/slm-training/resolve/main/report_ceiling.py
4.59 kB
| """Aggregate judged ceiling runs into the results table the brief asks for. | |
| Spec-adherence = judge pass rate on clean single-turn scenarios; Robustness = judge pass rate on adversarial, | |
| near-miss and multi-turn scenarios. Both are turn-level means over repeats (min-max shown). | |
| Usage: python report_ceiling.py [results/ceiling/judged] > results/ceiling/REPORT.md | |
| """ | |
| import json | |
| import sys | |
| from collections import Counter, defaultdict | |
| from pathlib import Path | |
| def load(folders): | |
| runs = defaultdict(list) # (model, strategy) -> list of row-lists, one per repeat | |
| for folder in folders: | |
| for f in sorted(Path(folder).glob("*.jsonl")): | |
| model, strategy, _ = f.stem.split("__") | |
| runs[(model, strategy)].append([json.loads(l) for l in open(f, encoding="utf-8")]) | |
| return runs | |
| def rate(rows, key="judge_spec_adherence"): | |
| rows = [r for r in rows if r[key] is not None] | |
| return sum(bool(r[key]) for r in rows) / len(rows) if rows else float("nan") | |
| def scenario_rate(rows): | |
| by_id = defaultdict(list) | |
| for r in rows: | |
| by_id[r["id"]].append(r["judge_spec_adherence"] == 1) | |
| return sum(all(v) for v in by_id.values()) / len(by_id) | |
| def spread(values): | |
| mean = sum(values) / len(values) | |
| return f"{mean:.1%}" + (f" ({min(values):.0%}-{max(values):.0%})" if len(values) > 1 else "") | |
| def main(): | |
| runs = load(sys.argv[1:] or ["results/ceiling/judged"]) | |
| rank = {"zero_shot": 0, "few_shot": 1, "cot": 2} | |
| order = sorted(runs, key=lambda k: (k[0] != "sonnet", k[0] != "mistral", k[0], rank.get(k[1], 9), k[1])) | |
| print("# Prompt-ceiling ablation: results\n") | |
| print("Judge columns are the LLM judge's score; exact-match columns compare the reply to the oracle line " | |
| "directly. Means over repeats, min-max in brackets.\n") | |
| print("| Model | Strategy | Repeats | Spec-adherence (judge) | Robustness (judge) | Spec-adherence (exact) " | |
| "| Robustness (exact) | Scenarios fully correct | Judge = exact match |") | |
| print("|---|---|---|---|---|---|---|---|---|") | |
| best = {} | |
| for m, s in order: | |
| reps = runs[(m, s)] | |
| adh = [rate([r for r in rows if r["metric"] == "adherence"]) for rows in reps] | |
| rob = [rate([r for r in rows if r["metric"] == "robustness"]) for rows in reps] | |
| adh_em = [rate([r for r in rows if r["metric"] == "adherence"], "exact_match") for rows in reps] | |
| rob_em = [rate([r for r in rows if r["metric"] == "robustness"], "exact_match") for rows in reps] | |
| scen = [scenario_rate(rows) for rows in reps] | |
| agree = [sum((r["judge_spec_adherence"] == 1) == r["exact_match"] for r in rows) / len(rows) for rows in reps] | |
| print(f"| {m} | {s} | {len(reps)} | {spread(adh)} | {spread(rob)} | {spread(adh_em)} | {spread(rob_em)} " | |
| f"| {spread(scen)} | {spread(agree)} |") | |
| overall = sum(adh) / len(adh) + sum(rob) / len(rob) | |
| if m not in best or overall > best[m][1]: | |
| best[m] = (s, overall) | |
| print("\n## Pass rate by slice (all repeats pooled)\n") | |
| slices = sorted({r["slice"] for reps in runs.values() for rows in reps for r in rows}) | |
| print("| Slice | " + " | ".join(f"{m} {s}" for m, s in order) + " |") | |
| print("|---|" + "---|" * len(order)) | |
| for sl in slices: | |
| cells = [rate([r for rows in runs[k] for r in rows if r["slice"] == sl]) for k in order] | |
| print(f"| {sl} | " + " | ".join(f"{c:.0%}" for c in cells) + " |") | |
| print("\n## Failure modes at each model's best strategy (all repeats pooled)\n") | |
| for m, (s, _) in best.items(): | |
| rows = [r for rows in runs[(m, s)] for r in rows] | |
| fails = Counter(r["judge_failure_mode"] for r in rows if r["judge_spec_adherence"] == 0) | |
| total = sum(fails.values()) | |
| print(f"**{m} / {s}**: {total} failed turns of {len(rows)}: " | |
| + ", ".join(f"{k} {v}" for k, v in fails.most_common())) | |
| print() | |
| print("## Long sessions: pass rate by turn number (best strategy, pooled)\n") | |
| print("| Turn | " + " | ".join(f"{m} {s}" for m, (s, _) in best.items()) + " |") | |
| print("|---|" + "---|" * len(best)) | |
| for t in range(12): | |
| cells = [] | |
| for m, (s, _) in best.items(): | |
| rows = [r for rows in runs[(m, s)] for r in rows if r["slice"] == "long_session" and r["turn"] == t] | |
| cells.append(f"{rate(rows):.0%}" if rows else "-") | |
| print(f"| {t + 1} | " + " | ".join(cells) + " |") | |
| if __name__ == "__main__": | |
| main() | |