slm-training / report_ceiling.py
debudash's picture
Mirror slm-training at 4376283
4643f62 verified
Raw History Blame Contribute Delete
4.59 kB
"""Aggregate judged ceiling runs into the results table the brief asks for.
Spec-adherence = judge pass rate on clean single-turn scenarios; Robustness = judge pass rate on adversarial,
near-miss and multi-turn scenarios. Both are turn-level means over repeats (min-max shown).
Usage: python report_ceiling.py [results/ceiling/judged] > results/ceiling/REPORT.md
"""
import json
import sys
from collections import Counter, defaultdict
from pathlib import Path
def load(folders):
runs = defaultdict(list) # (model, strategy) -> list of row-lists, one per repeat
for folder in folders:
for f in sorted(Path(folder).glob("*.jsonl")):
model, strategy, _ = f.stem.split("__")
runs[(model, strategy)].append([json.loads(l) for l in open(f, encoding="utf-8")])
return runs
def rate(rows, key="judge_spec_adherence"):
rows = [r for r in rows if r[key] is not None]
return sum(bool(r[key]) for r in rows) / len(rows) if rows else float("nan")
def scenario_rate(rows):
by_id = defaultdict(list)
for r in rows:
by_id[r["id"]].append(r["judge_spec_adherence"] == 1)
return sum(all(v) for v in by_id.values()) / len(by_id)
def spread(values):
mean = sum(values) / len(values)
return f"{mean:.1%}" + (f" ({min(values):.0%}-{max(values):.0%})" if len(values) > 1 else "")
def main():
runs = load(sys.argv[1:] or ["results/ceiling/judged"])
rank = {"zero_shot": 0, "few_shot": 1, "cot": 2}
order = sorted(runs, key=lambda k: (k[0] != "sonnet", k[0] != "mistral", k[0], rank.get(k[1], 9), k[1]))
print("# Prompt-ceiling ablation: results\n")
print("Judge columns are the LLM judge's score; exact-match columns compare the reply to the oracle line "
"directly. Means over repeats, min-max in brackets.\n")
print("| Model | Strategy | Repeats | Spec-adherence (judge) | Robustness (judge) | Spec-adherence (exact) "
"| Robustness (exact) | Scenarios fully correct | Judge = exact match |")
print("|---|---|---|---|---|---|---|---|---|")
best = {}
for m, s in order:
reps = runs[(m, s)]
adh = [rate([r for r in rows if r["metric"] == "adherence"]) for rows in reps]
rob = [rate([r for r in rows if r["metric"] == "robustness"]) for rows in reps]
adh_em = [rate([r for r in rows if r["metric"] == "adherence"], "exact_match") for rows in reps]
rob_em = [rate([r for r in rows if r["metric"] == "robustness"], "exact_match") for rows in reps]
scen = [scenario_rate(rows) for rows in reps]
agree = [sum((r["judge_spec_adherence"] == 1) == r["exact_match"] for r in rows) / len(rows) for rows in reps]
print(f"| {m} | {s} | {len(reps)} | {spread(adh)} | {spread(rob)} | {spread(adh_em)} | {spread(rob_em)} "
f"| {spread(scen)} | {spread(agree)} |")
overall = sum(adh) / len(adh) + sum(rob) / len(rob)
if m not in best or overall > best[m][1]:
best[m] = (s, overall)
print("\n## Pass rate by slice (all repeats pooled)\n")
slices = sorted({r["slice"] for reps in runs.values() for rows in reps for r in rows})
print("| Slice | " + " | ".join(f"{m} {s}" for m, s in order) + " |")
print("|---|" + "---|" * len(order))
for sl in slices:
cells = [rate([r for rows in runs[k] for r in rows if r["slice"] == sl]) for k in order]
print(f"| {sl} | " + " | ".join(f"{c:.0%}" for c in cells) + " |")
print("\n## Failure modes at each model's best strategy (all repeats pooled)\n")
for m, (s, _) in best.items():
rows = [r for rows in runs[(m, s)] for r in rows]
fails = Counter(r["judge_failure_mode"] for r in rows if r["judge_spec_adherence"] == 0)
total = sum(fails.values())
print(f"**{m} / {s}**: {total} failed turns of {len(rows)}: "
+ ", ".join(f"{k} {v}" for k, v in fails.most_common()))
print()
print("## Long sessions: pass rate by turn number (best strategy, pooled)\n")
print("| Turn | " + " | ".join(f"{m} {s}" for m, (s, _) in best.items()) + " |")
print("|---|" + "---|" * len(best))
for t in range(12):
cells = []
for m, (s, _) in best.items():
rows = [r for rows in runs[(m, s)] for r in rows if r["slice"] == "long_session" and r["turn"] == t]
cells.append(f"{rate(rows):.0%}" if rows else "-")
print(f"| {t + 1} | " + " | ".join(cells) + " |")
if __name__ == "__main__":
main()