"""Aggregate judged ceiling runs into the results table the brief asks for. Spec-adherence = judge pass rate on clean single-turn scenarios; Robustness = judge pass rate on adversarial, near-miss and multi-turn scenarios. Both are turn-level means over repeats (min-max shown). Usage: python report_ceiling.py [results/ceiling/judged] > results/ceiling/REPORT.md """ import json import sys from collections import Counter, defaultdict from pathlib import Path def load(folders): runs = defaultdict(list) # (model, strategy) -> list of row-lists, one per repeat for folder in folders: for f in sorted(Path(folder).glob("*.jsonl")): model, strategy, _ = f.stem.split("__") runs[(model, strategy)].append([json.loads(l) for l in open(f, encoding="utf-8")]) return runs def rate(rows, key="judge_spec_adherence"): rows = [r for r in rows if r[key] is not None] return sum(bool(r[key]) for r in rows) / len(rows) if rows else float("nan") def scenario_rate(rows): by_id = defaultdict(list) for r in rows: by_id[r["id"]].append(r["judge_spec_adherence"] == 1) return sum(all(v) for v in by_id.values()) / len(by_id) def spread(values): mean = sum(values) / len(values) return f"{mean:.1%}" + (f" ({min(values):.0%}-{max(values):.0%})" if len(values) > 1 else "") def main(): runs = load(sys.argv[1:] or ["results/ceiling/judged"]) rank = {"zero_shot": 0, "few_shot": 1, "cot": 2} order = sorted(runs, key=lambda k: (k[0] != "sonnet", k[0] != "mistral", k[0], rank.get(k[1], 9), k[1])) print("# Prompt-ceiling ablation: results\n") print("Judge columns are the LLM judge's score; exact-match columns compare the reply to the oracle line " "directly. Means over repeats, min-max in brackets.\n") print("| Model | Strategy | Repeats | Spec-adherence (judge) | Robustness (judge) | Spec-adherence (exact) " "| Robustness (exact) | Scenarios fully correct | Judge = exact match |") print("|---|---|---|---|---|---|---|---|---|") best = {} for m, s in order: reps = runs[(m, s)] adh = [rate([r for r in rows if r["metric"] == "adherence"]) for rows in reps] rob = [rate([r for r in rows if r["metric"] == "robustness"]) for rows in reps] adh_em = [rate([r for r in rows if r["metric"] == "adherence"], "exact_match") for rows in reps] rob_em = [rate([r for r in rows if r["metric"] == "robustness"], "exact_match") for rows in reps] scen = [scenario_rate(rows) for rows in reps] agree = [sum((r["judge_spec_adherence"] == 1) == r["exact_match"] for r in rows) / len(rows) for rows in reps] print(f"| {m} | {s} | {len(reps)} | {spread(adh)} | {spread(rob)} | {spread(adh_em)} | {spread(rob_em)} " f"| {spread(scen)} | {spread(agree)} |") overall = sum(adh) / len(adh) + sum(rob) / len(rob) if m not in best or overall > best[m][1]: best[m] = (s, overall) print("\n## Pass rate by slice (all repeats pooled)\n") slices = sorted({r["slice"] for reps in runs.values() for rows in reps for r in rows}) print("| Slice | " + " | ".join(f"{m} {s}" for m, s in order) + " |") print("|---|" + "---|" * len(order)) for sl in slices: cells = [rate([r for rows in runs[k] for r in rows if r["slice"] == sl]) for k in order] print(f"| {sl} | " + " | ".join(f"{c:.0%}" for c in cells) + " |") print("\n## Failure modes at each model's best strategy (all repeats pooled)\n") for m, (s, _) in best.items(): rows = [r for rows in runs[(m, s)] for r in rows] fails = Counter(r["judge_failure_mode"] for r in rows if r["judge_spec_adherence"] == 0) total = sum(fails.values()) print(f"**{m} / {s}**: {total} failed turns of {len(rows)}: " + ", ".join(f"{k} {v}" for k, v in fails.most_common())) print() print("## Long sessions: pass rate by turn number (best strategy, pooled)\n") print("| Turn | " + " | ".join(f"{m} {s}" for m, (s, _) in best.items()) + " |") print("|---|" + "---|" * len(best)) for t in range(12): cells = [] for m, (s, _) in best.items(): rows = [r for rows in runs[(m, s)] for r in rows if r["slice"] == "long_session" and r["turn"] == t] cells.append(f"{rate(rows):.0%}" if rows else "-") print(f"| {t + 1} | " + " | ".join(cells) + " |") if __name__ == "__main__": main()