repro-bundle-v2/repro/analyze_v2.py

#!/usr/bin/env python3
"""Evaluate the PRE-STATED predicates in repro/ddsvm_v2.py against results/v2/.

Emits results/v2/analysis_v2.json plus three figures under results/v2/.
All predicates were fixed in the ddsvm_v2.py docstring before the run.
"""
from __future__ import annotations

import json
import os

import numpy as np
from scipy import stats

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
V2 = os.path.join(ROOT, "results", "v2")
DATASETS = ["moons-hard", "rings", "gauss-xor", "digits-3v8"]
METHODS = ["linear-svm", "rbf-svm", "deep-ce", "deep-svm", "ddsvm", "ddsvm-rand"]


def ci95(x):
    """Mean and 95% CI half-width via t-distribution."""
    x = np.asarray(x, dtype=float)
    x = x[np.isfinite(x)]
    n = len(x)
    if n < 2:
        return float(x.mean()) if n else float("nan"), float("nan"), n
    m = x.mean()
    hw = stats.t.ppf(0.975, n - 1) * x.std(ddof=1) / np.sqrt(n)
    return float(m), float(hw), n


def paired(a, b):
    """Paired difference a-b: mean, CI half-width, t-test p, wilcoxon p, n."""
    a, b = np.asarray(a, float), np.asarray(b, float)
    d = a - b
    m, hw, n = ci95(d)
    tp = float(stats.ttest_rel(a, b).pvalue)
    try:
        wp = float(stats.wilcoxon(a, b).pvalue)
    except Exception:
        wp = float("nan")
    return dict(mean=m, ci_hw=hw, lo=m - hw, hi=m + hw, p_ttest=tp,
                p_wilcoxon=wp, n=n, ci_excludes_zero=bool((m - hw) * (m + hw) > 0))


def load(ds):
    with open(os.path.join(V2, f"ddsvm_v2_{ds}.json")) as f:
        return json.load(f)


def main():
    data = {ds: load(ds) for ds in DATASETS}
    out = {"datasets": {}, "integrity": {}, "claims": {}}

    # ------------------------------------------------------- run integrity --
    for ds in DATASETS:
        integ = data[ds]["integrity"]
        runs = data[ds]["runs"]
        # extra check: cycle traces genuinely differ across seeds
        first_cycle_hinge = [r["ddsvm"]["cycles"][0]["hinge_end"] for r in runs]
        out["integrity"][ds] = dict(
            n_seeds=integ["n_seeds"],
            unique_data_hashes=integ["n_unique_data_hashes"],
            all_hashes_distinct=integ["all_hashes_distinct"],
            acc_std_by_method=integ["acc_std_by_method"],
            unique_cycle1_hinge=len(set(round(v, 10) for v in first_cycle_hinge)),
            wall_time_sec=data[ds]["wall_time_sec"],
            n_train=runs[0]["n_train"], n_test=runs[0]["n_test"], d=runs[0]["d"])

    # ------------------------------------------------- accuracy comparison --
    for ds in DATASETS:
        runs = data[ds]["runs"]
        acc = {m: [r[m]["acc"] for r in runs] for m in METHODS}
        row = {"acc": {}}
        for m in METHODS:
            mu, hw, n = ci95(acc[m])
            row["acc"][m] = dict(mean=mu, ci_hw=hw, lo=mu - hw, hi=mu + hw, n=n)
        row["paired"] = {
            "ddsvm_vs_deep-ce": paired(acc["ddsvm"], acc["deep-ce"]),
            "ddsvm_vs_deep-svm": paired(acc["ddsvm"], acc["deep-svm"]),
            "ddsvm_vs_rbf-svm": paired(acc["ddsvm"], acc["rbf-svm"]),
            "ddsvm_vs_linear-svm": paired(acc["ddsvm"], acc["linear-svm"]),
            "ddsvm_vs_ddsvm-rand": paired(acc["ddsvm"], acc["ddsvm-rand"]),
        }
        # normalized test margin (ddsvm vs deep-svm vs ddsvm-rand)
        nm = {m: [r[m]["test_norm_margin_mean"] for r in runs]
              for m in ["deep-svm", "ddsvm", "ddsvm-rand"]}
        row["test_norm_margin"] = {m: dict(zip(("mean", "ci_hw", "n"), ci95(v)))
                                   for m, v in nm.items()}
        row["margin_paired"] = {
            "ddsvm_vs_deep-svm": paired(nm["ddsvm"], nm["deep-svm"]),
            "ddsvm_vs_ddsvm-rand": paired(nm["ddsvm"], nm["ddsvm-rand"]),
        }
        out["datasets"][ds] = row

    # ================================================================ CLAIM 1 =
    c1 = {"per_dataset": {}, "checks": []}
    for ds in DATASETS:
        runs = data[ds]["runs"]
        # P1a structural: head frozen in A, features frozen in B, C displaces
        head_drift = [c["head_drift_A"] for r in runs for c in r["ddsvm"]["cycles"]]
        feat_drift = [c["feat_drift_B"] for r in runs for c in r["ddsvm"]["cycles"]]
        feat_move_A = [c["feat_move_A"] for r in runs for c in r["ddsvm"]["cycles"]]
        head_move_B = [c["head_move_B"] for r in runs for c in r["ddsvm"]["cycles"]]
        disp_C = [c["mean_disp_C"] for r in runs for c in r["ddsvm"]["cycles"]]
        p1a = dict(
            max_head_drift_in_phaseA=float(np.max(head_drift)),
            max_feat_drift_in_phaseB=float(np.max(feat_drift)),
            min_feat_move_in_phaseA=float(np.min(feat_move_A)),
            min_head_move_in_phaseB=float(np.min(head_move_B)),
            min_mean_disp_phaseC=float(np.min(disp_C)),
            n_cycle_observations=len(disp_C))
        p1a["pass"] = bool(p1a["max_head_drift_in_phaseA"] == 0.0
                           and p1a["max_feat_drift_in_phaseB"] == 0.0
                           and p1a["min_feat_move_in_phaseA"] > 0
                           and p1a["min_head_move_in_phaseB"] > 0
                           and p1a["min_mean_disp_phaseC"] > 0)

        # P1b convergence: hinge ratio cycle15/cycle1 per seed
        ratios, finals = [], []
        for r in runs:
            cy = r["ddsvm"]["cycles"]
            h1, h15 = cy[0]["hinge_end"], cy[-1]["hinge_end"]
            ratios.append(h15 / max(h1, 1e-300))
            finals.append(h15)
        rm, rhw, rn = ci95(ratios)
        fm, fhw, fn = ci95(finals)
        p1b = dict(ratio_mean=rm, ratio_ci_hw=rhw, ratio_lo=rm - rhw,
                   ratio_hi=rm + rhw, final_hinge_mean=fm, final_hinge_ci_hw=fhw,
                   n=rn)
        p1b["pass"] = bool((rm + rhw) <= 0.5 and fm > 1e-8)

        # P1c trend: OLS log(mean hinge) vs cycle, on the seed-averaged curve
        curve = np.array([[c["hinge_end"] for c in r["ddsvm"]["cycles"]]
                          for r in runs])          # (seeds, cycles)
        mean_curve = curve.mean(0)
        cycles = np.arange(1, len(mean_curve) + 1)
        lr = stats.linregress(cycles, np.log(np.maximum(mean_curve, 1e-300)))
        # Degeneracy guard: if the mean hinge reaches EXACTLY 0 the log-fit is an
        # artifact of the 1e-300 floor, not a rate. Report it, never quote it.
        n_exact_zero = int((curve == 0).sum())
        degenerate = bool((mean_curve == 0).any())
        p1c = dict(slope=float(lr.slope), intercept=float(lr.intercept),
                   r2=float(lr.rvalue ** 2), p_value=float(lr.pvalue),
                   degenerate_exact_zero_hinge=degenerate,
                   n_exact_zero_observations=n_exact_zero,
                   n_observations=int(curve.size),
                   first_zero_cycle=(int(np.argmax(mean_curve == 0)) + 1
                                     if degenerate else None),
                   slope_is_floor_artifact=degenerate,
                   mean_curve=[float(v) for v in mean_curve],
                   curve_ci_hw=[float(ci95(curve[:, j])[1])
                                for j in range(curve.shape[1])])
        # A degenerate curve cannot pass a trend predicate.
        p1c["pass"] = bool(not degenerate
                           and lr.slope <= -0.05 and lr.rvalue ** 2 >= 0.70)

        c1["per_dataset"][ds] = dict(P1a=p1a, P1b=p1b, P1c=p1c)
        c1["checks"] += [p1a["pass"], p1b["pass"], p1c["pass"]]

    n_pass1 = sum(c1["checks"])
    all_p1a = all(c1["per_dataset"][d]["P1a"]["pass"] for d in DATASETS)
    c1["n_pass"] = int(n_pass1)
    c1["n_checks"] = len(c1["checks"])
    c1["verdict"] = ("VERIFIED" if (n_pass1 >= 10 and all_p1a)
                     else "PARTIAL" if n_pass1 >= 6 else "NOT REPRODUCED")

    # ================================================================ CLAIM 2 =
    c2 = {"per_dataset": {}, "checks": []}
    for ds in DATASETS:
        runs = data[ds]["runs"]
        # P2a cosine alignment (per-seed mean over cycles)
        cos_seed = [float(np.nanmean([c["cos_active"] for c in r["ddsvm"]["cycles"]]))
                    for r in runs]
        cm, chw, cn = ci95(cos_seed)
        p2a = dict(mean=cm, ci_hw=chw, lo=cm - chw, hi=cm + chw, n=cn)
        p2a["pass"] = bool((cm - chw) >= 0.5)

        # P2b per-cycle active-set margin change
        dact = [float(np.nanmean([c["delta_margin_active"]
                                  for c in r["ddsvm"]["cycles"]])) for r in runs]
        dm, dhw, dn = ci95(dact)
        p2b = dict(mean=dm, ci_hw=dhw, lo=dm - dhw, hi=dm + dhw, n=dn)
        p2b["pass"] = bool(dm > 0 and (dm - dhw) > 0)

        # P2c end-to-end margin growth + violator reduction
        growth, viol_down = [], []
        for r in runs:
            cy = r["ddsvm"]["cycles"]
            growth.append(cy[-1]["margin_post_mean"] - cy[0]["margin_pre_mean"])
            viol_down.append(cy[-1]["viol_post"] < cy[0]["viol_pre"])
        gm, ghw, gn = ci95(growth)
        frac_down = float(np.mean(viol_down))
        p2c = dict(growth_mean=gm, growth_ci_hw=ghw, growth_lo=gm - ghw,
                   growth_hi=gm + ghw, frac_seeds_violators_down=frac_down, n=gn)
        p2c["pass"] = bool(gm > 0 and (gm - ghw) > 0 and frac_down >= 0.80)

        # P2d geometry-aware ablation on test normalized margin
        nm_d = [r["ddsvm"]["test_norm_margin_mean"] for r in runs]
        nm_r = [r["ddsvm-rand"]["test_norm_margin_mean"] for r in runs]
        pr = paired(nm_d, nm_r)
        p2d = dict(**pr)
        p2d["pass"] = bool(pr["mean"] > 0 and pr["lo"] > 0)

        # diagnostic: cosine for the random-push control (should be ~0)
        cos_rand = [float(np.nanmean([c["cos_active"]
                                      for c in r["ddsvm-rand"]["cycles"]]))
                    for r in runs]
        rm_, rhw_, _ = ci95(cos_rand)
        p2a["control_random_push_cos_mean"] = rm_
        p2a["control_random_push_cos_ci_hw"] = rhw_
        # POST-HOC DIAGNOSTIC (declared after seeing P2a; explicitly NOT part of
        # any verdict). Does the boundary-normal push align better than a random
        # push? This measures whether the mechanism executes as described, which
        # is separate from whether it helps (P2d).
        p2a["posthoc_cos_vs_random_paired"] = paired(cos_seed, cos_rand)

        # active-set (support-vector) fraction trace
        af = np.array([[c["active_frac"] for c in r["ddsvm"]["cycles"]]
                       for r in runs])
        vio = np.array([[c["viol_post"] for c in r["ddsvm"]["cycles"]]
                        for r in runs], dtype=float)
        c2["per_dataset"][ds] = dict(
            P2a=p2a, P2b=p2b, P2c=p2c, P2d=p2d,
            active_frac_curve=[float(v) for v in af.mean(0)],
            active_frac_ci=[float(ci95(af[:, j])[1]) for j in range(af.shape[1])],
            violators_curve=[float(v) for v in vio.mean(0)],
            margin_pre_curve=[float(np.mean([r["ddsvm"]["cycles"][j]["margin_pre_mean"]
                                             for r in runs])) for j in range(15)],
            margin_post_curve=[float(np.mean([r["ddsvm"]["cycles"][j]["margin_post_mean"]
                                              for r in runs])) for j in range(15)])
        c2["checks"] += [p2a["pass"], p2b["pass"], p2c["pass"], p2d["pass"]]

    n_pass2 = sum(c2["checks"])
    c2["n_pass"] = int(n_pass2)
    c2["n_checks"] = len(c2["checks"])
    c2["verdict"] = ("VERIFIED" if n_pass2 >= 13
                     else "PARTIAL" if n_pass2 >= 8 else "NOT REPRODUCED")

    # ================================================================ CLAIM 3 =
    c3 = {"per_dataset": {}}
    n_both_better = 0
    n_regress = 0
    n_any_better = 0
    for ds in DATASETS:
        pr = out["datasets"][ds]["paired"]
        ce = pr["ddsvm_vs_deep-ce"]
        sv = pr["ddsvm_vs_deep-svm"]
        both = bool(ce["mean"] > 0 and ce["lo"] > 0 and sv["mean"] > 0 and sv["lo"] > 0)
        any_b = bool((ce["mean"] > 0 and ce["lo"] > 0)
                     or (sv["mean"] > 0 and sv["lo"] > 0))
        regress = bool(ce["hi"] < 0 or sv["hi"] < 0)
        n_both_better += both
        n_any_better += any_b
        n_regress += regress
        c3["per_dataset"][ds] = dict(both_baselines_beaten=both,
                                     any_baseline_beaten=any_b,
                                     regression=regress,
                                     vs_deep_ce=ce, vs_deep_svm=sv,
                                     vs_rbf_svm=pr["ddsvm_vs_rbf-svm"],
                                     vs_linear_svm=pr["ddsvm_vs_linear-svm"])
    c3["n_datasets_both_baselines_beaten"] = int(n_both_better)
    c3["n_datasets_any_baseline_beaten"] = int(n_any_better)
    c3["n_datasets_regression"] = int(n_regress)
    if n_both_better >= 2 and n_regress == 0:
        c3["verdict"] = "VERIFIED"
    elif n_any_better == 0 or n_regress >= 2:
        c3["verdict"] = "NOT REPRODUCED"
    else:
        c3["verdict"] = "PARTIAL"

    out["claims"] = {"claim1": c1, "claim2": c2, "claim3": c3}

    with open(os.path.join(V2, "analysis_v2.json"), "w") as f:
        json.dump(out, f, indent=2)

    # ------------------------------------------------------------- figures --
    _fig_claim1(out, c1)
    _fig_claim2(out, c2)
    _fig_claim2b(out, c2)
    _fig_claim3(out)

    # ---------------------------------------------------------- console -----
    print("=" * 72)
    for k, c in out["claims"].items():
        print(f"{k}: {c['verdict']}"
              + (f"  ({c.get('n_pass')}/{c.get('n_checks')} checks)"
                 if "n_pass" in c else ""))
    print("=" * 72)
    for ds in DATASETS:
        a = out["datasets"][ds]["acc"]
        print(f"\n{ds}  (n_train={out['integrity'][ds]['n_train']}, "
              f"d={out['integrity'][ds]['d']}, "
              f"{out['integrity'][ds]['unique_data_hashes']}/25 unique hashes)")
        for m in METHODS:
            print(f"   {m:12s} {a[m]['mean']*100:6.2f} +/- {a[m]['ci_hw']*100:.2f}")
        for k, v in out["datasets"][ds]["paired"].items():
            print(f"   {k:24s} d={v['mean']*100:+6.3f} "
                  f"[{v['lo']*100:+.3f},{v['hi']*100:+.3f}] p={v['p_ttest']:.4f}")
    print("\nCLAIM 1 per-dataset:")
    for ds in DATASETS:
        d = c1["per_dataset"][ds]
        print(f"  {ds:12s} P1a={d['P1a']['pass']} P1b={d['P1b']['pass']}"
              f"(ratio {d['P1b']['ratio_mean']:.4f}+/-{d['P1b']['ratio_ci_hw']:.4f}) "
              f"P1c={d['P1c']['pass']}(slope {d['P1c']['slope']:.4f}, "
              f"R2={d['P1c']['r2']:.4f})")
    print("\nCLAIM 2 per-dataset:")
    for ds in DATASETS:
        d = c2["per_dataset"][ds]
        print(f"  {ds:12s} P2a={d['P2a']['pass']}(cos {d['P2a']['mean']:.4f}"
              f"+/-{d['P2a']['ci_hw']:.4f}; rand-ctrl "
              f"{d['P2a']['control_random_push_cos_mean']:.4f}) "
              f"P2b={d['P2b']['pass']}({d['P2b']['mean']:+.4f}) "
              f"P2c={d['P2c']['pass']}({d['P2c']['growth_mean']:+.3f}, "
              f"viol_down {d['P2c']['frac_seeds_violators_down']:.2f}) "
              f"P2d={d['P2d']['pass']}({d['P2d']['mean']:+.5f} "
              f"[{d['P2d']['lo']:+.5f},{d['P2d']['hi']:+.5f}])")
    print(f"\nwritten -> {os.path.join(V2, 'analysis_v2.json')}")


def _fig_claim1(out, c1):
    fig, axes = plt.subplots(1, 4, figsize=(18, 4.2))
    for ax, ds in zip(axes, DATASETS):
        d = c1["per_dataset"][ds]["P1c"]
        y = np.array(d["mean_curve"])
        e = np.array(d["curve_ci_hw"])
        x = np.arange(1, len(y) + 1)
        ax.errorbar(x, y, yerr=e, marker="o", ms=4, lw=1.6, capsize=3,
                    color="#1f77b4", label="mean train hinge (95% CI, 25 seeds)")
        if d["degenerate_exact_zero_hinge"]:
            # Hinge hits EXACTLY 0 -> a log fit is a floor artifact. Plot linear
            # and refuse to draw the fitted line at all.
            ax.axvline(d["first_zero_cycle"], color="#d62728", ls="--", lw=1.8,
                       label=f"hinge = 0 exactly from cycle {d['first_zero_cycle']}")
            ax.set_ylim(bottom=-0.01)
            ax.set_title(f"{ds}  [P1c FAIL - DEGENERATE]\n"
                         f"train set separated; log-slope "
                         f"({d['slope']:.1f}) is a 1e-300 floor artifact, not a rate",
                         fontsize=9)
        else:
            fit = np.exp(d["intercept"] + d["slope"] * x)
            ax.plot(x, fit, "--", color="#d62728", lw=1.8,
                    label=f"OLS log-fit: slope={d['slope']:.3f}, R2={d['r2']:.3f}")
            ax.set_yscale("log")
            ax.set_title(f"{ds}  [P1c {'PASS' if d['pass'] else 'FAIL'}]")
        ax.set_xlabel("alternating cycle")
        ax.set_ylabel("end-of-cycle train hinge loss")
        ax.grid(alpha=0.35, ls="--")
        ax.legend(fontsize=7.5)
    fig.suptitle("Claim 1: DDSVM alternating-cycle convergence, 25 seeds/dataset, "
                 "OLS trend on log(mean hinge) vs cycle", fontsize=11)
    fig.tight_layout()
    fig.savefig(os.path.join(V2, "claim1_convergence.png"), dpi=110)
    plt.close(fig)


def _fig_claim2(out, c2):
    fig, axes = plt.subplots(2, 4, figsize=(18, 8))
    for j, ds in enumerate(DATASETS):
        d = c2["per_dataset"][ds]
        x = np.arange(1, 16)
        ax = axes[0, j]
        ax.plot(x, d["margin_pre_curve"], "o--", ms=4, label="pre-refinement mean margin")
        ax.plot(x, d["margin_post_curve"], "s-", ms=4, label="post-refinement mean margin")
        ax.axhline(1.0, color="r", ls=":", lw=1.2, label="target margin = 1.0")
        ax.set_xlabel("cycle")
        ax.set_ylabel("train geometric margin")
        ax.set_title(f"{ds}: margin per cycle")
        ax.grid(alpha=0.35, ls="--")
        ax.legend(fontsize=7.5)
        ax = axes[1, j]
        ax.plot(x, np.array(d["active_frac_curve"]) * 100, "^-", ms=4,
                color="#9467bd", label="active set (gamma<1) % of train")
        ax.set_xlabel("cycle")
        ax.set_ylabel("active / support fraction (%)")
        ax.set_title(f"{ds}: support set shrinks\n"
                     f"cos(dz, y*n)={d['P2a']['mean']:.3f} vs random ctrl "
                     f"{d['P2a']['control_random_push_cos_mean']:.3f}", fontsize=9)
        ax.grid(alpha=0.35, ls="--")
        ax.legend(fontsize=7.5)
    fig.suptitle("Claim 2: geometry-aware push -- margin growth, support-vector "
                 "(active-set) shrinkage, and direction alignment (25 seeds)",
                 fontsize=11)
    fig.tight_layout()
    fig.savefig(os.path.join(V2, "claim2_geometry.png"), dpi=110)
    plt.close(fig)


def _fig_claim2b(out, c2):
    """The ablation that makes 'geometry-aware' falsifiable, in two panels:
    (left) does the push follow the boundary normal?  (right) does it help?"""
    fig, axes = plt.subplots(1, 2, figsize=(13, 4.8))
    x = np.arange(len(DATASETS))
    w = 0.36

    ax = axes[0]
    cn = [c2["per_dataset"][d]["P2a"]["mean"] for d in DATASETS]
    cne = [c2["per_dataset"][d]["P2a"]["ci_hw"] for d in DATASETS]
    cr = [c2["per_dataset"][d]["P2a"]["control_random_push_cos_mean"] for d in DATASETS]
    cre = [c2["per_dataset"][d]["P2a"]["control_random_push_cos_ci_hw"] for d in DATASETS]
    ax.bar(x - w / 2, cn, w, yerr=cne, capsize=4, color="#2ca02c",
           label="push along boundary normal n = w/||w||")
    ax.bar(x + w / 2, cr, w, yerr=cre, capsize=4, color="#c5b0d5",
           label="control: push along a random unit vector")
    ax.axhline(0.5, color="r", ls="--", lw=1.5,
               label="pre-stated P2a threshold (CI lower bound >= 0.5)")
    ax.set_xticks(x)
    ax.set_xticklabels(DATASETS, rotation=15, fontsize=9)
    ax.set_ylabel("cos(achieved displacement, y_i * n) on active set")
    ax.set_title("MECHANISM: the push does follow the normal\n"
                 "(every dataset beats its random control, p < 1e-7)", fontsize=10)
    ax.grid(axis="y", alpha=0.35, ls="--")
    ax.legend(fontsize=8)

    ax = axes[1]
    dm = [c2["per_dataset"][d]["P2d"]["mean"] for d in DATASETS]
    de = [c2["per_dataset"][d]["P2d"]["ci_hw"] for d in DATASETS]
    cols = ["#2ca02c" if m - e > 0 else "#d62728" if m + e < 0 else "#7f7f7f"
            for m, e in zip(dm, de)]
    ax.bar(x, dm, 0.55, yerr=de, capsize=5, color=cols)
    ax.axhline(0, color="k", lw=1.2)
    ax.set_xticks(x)
    ax.set_xticklabels(DATASETS, rotation=15, fontsize=9)
    ax.set_ylabel("paired dTest normalized margin (ddsvm - ddsvm-rand)")
    ax.set_title("EFFECT: but it buys no test margin over a random push\n"
                 "(all four 95% CIs straddle 0 -> P2d FAILS 4/4)", fontsize=10)
    ax.grid(axis="y", alpha=0.35, ls="--")

    fig.suptitle("Claim 2 ablation: geometry-aware push vs random-direction push, "
                 "25 paired seeds per dataset", fontsize=11)
    fig.tight_layout()
    fig.savefig(os.path.join(V2, "claim2b_ablation.png"), dpi=110)
    plt.close(fig)


def _fig_claim3(out):
    fig, axes = plt.subplots(1, 4, figsize=(18, 4.4))
    for ax, ds in zip(axes, DATASETS):
        a = out["datasets"][ds]["acc"]
        ms = METHODS
        mu = [a[m]["mean"] * 100 for m in ms]
        hw = [a[m]["ci_hw"] * 100 for m in ms]
        cols = ["#7f7f7f", "#8c564b", "#ff7f0e", "#1f77b4", "#2ca02c", "#c5b0d5"]
        ax.bar(range(len(ms)), mu, yerr=hw, capsize=4, color=cols)
        ax.set_xticks(range(len(ms)))
        ax.set_xticklabels(ms, rotation=35, ha="right", fontsize=8)
        ax.set_ylabel("test accuracy (%)")
        lo = min(m - h for m, h in zip(mu, hw))
        ax.set_ylim(max(0, lo - 5), 101)
        pr = out["datasets"][ds]["paired"]
        ax.set_title(f"{ds}\nddsvm-ce {pr['ddsvm_vs_deep-ce']['mean']*100:+.2f}pp, "
                     f"ddsvm-dsvm {pr['ddsvm_vs_deep-svm']['mean']*100:+.2f}pp",
                     fontsize=9)
        ax.grid(axis="y", alpha=0.35, ls="--")
    fig.suptitle("Claim 3: head-to-head test accuracy, mean +/- 95% CI over 25 "
                 "paired seeds per dataset", fontsize=11)
    fig.tight_layout()
    fig.savefig(os.path.join(V2, "claim3_baselines.png"), dpi=110)
    plt.close(fig)


if __name__ == "__main__":
    main()