eval/figures.py

#!/usr/bin/env python3
"""Build the paper's two figures and their underlying data from
`results/reproduced_headline.json` (or any headline JSON of the same schema).

    python3 eval/figures.py [--headline results/published/reproduced_headline.json]

Writes results/figure_headline.{png,pdf}, results/figure_causal.{png,pdf} and
results/figure_headline_data.csv — the numbers the headline figure plots, so the
figure is checkable without rendering it.
"""
import argparse
import csv
import json
from pathlib import Path

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt  # noqa: E402
import numpy as np  # noqa: E402

ROOT = Path(__file__).resolve().parent.parent
E1_ORDER = [("PNSR_K4_fin", "PNSR s1"), ("PNSR_K4_s2", "PNSR s2"),
            ("PNSR_K4_s3", "PNSR s3"), ("G_TAU512", "τ=512"),
            ("G_CONVEX_FREE", "gated free"), ("G_CONVEX_RETAIN", "gated retain"),
            ("RMT_s1", "RMT")]
HANDLES = [("J_self_lesion_SELF_REF", "exact store: J_self lesion", "#1b6f4a", "exact"),
           ("J_obs_lesion_pointer_families", "exact store: J_obs lesion", "#3aa06b", "exact"),
           ("delta_cross_lifetime_swap", "learned state: cross-lifetime swap", "#7b2d5e", "learned"),
           ("delta_reset64", "learned state: reset every 64", "#b8547f", "learned"),
           ("delta_state_zero", "learned state: zero + freeze writes", "#d98cae", "learned")]


def load(path):
    return json.loads(Path(path).read_text())


def headline(H, out):
    e1m = H["experiment1_generic_recurrent_state"]["models"]
    e2 = H["experiment2_protected_bindings"]
    e1 = [(lab, e1m[k]["learned_state_causal"]["delta_cross_lifetime_swap"])
          for k, lab in E1_ORDER if k in e1m]
    unb = [(f"unbound s{s}", e2["capacity_matched_unbound"][f"seed{s}"]["delta_payload_swap"])
           for s in (1, 2, 3) if f"seed{s}" in e2.get("capacity_matched_unbound", {})]
    bnd = [(f"bind s{s}", e2["bind"][f"seed{s}"]["delta_payload_swap"])
           for s in (1, 2, 3) if f"seed{s}" in e2.get("bind", {})]

    with open(out / "figure_headline_data.csv", "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["group", "label", "delta_cross_lifetime_swap"])
        for g, items in (("experiment1_unstructured_learned_state", e1),
                         ("experiment2_capacity_matched_unstructured_control", unb),
                         ("experiment2_binding_local_writes", bnd)):
            for lab, v in items:
                w.writerow([g, lab, v])

    fig, ax = plt.subplots(figsize=(12.5, 5.6), constrained_layout=True)
    groups = [("Experiment 1\nunstructured learned state\n(4 mechanisms, 3 seeds)", e1, "#8a8f98"),
              ("Experiment 2 control\ncapacity-matched,\nunstructured writes", unb, "#c8642a"),
              ("Experiment 2\nbinding-local writes", bnd, "#1b6f4a")]
    x, ticks, labels, centres = 0, [], [], []
    for gname, items, col in groups:
        xs = []
        for n, v in items:
            ax.bar(x, v, 0.72, color=col, edgecolor="black", linewidth=0.5)
            ticks.append(x)
            labels.append(n)
            xs.append(x)
            x += 1
        if xs:
            centres.append((float(np.mean(xs)), gname))
        x += 1.1
    ax.axhline(0, color="black", lw=0.9)
    ax.axhline(0.10, color="#1b6f4a", ls="--", lw=1.1,
               label="preregistered causal threshold (0.10)")
    ax.set_xticks(ticks)
    ax.set_xticklabels(labels, fontsize=9, rotation=35, ha="right")
    ax.set_ylabel("causal effect of the learned state\n(warm − cross-lifetime swap)", fontsize=11)
    ax.set_title("A learned recurrent state becomes causally load-bearing only when "
                 "entity bindings are structurally protected", fontsize=13, weight="bold")
    ax.legend(fontsize=9, loc="upper left")
    ax.grid(axis="y", alpha=0.25)
    ax.set_ylim(-0.02, 0.20)
    for cx, gname in centres:
        ax.annotate(gname, xy=(cx, -0.0165), xycoords=("data", "data"), ha="center",
                    fontsize=9.5, weight="bold", annotation_clip=False)
    fig.savefig(out / "figure_headline.png", dpi=200)
    fig.savefig(out / "figure_headline.pdf")
    plt.close(fig)
    print("  figure_headline: E1", [round(v, 4) for _, v in e1])
    print("                   unbound", [round(v, 4) for _, v in unb],
          "bind", [round(v, 4) for _, v in bnd])


def causal(H, out):
    m = H["experiment1_generic_recurrent_state"]["models"]
    runs = [(k, lab) for k, lab in E1_ORDER if k in m]
    fig, (a, b) = plt.subplots(1, 2, figsize=(14, 5.4), constrained_layout=True,
                               gridspec_kw={"width_ratios": [2.05, 1]})
    n, w = len(HANDLES), 0.16
    for i, (key, lab, col, _) in enumerate(HANDLES):
        xs = np.arange(len(runs)) + (i - (n - 1) / 2) * w
        ys = [(m[k].get("exact_store_lesions", {}) | m[k].get("learned_state_causal", {})
               ).get(key) or 0.0 for k, _ in runs]
        a.bar(xs, ys, w, label=lab, color=col, edgecolor="black", linewidth=0.4)
    a.axhline(0, color="black", lw=0.9)
    a.set_xticks(np.arange(len(runs)))
    a.set_xticklabels([lab for _, lab in runs], fontsize=9, rotation=20, ha="right")
    a.set_ylabel("causal effect size (warm − intervened accuracy)", fontsize=11)
    a.set_title("Exact stores are causally necessary; the learned state is not",
                fontsize=12, weight="bold")
    a.legend(fontsize=8.5, ncol=2)
    a.grid(axis="y", alpha=0.25)

    cos = [m[k].get("state_statistics", {}).get("mean_cross_lifetime_cosine") for k, _ in runs]
    nrm = [m[k].get("state_statistics", {}).get("max_slot_norm") for k, _ in runs]
    b.scatter(cos, nrm, s=90, c="#7b2d5e", edgecolor="black", zorder=3)
    for (k, lab), c, nn in zip(runs, cos, nrm):
        if c is not None and nn is not None:
            b.annotate(lab.replace("\n", " "), (c, nn), fontsize=8,
                       xytext=(4, 4), textcoords="offset points")
    b.set_yscale("log")
    b.set_xlabel("mean cross-lifetime cosine\n(is the state lifetime-specific?)", fontsize=10)
    b.set_ylabel("max slot norm (log)", fontsize=10)
    b.set_title("States differ enormously; none is causally used", fontsize=12, weight="bold")
    b.grid(alpha=0.25)
    fig.savefig(out / "figure_causal.png", dpi=200)
    fig.savefig(out / "figure_causal.pdf")
    plt.close(fig)
    print("  figure_causal: cosine", [c for c in cos], "norms", [nn for nn in nrm])


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--headline", default=str(ROOT / "results" / "published" /
                                              "reproduced_headline.json"))
    ap.add_argument("--outdir", default=str(ROOT / "results"))
    args = ap.parse_args()
    out = Path(args.outdir)
    out.mkdir(parents=True, exist_ok=True)
    H = load(args.headline)
    headline(H, out)
    causal(H, out)
    print("->", out)


if __name__ == "__main__":
    main()