File size: 4,788 Bytes
09af6cc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 | """Engram-Figure-4-style summary figure from exp/analysis/repr_summary.json.
Panel layout mirrors the Engram paper's Fig.4 (LogitLens convergence + CKA
alignment with best-match trajectory), plus our shuffled-delta control and the
draft-vs-target alignment:
(a) LogitLens: own-slot accuracy vs draft layer, all-mask / soft / shuf.
Dashed line = all-mask final-layer accuracy ("soft L1 即达到基线终点").
(b) CKA(soft layer i, all-mask layer j) with per-row best-match trajectory
(white) and diagonal reference — the engram-style "+1 layer" shift.
(c) CKA(shuf, all-mask), same trajectory — identical shift with WRONG
content: the geometric shift alone is not evidence of effective depth.
(d) CKA(draft, target 36 layers) heatmap with best-match trajectories for
all-mask (white) and soft (red): the soft INPUT aligns to target L22.
No GPU needed; reads only the JSON.
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
TENSOR_NAMES = ["emb_in", "L0", "L1", "L2", "L3", "L4", "norm_out"]
def cka_panel(ax, mat, title, traj_color="white", extra_traj=None):
n = mat.shape[0]
im = ax.imshow(mat, vmin=0, vmax=1, cmap="viridis", aspect="auto")
best = mat.argmax(axis=1)
ax.plot(best, np.arange(n), color=traj_color, marker="o", ms=5, lw=2,
label="best match (row argmax)")
ax.plot(np.arange(n), np.arange(n), "w--", lw=1, alpha=0.6, label="diagonal")
if extra_traj is not None:
ax.plot(extra_traj, np.arange(n), color="red", marker="s", ms=5, lw=2,
label="soft best match")
ax.set_xticks(range(n), TENSOR_NAMES, rotation=45, fontsize=8)
ax.set_yticks(range(n), TENSOR_NAMES, fontsize=8)
ax.set_xlabel("all-mask draft layer")
ax.set_title(title, fontsize=10)
for i in range(n):
for j in range(n):
ax.text(j, i, f"{mat[i, j]:.2f}", ha="center", va="center", fontsize=6,
color="w" if mat[i, j] < 0.55 else "k")
ax.legend(fontsize=7, loc="upper left")
return im
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--summary", default=str(Path(__file__).parent / "analysis" / "repr_summary.json"))
ap.add_argument("--fig-dir", default=str(Path(__file__).parent / "figs"))
args = ap.parse_args()
s = json.load(open(args.summary))
fig_dir = Path(args.fig_dir)
fig_dir.mkdir(parents=True, exist_ok=True)
lens = {c: np.array(s["lens_acc"][c]) for c in ("allmask", "soft", "shuf", "half")}
cross_soft = np.array(s["cross_cka_soft"])
cross_shuf = np.array(s["cross_cka_shuf"])
d2t_all = np.array(s["d2t_allmask"])
d2t_soft = np.array(s["d2t_soft"])
fig, axes = plt.subplots(1, 4, figsize=(21, 4.6))
# (a) LogitLens convergence
ax = axes[0]
styles = {"allmask": ("k--", "all-mask"), "soft": ("r-", "soft (oracle, slots 1-15)"),
"shuf": ("g:", "shuffled control"), "half": ("b-.", "half (oracle, slots 1-8)")}
for cond, (st, lab) in styles.items():
acc = lens[cond][:, 1:].mean(axis=1)
ax.plot(range(len(TENSOR_NAMES)), acc, st, marker="o", ms=5, label=lab)
final_all = lens["allmask"][-1, 1:].mean()
ax.axhline(final_all, color="k", ls=":", lw=1, alpha=0.6)
ax.annotate(f"all-mask final = {final_all:.3f}", (0.4, final_all + 0.002), fontsize=8)
ax.set_xticks(range(len(TENSOR_NAMES)), TENSOR_NAMES, rotation=45, fontsize=8)
ax.set_xlabel("draft layer")
ax.set_ylabel("own-slot top-1 acc (slots 1-15)")
ax.set_title("(a) LogitLens: faster convergence", fontsize=10)
ax.grid(alpha=0.3)
ax.legend(fontsize=8)
# (b)(c) draft-draft CKA with best-match trajectory
cka_panel(axes[1], cross_soft, "(b) CKA: soft → all-mask (rows=soft)")
cka_panel(axes[2], cross_shuf, "(c) CKA: shuffled → all-mask (control)")
# (d) draft-target CKA with two trajectories
ax = axes[3]
im = ax.imshow(d2t_soft, vmin=0, vmax=0.6, cmap="viridis", aspect="auto")
ax.plot(d2t_all.argmax(axis=1), np.arange(7), "w-o", ms=5, lw=2, label="all-mask best match")
ax.plot(d2t_soft.argmax(axis=1), np.arange(7), "r-s", ms=5, lw=2, label="soft best match")
ax.set_xticks(range(0, 37, 3), [str(i) for i in range(0, 37, 3)], fontsize=8)
ax.set_yticks(range(7), TENSOR_NAMES, fontsize=8)
ax.set_xlabel("target layer")
ax.set_title("(d) CKA: draft → target layers (heatmap = soft)", fontsize=10)
ax.legend(fontsize=7, loc="upper left")
fig.tight_layout()
out = fig_dir / "fig4_engram_style.png"
fig.savefig(out, dpi=200)
plt.close(fig)
print(f"saved {out}")
if __name__ == "__main__":
main()
|