File size: 4,788 Bytes
09af6cc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
"""Engram-Figure-4-style summary figure from exp/analysis/repr_summary.json.

Panel layout mirrors the Engram paper's Fig.4 (LogitLens convergence + CKA
alignment with best-match trajectory), plus our shuffled-delta control and the
draft-vs-target alignment:

  (a) LogitLens: own-slot accuracy vs draft layer, all-mask / soft / shuf.
      Dashed line = all-mask final-layer accuracy ("soft L1 即达到基线终点").
  (b) CKA(soft layer i, all-mask layer j) with per-row best-match trajectory
      (white) and diagonal reference — the engram-style "+1 layer" shift.
  (c) CKA(shuf, all-mask), same trajectory — identical shift with WRONG
      content: the geometric shift alone is not evidence of effective depth.
  (d) CKA(draft, target 36 layers) heatmap with best-match trajectories for
      all-mask (white) and soft (red): the soft INPUT aligns to target L22.

No GPU needed; reads only the JSON.
"""

from __future__ import annotations

import argparse
import json
from pathlib import Path

import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np

TENSOR_NAMES = ["emb_in", "L0", "L1", "L2", "L3", "L4", "norm_out"]


def cka_panel(ax, mat, title, traj_color="white", extra_traj=None):
    n = mat.shape[0]
    im = ax.imshow(mat, vmin=0, vmax=1, cmap="viridis", aspect="auto")
    best = mat.argmax(axis=1)
    ax.plot(best, np.arange(n), color=traj_color, marker="o", ms=5, lw=2,
            label="best match (row argmax)")
    ax.plot(np.arange(n), np.arange(n), "w--", lw=1, alpha=0.6, label="diagonal")
    if extra_traj is not None:
        ax.plot(extra_traj, np.arange(n), color="red", marker="s", ms=5, lw=2,
                label="soft best match")
    ax.set_xticks(range(n), TENSOR_NAMES, rotation=45, fontsize=8)
    ax.set_yticks(range(n), TENSOR_NAMES, fontsize=8)
    ax.set_xlabel("all-mask draft layer")
    ax.set_title(title, fontsize=10)
    for i in range(n):
        for j in range(n):
            ax.text(j, i, f"{mat[i, j]:.2f}", ha="center", va="center", fontsize=6,
                    color="w" if mat[i, j] < 0.55 else "k")
    ax.legend(fontsize=7, loc="upper left")
    return im


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--summary", default=str(Path(__file__).parent / "analysis" / "repr_summary.json"))
    ap.add_argument("--fig-dir", default=str(Path(__file__).parent / "figs"))
    args = ap.parse_args()
    s = json.load(open(args.summary))
    fig_dir = Path(args.fig_dir)
    fig_dir.mkdir(parents=True, exist_ok=True)

    lens = {c: np.array(s["lens_acc"][c]) for c in ("allmask", "soft", "shuf", "half")}
    cross_soft = np.array(s["cross_cka_soft"])
    cross_shuf = np.array(s["cross_cka_shuf"])
    d2t_all = np.array(s["d2t_allmask"])
    d2t_soft = np.array(s["d2t_soft"])

    fig, axes = plt.subplots(1, 4, figsize=(21, 4.6))

    # (a) LogitLens convergence
    ax = axes[0]
    styles = {"allmask": ("k--", "all-mask"), "soft": ("r-", "soft (oracle, slots 1-15)"),
              "shuf": ("g:", "shuffled control"), "half": ("b-.", "half (oracle, slots 1-8)")}
    for cond, (st, lab) in styles.items():
        acc = lens[cond][:, 1:].mean(axis=1)
        ax.plot(range(len(TENSOR_NAMES)), acc, st, marker="o", ms=5, label=lab)
    final_all = lens["allmask"][-1, 1:].mean()
    ax.axhline(final_all, color="k", ls=":", lw=1, alpha=0.6)
    ax.annotate(f"all-mask final = {final_all:.3f}", (0.4, final_all + 0.002), fontsize=8)
    ax.set_xticks(range(len(TENSOR_NAMES)), TENSOR_NAMES, rotation=45, fontsize=8)
    ax.set_xlabel("draft layer")
    ax.set_ylabel("own-slot top-1 acc (slots 1-15)")
    ax.set_title("(a) LogitLens: faster convergence", fontsize=10)
    ax.grid(alpha=0.3)
    ax.legend(fontsize=8)

    # (b)(c) draft-draft CKA with best-match trajectory
    cka_panel(axes[1], cross_soft, "(b) CKA: soft → all-mask  (rows=soft)")
    cka_panel(axes[2], cross_shuf, "(c) CKA: shuffled → all-mask (control)")

    # (d) draft-target CKA with two trajectories
    ax = axes[3]
    im = ax.imshow(d2t_soft, vmin=0, vmax=0.6, cmap="viridis", aspect="auto")
    ax.plot(d2t_all.argmax(axis=1), np.arange(7), "w-o", ms=5, lw=2, label="all-mask best match")
    ax.plot(d2t_soft.argmax(axis=1), np.arange(7), "r-s", ms=5, lw=2, label="soft best match")
    ax.set_xticks(range(0, 37, 3), [str(i) for i in range(0, 37, 3)], fontsize=8)
    ax.set_yticks(range(7), TENSOR_NAMES, fontsize=8)
    ax.set_xlabel("target layer")
    ax.set_title("(d) CKA: draft → target layers (heatmap = soft)", fontsize=10)
    ax.legend(fontsize=7, loc="upper left")

    fig.tight_layout()
    out = fig_dir / "fig4_engram_style.png"
    fig.savefig(out, dpi=200)
    plt.close(fig)
    print(f"saved {out}")


if __name__ == "__main__":
    main()