File size: 8,622 Bytes
788ee34
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33dbd59
788ee34
 
 
 
 
33dbd59
 
 
788ee34
 
33dbd59
 
 
 
 
 
788ee34
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f473997
 
 
 
788ee34
 
 
 
 
 
 
 
884986c
788ee34
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
"""The latent bridge: TinyChess trajectory -> continuous Gemma prefix.

Two stages (per the brief):
  1. LatentBridge (baseline): per-step linear projections of a few pooled
     signals, preserving step ordering and slot identity where cheap.
  2. ResamplerBridge: learned queries cross-attend over the full [64+d_mem]
     token set per step (Perceiver-style), k queries per step.

Both emit [B, P, 640] already multiplied by Gemma's embedding scale
sqrt(hidden_size) because inputs_embeds BYPASSES Gemma's internal
embedding scaling (verified in transformers 5.16.1 - see
gemma_decoder_investigation.md).

Inputs consumed per step (all from capture_with_hooks, all detacheable):
  h [B,64,d], h_prev [B,64,d], router [B,n_ops], p_halt [B],
  mem_attn [B,64,n_mem], memory [B,n_mem,d]
Defaults keep it small: 8 queries per step in the resampler.

The 4 metadata channels (value/margin/halt/n_moves) are engineered
observables, encoded through a tiny MLP into 2 extra tokens per step.
"""
from __future__ import annotations
import math
import torch
import torch.nn as nn
import torch.nn.functional as F

GEMMA_HIDDEN = 640          # gemma-3-270m text_config.hidden_size (verified)
GEMMA_SCALE = GEMMA_HIDDEN ** 0.5


class StepDescriptor(nn.Module):
    """Per-step engineered observable vector fed to both bridge variants.

    All quantities measured from the trace, cheap, and informative:
      ||dh|| (mean+max over squares), dh concentration (top-square share),
      ||dh|| momentum (vs previous delta), router entropy,
      router top-1 identity onehot, halting probability, aggregate ||dh||,
      memory attention entropy, memory attention top-slot share.
    Order fixed by FEATURE_KEYS.
    """
    FEATURE_KEYS = ["dh_norm_mean", "dh_norm_max", "dh_top_share",
                    "dh_momentum", "router_entropy", "router_toponehot",
                    "p_halt", "memat_entropy", "memat_top_share"]

    def __init__(self, n_ops=4, n_mem=8):
        super().__init__()
        self.n_ops, self.n_mem = n_ops, n_mem
        # feature count: dn_mean, dn_max, top_share, momentum (4) +
        # router entropy (1) + top-1 onehot (n_ops) + p_halt (1) +
        # memat entropy (1) + memat top-share (1)
        self.d = 8 + n_ops

    def forward(self, s: dict) -> torch.Tensor:
        h, hp = s["h"], s.get("h_prev", None)
        B = h.shape[0]
        dev = h.device
        feats = []
        if hp is not None:
            dh = (h - hp).abs().mean(-1)                 # [B,64]
            dn_mean = dh.mean(-1, keepdim=True)
            dn_max = dh.amax(-1, keepdim=True)
            top_share = ((dh ** 2).sum(-1, keepdim=True)
                         / (dn_mean ** 2 * 64 + 1e-6)).clamp(0, 1)
            prev_mean = hp.abs().mean(-1).mean(-1, keepdim=True) * 0.5  # no t-2 delta: neutral prior
            moment = dn_mean / (prev_mean + 1e-6)
        else:
            abs_mean = h.abs().mean(-1)                   # [B,64]
            dn_mean = abs_mean.mean(-1, keepdim=True)     # magnitude proxy at t=0
            dn_max = abs_mean.amax(-1, keepdim=True)
            top_share = ((abs_mean ** 2).sum(-1, keepdim=True)
                         / (dn_mean ** 2 * 64 + 1e-6)).clamp(0, 1)
            moment = torch.ones(B, 1, device=dev)         # neutral at t=0
        feats += [dn_mean, dn_max, top_share, moment]
        r = s.get("router")
        if r is not None and r.shape[-1] > 1:
            ent = -(r * (r + 1e-9).log()).sum(-1, keepdim=True)
            top = F.one_hot(r.argmax(-1), r.shape[-1]).float()
            feats += [ent, top]
        else:
            feats += [torch.zeros(B, 1, device=h.device),
                      torch.zeros(B, self.n_ops, device=h.device)]
        feats.append(s["p_halt"][:, None] if s.get("p_halt") is not None
                     else torch.zeros(B, 1, device=h.device))
        ma = s.get("mem_attn")
        if ma is not None:
            ment = -(ma * (ma + 1e-9).log()).sum(-1)          # [B,64]
            share = ma.amax(-1)                               # [B,64] -> use mean of max
            feats += [ment.mean(-1, keepdim=True), share.mean(-1, keepdim=True)]
        else:
            feats += [torch.zeros(B, 1, device=h.device), torch.zeros(B, 1, device=h.device)]
        return torch.cat(feats, -1)                          # [B, d_desc]


class LatentBridge(nn.Module):
    """Baseline: per-step linear mapping [per-step summary] -> k tokens."""

    def __init__(self, d_model=64, n_mem=8, n_ops=4, tokens_per_step=4):
        super().__init__()
        self.desc = StepDescriptor(n_ops, n_mem)
        d_in = self.desc.d
        self.proj = nn.Sequential(nn.Linear(d_in, 256), nn.GELU(),
                                  nn.Linear(256, tokens_per_step * GEMMA_HIDDEN))
        self.tps = tokens_per_step
        # learned BOS-anchored opener helps Gemma treat the prefix as content
        self.opener = nn.Parameter(torch.randn(1, GEMMA_HIDDEN) * 0.02)

    @staticmethod
    def _detach_step(s: dict) -> dict:
        """One-way boundary: the bridge consumes DETACHED latents only.
        Gradients can never flow back through the bridge into the chess
        model, by construction (mirrors thought.py's assert pattern)."""
        return {k: (v.detach() if torch.is_tensor(v) else v) for k, v in s.items()}

    def forward(self, cap: dict) -> torch.Tensor:
        B = cap["steps"][0]["h"].shape[0]
        toks = []
        toks.append(self.opener.expand(B, 1, -1))
        for s in cap["steps"]:
            v = self.desc(self._detach_step(s))                # [B, d_desc]
            e = self.proj(v).view(B, self.tps, GEMMA_HIDDEN)
            toks.append(e)
        return torch.cat(toks, 1) * GEMMA_SCALE               # [B, P, 640]


class ResamplerBridge(nn.Module):
    """Perceiver-style: k learned queries attend over ALL tokens of a step
    (64 square states + 8 memory slots + descriptor) preserving square and
    slot identity inside attention, positional square identities added."""

    def __init__(self, d_model=64, n_mem=8, n_ops=4, queries_per_step=8,
                 n_heads=4):
        super().__init__()
        self.d = d_model
        d_in = d_model + n_mem * d_model and d_model  # square/h slots flattened
        self.desc = StepDescriptor(n_ops, n_mem)
        d_tok = d_model
        self.tokens = nn.ModuleList()
        # per-step query bank (temporal identity)
        self.n_steps_bank = 12
        self.q_bank = nn.Parameter(torch.randn(self.n_steps_bank, queries_per_step, d_model) * 0.05)
        self.kv = nn.Linear(d_tok, 2 * d_tok)
        self.qp = nn.Linear(d_model, d_tok)
        self.out = nn.Linear(d_tok, GEMMA_HIDDEN)
        self.mem_proj = nn.Linear(d_model, d_tok)
        self.desc_proj = nn.Linear(self.desc.d, d_tok)
        self.n_heads = n_heads
        self.tps = queries_per_step

    def forward(self, cap: dict) -> torch.Tensor:
        B = cap["steps"][0]["h"].shape[0]
        dev = cap["steps"][0]["h"].device
        S = cap["memory"].shape[1] if cap.get("memory") is not None else 0
        outs = []
        for t, s in enumerate(cap["steps"]):
            sh = {k: (v.detach() if torch.is_tensor(v) else v) for k, v in s.items()}  # one-way boundary
            h = sh["h"]                                        # [B,64,d]
            x = torch.cat([h, self.mem_proj(cap["memory"].detach())], 1) if cap.get("memory") is not None else h
            dscr = self.desc_proj(self.desc(sh))[:, None, :]   # [B,1,d]
            kv_src = torch.cat([x, dscr], 1)                   # [B, 64+8+1, d]
            q_idx = min(t, self.n_steps_bank - 1)
            q = self.q_bank[q_idx].unsqueeze(0).expand(B, -1, -1)  # [B,k,d]
            q = self.qp(q)
            k, v = self.kv(kv_src).chunk(2, -1)
            # numeric safety (torch-2.13 sdpa backward produced NaN grads with
            # unnormalised h magnitudes ~4x from deep recurrences):
            k = k * torch.rsqrt(k.pow(2).mean(-1, keepdim=True) + 1e-6)
            v = v * torch.rsqrt(v.pow(2).mean(-1, keepdim=True) + 1e-6)
            per = self.d // self.n_heads
            q_ = q.view(B, self.tps, self.n_heads, per).transpose(1, 2)
            k_ = k.view(B, -1, self.n_heads, per).transpose(1, 2)
            v_ = v.view(B, -1, self.n_heads, per).transpose(1, 2)
            a = F.scaled_dot_product_attention(q_, k_, v_)
            a = a.transpose(1, 2).reshape(B, self.tps, self.d)
            outs.append(self.out(a))
        emb = torch.cat(outs, 1)
        opener = torch.full((B, 1, GEMMA_HIDDEN), 1e-4, device=dev)
        return torch.cat([opener, emb], 1) * GEMMA_SCALE