File size: 8,622 Bytes
788ee34 33dbd59 788ee34 33dbd59 788ee34 33dbd59 788ee34 f473997 788ee34 884986c 788ee34 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 | """The latent bridge: TinyChess trajectory -> continuous Gemma prefix.
Two stages (per the brief):
1. LatentBridge (baseline): per-step linear projections of a few pooled
signals, preserving step ordering and slot identity where cheap.
2. ResamplerBridge: learned queries cross-attend over the full [64+d_mem]
token set per step (Perceiver-style), k queries per step.
Both emit [B, P, 640] already multiplied by Gemma's embedding scale
sqrt(hidden_size) because inputs_embeds BYPASSES Gemma's internal
embedding scaling (verified in transformers 5.16.1 - see
gemma_decoder_investigation.md).
Inputs consumed per step (all from capture_with_hooks, all detacheable):
h [B,64,d], h_prev [B,64,d], router [B,n_ops], p_halt [B],
mem_attn [B,64,n_mem], memory [B,n_mem,d]
Defaults keep it small: 8 queries per step in the resampler.
The 4 metadata channels (value/margin/halt/n_moves) are engineered
observables, encoded through a tiny MLP into 2 extra tokens per step.
"""
from __future__ import annotations
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
GEMMA_HIDDEN = 640 # gemma-3-270m text_config.hidden_size (verified)
GEMMA_SCALE = GEMMA_HIDDEN ** 0.5
class StepDescriptor(nn.Module):
"""Per-step engineered observable vector fed to both bridge variants.
All quantities measured from the trace, cheap, and informative:
||dh|| (mean+max over squares), dh concentration (top-square share),
||dh|| momentum (vs previous delta), router entropy,
router top-1 identity onehot, halting probability, aggregate ||dh||,
memory attention entropy, memory attention top-slot share.
Order fixed by FEATURE_KEYS.
"""
FEATURE_KEYS = ["dh_norm_mean", "dh_norm_max", "dh_top_share",
"dh_momentum", "router_entropy", "router_toponehot",
"p_halt", "memat_entropy", "memat_top_share"]
def __init__(self, n_ops=4, n_mem=8):
super().__init__()
self.n_ops, self.n_mem = n_ops, n_mem
# feature count: dn_mean, dn_max, top_share, momentum (4) +
# router entropy (1) + top-1 onehot (n_ops) + p_halt (1) +
# memat entropy (1) + memat top-share (1)
self.d = 8 + n_ops
def forward(self, s: dict) -> torch.Tensor:
h, hp = s["h"], s.get("h_prev", None)
B = h.shape[0]
dev = h.device
feats = []
if hp is not None:
dh = (h - hp).abs().mean(-1) # [B,64]
dn_mean = dh.mean(-1, keepdim=True)
dn_max = dh.amax(-1, keepdim=True)
top_share = ((dh ** 2).sum(-1, keepdim=True)
/ (dn_mean ** 2 * 64 + 1e-6)).clamp(0, 1)
prev_mean = hp.abs().mean(-1).mean(-1, keepdim=True) * 0.5 # no t-2 delta: neutral prior
moment = dn_mean / (prev_mean + 1e-6)
else:
abs_mean = h.abs().mean(-1) # [B,64]
dn_mean = abs_mean.mean(-1, keepdim=True) # magnitude proxy at t=0
dn_max = abs_mean.amax(-1, keepdim=True)
top_share = ((abs_mean ** 2).sum(-1, keepdim=True)
/ (dn_mean ** 2 * 64 + 1e-6)).clamp(0, 1)
moment = torch.ones(B, 1, device=dev) # neutral at t=0
feats += [dn_mean, dn_max, top_share, moment]
r = s.get("router")
if r is not None and r.shape[-1] > 1:
ent = -(r * (r + 1e-9).log()).sum(-1, keepdim=True)
top = F.one_hot(r.argmax(-1), r.shape[-1]).float()
feats += [ent, top]
else:
feats += [torch.zeros(B, 1, device=h.device),
torch.zeros(B, self.n_ops, device=h.device)]
feats.append(s["p_halt"][:, None] if s.get("p_halt") is not None
else torch.zeros(B, 1, device=h.device))
ma = s.get("mem_attn")
if ma is not None:
ment = -(ma * (ma + 1e-9).log()).sum(-1) # [B,64]
share = ma.amax(-1) # [B,64] -> use mean of max
feats += [ment.mean(-1, keepdim=True), share.mean(-1, keepdim=True)]
else:
feats += [torch.zeros(B, 1, device=h.device), torch.zeros(B, 1, device=h.device)]
return torch.cat(feats, -1) # [B, d_desc]
class LatentBridge(nn.Module):
"""Baseline: per-step linear mapping [per-step summary] -> k tokens."""
def __init__(self, d_model=64, n_mem=8, n_ops=4, tokens_per_step=4):
super().__init__()
self.desc = StepDescriptor(n_ops, n_mem)
d_in = self.desc.d
self.proj = nn.Sequential(nn.Linear(d_in, 256), nn.GELU(),
nn.Linear(256, tokens_per_step * GEMMA_HIDDEN))
self.tps = tokens_per_step
# learned BOS-anchored opener helps Gemma treat the prefix as content
self.opener = nn.Parameter(torch.randn(1, GEMMA_HIDDEN) * 0.02)
@staticmethod
def _detach_step(s: dict) -> dict:
"""One-way boundary: the bridge consumes DETACHED latents only.
Gradients can never flow back through the bridge into the chess
model, by construction (mirrors thought.py's assert pattern)."""
return {k: (v.detach() if torch.is_tensor(v) else v) for k, v in s.items()}
def forward(self, cap: dict) -> torch.Tensor:
B = cap["steps"][0]["h"].shape[0]
toks = []
toks.append(self.opener.expand(B, 1, -1))
for s in cap["steps"]:
v = self.desc(self._detach_step(s)) # [B, d_desc]
e = self.proj(v).view(B, self.tps, GEMMA_HIDDEN)
toks.append(e)
return torch.cat(toks, 1) * GEMMA_SCALE # [B, P, 640]
class ResamplerBridge(nn.Module):
"""Perceiver-style: k learned queries attend over ALL tokens of a step
(64 square states + 8 memory slots + descriptor) preserving square and
slot identity inside attention, positional square identities added."""
def __init__(self, d_model=64, n_mem=8, n_ops=4, queries_per_step=8,
n_heads=4):
super().__init__()
self.d = d_model
d_in = d_model + n_mem * d_model and d_model # square/h slots flattened
self.desc = StepDescriptor(n_ops, n_mem)
d_tok = d_model
self.tokens = nn.ModuleList()
# per-step query bank (temporal identity)
self.n_steps_bank = 12
self.q_bank = nn.Parameter(torch.randn(self.n_steps_bank, queries_per_step, d_model) * 0.05)
self.kv = nn.Linear(d_tok, 2 * d_tok)
self.qp = nn.Linear(d_model, d_tok)
self.out = nn.Linear(d_tok, GEMMA_HIDDEN)
self.mem_proj = nn.Linear(d_model, d_tok)
self.desc_proj = nn.Linear(self.desc.d, d_tok)
self.n_heads = n_heads
self.tps = queries_per_step
def forward(self, cap: dict) -> torch.Tensor:
B = cap["steps"][0]["h"].shape[0]
dev = cap["steps"][0]["h"].device
S = cap["memory"].shape[1] if cap.get("memory") is not None else 0
outs = []
for t, s in enumerate(cap["steps"]):
sh = {k: (v.detach() if torch.is_tensor(v) else v) for k, v in s.items()} # one-way boundary
h = sh["h"] # [B,64,d]
x = torch.cat([h, self.mem_proj(cap["memory"].detach())], 1) if cap.get("memory") is not None else h
dscr = self.desc_proj(self.desc(sh))[:, None, :] # [B,1,d]
kv_src = torch.cat([x, dscr], 1) # [B, 64+8+1, d]
q_idx = min(t, self.n_steps_bank - 1)
q = self.q_bank[q_idx].unsqueeze(0).expand(B, -1, -1) # [B,k,d]
q = self.qp(q)
k, v = self.kv(kv_src).chunk(2, -1)
# numeric safety (torch-2.13 sdpa backward produced NaN grads with
# unnormalised h magnitudes ~4x from deep recurrences):
k = k * torch.rsqrt(k.pow(2).mean(-1, keepdim=True) + 1e-6)
v = v * torch.rsqrt(v.pow(2).mean(-1, keepdim=True) + 1e-6)
per = self.d // self.n_heads
q_ = q.view(B, self.tps, self.n_heads, per).transpose(1, 2)
k_ = k.view(B, -1, self.n_heads, per).transpose(1, 2)
v_ = v.view(B, -1, self.n_heads, per).transpose(1, 2)
a = F.scaled_dot_product_attention(q_, k_, v_)
a = a.transpose(1, 2).reshape(B, self.tps, self.d)
outs.append(self.out(a))
emb = torch.cat(outs, 1)
opener = torch.full((B, 1, GEMMA_HIDDEN), 1e-4, device=dev)
return torch.cat([opener, emb], 1) * GEMMA_SCALE
|