agent
NaN fix v2 (root cause: exactly-zero embedding rows trigger gemma-backward NaN on torch 2.13, localised to layer-12 mlp grad_out via hooks; A-probe with zero token reproduced it): no zero rows anywhere - B uses 1e-4 no-info epsilon, resampler opener 1e-4
884986c Download experiments/gemma_decoder/bridge.py from cazyundee/training: direct link, hf CLI and curl.
- Browser
- Download file 8.62 kB
-
https://huggingface.co/spaces/cazyundee/training/resolve/main/experiments/gemma_decoder/bridge.py
- Command line
-
hf download hf://spaces/cazyundee/training/experiments/gemma_decoder/bridge.py
-
curl -L -o bridge.py https://huggingface.co/spaces/cazyundee/training/resolve/main/experiments/gemma_decoder/bridge.py
8.62 kB
| """The latent bridge: TinyChess trajectory -> continuous Gemma prefix. | |
| Two stages (per the brief): | |
| 1. LatentBridge (baseline): per-step linear projections of a few pooled | |
| signals, preserving step ordering and slot identity where cheap. | |
| 2. ResamplerBridge: learned queries cross-attend over the full [64+d_mem] | |
| token set per step (Perceiver-style), k queries per step. | |
| Both emit [B, P, 640] already multiplied by Gemma's embedding scale | |
| sqrt(hidden_size) because inputs_embeds BYPASSES Gemma's internal | |
| embedding scaling (verified in transformers 5.16.1 - see | |
| gemma_decoder_investigation.md). | |
| Inputs consumed per step (all from capture_with_hooks, all detacheable): | |
| h [B,64,d], h_prev [B,64,d], router [B,n_ops], p_halt [B], | |
| mem_attn [B,64,n_mem], memory [B,n_mem,d] | |
| Defaults keep it small: 8 queries per step in the resampler. | |
| The 4 metadata channels (value/margin/halt/n_moves) are engineered | |
| observables, encoded through a tiny MLP into 2 extra tokens per step. | |
| """ | |
| from __future__ import annotations | |
| import math | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| GEMMA_HIDDEN = 640 # gemma-3-270m text_config.hidden_size (verified) | |
| GEMMA_SCALE = GEMMA_HIDDEN ** 0.5 | |
| class StepDescriptor(nn.Module): | |
| """Per-step engineered observable vector fed to both bridge variants. | |
| All quantities measured from the trace, cheap, and informative: | |
| ||dh|| (mean+max over squares), dh concentration (top-square share), | |
| ||dh|| momentum (vs previous delta), router entropy, | |
| router top-1 identity onehot, halting probability, aggregate ||dh||, | |
| memory attention entropy, memory attention top-slot share. | |
| Order fixed by FEATURE_KEYS. | |
| """ | |
| FEATURE_KEYS = ["dh_norm_mean", "dh_norm_max", "dh_top_share", | |
| "dh_momentum", "router_entropy", "router_toponehot", | |
| "p_halt", "memat_entropy", "memat_top_share"] | |
| def __init__(self, n_ops=4, n_mem=8): | |
| super().__init__() | |
| self.n_ops, self.n_mem = n_ops, n_mem | |
| # feature count: dn_mean, dn_max, top_share, momentum (4) + | |
| # router entropy (1) + top-1 onehot (n_ops) + p_halt (1) + | |
| # memat entropy (1) + memat top-share (1) | |
| self.d = 8 + n_ops | |
| def forward(self, s: dict) -> torch.Tensor: | |
| h, hp = s["h"], s.get("h_prev", None) | |
| B = h.shape[0] | |
| dev = h.device | |
| feats = [] | |
| if hp is not None: | |
| dh = (h - hp).abs().mean(-1) # [B,64] | |
| dn_mean = dh.mean(-1, keepdim=True) | |
| dn_max = dh.amax(-1, keepdim=True) | |
| top_share = ((dh ** 2).sum(-1, keepdim=True) | |
| / (dn_mean ** 2 * 64 + 1e-6)).clamp(0, 1) | |
| prev_mean = hp.abs().mean(-1).mean(-1, keepdim=True) * 0.5 # no t-2 delta: neutral prior | |
| moment = dn_mean / (prev_mean + 1e-6) | |
| else: | |
| abs_mean = h.abs().mean(-1) # [B,64] | |
| dn_mean = abs_mean.mean(-1, keepdim=True) # magnitude proxy at t=0 | |
| dn_max = abs_mean.amax(-1, keepdim=True) | |
| top_share = ((abs_mean ** 2).sum(-1, keepdim=True) | |
| / (dn_mean ** 2 * 64 + 1e-6)).clamp(0, 1) | |
| moment = torch.ones(B, 1, device=dev) # neutral at t=0 | |
| feats += [dn_mean, dn_max, top_share, moment] | |
| r = s.get("router") | |
| if r is not None and r.shape[-1] > 1: | |
| ent = -(r * (r + 1e-9).log()).sum(-1, keepdim=True) | |
| top = F.one_hot(r.argmax(-1), r.shape[-1]).float() | |
| feats += [ent, top] | |
| else: | |
| feats += [torch.zeros(B, 1, device=h.device), | |
| torch.zeros(B, self.n_ops, device=h.device)] | |
| feats.append(s["p_halt"][:, None] if s.get("p_halt") is not None | |
| else torch.zeros(B, 1, device=h.device)) | |
| ma = s.get("mem_attn") | |
| if ma is not None: | |
| ment = -(ma * (ma + 1e-9).log()).sum(-1) # [B,64] | |
| share = ma.amax(-1) # [B,64] -> use mean of max | |
| feats += [ment.mean(-1, keepdim=True), share.mean(-1, keepdim=True)] | |
| else: | |
| feats += [torch.zeros(B, 1, device=h.device), torch.zeros(B, 1, device=h.device)] | |
| return torch.cat(feats, -1) # [B, d_desc] | |
| class LatentBridge(nn.Module): | |
| """Baseline: per-step linear mapping [per-step summary] -> k tokens.""" | |
| def __init__(self, d_model=64, n_mem=8, n_ops=4, tokens_per_step=4): | |
| super().__init__() | |
| self.desc = StepDescriptor(n_ops, n_mem) | |
| d_in = self.desc.d | |
| self.proj = nn.Sequential(nn.Linear(d_in, 256), nn.GELU(), | |
| nn.Linear(256, tokens_per_step * GEMMA_HIDDEN)) | |
| self.tps = tokens_per_step | |
| # learned BOS-anchored opener helps Gemma treat the prefix as content | |
| self.opener = nn.Parameter(torch.randn(1, GEMMA_HIDDEN) * 0.02) | |
| def _detach_step(s: dict) -> dict: | |
| """One-way boundary: the bridge consumes DETACHED latents only. | |
| Gradients can never flow back through the bridge into the chess | |
| model, by construction (mirrors thought.py's assert pattern).""" | |
| return {k: (v.detach() if torch.is_tensor(v) else v) for k, v in s.items()} | |
| def forward(self, cap: dict) -> torch.Tensor: | |
| B = cap["steps"][0]["h"].shape[0] | |
| toks = [] | |
| toks.append(self.opener.expand(B, 1, -1)) | |
| for s in cap["steps"]: | |
| v = self.desc(self._detach_step(s)) # [B, d_desc] | |
| e = self.proj(v).view(B, self.tps, GEMMA_HIDDEN) | |
| toks.append(e) | |
| return torch.cat(toks, 1) * GEMMA_SCALE # [B, P, 640] | |
| class ResamplerBridge(nn.Module): | |
| """Perceiver-style: k learned queries attend over ALL tokens of a step | |
| (64 square states + 8 memory slots + descriptor) preserving square and | |
| slot identity inside attention, positional square identities added.""" | |
| def __init__(self, d_model=64, n_mem=8, n_ops=4, queries_per_step=8, | |
| n_heads=4): | |
| super().__init__() | |
| self.d = d_model | |
| d_in = d_model + n_mem * d_model and d_model # square/h slots flattened | |
| self.desc = StepDescriptor(n_ops, n_mem) | |
| d_tok = d_model | |
| self.tokens = nn.ModuleList() | |
| # per-step query bank (temporal identity) | |
| self.n_steps_bank = 12 | |
| self.q_bank = nn.Parameter(torch.randn(self.n_steps_bank, queries_per_step, d_model) * 0.05) | |
| self.kv = nn.Linear(d_tok, 2 * d_tok) | |
| self.qp = nn.Linear(d_model, d_tok) | |
| self.out = nn.Linear(d_tok, GEMMA_HIDDEN) | |
| self.mem_proj = nn.Linear(d_model, d_tok) | |
| self.desc_proj = nn.Linear(self.desc.d, d_tok) | |
| self.n_heads = n_heads | |
| self.tps = queries_per_step | |
| def forward(self, cap: dict) -> torch.Tensor: | |
| B = cap["steps"][0]["h"].shape[0] | |
| dev = cap["steps"][0]["h"].device | |
| S = cap["memory"].shape[1] if cap.get("memory") is not None else 0 | |
| outs = [] | |
| for t, s in enumerate(cap["steps"]): | |
| sh = {k: (v.detach() if torch.is_tensor(v) else v) for k, v in s.items()} # one-way boundary | |
| h = sh["h"] # [B,64,d] | |
| x = torch.cat([h, self.mem_proj(cap["memory"].detach())], 1) if cap.get("memory") is not None else h | |
| dscr = self.desc_proj(self.desc(sh))[:, None, :] # [B,1,d] | |
| kv_src = torch.cat([x, dscr], 1) # [B, 64+8+1, d] | |
| q_idx = min(t, self.n_steps_bank - 1) | |
| q = self.q_bank[q_idx].unsqueeze(0).expand(B, -1, -1) # [B,k,d] | |
| q = self.qp(q) | |
| k, v = self.kv(kv_src).chunk(2, -1) | |
| # numeric safety (torch-2.13 sdpa backward produced NaN grads with | |
| # unnormalised h magnitudes ~4x from deep recurrences): | |
| k = k * torch.rsqrt(k.pow(2).mean(-1, keepdim=True) + 1e-6) | |
| v = v * torch.rsqrt(v.pow(2).mean(-1, keepdim=True) + 1e-6) | |
| per = self.d // self.n_heads | |
| q_ = q.view(B, self.tps, self.n_heads, per).transpose(1, 2) | |
| k_ = k.view(B, -1, self.n_heads, per).transpose(1, 2) | |
| v_ = v.view(B, -1, self.n_heads, per).transpose(1, 2) | |
| a = F.scaled_dot_product_attention(q_, k_, v_) | |
| a = a.transpose(1, 2).reshape(B, self.tps, self.d) | |
| outs.append(self.out(a)) | |
| emb = torch.cat(outs, 1) | |
| opener = torch.full((B, 1, GEMMA_HIDDEN), 1e-4, device=dev) | |
| return torch.cat([opener, emb], 1) * GEMMA_SCALE | |