"""Locate the NaN source in D-condition backward: backward hooks on all ResamplerBridge submodules AND all gemma layers record the first module whose grad output contains NaN. Also probes: (1) ConstantPrefix-with-a-zero-token (A-style) through gemma backward (does A still work with zeros inserted?), (2) plain sdpa-gradient sanity at the bridge's exact shapes. """ import os, sys sys.path.insert(0, "/app") import torch import torch.nn.functional as F from huggingface_hub import hf_hub_download import shutil from tinychess.checkpoint import load_model_only from experiments.gemma_decoder.run_pilot import (make_positions, build_prefix, heldout_ce, GEMMA_SCALE) from experiments.gemma_decoder.capture import capture_with_hooks from experiments.gemma_decoder.bridge import ResamplerBridge from experiments.gemma_decoder.targets import targets_from_capture from experiments.gemma_decoder import run_pilot as rp def getnan(t): if t is None: return False if not torch.is_tensor(t): return False return bool(torch.isnan(t.float()).any().item()) print("torch", torch.__version__, flush=True) src_pt = hf_hub_download("cazyundee/tinychess-ckpt", "fast_wide/latest.pt", token=os.environ.get("HF_TOKEN")) os.makedirs("runs/_pilot_ckpts", exist_ok=True) shutil.copy(src_pt, "runs/_pilot_ckpts/fast_wide.pt") m, cfg = load_model_only("runs/_pilot_ckpts/fast_wide.pt") poss = make_positions(6, seed=7) caps = [capture_with_hooks(m, b, steps=6, adaptive=True) for b in poss] items = [(c, targets_from_capture(c, ops=tuple(m.core.ops))["text"]) for c in caps] tok, gemma = rp.load_gemma("google/gemma-3-270m", "cpu") def localise(tag, make_prefix_fn, text): """Run one forward+backward with NaN-locating hooks.""" nan_report = [] hooks = [] def mk_hook(name): def h(mod, gin, gout): bad = any(getnan(t) for t in (gout if isinstance(gout, tuple) else (gout,))) gonan = any(getnan(t) for t in gin) if bad or gonan: nan_report.append((name, "grad_out" if bad else "grad_in", gonan, bad)) return h # bridge submodules for name, mod in bridge.named_modules(): if name: hooks.append(mod.register_full_backward_hook(mk_hook("B:" + name))) # gemma layers for name, mod in gemma.named_modules(): nm = name.split(".") if len(nm) >= 2 and (nm[-1].isdigit() or nm[-1] in ("self_attn", "mlp")): hooks.append(mod.register_full_backward_hook(mk_hook("G:" + name))) try: prefix = make_prefix_fn() ids = tok(text, return_tensors="pt").input_ids eos = tok.eos_token_id eos_list = eos if isinstance(eos, list) else [eos] body = ids[0, 1:] end = next((i for i, t in enumerate(body) if int(t) in eos_list), len(body)) tgt = body[:end + 1] emb = gemma.model.embed_tokens(ids) * GEMMA_SCALE full = torch.cat([prefix, emb], 1) out = gemma(inputs_embeds=full) logits = out.logits[0, prefix.shape[1]-1:-1] loss = F.cross_entropy(logits[:len(tgt)], tgt) loss.backward() print(f"{tag}: loss={float(loss):.3f} " f"logits_nan={getnan(logits)} prefix_nan={getnan(prefix.detach())}", flush=True) # forward activations of gemma: check nan in hidden states if nan_report: print(f"{tag}: FIRST NaN sites (in hook order):", flush=True) for r in nan_report[:8]: print(" ", r, flush=True) else: print(f"{tag}: NO NaN in any module grad", flush=True) finally: for h in hooks: h.remove() return loss cap, text = items[0] cap = dict(cap); cap["condition"] = "D" bridge = ResamplerBridge(d_model=cfg.d_model, n_mem=cfg.n_mem, n_ops=len(m.core.ops)) for n, p in bridge.named_parameters(): p.grad = None localise("D-full", lambda: build_prefix(bridge, cap, "D", caps), text) # Probe 2: A-style constant prefix with one zero token - does gemma backward survive zeros? cp = torch.nn.Parameter(torch.randn(1, 48, 640) * 0.02) cp.data[:, 0, :] = 0.0 def const_prefix(): return cp * GEMMA_SCALE localise("A-const-with-zero-token", const_prefix, text) # Probe 3: raw sdpa backward at the bridge's exact shape q = torch.randn(1, 8, 96, requires_grad=True) k = torch.randn(1, 65, 96, requires_grad=True) v = torch.randn(1, 65, 96, requires_grad=True) qq = q.view(1, 8, 4, 24).transpose(1, 2) kk = k.view(1, 65, 4, 24).transpose(1, 2) vv = v.view(1, 65, 4, 24).transpose(1, 2) a = F.scaled_dot_product_attention(qq, kk, vv) a.sum().backward() print("sdpa-shape-probe: q_nan", getnan(q.grad), "k_nan", getnan(k.grad), "v_nan", getnan(v.grad), flush=True) print("NAN LOCALISATION COMPLETE", flush=True)