agent
debug_nan: hook-based NaN localisation (bridge + all gemma layers) + zero-token and sdpa-shape probes
b5af418 Download experiments/gemma_decoder/debug_nan.py from cazyundee/training: direct link, hf CLI and curl.
- Browser
- Download file 4.81 kB
-
https://huggingface.co/spaces/cazyundee/training/resolve/main/experiments/gemma_decoder/debug_nan.py
- Command line
-
hf download hf://spaces/cazyundee/training/experiments/gemma_decoder/debug_nan.py
-
curl -L -o debug_nan.py https://huggingface.co/spaces/cazyundee/training/resolve/main/experiments/gemma_decoder/debug_nan.py
4.81 kB
| """Locate the NaN source in D-condition backward: backward hooks on all | |
| ResamplerBridge submodules AND all gemma layers record the first module whose | |
| grad output contains NaN. Also probes: (1) ConstantPrefix-with-a-zero-token | |
| (A-style) through gemma backward (does A still work with zeros inserted?), | |
| (2) plain sdpa-gradient sanity at the bridge's exact shapes. | |
| """ | |
| import os, sys | |
| sys.path.insert(0, "/app") | |
| import torch | |
| import torch.nn.functional as F | |
| from huggingface_hub import hf_hub_download | |
| import shutil | |
| from tinychess.checkpoint import load_model_only | |
| from experiments.gemma_decoder.run_pilot import (make_positions, build_prefix, | |
| heldout_ce, GEMMA_SCALE) | |
| from experiments.gemma_decoder.capture import capture_with_hooks | |
| from experiments.gemma_decoder.bridge import ResamplerBridge | |
| from experiments.gemma_decoder.targets import targets_from_capture | |
| from experiments.gemma_decoder import run_pilot as rp | |
| def getnan(t): | |
| if t is None: return False | |
| if not torch.is_tensor(t): return False | |
| return bool(torch.isnan(t.float()).any().item()) | |
| print("torch", torch.__version__, flush=True) | |
| src_pt = hf_hub_download("cazyundee/tinychess-ckpt", "fast_wide/latest.pt", | |
| token=os.environ.get("HF_TOKEN")) | |
| os.makedirs("runs/_pilot_ckpts", exist_ok=True) | |
| shutil.copy(src_pt, "runs/_pilot_ckpts/fast_wide.pt") | |
| m, cfg = load_model_only("runs/_pilot_ckpts/fast_wide.pt") | |
| poss = make_positions(6, seed=7) | |
| caps = [capture_with_hooks(m, b, steps=6, adaptive=True) for b in poss] | |
| items = [(c, targets_from_capture(c, ops=tuple(m.core.ops))["text"]) for c in caps] | |
| tok, gemma = rp.load_gemma("google/gemma-3-270m", "cpu") | |
| def localise(tag, make_prefix_fn, text): | |
| """Run one forward+backward with NaN-locating hooks.""" | |
| nan_report = [] | |
| hooks = [] | |
| def mk_hook(name): | |
| def h(mod, gin, gout): | |
| bad = any(getnan(t) for t in (gout if isinstance(gout, tuple) else (gout,))) | |
| gonan = any(getnan(t) for t in gin) | |
| if bad or gonan: | |
| nan_report.append((name, "grad_out" if bad else "grad_in", gonan, bad)) | |
| return h | |
| # bridge submodules | |
| for name, mod in bridge.named_modules(): | |
| if name: | |
| hooks.append(mod.register_full_backward_hook(mk_hook("B:" + name))) | |
| # gemma layers | |
| for name, mod in gemma.named_modules(): | |
| nm = name.split(".") | |
| if len(nm) >= 2 and (nm[-1].isdigit() or nm[-1] in ("self_attn", "mlp")): | |
| hooks.append(mod.register_full_backward_hook(mk_hook("G:" + name))) | |
| try: | |
| prefix = make_prefix_fn() | |
| ids = tok(text, return_tensors="pt").input_ids | |
| eos = tok.eos_token_id | |
| eos_list = eos if isinstance(eos, list) else [eos] | |
| body = ids[0, 1:] | |
| end = next((i for i, t in enumerate(body) if int(t) in eos_list), len(body)) | |
| tgt = body[:end + 1] | |
| emb = gemma.model.embed_tokens(ids) * GEMMA_SCALE | |
| full = torch.cat([prefix, emb], 1) | |
| out = gemma(inputs_embeds=full) | |
| logits = out.logits[0, prefix.shape[1]-1:-1] | |
| loss = F.cross_entropy(logits[:len(tgt)], tgt) | |
| loss.backward() | |
| print(f"{tag}: loss={float(loss):.3f} " | |
| f"logits_nan={getnan(logits)} prefix_nan={getnan(prefix.detach())}", flush=True) | |
| # forward activations of gemma: check nan in hidden states | |
| if nan_report: | |
| print(f"{tag}: FIRST NaN sites (in hook order):", flush=True) | |
| for r in nan_report[:8]: | |
| print(" ", r, flush=True) | |
| else: | |
| print(f"{tag}: NO NaN in any module grad", flush=True) | |
| finally: | |
| for h in hooks: | |
| h.remove() | |
| return loss | |
| cap, text = items[0] | |
| cap = dict(cap); cap["condition"] = "D" | |
| bridge = ResamplerBridge(d_model=cfg.d_model, n_mem=cfg.n_mem, | |
| n_ops=len(m.core.ops)) | |
| for n, p in bridge.named_parameters(): | |
| p.grad = None | |
| localise("D-full", lambda: build_prefix(bridge, cap, "D", caps), text) | |
| # Probe 2: A-style constant prefix with one zero token - does gemma backward survive zeros? | |
| cp = torch.nn.Parameter(torch.randn(1, 48, 640) * 0.02) | |
| cp.data[:, 0, :] = 0.0 | |
| def const_prefix(): | |
| return cp * GEMMA_SCALE | |
| localise("A-const-with-zero-token", const_prefix, text) | |
| # Probe 3: raw sdpa backward at the bridge's exact shape | |
| q = torch.randn(1, 8, 96, requires_grad=True) | |
| k = torch.randn(1, 65, 96, requires_grad=True) | |
| v = torch.randn(1, 65, 96, requires_grad=True) | |
| qq = q.view(1, 8, 4, 24).transpose(1, 2) | |
| kk = k.view(1, 65, 4, 24).transpose(1, 2) | |
| vv = v.view(1, 65, 4, 24).transpose(1, 2) | |
| a = F.scaled_dot_product_attention(qq, kk, vv) | |
| a.sum().backward() | |
| print("sdpa-shape-probe: q_nan", getnan(q.grad), "k_nan", getnan(k.grad), | |
| "v_nan", getnan(v.grad), flush=True) | |
| print("NAN LOCALISATION COMPLETE", flush=True) | |