File size: 4,808 Bytes
b5af418 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 | """Locate the NaN source in D-condition backward: backward hooks on all
ResamplerBridge submodules AND all gemma layers record the first module whose
grad output contains NaN. Also probes: (1) ConstantPrefix-with-a-zero-token
(A-style) through gemma backward (does A still work with zeros inserted?),
(2) plain sdpa-gradient sanity at the bridge's exact shapes.
"""
import os, sys
sys.path.insert(0, "/app")
import torch
import torch.nn.functional as F
from huggingface_hub import hf_hub_download
import shutil
from tinychess.checkpoint import load_model_only
from experiments.gemma_decoder.run_pilot import (make_positions, build_prefix,
heldout_ce, GEMMA_SCALE)
from experiments.gemma_decoder.capture import capture_with_hooks
from experiments.gemma_decoder.bridge import ResamplerBridge
from experiments.gemma_decoder.targets import targets_from_capture
from experiments.gemma_decoder import run_pilot as rp
def getnan(t):
if t is None: return False
if not torch.is_tensor(t): return False
return bool(torch.isnan(t.float()).any().item())
print("torch", torch.__version__, flush=True)
src_pt = hf_hub_download("cazyundee/tinychess-ckpt", "fast_wide/latest.pt",
token=os.environ.get("HF_TOKEN"))
os.makedirs("runs/_pilot_ckpts", exist_ok=True)
shutil.copy(src_pt, "runs/_pilot_ckpts/fast_wide.pt")
m, cfg = load_model_only("runs/_pilot_ckpts/fast_wide.pt")
poss = make_positions(6, seed=7)
caps = [capture_with_hooks(m, b, steps=6, adaptive=True) for b in poss]
items = [(c, targets_from_capture(c, ops=tuple(m.core.ops))["text"]) for c in caps]
tok, gemma = rp.load_gemma("google/gemma-3-270m", "cpu")
def localise(tag, make_prefix_fn, text):
"""Run one forward+backward with NaN-locating hooks."""
nan_report = []
hooks = []
def mk_hook(name):
def h(mod, gin, gout):
bad = any(getnan(t) for t in (gout if isinstance(gout, tuple) else (gout,)))
gonan = any(getnan(t) for t in gin)
if bad or gonan:
nan_report.append((name, "grad_out" if bad else "grad_in", gonan, bad))
return h
# bridge submodules
for name, mod in bridge.named_modules():
if name:
hooks.append(mod.register_full_backward_hook(mk_hook("B:" + name)))
# gemma layers
for name, mod in gemma.named_modules():
nm = name.split(".")
if len(nm) >= 2 and (nm[-1].isdigit() or nm[-1] in ("self_attn", "mlp")):
hooks.append(mod.register_full_backward_hook(mk_hook("G:" + name)))
try:
prefix = make_prefix_fn()
ids = tok(text, return_tensors="pt").input_ids
eos = tok.eos_token_id
eos_list = eos if isinstance(eos, list) else [eos]
body = ids[0, 1:]
end = next((i for i, t in enumerate(body) if int(t) in eos_list), len(body))
tgt = body[:end + 1]
emb = gemma.model.embed_tokens(ids) * GEMMA_SCALE
full = torch.cat([prefix, emb], 1)
out = gemma(inputs_embeds=full)
logits = out.logits[0, prefix.shape[1]-1:-1]
loss = F.cross_entropy(logits[:len(tgt)], tgt)
loss.backward()
print(f"{tag}: loss={float(loss):.3f} "
f"logits_nan={getnan(logits)} prefix_nan={getnan(prefix.detach())}", flush=True)
# forward activations of gemma: check nan in hidden states
if nan_report:
print(f"{tag}: FIRST NaN sites (in hook order):", flush=True)
for r in nan_report[:8]:
print(" ", r, flush=True)
else:
print(f"{tag}: NO NaN in any module grad", flush=True)
finally:
for h in hooks:
h.remove()
return loss
cap, text = items[0]
cap = dict(cap); cap["condition"] = "D"
bridge = ResamplerBridge(d_model=cfg.d_model, n_mem=cfg.n_mem,
n_ops=len(m.core.ops))
for n, p in bridge.named_parameters():
p.grad = None
localise("D-full", lambda: build_prefix(bridge, cap, "D", caps), text)
# Probe 2: A-style constant prefix with one zero token - does gemma backward survive zeros?
cp = torch.nn.Parameter(torch.randn(1, 48, 640) * 0.02)
cp.data[:, 0, :] = 0.0
def const_prefix():
return cp * GEMMA_SCALE
localise("A-const-with-zero-token", const_prefix, text)
# Probe 3: raw sdpa backward at the bridge's exact shape
q = torch.randn(1, 8, 96, requires_grad=True)
k = torch.randn(1, 65, 96, requires_grad=True)
v = torch.randn(1, 65, 96, requires_grad=True)
qq = q.view(1, 8, 4, 24).transpose(1, 2)
kk = k.view(1, 65, 4, 24).transpose(1, 2)
vv = v.view(1, 65, 4, 24).transpose(1, 2)
a = F.scaled_dot_product_attention(qq, kk, vv)
a.sum().backward()
print("sdpa-shape-probe: q_nan", getnan(q.grad), "k_nan", getnan(k.grad),
"v_nan", getnan(v.grad), flush=True)
print("NAN LOCALISATION COMPLETE", flush=True)
|