File size: 4,808 Bytes
b5af418
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
"""Locate the NaN source in D-condition backward: backward hooks on all
ResamplerBridge submodules AND all gemma layers record the first module whose
grad output contains NaN. Also probes: (1) ConstantPrefix-with-a-zero-token
(A-style) through gemma backward (does A still work with zeros inserted?),
(2) plain sdpa-gradient sanity at the bridge's exact shapes.
"""
import os, sys
sys.path.insert(0, "/app")
import torch
import torch.nn.functional as F
from huggingface_hub import hf_hub_download
import shutil
from tinychess.checkpoint import load_model_only
from experiments.gemma_decoder.run_pilot import (make_positions, build_prefix,
    heldout_ce, GEMMA_SCALE)
from experiments.gemma_decoder.capture import capture_with_hooks
from experiments.gemma_decoder.bridge import ResamplerBridge
from experiments.gemma_decoder.targets import targets_from_capture
from experiments.gemma_decoder import run_pilot as rp

def getnan(t):
    if t is None: return False
    if not torch.is_tensor(t): return False
    return bool(torch.isnan(t.float()).any().item())

print("torch", torch.__version__, flush=True)
src_pt = hf_hub_download("cazyundee/tinychess-ckpt", "fast_wide/latest.pt",
                         token=os.environ.get("HF_TOKEN"))
os.makedirs("runs/_pilot_ckpts", exist_ok=True)
shutil.copy(src_pt, "runs/_pilot_ckpts/fast_wide.pt")
m, cfg = load_model_only("runs/_pilot_ckpts/fast_wide.pt")
poss = make_positions(6, seed=7)
caps = [capture_with_hooks(m, b, steps=6, adaptive=True) for b in poss]
items = [(c, targets_from_capture(c, ops=tuple(m.core.ops))["text"]) for c in caps]

tok, gemma = rp.load_gemma("google/gemma-3-270m", "cpu")

def localise(tag, make_prefix_fn, text):
    """Run one forward+backward with NaN-locating hooks."""
    nan_report = []
    hooks = []
    def mk_hook(name):
        def h(mod, gin, gout):
            bad = any(getnan(t) for t in (gout if isinstance(gout, tuple) else (gout,)))
            gonan = any(getnan(t) for t in gin)
            if bad or gonan:
                nan_report.append((name, "grad_out" if bad else "grad_in", gonan, bad))
        return h
    # bridge submodules
    for name, mod in bridge.named_modules():
        if name:
            hooks.append(mod.register_full_backward_hook(mk_hook("B:" + name)))
    # gemma layers
    for name, mod in gemma.named_modules():
        nm = name.split(".")
        if len(nm) >= 2 and (nm[-1].isdigit() or nm[-1] in ("self_attn", "mlp")):
            hooks.append(mod.register_full_backward_hook(mk_hook("G:" + name)))
    try:
        prefix = make_prefix_fn()
        ids = tok(text, return_tensors="pt").input_ids
        eos = tok.eos_token_id
        eos_list = eos if isinstance(eos, list) else [eos]
        body = ids[0, 1:]
        end = next((i for i, t in enumerate(body) if int(t) in eos_list), len(body))
        tgt = body[:end + 1]
        emb = gemma.model.embed_tokens(ids) * GEMMA_SCALE
        full = torch.cat([prefix, emb], 1)
        out = gemma(inputs_embeds=full)
        logits = out.logits[0, prefix.shape[1]-1:-1]
        loss = F.cross_entropy(logits[:len(tgt)], tgt)
        loss.backward()
        print(f"{tag}: loss={float(loss):.3f} "
              f"logits_nan={getnan(logits)} prefix_nan={getnan(prefix.detach())}", flush=True)
        # forward activations of gemma: check nan in hidden states
        if nan_report:
            print(f"{tag}: FIRST NaN sites (in hook order):", flush=True)
            for r in nan_report[:8]:
                print("   ", r, flush=True)
        else:
            print(f"{tag}: NO NaN in any module grad", flush=True)
    finally:
        for h in hooks:
            h.remove()
    return loss

cap, text = items[0]
cap = dict(cap); cap["condition"] = "D"
bridge = ResamplerBridge(d_model=cfg.d_model, n_mem=cfg.n_mem,
                         n_ops=len(m.core.ops))
for n, p in bridge.named_parameters():
    p.grad = None

localise("D-full", lambda: build_prefix(bridge, cap, "D", caps), text)

# Probe 2: A-style constant prefix with one zero token - does gemma backward survive zeros?
cp = torch.nn.Parameter(torch.randn(1, 48, 640) * 0.02)
cp.data[:, 0, :] = 0.0
def const_prefix():
    return cp * GEMMA_SCALE
localise("A-const-with-zero-token", const_prefix, text)

# Probe 3: raw sdpa backward at the bridge's exact shape
q = torch.randn(1, 8, 96, requires_grad=True)
k = torch.randn(1, 65, 96, requires_grad=True)
v = torch.randn(1, 65, 96, requires_grad=True)
qq = q.view(1, 8, 4, 24).transpose(1, 2)
kk = k.view(1, 65, 4, 24).transpose(1, 2)
vv = v.view(1, 65, 4, 24).transpose(1, 2)
a = F.scaled_dot_product_attention(qq, kk, vv)
a.sum().backward()
print("sdpa-shape-probe: q_nan", getnan(q.grad), "k_nan", getnan(k.grad),
      "v_nan", getnan(v.grad), flush=True)
print("NAN LOCALISATION COMPLETE", flush=True)