File size: 2,586 Bytes
bb51fdf b052374 bb51fdf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | """Operator-requested diagnostics (steps 1-4) for the cuDNN SDPA NaN bug:
1) GPU + cudnn version + backend context
2) failing seq_len % 128 check
3) A-const-with-zero-token repro with cuDNN SDPA disabled
4) per-param grad finiteness both backends
"""
import os, sys
sys.path.insert(0, "/app")
import torch
import torch.nn.functional as F
print("torch", torch.__version__, flush=True)
print("cuda available:", torch.cuda.is_available(), flush=True)
if torch.cuda.is_available():
print("GPU:", torch.cuda.get_device_name(0), flush=True)
print("capability:", torch.cuda.get_device_capability(0), flush=True)
try:
import torch.backends.cudnn as _c
print("cudnn version:", _c.version(), "enabled:", _c.is_enabled(), flush=True)
except Exception as e:
print("cudnn import fail:", e, flush=True)
from experiments.gemma_decoder import run_pilot as rp
tok, gemma = rp.load_gemma("google/gemma-3-270m", "cpu")
text = ("Some passes barely move the representation while others rework it "
"substantly.")
def const_with_zero_token(zero=True):
cp = torch.nn.Parameter(torch.randn(1, 48, 640) * 0.02)
if zero:
cp.data[:, 0, :] = 0.0
def f():
return cp * rp.GEMMA_SCALE
return cp, f
def run(tag, zero=True, disable_cudnn=False):
if disable_cudnn and torch.cuda.is_available():
torch.backends.cuda.enable_cudnn_sdp(False)
try:
cp, f = const_with_zero_token(zero)
ids = tok(text, return_tensors="pt").input_ids
emb = gemma.model.embed_tokens(ids) * rp.GEMMA_SCALE
prefix = f()
full = torch.cat([prefix, emb], 1)
L = full.shape[1]
out = gemma(inputs_embeds=full)
logits = out.logits[0, prefix.shape[1]-1:-1]
tgt = ids[0, 1:]
n = min(logits.shape[0], tgt.shape[0])
loss = F.cross_entropy(logits[:n], tgt[:n])
loss.backward()
gnorm = float(cp.grad.abs().sum())
import math
print(f"{tag}: seq_len={L} (mod128={L%128}) loss={float(loss):.3f} "
f"finite={math.isfinite(gnorm)} gnorm={gnorm:.4e}", flush=True)
return math.isfinite(gnorm)
finally:
if torch.cuda.is_available():
torch.backends.cuda.enable_cudnn_sdp(True)
r1 = run("zero-token DEFAULT backend", zero=True, disable_cudnn=False)
r2 = run("zero-token NO-cuDNN-SDPA", zero=True, disable_cudnn=True)
r3 = run("nonzero-token DEFAULT backend", zero=False, disable_cudnn=False)
print(f"RESULTS: default+zero={r1} nocudnn+zero={r2} default+nonzero={r3}", flush=True)
print("BACKEND DIAG COMPLETE", flush=True)
|