"""Operator-requested diagnostics (steps 1-4) for the cuDNN SDPA NaN bug: 1) GPU + cudnn version + backend context 2) failing seq_len % 128 check 3) A-const-with-zero-token repro with cuDNN SDPA disabled 4) per-param grad finiteness both backends """ import os, sys sys.path.insert(0, "/app") import torch import torch.nn.functional as F print("torch", torch.__version__, flush=True) print("cuda available:", torch.cuda.is_available(), flush=True) if torch.cuda.is_available(): print("GPU:", torch.cuda.get_device_name(0), flush=True) print("capability:", torch.cuda.get_device_capability(0), flush=True) try: import torch.backends.cudnn as _c print("cudnn version:", _c.version(), "enabled:", _c.is_enabled(), flush=True) except Exception as e: print("cudnn import fail:", e, flush=True) from experiments.gemma_decoder import run_pilot as rp tok, gemma = rp.load_gemma("google/gemma-3-270m", "cpu") text = ("Some passes barely move the representation while others rework it " "substantly.") def const_with_zero_token(zero=True): cp = torch.nn.Parameter(torch.randn(1, 48, 640) * 0.02) if zero: cp.data[:, 0, :] = 0.0 def f(): return cp * rp.GEMMA_SCALE return cp, f def run(tag, zero=True, disable_cudnn=False): if disable_cudnn and torch.cuda.is_available(): torch.backends.cuda.enable_cudnn_sdp(False) try: cp, f = const_with_zero_token(zero) ids = tok(text, return_tensors="pt").input_ids emb = gemma.model.embed_tokens(ids) * rp.GEMMA_SCALE prefix = f() full = torch.cat([prefix, emb], 1) L = full.shape[1] out = gemma(inputs_embeds=full) logits = out.logits[0, prefix.shape[1]-1:-1] tgt = ids[0, 1:] n = min(logits.shape[0], tgt.shape[0]) loss = F.cross_entropy(logits[:n], tgt[:n]) loss.backward() gnorm = float(cp.grad.abs().sum()) import math print(f"{tag}: seq_len={L} (mod128={L%128}) loss={float(loss):.3f} " f"finite={math.isfinite(gnorm)} gnorm={gnorm:.4e}", flush=True) return math.isfinite(gnorm) finally: if torch.cuda.is_available(): torch.backends.cuda.enable_cudnn_sdp(True) r1 = run("zero-token DEFAULT backend", zero=True, disable_cudnn=False) r2 = run("zero-token NO-cuDNN-SDPA", zero=True, disable_cudnn=True) r3 = run("nonzero-token DEFAULT backend", zero=False, disable_cudnn=False) print(f"RESULTS: default+zero={r1} nocudnn+zero={r2} default+nonzero={r3}", flush=True) print("BACKEND DIAG COMPLETE", flush=True)