Download experiments/gemma_decoder/debug_backend.py from cazyundee/training: direct link, hf CLI and curl.
- Browser
- Download file 2.59 kB
-
https://huggingface.co/spaces/cazyundee/training/resolve/main/experiments/gemma_decoder/debug_backend.py
- Command line
-
hf download hf://spaces/cazyundee/training/experiments/gemma_decoder/debug_backend.py
-
curl -L -o debug_backend.py https://huggingface.co/spaces/cazyundee/training/resolve/main/experiments/gemma_decoder/debug_backend.py
2.59 kB
| """Operator-requested diagnostics (steps 1-4) for the cuDNN SDPA NaN bug: | |
| 1) GPU + cudnn version + backend context | |
| 2) failing seq_len % 128 check | |
| 3) A-const-with-zero-token repro with cuDNN SDPA disabled | |
| 4) per-param grad finiteness both backends | |
| """ | |
| import os, sys | |
| sys.path.insert(0, "/app") | |
| import torch | |
| import torch.nn.functional as F | |
| print("torch", torch.__version__, flush=True) | |
| print("cuda available:", torch.cuda.is_available(), flush=True) | |
| if torch.cuda.is_available(): | |
| print("GPU:", torch.cuda.get_device_name(0), flush=True) | |
| print("capability:", torch.cuda.get_device_capability(0), flush=True) | |
| try: | |
| import torch.backends.cudnn as _c | |
| print("cudnn version:", _c.version(), "enabled:", _c.is_enabled(), flush=True) | |
| except Exception as e: | |
| print("cudnn import fail:", e, flush=True) | |
| from experiments.gemma_decoder import run_pilot as rp | |
| tok, gemma = rp.load_gemma("google/gemma-3-270m", "cpu") | |
| text = ("Some passes barely move the representation while others rework it " | |
| "substantly.") | |
| def const_with_zero_token(zero=True): | |
| cp = torch.nn.Parameter(torch.randn(1, 48, 640) * 0.02) | |
| if zero: | |
| cp.data[:, 0, :] = 0.0 | |
| def f(): | |
| return cp * rp.GEMMA_SCALE | |
| return cp, f | |
| def run(tag, zero=True, disable_cudnn=False): | |
| if disable_cudnn and torch.cuda.is_available(): | |
| torch.backends.cuda.enable_cudnn_sdp(False) | |
| try: | |
| cp, f = const_with_zero_token(zero) | |
| ids = tok(text, return_tensors="pt").input_ids | |
| emb = gemma.model.embed_tokens(ids) * rp.GEMMA_SCALE | |
| prefix = f() | |
| full = torch.cat([prefix, emb], 1) | |
| L = full.shape[1] | |
| out = gemma(inputs_embeds=full) | |
| logits = out.logits[0, prefix.shape[1]-1:-1] | |
| tgt = ids[0, 1:] | |
| n = min(logits.shape[0], tgt.shape[0]) | |
| loss = F.cross_entropy(logits[:n], tgt[:n]) | |
| loss.backward() | |
| gnorm = float(cp.grad.abs().sum()) | |
| import math | |
| print(f"{tag}: seq_len={L} (mod128={L%128}) loss={float(loss):.3f} " | |
| f"finite={math.isfinite(gnorm)} gnorm={gnorm:.4e}", flush=True) | |
| return math.isfinite(gnorm) | |
| finally: | |
| if torch.cuda.is_available(): | |
| torch.backends.cuda.enable_cudnn_sdp(True) | |
| r1 = run("zero-token DEFAULT backend", zero=True, disable_cudnn=False) | |
| r2 = run("zero-token NO-cuDNN-SDPA", zero=True, disable_cudnn=True) | |
| r3 = run("nonzero-token DEFAULT backend", zero=False, disable_cudnn=False) | |
| print(f"RESULTS: default+zero={r1} nocudnn+zero={r2} default+nonzero={r3}", flush=True) | |
| print("BACKEND DIAG COMPLETE", flush=True) | |