Download experiments/gemma_decoder/debug_d.py from cazyundee/training: direct link, hf CLI and curl.
- Browser
- Download file 2.59 kB
-
https://huggingface.co/spaces/cazyundee/training/resolve/main/experiments/gemma_decoder/debug_d.py
- Command line
-
hf download hf://spaces/cazyundee/training/experiments/gemma_decoder/debug_d.py
-
curl -L -o debug_d.py https://huggingface.co/spaces/cazyundee/training/resolve/main/experiments/gemma_decoder/debug_d.py
2.59 kB
| """Minimal Space-side repro of condition D's zero-grad crash.""" | |
| import os, sys | |
| sys.path.insert(0, "/app") | |
| import torch | |
| from huggingface_hub import hf_hub_download | |
| from tinychess.checkpoint import load_model_only | |
| from experiments.gemma_decoder.run_pilot import (make_positions, build_prefix, | |
| heldout_ce) | |
| from experiments.gemma_decoder.capture import capture_with_hooks | |
| from experiments.gemma_decoder.bridge import ResamplerBridge | |
| from experiments.gemma_decoder import run_pilot as rp | |
| print("torch", torch.__version__, flush=True) | |
| from huggingface_hub import hf_hub_download | |
| import shutil, os as _os | |
| src_pt = hf_hub_download("cazyundee/tinychess-ckpt", "fast_wide/latest.pt", | |
| token=_os.environ.get("HF_TOKEN")) | |
| _os.makedirs("runs/_pilot_ckpts", exist_ok=True) | |
| shutil.copy(src_pt, "runs/_pilot_ckpts/fast_wide.pt") | |
| m, cfg = load_model_only("runs/_pilot_ckpts/fast_wide.pt") | |
| poss = make_positions(6, seed=7) | |
| caps = [capture_with_hooks(m, b, steps=6, adaptive=True) for b in poss] | |
| from experiments.gemma_decoder.targets import targets_from_capture | |
| items = [(c, targets_from_capture(c, ops=tuple(m.core.ops))["text"]) for c in caps] | |
| # load REAL gemma | |
| tok, gemma = rp.load_gemma("google/gemma-3-270m", "cpu") | |
| bridge = ResamplerBridge(d_model=cfg.d_model, n_mem=cfg.n_mem, | |
| n_ops=len(m.core.ops)) | |
| opt = torch.optim.Adam(bridge.parameters(), 1e-3) | |
| for it in range(4): | |
| cap, text = items[it % len(items)] | |
| cap = dict(cap); cap["condition"] = "D" | |
| prefix = build_prefix(bridge, cap, "D", caps) | |
| print(f"it{it} prefix requires_grad={prefix.requires_grad} " | |
| f"fn={float(prefix.abs().mean()):.4e}", flush=True) | |
| ids = tok(text, return_tensors="pt").input_ids | |
| eos = tok.eos_token_id | |
| eos_list = eos if isinstance(eos, list) else [eos] | |
| body = ids[0, 1:] | |
| end = next((i for i, t in enumerate(body) if int(t) in eos_list), len(body)) | |
| tgt = body[:end + 1] | |
| emb = gemma.model.embed_tokens(ids) * rp.GEMMA_SCALE | |
| full = torch.cat([prefix, emb], 1) | |
| out = gemma(inputs_embeds=full) | |
| logits = out.logits[0, prefix.shape[1]-1:-1] | |
| loss = torch.nn.functional.cross_entropy(logits[:len(tgt)], tgt) | |
| print(f"it{it} loss={float(loss):.3f} finite={torch.isfinite(loss).item()}", flush=True) | |
| opt.zero_grad() | |
| loss.backward() | |
| pg = {n: (float(p.grad.abs().sum()) if p.grad is not None else None) | |
| for n, p in bridge.named_parameters()} | |
| print(f"it{it} grads={ {k: (round(v,5) if v is not None else None) for k,v in pg.items()} }", flush=True) | |
| print("DEBUG D COMPLETE", flush=True) | |