"""Minimal Space-side repro of condition D's zero-grad crash.""" import os, sys sys.path.insert(0, "/app") import torch from huggingface_hub import hf_hub_download from tinychess.checkpoint import load_model_only from experiments.gemma_decoder.run_pilot import (make_positions, build_prefix, heldout_ce) from experiments.gemma_decoder.capture import capture_with_hooks from experiments.gemma_decoder.bridge import ResamplerBridge from experiments.gemma_decoder import run_pilot as rp print("torch", torch.__version__, flush=True) from huggingface_hub import hf_hub_download import shutil, os as _os src_pt = hf_hub_download("cazyundee/tinychess-ckpt", "fast_wide/latest.pt", token=_os.environ.get("HF_TOKEN")) _os.makedirs("runs/_pilot_ckpts", exist_ok=True) shutil.copy(src_pt, "runs/_pilot_ckpts/fast_wide.pt") m, cfg = load_model_only("runs/_pilot_ckpts/fast_wide.pt") poss = make_positions(6, seed=7) caps = [capture_with_hooks(m, b, steps=6, adaptive=True) for b in poss] from experiments.gemma_decoder.targets import targets_from_capture items = [(c, targets_from_capture(c, ops=tuple(m.core.ops))["text"]) for c in caps] # load REAL gemma tok, gemma = rp.load_gemma("google/gemma-3-270m", "cpu") bridge = ResamplerBridge(d_model=cfg.d_model, n_mem=cfg.n_mem, n_ops=len(m.core.ops)) opt = torch.optim.Adam(bridge.parameters(), 1e-3) for it in range(4): cap, text = items[it % len(items)] cap = dict(cap); cap["condition"] = "D" prefix = build_prefix(bridge, cap, "D", caps) print(f"it{it} prefix requires_grad={prefix.requires_grad} " f"fn={float(prefix.abs().mean()):.4e}", flush=True) ids = tok(text, return_tensors="pt").input_ids eos = tok.eos_token_id eos_list = eos if isinstance(eos, list) else [eos] body = ids[0, 1:] end = next((i for i, t in enumerate(body) if int(t) in eos_list), len(body)) tgt = body[:end + 1] emb = gemma.model.embed_tokens(ids) * rp.GEMMA_SCALE full = torch.cat([prefix, emb], 1) out = gemma(inputs_embeds=full) logits = out.logits[0, prefix.shape[1]-1:-1] loss = torch.nn.functional.cross_entropy(logits[:len(tgt)], tgt) print(f"it{it} loss={float(loss):.3f} finite={torch.isfinite(loss).item()}", flush=True) opt.zero_grad() loss.backward() pg = {n: (float(p.grad.abs().sum()) if p.grad is not None else None) for n, p in bridge.named_parameters()} print(f"it{it} grads={ {k: (round(v,5) if v is not None else None) for k,v in pg.items()} }", flush=True) print("DEBUG D COMPLETE", flush=True)