agent
debug_d: restore m,cfg assignment lost in the hub-pull patch
c58f45e
Raw History Blame Contribute Delete
2.59 kB
"""Minimal Space-side repro of condition D's zero-grad crash."""
import os, sys
sys.path.insert(0, "/app")
import torch
from huggingface_hub import hf_hub_download
from tinychess.checkpoint import load_model_only
from experiments.gemma_decoder.run_pilot import (make_positions, build_prefix,
heldout_ce)
from experiments.gemma_decoder.capture import capture_with_hooks
from experiments.gemma_decoder.bridge import ResamplerBridge
from experiments.gemma_decoder import run_pilot as rp
print("torch", torch.__version__, flush=True)
from huggingface_hub import hf_hub_download
import shutil, os as _os
src_pt = hf_hub_download("cazyundee/tinychess-ckpt", "fast_wide/latest.pt",
token=_os.environ.get("HF_TOKEN"))
_os.makedirs("runs/_pilot_ckpts", exist_ok=True)
shutil.copy(src_pt, "runs/_pilot_ckpts/fast_wide.pt")
m, cfg = load_model_only("runs/_pilot_ckpts/fast_wide.pt")
poss = make_positions(6, seed=7)
caps = [capture_with_hooks(m, b, steps=6, adaptive=True) for b in poss]
from experiments.gemma_decoder.targets import targets_from_capture
items = [(c, targets_from_capture(c, ops=tuple(m.core.ops))["text"]) for c in caps]
# load REAL gemma
tok, gemma = rp.load_gemma("google/gemma-3-270m", "cpu")
bridge = ResamplerBridge(d_model=cfg.d_model, n_mem=cfg.n_mem,
n_ops=len(m.core.ops))
opt = torch.optim.Adam(bridge.parameters(), 1e-3)
for it in range(4):
cap, text = items[it % len(items)]
cap = dict(cap); cap["condition"] = "D"
prefix = build_prefix(bridge, cap, "D", caps)
print(f"it{it} prefix requires_grad={prefix.requires_grad} "
f"fn={float(prefix.abs().mean()):.4e}", flush=True)
ids = tok(text, return_tensors="pt").input_ids
eos = tok.eos_token_id
eos_list = eos if isinstance(eos, list) else [eos]
body = ids[0, 1:]
end = next((i for i, t in enumerate(body) if int(t) in eos_list), len(body))
tgt = body[:end + 1]
emb = gemma.model.embed_tokens(ids) * rp.GEMMA_SCALE
full = torch.cat([prefix, emb], 1)
out = gemma(inputs_embeds=full)
logits = out.logits[0, prefix.shape[1]-1:-1]
loss = torch.nn.functional.cross_entropy(logits[:len(tgt)], tgt)
print(f"it{it} loss={float(loss):.3f} finite={torch.isfinite(loss).item()}", flush=True)
opt.zero_grad()
loss.backward()
pg = {n: (float(p.grad.abs().sum()) if p.grad is not None else None)
for n, p in bridge.named_parameters()}
print(f"it{it} grads={ {k: (round(v,5) if v is not None else None) for k,v in pg.items()} }", flush=True)
print("DEBUG D COMPLETE", flush=True)