File size: 2,590 Bytes
697943f
 
 
 
 
 
 
 
 
 
 
 
 
98f084f
 
 
 
 
 
c58f45e
697943f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
"""Minimal Space-side repro of condition D's zero-grad crash."""
import os, sys
sys.path.insert(0, "/app")
import torch
from huggingface_hub import hf_hub_download
from tinychess.checkpoint import load_model_only
from experiments.gemma_decoder.run_pilot import (make_positions, build_prefix,
    heldout_ce)
from experiments.gemma_decoder.capture import capture_with_hooks
from experiments.gemma_decoder.bridge import ResamplerBridge
from experiments.gemma_decoder import run_pilot as rp

print("torch", torch.__version__, flush=True)
from huggingface_hub import hf_hub_download
import shutil, os as _os
src_pt = hf_hub_download("cazyundee/tinychess-ckpt", "fast_wide/latest.pt",
                         token=_os.environ.get("HF_TOKEN"))
_os.makedirs("runs/_pilot_ckpts", exist_ok=True)
shutil.copy(src_pt, "runs/_pilot_ckpts/fast_wide.pt")
m, cfg = load_model_only("runs/_pilot_ckpts/fast_wide.pt")
poss = make_positions(6, seed=7)
caps = [capture_with_hooks(m, b, steps=6, adaptive=True) for b in poss]
from experiments.gemma_decoder.targets import targets_from_capture
items = [(c, targets_from_capture(c, ops=tuple(m.core.ops))["text"]) for c in caps]

# load REAL gemma
tok, gemma = rp.load_gemma("google/gemma-3-270m", "cpu")

bridge = ResamplerBridge(d_model=cfg.d_model, n_mem=cfg.n_mem,
                         n_ops=len(m.core.ops))
opt = torch.optim.Adam(bridge.parameters(), 1e-3)
for it in range(4):
    cap, text = items[it % len(items)]
    cap = dict(cap); cap["condition"] = "D"
    prefix = build_prefix(bridge, cap, "D", caps)
    print(f"it{it} prefix requires_grad={prefix.requires_grad} "
          f"fn={float(prefix.abs().mean()):.4e}", flush=True)
    ids = tok(text, return_tensors="pt").input_ids
    eos = tok.eos_token_id
    eos_list = eos if isinstance(eos, list) else [eos]
    body = ids[0, 1:]
    end = next((i for i, t in enumerate(body) if int(t) in eos_list), len(body))
    tgt = body[:end + 1]
    emb = gemma.model.embed_tokens(ids) * rp.GEMMA_SCALE
    full = torch.cat([prefix, emb], 1)
    out = gemma(inputs_embeds=full)
    logits = out.logits[0, prefix.shape[1]-1:-1]
    loss = torch.nn.functional.cross_entropy(logits[:len(tgt)], tgt)
    print(f"it{it} loss={float(loss):.3f} finite={torch.isfinite(loss).item()}", flush=True)
    opt.zero_grad()
    loss.backward()
    pg = {n: (float(p.grad.abs().sum()) if p.grad is not None else None)
          for n, p in bridge.named_parameters()}
    print(f"it{it} grads={ {k: (round(v,5) if v is not None else None) for k,v in pg.items()} }", flush=True)
print("DEBUG D COMPLETE", flush=True)