File size: 5,117 Bytes
788ee34 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | """Tests for the gemma_decoder prototype:
1. capture_with_hooks is a FAITHFUL mirror: aggregate == norm_out == h of the
unmodified forward; n_steps equal; trajectory[-1] == last captured state;
value identical; logits/argmax identical.
2. Hooks capture router [B,4] and mem_attn [B,64,8] per step, matching the
collect-based internal trace in distribution.
3. Defaults unchanged: running capture twice does not change model outputs.
Run: python experiments/gemma_decoder/tests_gemma.py
"""
import sys, os
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
import chess, torch
from tinychess.config import get_config
from tinychess.model import TinyChess
from experiments.gemma_decoder.capture import capture_with_hooks, reference_outputs
from experiments.gemma_decoder.targets import targets_from_capture
from experiments.gemma_decoder.bridge import LatentBridge, ResamplerBridge
def main():
torch.manual_seed(0)
cfg = get_config("full", max_steps=8)
m = TinyChess(cfg)
b = chess.Board()
# --- 1. faithful mirror ---
for adaptive in (True, False):
cap = capture_with_hooks(m, b, steps=6, adaptive=adaptive)
ref = reference_outputs(m, b, steps=6, adaptive=adaptive)
assert torch.allclose(cap["aggregate"], ref["h"], atol=1e-6), "aggregate != h"
assert torch.allclose(cap["n_steps"], ref["n_steps"]), "n_steps mismatch"
assert torch.allclose(cap["steps"][-1]["h"], ref["trajectory"][-1], atol=1e-6), "last state mismatch"
assert abs(cap["value"].item() - ref["value"].item()) < 1e-6, "value mismatch"
print("ok 1: capture mirrors unmodified forward (adaptive + fixed)")
# logits identity: capture path + real head math must reproduce ref logits
cap = capture_with_hooks(m, b, steps=6, adaptive=True)
ref = reference_outputs(m, b, steps=6, adaptive=True)
normed = cap["aggregate"]
pooled = torch.cat([normed.mean(1), normed.amax(1)], -1)
# reproduce logits pipeline
pb = __import__("tinychess.batching", fromlist=["encode_batch"]).encode_batch([b], [b.peek() if b.move_stack else None])
z = m.readout(pooled[:, :cfg.d_model])
cand = m.move_emb(pb.cand_fields)
sq_m = m.sq_to_move(normed)
import torch.nn.functional as F
idx_f = pb.cand_fields[..., 0].clamp(0, 63)
idx_t = pb.cand_fields[..., 1].clamp(0, 63)
cand = cand + torch.gather(sq_m, 1, idx_f[..., None].expand(-1, -1, sq_m.shape[-1]))
cand = cand + torch.gather(sq_m, 1, idx_t[..., None].expand(-1, -1, sq_m.shape[-1]))
ctx = z
for _ in range(cfg.n_refine):
z = m.refine(ctx, z)
logits = torch.einsum("bd,bld->bl", z, cand) * m.move_scale / (cfg.d_move ** 0.5)
logits = logits.masked_fill(~pb.cand_mask, float("-inf"))
assert torch.allclose(logits, ref["logits"], atol=1e-5), "logits mismatch"
assert logits.argmax(-1).equal(ref["logits"].argmax(-1))
print("ok 2: logits and argmax reproduce exactly from captured state")
# --- 2. hook data sanity ---
for s in cap["steps"]:
r = s["router"]
ma = s["mem_attn"]
assert r is not None and r.shape[-1] == len(m.core.ops) and abs(r.sum(-1).item() - 1) < 1e-5
assert ma is not None and ma.shape == (1, 64, 8) and abs(ma.sum(-1).mean().item() - 1) < 1e-4
print("ok 3: router weights + memory attention captured per step (normalised)")
# --- 3. determinism / no mutation ---
ref2 = reference_outputs(m, b, steps=6, adaptive=True)
assert torch.equal(ref["logits"], ref2["logits"])
print("ok 4: model outputs unchanged after instrumented runs")
# --- 4. targets from a real trajectory ---
cap = capture_with_hooks(m, b, steps=6, adaptive=True)
meta = targets_from_capture(cap, ops=tuple(m.core.ops))
assert meta["text"] and isinstance(meta["text"], str)
print("ok 5: grounded target: ", meta["text"][:180])
# --- 5. bridges produce correctly scaled prefix embeddings ---
lb = LatentBridge(d_model=cfg.d_model, n_mem=cfg.n_mem, n_ops=len(m.core.ops))
emb = lb(cap)
assert emb.shape[1] == 1 + len(cap["steps"]) * lb.tps
scale = 640 ** 0.5
unscaled = emb / scale
print(f"ok 6: LatentBridge prefix {tuple(emb.shape)}; token RMS={unscaled.pow(2).mean().sqrt():.3f} (target ~1 for embed-magnitude parity)")
rb = ResamplerBridge(d_model=cfg.d_model, n_mem=cfg.n_mem, n_ops=len(m.core.ops))
emb2 = rb(cap)
print(f"ok 7: ResamplerBridge prefix {tuple(emb2.shape)}")
# gradient flows into bridge but model params untouched
opt = torch.optim.Adam(lb.parameters(), 1e-3)
out = lb(cap).sum()
out.backward()
gs = {n: p.grad.abs().sum().item() for n, p in lb.named_parameters()}
assert all(v > 0 for v in gs.values()), f"null bridge grads: {gs}"
mg = [n for n, p in m.named_parameters() if p.grad is not None and p.grad.abs().sum() > 0]
assert not mg, f"gradient leaked into chess model: {mg}"
print("ok 8: gradients reach bridge only; zero into TinyChess")
print("ALL GEMMA PROTOTYPE TESTS PASSED")
if __name__ == "__main__":
main()
|