File size: 5,117 Bytes
788ee34
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
"""Tests for the gemma_decoder prototype:

1. capture_with_hooks is a FAITHFUL mirror: aggregate == norm_out == h of the
   unmodified forward; n_steps equal; trajectory[-1] == last captured state;
   value identical; logits/argmax identical.
2. Hooks capture router [B,4] and mem_attn [B,64,8] per step, matching the
   collect-based internal trace in distribution.
3. Defaults unchanged: running capture twice does not change model outputs.

Run: python experiments/gemma_decoder/tests_gemma.py
"""
import sys, os
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
import chess, torch
from tinychess.config import get_config
from tinychess.model import TinyChess
from experiments.gemma_decoder.capture import capture_with_hooks, reference_outputs
from experiments.gemma_decoder.targets import targets_from_capture
from experiments.gemma_decoder.bridge import LatentBridge, ResamplerBridge


def main():
    torch.manual_seed(0)
    cfg = get_config("full", max_steps=8)
    m = TinyChess(cfg)
    b = chess.Board()

    # --- 1. faithful mirror ---
    for adaptive in (True, False):
        cap = capture_with_hooks(m, b, steps=6, adaptive=adaptive)
        ref = reference_outputs(m, b, steps=6, adaptive=adaptive)
        assert torch.allclose(cap["aggregate"], ref["h"], atol=1e-6), "aggregate != h"
        assert torch.allclose(cap["n_steps"], ref["n_steps"]), "n_steps mismatch"
        assert torch.allclose(cap["steps"][-1]["h"], ref["trajectory"][-1], atol=1e-6), "last state mismatch"
        assert abs(cap["value"].item() - ref["value"].item()) < 1e-6, "value mismatch"
    print("ok 1: capture mirrors unmodified forward (adaptive + fixed)")

    # logits identity: capture path + real head math must reproduce ref logits
    cap = capture_with_hooks(m, b, steps=6, adaptive=True)
    ref = reference_outputs(m, b, steps=6, adaptive=True)
    normed = cap["aggregate"]
    pooled = torch.cat([normed.mean(1), normed.amax(1)], -1)
    # reproduce logits pipeline
    pb = __import__("tinychess.batching", fromlist=["encode_batch"]).encode_batch([b], [b.peek() if b.move_stack else None])
    z = m.readout(pooled[:, :cfg.d_model])
    cand = m.move_emb(pb.cand_fields)
    sq_m = m.sq_to_move(normed)
    import torch.nn.functional as F
    idx_f = pb.cand_fields[..., 0].clamp(0, 63)
    idx_t = pb.cand_fields[..., 1].clamp(0, 63)
    cand = cand + torch.gather(sq_m, 1, idx_f[..., None].expand(-1, -1, sq_m.shape[-1]))
    cand = cand + torch.gather(sq_m, 1, idx_t[..., None].expand(-1, -1, sq_m.shape[-1]))
    ctx = z
    for _ in range(cfg.n_refine):
        z = m.refine(ctx, z)
    logits = torch.einsum("bd,bld->bl", z, cand) * m.move_scale / (cfg.d_move ** 0.5)
    logits = logits.masked_fill(~pb.cand_mask, float("-inf"))
    assert torch.allclose(logits, ref["logits"], atol=1e-5), "logits mismatch"
    assert logits.argmax(-1).equal(ref["logits"].argmax(-1))
    print("ok 2: logits and argmax reproduce exactly from captured state")

    # --- 2. hook data sanity ---
    for s in cap["steps"]:
        r = s["router"]
        ma = s["mem_attn"]
        assert r is not None and r.shape[-1] == len(m.core.ops) and abs(r.sum(-1).item() - 1) < 1e-5
        assert ma is not None and ma.shape == (1, 64, 8) and abs(ma.sum(-1).mean().item() - 1) < 1e-4
    print("ok 3: router weights + memory attention captured per step (normalised)")

    # --- 3. determinism / no mutation ---
    ref2 = reference_outputs(m, b, steps=6, adaptive=True)
    assert torch.equal(ref["logits"], ref2["logits"])
    print("ok 4: model outputs unchanged after instrumented runs")

    # --- 4. targets from a real trajectory ---
    cap = capture_with_hooks(m, b, steps=6, adaptive=True)
    meta = targets_from_capture(cap, ops=tuple(m.core.ops))
    assert meta["text"] and isinstance(meta["text"], str)
    print("ok 5: grounded target: ", meta["text"][:180])

    # --- 5. bridges produce correctly scaled prefix embeddings ---
    lb = LatentBridge(d_model=cfg.d_model, n_mem=cfg.n_mem, n_ops=len(m.core.ops))
    emb = lb(cap)
    assert emb.shape[1] == 1 + len(cap["steps"]) * lb.tps
    scale = 640 ** 0.5
    unscaled = emb / scale
    print(f"ok 6: LatentBridge prefix {tuple(emb.shape)}; token RMS={unscaled.pow(2).mean().sqrt():.3f} (target ~1 for embed-magnitude parity)")
    rb = ResamplerBridge(d_model=cfg.d_model, n_mem=cfg.n_mem, n_ops=len(m.core.ops))
    emb2 = rb(cap)
    print(f"ok 7: ResamplerBridge prefix {tuple(emb2.shape)}")

    # gradient flows into bridge but model params untouched
    opt = torch.optim.Adam(lb.parameters(), 1e-3)
    out = lb(cap).sum()
    out.backward()
    gs = {n: p.grad.abs().sum().item() for n, p in lb.named_parameters()}
    assert all(v > 0 for v in gs.values()), f"null bridge grads: {gs}"
    mg = [n for n, p in m.named_parameters() if p.grad is not None and p.grad.abs().sum() > 0]
    assert not mg, f"gradient leaked into chess model: {mg}"
    print("ok 8: gradients reach bridge only; zero into TinyChess")
    print("ALL GEMMA PROTOTYPE TESTS PASSED")


if __name__ == "__main__":
    main()