File size: 15,412 Bytes
788ee34
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dd20003
 
 
 
 
 
 
 
 
 
 
 
 
 
 
788ee34
 
 
dd20003
 
 
 
 
884986c
 
 
 
788ee34
884986c
 
788ee34
 
 
 
 
 
dd20003
 
 
788ee34
 
 
 
 
 
 
 
 
 
33dbd59
788ee34
 
 
 
 
 
 
6341f4d
 
 
 
 
788ee34
 
 
 
 
 
 
33dbd59
 
 
788ee34
33dbd59
f473997
33dbd59
f473997
494b63d
 
 
 
 
f473997
33dbd59
 
 
788ee34
 
33dbd59
 
788ee34
 
 
 
 
 
2e7e49c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
788ee34
 
 
6341f4d
 
788ee34
 
 
 
33dbd59
788ee34
 
33dbd59
788ee34
33dbd59
 
 
6341f4d
 
 
788ee34
 
33dbd59
788ee34
 
 
 
 
 
 
 
 
 
 
9ac5f43
 
 
f7cd607
 
 
788ee34
 
 
 
dd20003
788ee34
f7cd607
 
 
 
 
 
 
 
 
9ac5f43
 
 
 
 
 
 
788ee34
 
2e7e49c
 
788ee34
 
 
 
2e7e49c
 
 
 
788ee34
 
 
 
 
 
 
 
 
 
dd20003
 
 
 
 
 
 
 
788ee34
 
2e7e49c
788ee34
 
 
33dbd59
2e7e49c
 
 
 
 
 
 
788ee34
 
 
 
2e7e49c
788ee34
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
"""Pilot experiment: does a trainable bridge make frozen Gemma COMMUNICATE
TinyChess latent computation? Controls A-G from the brief, PT vs IT variants.

Design:
  - Dataset: N_pos positions from random opening plies of real games feeding
    capture_with_hooks; each becomes (cap_controls, grounded_text).
  - Conditions (prefix construction):
      A constant  : learned prefix, identical for every input (bridge sees zeros)
      B zero      : all-zero prefix embeddings (scaled)
      C shuffled  : prefix from an UNRELATED position (mismatched text)
      D final     : only final aggregate state (no trajectory)
      E full      : whole trajectory + dynamics (the real thing)
      F metadata  : engineered observables only (StepDescriptor, no raw h)
      G combined  : metadata + raw latents (resampler)
  - Train: teacher-forced next-token loss through FROZEN gemma (grad on),
    loss only on the target text tokens.
  - Eval: (1) held-out loss; (2) generation + claim-extraction check: how many
    generated sentences' binned claims match the ground-truth observables;
    (3) EOS termination rate.

Frozen-Gemma contract (asserted): all params requires_grad=False; model.eval();
no torch.no_grad() around the forward; bridge grads nonzero; gemma grads absent.

Usage (Space): python experiments/gemma_decoder/run_pilot.py --model google/gemma-3-270m --iters 40
Local CPU smoke possible with --iters 2 --positions 4 (no download).
"""
from __future__ import annotations
import argparse, json, os, random, sys, time

ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.insert(0, ROOT)
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

import chess
import torch
import torch.nn.functional as F

from tinychess.config import get_config
from tinychess.model import TinyChess
from experiments.gemma_decoder.capture import capture_with_hooks
from experiments.gemma_decoder.targets import targets_from_capture
from experiments.gemma_decoder.bridge import LatentBridge, ResamplerBridge, GEMMA_SCALE


def make_positions(n, seed=7):
    rng = random.Random(seed)
    out = []
    while len(out) < n:
        b = chess.Board()
        k = rng.randint(4, 24)
        for _ in range(k):
            ms = list(b.legal_moves)
            if not ms or b.is_game_over():
                break
            b.push(rng.choice(ms))
        if not b.is_game_over():
            out.append(b.copy(stack=False))
    return out


def load_gemma(model_id, device):
    from transformers import AutoTokenizer, Gemma3ForCausalLM
    tok = AutoTokenizer.from_pretrained(model_id)
    gemma = Gemma3ForCausalLM.from_pretrained(model_id, torch_dtype=torch.float32)
    gemma.eval()
    for p in gemma.parameters():
        p.requires_grad_(False)
    n_gemma = sum(p.numel() for p in gemma.parameters())
    n_frozen = sum(1 for p in gemma.parameters() if not p.requires_grad)
    assert n_gemma == sum(p.numel() for p in gemma.parameters())
    gemma = gemma.to(device)
    return tok, gemma


class ConstantPrefix(torch.nn.Module):
    """Condition A done right: a LEARNED prefix (soft prompt) disconnected
    from the input entirely - trainable, so gradients exist by construction."""
    def __init__(self, n_tokens, d=640):
        super().__init__()
        self.emb = torch.nn.Parameter(torch.randn(n_tokens, d) * 0.02)

    def forward(self, cap):
        B = cap["steps"][0]["h"].shape[0]
        return self.emb[None].expand(B, -1, -1) * GEMMA_SCALE


CONSTANT_A = None  # set in main(); condition-A learned soft prompt


def build_prefix(bridge, cap, condition, pool):
    """Returns embedded prefix [1,P,640] (already Gemma-scaled by the bridge).
    Conditions swap or corrupt inputs; bridges themselves are unaffected."""
    if condition == "A":      # constant: LEARNED soft prompt, no input at all
        global CONSTANT_A
        if CONSTANT_A is None:
            CONSTANT_A = ConstantPrefix(n_tokens=1 + cap.get("n_steps_target", 6) * 8)
        return CONSTANT_A(cap)
    if condition == "B":      # zero prefix -> near-zero (exact-zero embedding
        # rows trigger a torch-2.13 gemma-backward NaN bug; 1e-4 constant
        # carries no information, preserving B's scientific role as the
        # no-information floor)
        P = 1 + min(len(cap["steps"]), 4) * bridge.tps
        eps = torch.full((1, P, 640), 1e-4, device=cap["steps"][0]["h"].device)
        return eps * GEMMA_SCALE
    if condition == "C":      # shuffled: prefix of an unrelated position
        other = random.choice(pool)
        return bridge(other)
    if condition == "D":      # final state only
        cap_d = {"steps": [{"h": cap["aggregate"].detach(),
                            "h_prev": cap["steps"][0]["h"].detach(),
                            "p_halt": (cap["steps"][-1]["p_halt"].detach()
                                       if cap["steps"][-1].get("p_halt") is not None
                                       else None),
                            "router": None, "mem_attn": None,
                            "active": cap["steps"][-1]["active"],
                            "stop_weight": cap["steps"][-1]["stop_weight"]}],
                 "memory": cap.get("memory"), "aggregate": cap["aggregate"],
                 "n_steps": cap["n_steps"]}
        return bridge(cap_d)
    return bridge(cap)        # E full, F metadata-only, G combined (bridge decides)


def train_condition(bridge, gemma, tok, items, device, iters, lr=1e-3,
                    max_text_tokens=96, log_every=5, trainable=True):
    opt = torch.optim.Adam(bridge.parameters(), lr)
    losses = []
    bos = tok.bos_token_id
    eos = tok.eos_token_id
    for it in range(iters):
        cap, text, pool = items(it)
        prefix = build_prefix(bridge, cap, cap.get("condition", "E"), pool).to(device)
        ids = tok(text, return_tensors="pt").input_ids.to(device)  # [1,T] incl bos
        eos_list = eos if isinstance(eos, list) else [eos]
        body = ids[0, 1:]
        end = next((i for i, t in enumerate(body) if int(t) in eos_list), len(body))
        tgt_ids = body[:end + 1]  # text tokens + the terminator (learn to stop)
        # embeddings for the text tokens, Gemma-scaled as the model expects
        emb = gemma.model.embed_tokens(ids) * GEMMA_SCALE          # [1,T,640]
        full = torch.cat([prefix, emb], 1)
        out = gemma(inputs_embeds=full)
        logits = out.logits[0, prefix.shape[1]-1:-1]                # predict text tokens
        loss = F.cross_entropy(logits[:len(tgt_ids)], tgt_ids)
        opt.zero_grad()
        if trainable:
            loss.backward()
        # else: B zero-prefix involves no parameters; loss is floor-only        # train - record loss as the floor measurement
        # assertions of the frozen contract every step (cheap on 270M)
        if trainable:
            import math
            gnorm = sum(p.grad.abs().sum().item() for p in bridge.parameters() if p.grad is not None)
            if not math.isfinite(gnorm) or gnorm == 0:
                pg = {n: (p.grad.abs().sum().item() if p.grad is not None else None)
                      for n, p in bridge.named_parameters()}
                print(f"GRAD-DEBUG cond={cap.get('condition')} it={it} loss={float(loss):.3f} "
                      f"finite={math.isfinite(float(loss))} prefix_fn={float(prefix.abs().mean()):.4e} "
                      f"grads={pg}", flush=True)
            assert math.isfinite(gnorm) and gnorm > 0,                 f"bridge grads invalid: {'NaN' if not math.isfinite(gnorm) else 'zero'}"
        leaked = [n for n, p in gemma.named_parameters()
                  if p.requires_grad and p.grad is not
                  None and p.grad.abs().sum() > 0]
        assert not leaked, f"grad into frozen gemma: {leaked[:3]}"
        torch.nn.utils.clip_grad_norm_(bridge.parameters(), 1.0)
        if trainable:
            opt.step()
        losses.append(float(loss))
        if log_every and (it + 1) % log_every == 0:
            print(f"    iter {it+1}: loss {sum(losses[-log_every:])/log_every:.3f}", flush=True)
    return losses


def heldout_ce(bridge, gemma, tok, cap, pool, text, device):
    """Teacher-forced CE of the target text under the bridge's prefix on an
    UNSEEN position. Pure measurement (no backward, no opt)."""
    with torch.no_grad():
        prefix = build_prefix(bridge, cap, cap.get("condition", "E"), pool).to(device)
        ids = tok(text, return_tensors="pt").input_ids.to(device)
        eos = tok.eos_token_id
        eos_list = eos if isinstance(eos, list) else [eos]
        body = ids[0, 1:]
        end = next((i for i, t in enumerate(body) if int(t) in eos_list), len(body))
        tgt_ids = body[:end + 1]
        emb = gemma.model.embed_tokens(ids[:, :1 + end]).to(device)
        if emb.shape[-1] != prefix.shape[-1]:
            emb = emb[..., :prefix.shape[-1]]
        full = torch.cat([prefix, emb], 1)
        out = gemma(inputs_embeds=full)
        logits = out.logits[0, prefix.shape[1]-1:prefix.shape[1]-1+len(tgt_ids)]
        return float(torch.nn.functional.cross_entropy(logits, tgt_ids.to(device)))


@torch.no_grad()
def generate(bridge, gemma, tok, cap, pool, device, max_new=60, temperature=0.7):
    prefix = build_prefix(bridge, cap, "E", pool).to(device)
    eos_ids = tok.eos_token_id
    eos_ids = set(eos_ids if isinstance(eos_ids, list) else [eos_ids])
    bos = torch.tensor([[tok.bos_token_id]], device=device)
    emb = gemma.model.embed_tokens(bos) * GEMMA_SCALE
    full = torch.cat([prefix, emb], 1)
    out = gemma(inputs_embeds=full)
    next_tok = torch.multinomial(F.softmax(out.logits[0, -1] / temperature, -1), 1).view(1, 1)  # [1,1]
    generated = [int(next_tok)]
    for _ in range(max_new - 1):
        e = gemma.model.embed_tokens(next_tok) * GEMMA_SCALE        # [1,1,640]
        full = torch.cat([full, e], 1)
        out = gemma(inputs_embeds=full)                            # re-feed (no cache in mock path; fine at pilot scale)
        cand = torch.multinomial(F.softmax(out.logits[0, -1] / temperature, -1), 1).view(1, 1)
        t = int(cand)
        _eos = tok.eos_token_id
        _eos_set = set(_eos if isinstance(_eos, list) else [_eos])
        if t in _eos_set:
            return tok.decode(generated), True
        generated.append(t)
        next_tok = cand
    return tok.decode(generated), False


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--model", default="google/gemma-3-270m")
    ap.add_argument("--positions", type=int, default=24)
    ap.add_argument("--iters", type=int, default=40)
    ap.add_argument("--steps", type=int, default=6)
    ap.add_argument("--seed", type=int, default=7)
    ap.add_argument("--device", default="cpu")
    ap.add_argument("--checkpoint", default=None,
                    help="trained TinyChess checkpoint (recommended: real "
                         "dynamics; random init gives near-constant latents)")
    ap.add_argument("--hf-checkpoint", default=None,
                    help="pull the TinyChess checkpoint from cazyundee/"
                         "tinychess-ckpt via HF_TOKEN (e.g. test_web_exfil/latest.pt)")
    ap.add_argument("--out", default=None)
    args = ap.parse_args()

    t_start = time.time()
    global CONSTANT_A
    torch.manual_seed(args.seed)
    if args.hf_checkpoint and not args.checkpoint:
        import shutil
        from huggingface_hub import hf_hub_download
        src_pt = hf_hub_download("cazyundee/tinychess-ckpt", args.hf_checkpoint,
                                 token=os.environ.get("HF_TOKEN"))
        os.makedirs("runs/_pilot_ckpts", exist_ok=True)
        args.checkpoint = "runs/_pilot_ckpts/" + os.path.basename(os.path.dirname(args.hf_checkpoint)) + ".pt"
        shutil.copy(src_pt, args.checkpoint)
        print(f"[hf-checkpoint] pulled {args.hf_checkpoint}")
    if args.checkpoint:
        from tinychess.checkpoint import load_model_only
        model, cfg_ = load_model_only(args.checkpoint)
        cfg = model.cfg
    else:
        cfg = get_config("full", max_steps=8)
        model = TinyChess(cfg)

    positions = make_positions(args.positions, seed=args.seed)
    caps = [capture_with_hooks(model, b, steps=args.steps, adaptive=True)
            for b in positions]

    # grounded targets
    items = [(cap, targets_from_capture(cap, ops=tuple(model.core.ops))["text"])
             for cap in caps]
    # held-out split: last 20% NEVER seen in training (brief requirement)
    n_hold = max(2, len(items) // 5)
    train_items, hold_items = items[:-n_hold], items[-n_hold:]
    print(f"dataset: {len(train_items)} train / {n_hold} held-out; example target:")
    print("   ", items[0][1][:160])

    results = {"model": args.model, "seed": args.seed, "positions": len(items),
               "conditions": {}, "time": time.strftime("%F %T"), "iters": args.iters}

    tok, gemma = load_gemma(args.model, args.device)

    for condition in ["A", "B", "D", "E", "F", "G"]:
        if condition == "F":
            continue  # metadata-only needs a dedicated bridge variant; phase 2
        # fresh bridge per condition (independent training); condition A uses
        # the learned constant prefix as ITS trainable module
        if condition == "A":
            CONSTANT_A = ConstantPrefix(n_tokens=1 + args.steps * 8).to(args.device)
            bridge = CONSTANT_A
        else:
            bridge = ResamplerBridge(d_model=cfg.d_model, n_mem=cfg.n_mem,
                                     n_ops=len(model.core.ops)).to(args.device)
        rng = random.Random(args.seed)
        def item_fn(it, cond=condition, rng=rng):
            cap, text = train_items[rng.randrange(len(train_items))]
            cap = dict(cap); cap["condition"] = cond
            return cap, text, caps
        losses = train_condition(bridge, gemma, tok, item_fn, args.device,
                                 args.iters, trainable=(condition != "B"))
        # held-out CE on unseen positions, then generation on a held-out item
        heldout_loss = None
        for cap_h, text_h in hold_items:
            cap_h = dict(cap_h); cap_h["condition"] = condition
            hl = heldout_ce(bridge, gemma, tok, cap_h, caps, text_h, args.device)
            heldout_loss = hl if heldout_loss is None else (heldout_loss + hl) / 2
        cap0, text0 = hold_items[0]
        cap0 = dict(cap0); cap0["condition"] = condition
        gen_text, hit_eos = generate(bridge, gemma, tok, cap0, caps, args.device)
        rec = {"final_loss": sum(losses[-5:]) / min(5, len(losses)),
               "first_iters_loss": sum(losses[:5]) / min(5, len(losses)),
               "heldout_loss": heldout_loss, "gen_on": "heldout",
               "gen_hit_eos": hit_eos,
               "gen_sample": gen_text[:300], "true_target": text0[:300]}
        results["conditions"][condition] = rec
        print(f"condition {condition}: loss {rec['final_loss']:.3f} (from {rec['first_iters_loss']:.3f}) "
              f"eos={hit_eos}")
        print("    gen:", gen_text[:180].replace("\n", " "))
        print("    tgt:", text0[:180].replace("\n", " "))

    if args.out:
        os.makedirs(os.path.dirname(args.out), exist_ok=True)
        json.dump(results, open(args.out, "w"), indent=2)
        print(f"[saved] {args.out}")
    print(f"PILOT COMPLETE in {time.time()-t_start:.0f}s")


if __name__ == "__main__":
    main()