#!/usr/bin/env python3 """Latent capture for the Gemma decoder prototype (gemma_decoder/). Design constraints (from the prototype brief): * TinyChess's default execution must be UNCHANGED. We do not touch the model: we run an instrumented SEPARATE forward pass that mirrors TinyChess.forward() for family='recurrent' step by step, using the SAME module calls, so the math (and hence outputs) is identical. * Capture BEFORE detach: the stock trace mode detaches on append. Here we keep live tensors (requires_grad respected) but hand out clones so caller code can detach at will. * ACT semantics preserved: which examples are still running at each step, per-example n_steps, halting-weighted aggregate vs raw per-step states, and the remainder-weight stop step. Usage: from experiments.gemma_decoder.capture import capture_trajectory, make_batch cap = capture_trajectory(model, board, steps=6, adaptive=True) cap["steps"][t]["h"] # [B,64,64] live tensor cap["steps"][t]["active"] # [B] bool: examples still computing at t ... Non-interference test lives in tests_gemma.py (test_capture_identical). """ from __future__ import annotations import sys, os sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) import chess import torch from tinychess.batching import encode_batch @torch.no_grad() def _identical_check_env(): pass def make_batch(board: chess.Board): pb = encode_batch([board], [board.peek() if board.move_stack else None]) return pb def capture_trajectory(model, board, steps: int = 6, adaptive: bool = True, device: str = "cpu"): """Mirror TinyChess.forward for family='recurrent' with live capture. Returns a dict: B steps; each step t dict: h (post-update), h_prev, router [B,4] (or None), p_halt [B], mem_attn [B,64,8] (or None), active [B] bool (ACT: still running at the START of step t), stop_weight [B] (remainder weight IF this is the element's stop step). Final: pooled [B,2d], value [B,1], memory [B,8,d], n_steps [B], aggregate [B,64,d] (halting-weighted mean == what the real forward computes when adaptive=True), and the model's actual final outputs from a plain (unmodified) forward call, for verification. """ cfg = model.cfg pb = encode_batch([board], [board.peek() if board.move_stack else None]) pb = pb.to(device) B = pb.squares.shape[0] d = cfg.d_model x = model.encoder(pb.squares, pb.extras, pb.glob) mem = model.core.mem0[None].expand(B, -1, -1).contiguous() if cfg.use_memory else None core = model.core cap = {"steps": [], "n_steps_target": steps, "adaptive": adaptive} x_out = None if adaptive and cfg.use_halting: # mirror _act_loop but capture per step still = torch.ones(B, device=device) acc = torch.zeros(B, device=device) x_out = torch.zeros_like(x) n_steps = torch.zeros(B, device=device) thr = cfg.halt_threshold mem_out = torch.zeros_like(mem) if mem is not None else None for t in range(steps): active_t = still.clone() x, mem, ph = core(x, mem, t) if ph is None: ph = torch.full((B,), 1.0 / steps, device=device) forced_min = t < (cfg.min_steps - 1) is_last = (t == steps - 1) new_acc = acc + ph finish = ((new_acc > thr) | is_last) & (~torch.tensor(forced_min, device=device)) finish = finish & (still > 0) w_stop = torch.where(finish, (1.0 - acc).clamp(min=0.0), ph) * still # capture BEFORE accumulating into x_out cap["steps"].append({ "h": x, "p_halt": ph, "active": active_t, "stop_weight": w_stop, "router": None, "mem_attn": None, # filled below if needed }) x_out = x_out + w_stop[:, None, None] * x if mem_out is not None: mem_out = mem_out + w_stop[:, None, None] * mem n_steps = n_steps + still acc = torch.where(finish, acc, new_acc) still = torch.where(finish, torch.zeros_like(still), still) if float(still.max()) == 0.0: break cap["n_steps"] = n_steps cap["aggregate"] = x_out cap["memory"] = mem_out if mem_out is not None else mem else: halts = [] for t in range(steps): x, mem, ph = core(x, mem, t) halts.append(ph) cap["steps"].append({ "h": x, "p_halt": ph, "active": torch.ones(B, dtype=torch.bool, device=device), "stop_weight": torch.zeros(B, device=device), "router": None, "mem_attn": None, }) cap["n_steps"] = torch.full((B,), float(steps), device=device) cap["aggregate"] = x cap["memory"] = mem # Router weights and mem_attn for verification/inspection: recompute via # the collect mechanism on a second mirrored pass? Cheaper: they are pure # functions of (x_in, step) inside core.forward; rather than duplicating # numerics we register ONE-TIME forward hooks during the mirrored pass. # (The main pass above already called core(x, mem, t) with default # collect=None, so no internal trace exists.) For the prototype, router # weights and mem_attn are captured by a small hook installed per step # in capture_with_hooks() below. return cap @torch.enable_grad() def capture_with_hooks(model, board, steps: int = 6, adaptive: bool = True, device: str = "cpu"): """Full capture INCLUDING router weights [B,4] and mem_attn [B,64,8]. Uses forward hooks on core._mlp (fires once per core step, giving the router-weighted delta decomposition) and core._mem_read (gives mem_attn). Hook payloads are cloned immediately, so upstream in-place writes cannot corrupt them. """ cfg = model.cfg pb = encode_batch([board], [board.peek() if board.move_stack else None]) pb = pb.to(device) B = pb.squares.shape[0] x = model.encoder(pb.squares, pb.extras, pb.glob) mem = model.core.mem0[None].expand(B, -1, -1).contiguous() if cfg.use_memory else None core = model.core router_ws, mem_attns = [], [] state = {"mem": None} # updated each step by the loop below hooks = [] rbuf, mbuf = [], [] def router_linear_hook(module, inp, out): rbuf.append(torch.softmax(out / core.cfg.router_temp, -1).detach().clone()) if cfg.use_router: hooks.append(core.router.register_forward_hook(router_linear_hook)) if cfg.use_memory: state = {"mem": None} def mem_r_hook(module, inp, out): # q = post-linear query [B,64,d]; recompute the read attention with # the same math as core._mem_read against the memory CURRENT at # this step (tracked in state["mem"] from the mirrored loop). q = out.detach() memv = state["mem"] if memv is not None and q.dim() == 3 and memv.dim() == 3: att = torch.einsum("bnd,bmd->bnm", q, memv) / (core.d ** 0.5) att = att + torch.log(core.mem_mask.clamp_min(1e-9))[None, None] mbuf.append(att.softmax(-1).detach().clone()) hooks.append(core.mem_r.register_forward_hook(mem_r_hook)) try: x_prev = x halts = [] step_states = [] if adaptive and cfg.use_halting: still = torch.ones(B, device=device) acc = torch.zeros(B, device=device) x_out = torch.zeros_like(x) n_steps = torch.zeros(B, device=device) mem_out = torch.zeros_like(mem) if mem is not None else None thr = cfg.halt_threshold for t in range(steps): active_t = still.clone() pre_r = len(rbuf) pre_m = len(mbuf) if cfg.use_memory: state["mem"] = mem x, mem, ph = core(x, mem, t) if ph is None: ph = torch.full((B,), 1.0 / steps, device=device) forced_min = t < (cfg.min_steps - 1) is_last = (t == steps - 1) new_acc = acc + ph finish = ((new_acc > thr) | is_last) & (~torch.tensor(forced_min, device=device)) finish = finish & (still > 0) w_stop = torch.where(finish, (1.0 - acc).clamp(min=0.0), ph) * still step_states.append({ "h": x, "p_halt": ph, "active": active_t, "stop_weight": w_stop, "router": (rbuf[pre_r] if len(rbuf) > pre_r else None), "mem_attn": (mbuf[pre_m] if len(mbuf) > pre_m else None), "h_prev": x_prev, }) x_out = x_out + w_stop[:, None, None] * x if mem_out is not None: mem_out = mem_out + w_stop[:, None, None] * mem n_steps = n_steps + still acc = torch.where(finish, acc, new_acc) still = torch.where(finish, torch.zeros_like(still), still) x_prev = x if float(still.max()) == 0.0: break else: for t in range(steps): pre_r = len(rbuf) pre_m = len(mbuf) x_prev = x if cfg.use_memory: state["mem"] = mem x, mem, ph = core(x, mem, t) step_states.append({ "h": x, "p_halt": ph, "active": torch.ones(B, dtype=torch.bool, device=device), "stop_weight": torch.zeros(B, device=device), "router": (rbuf[pre_r] if len(rbuf) > pre_r else None), "mem_attn": (mbuf[pre_m] if len(mbuf) > pre_m else None), "h_prev": x_prev, }) n_steps = (torch.full((B,), float(steps), device=device) if not (adaptive and cfg.use_halting) else n_steps) finally: for h in hooks: h.remove() pooled_source = x_out if (adaptive and cfg.use_halting) else x normed = model.norm_out(pooled_source) pooled = torch.cat([normed.mean(1), normed.amax(1)], -1) cap = {"steps": step_states, "n_steps": n_steps, "aggregate_raw": pooled_source, # pre-norm halting-weighted mean "aggregate": normed, # == TinyChess.forward's 'h' "memory": mem, "pooled": pooled, "value": model.value(pooled)} return cap @torch.no_grad() def reference_outputs(model, board, steps: int, adaptive: bool, device: str = "cpu"): """Plain unmodified forward - the ground truth for non-interference tests.""" pb = encode_batch([board], [board.peek() if board.move_stack else None]).to(device) return model(**pb.model_args(), steps=steps, adaptive=adaptive, trace=True)