senku21x's picture
Dory forward pass, J-lens / R-lens / J++ code, fit records
498b9c9 verified
Raw History Blame Contribute Delete
3.11 kB
"""A chain-lens analogue for Dory (after "The Chain Lens [interim]", vgel & Douglas 2026).
The published chain lens rolls a lens's top tokens out into short phrases with the model's multi-token-prediction
head. Dory has no such head. Here the model itself does the roll-out: the state under study is placed at one
"anchor" position at its own node, the rest of the network is run on it, and the chain tokens that follow may
attend only to the anchor (from that node on) and to each other. Nothing else of the original prompt is present,
so whatever the chain says about the prompt came through that one state. Decoding is contrastive against the
same roll-out from the node's mean state (Li et al. 2022: among tokens within a factor of the best, pick the one
whose log-probability rises most over the prior).
Restriction: a node inside loop t is rolled out with exactly t loops, so the anchor never needs the input-block
output of a prompt it no longer has.
"""
from __future__ import annotations
import torch
from .model import ATTN, Dory
@torch.no_grad()
def rollout_logits(model: Dory, anchors: torch.Tensor, node: tuple, k: int, chain: torch.Tensor) -> torch.Tensor:
"""Next-token logits after each chain. anchors [B, d] (states at `node`), chain [B, m] token ids, m >= 1."""
assert node[0] not in ("u", "rec", "s") or node[1] == k, "roll a loop-t node out with k = t loops"
B, m = chain.shape
ids = torch.cat([chain[:, :1], chain], dim=1) # position 0 is the anchor (its token is a placeholder)
block = torch.zeros(m + 1, dtype=torch.bool); block[0] = True
ATTN.block = block
def tap(name, h):
if name == node:
h = h.clone(); h[:, 0] = anchors.to(h.dtype)
ATTN.block = None # from here on the chain can see the anchor
return h
try:
final = model.forward(ids, n_loops=k, tap=tap)
finally:
ATTN.block = None
return model.unembed(final[:, -1])
@torch.no_grad()
def chains(model: Dory, h: torch.Tensor, node: tuple, k: int, seeds: torch.Tensor, steps: int = 4,
prior: torch.Tensor | None = None, alpha: float = 0.1, ban: torch.Tensor | None = None) -> torch.Tensor:
"""Extend each seed token by `steps` tokens. h [d]; seeds [K]; prior [d] = the mean state at `node` for
contrastive decoding (None = plain greedy). Returns token ids [K, 1 + steps]."""
K = seeds.shape[0]
chain = seeds[:, None].clone()
A = h[None].expand(K, -1)
for _ in range(steps):
lp = torch.log_softmax(rollout_logits(model, A, node, k, chain), -1)
if ban is not None:
lp = lp.masked_fill(ban, -1e9)
if prior is None:
nxt = lp.argmax(-1)
else:
lq = torch.log_softmax(rollout_logits(model, prior[None].expand(K, -1), node, k, chain), -1)
ok = lp >= lp.max(-1, keepdim=True).values + torch.log(torch.tensor(alpha))
nxt = (lp - lq).masked_fill(~ok, -1e9).argmax(-1)
chain = torch.cat([chain, nxt[:, None]], dim=1)
return chain