File size: 3,106 Bytes
498b9c9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
"""A chain-lens analogue for Dory (after "The Chain Lens [interim]", vgel & Douglas 2026).

The published chain lens rolls a lens's top tokens out into short phrases with the model's multi-token-prediction
head. Dory has no such head. Here the model itself does the roll-out: the state under study is placed at one
"anchor" position at its own node, the rest of the network is run on it, and the chain tokens that follow may
attend only to the anchor (from that node on) and to each other. Nothing else of the original prompt is present,
so whatever the chain says about the prompt came through that one state. Decoding is contrastive against the
same roll-out from the node's mean state (Li et al. 2022: among tokens within a factor of the best, pick the one
whose log-probability rises most over the prior).

Restriction: a node inside loop t is rolled out with exactly t loops, so the anchor never needs the input-block
output of a prompt it no longer has.
"""
from __future__ import annotations

import torch

from .model import ATTN, Dory


@torch.no_grad()
def rollout_logits(model: Dory, anchors: torch.Tensor, node: tuple, k: int, chain: torch.Tensor) -> torch.Tensor:
    """Next-token logits after each chain. anchors [B, d] (states at `node`), chain [B, m] token ids, m >= 1."""
    assert node[0] not in ("u", "rec", "s") or node[1] == k, "roll a loop-t node out with k = t loops"
    B, m = chain.shape
    ids = torch.cat([chain[:, :1], chain], dim=1)                      # position 0 is the anchor (its token is a placeholder)
    block = torch.zeros(m + 1, dtype=torch.bool); block[0] = True
    ATTN.block = block

    def tap(name, h):
        if name == node:
            h = h.clone(); h[:, 0] = anchors.to(h.dtype)
            ATTN.block = None                                          # from here on the chain can see the anchor
            return h

    try:
        final = model.forward(ids, n_loops=k, tap=tap)
    finally:
        ATTN.block = None
    return model.unembed(final[:, -1])


@torch.no_grad()
def chains(model: Dory, h: torch.Tensor, node: tuple, k: int, seeds: torch.Tensor, steps: int = 4,
           prior: torch.Tensor | None = None, alpha: float = 0.1, ban: torch.Tensor | None = None) -> torch.Tensor:
    """Extend each seed token by `steps` tokens. h [d]; seeds [K]; prior [d] = the mean state at `node` for
    contrastive decoding (None = plain greedy). Returns token ids [K, 1 + steps]."""
    K = seeds.shape[0]
    chain = seeds[:, None].clone()
    A = h[None].expand(K, -1)
    for _ in range(steps):
        lp = torch.log_softmax(rollout_logits(model, A, node, k, chain), -1)
        if ban is not None:
            lp = lp.masked_fill(ban, -1e9)
        if prior is None:
            nxt = lp.argmax(-1)
        else:
            lq = torch.log_softmax(rollout_logits(model, prior[None].expand(K, -1), node, k, chain), -1)
            ok = lp >= lp.max(-1, keepdim=True).values + torch.log(torch.tensor(alpha))
            nxt = (lp - lq).masked_fill(~ok, -1e9).argmax(-1)
        chain = torch.cat([chain, nxt[:, None]], dim=1)
    return chain