File size: 1,948 Bytes
498b9c9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
"""Apply a fitted Dory J-lens to a prompt: top tokens at every source node for one position.

    python readout_example.py --lens /path/to/jlens_k2.safetensors --prompt "Fact: ... is" [--position -1] [--every 4]

The lens file fixes the loop count (meta["n_loops"]); the model is run with that many loops.
"""
import argparse, sys, pathlib, torch
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2] / "src"))
from dory.model import Dory, snapshot_path, load_tokenizer
from dory.jlens import JLens, logit_lens, node_key

ap = argparse.ArgumentParser()
ap.add_argument("--lens", required=True); ap.add_argument("--prompt", required=True)
ap.add_argument("--position", type=int, default=-1); ap.add_argument("--every", type=int, default=4)
ap.add_argument("--topk", type=int, default=8); ap.add_argument("--chat", action="store_true")
args = ap.parse_args()

snap = snapshot_path(); tok = load_tokenizer(snap)
model = Dory.from_snapshot(snap, device="cuda", dtype=torch.float32)
lens = JLens.load(args.lens, device="cuda")
text = args.prompt
if args.chat:
    text = tok.apply_chat_template([{"role": "user", "content": text}], tokenize=False, add_generation_prompt=True, enable_thinking=False)
ids = tok(text, return_tensors="pt", add_special_tokens=not args.chat).input_ids.cuda()
states = {}
with torch.no_grad():
    final = model.forward(ids, n_loops=lens.k, tap=lambda n, h: states.__setitem__(n, h))
dec = lambda lg: [tok.decode([t]) for t in lg.topk(args.topk).indices.tolist()]
print(f"loops {lens.k}; reading token {tok.decode([ids[0, args.position].item()])!r}; model's next-token top-{args.topk}: {dec(model.unembed(final)[0, args.position])}")
for i, node in enumerate(lens.nodes()):
    if i % args.every and node[0] not in ("x", "u", "s"):
        continue
    h = states[node][0, args.position]
    print(f"{node_key(node):10s} J-lens {dec(lens.logits(model, h, node))}   | logit lens {dec(logit_lens(model, h))[:4]}")