File size: 1,948 Bytes
498b9c9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | """Apply a fitted Dory J-lens to a prompt: top tokens at every source node for one position.
python readout_example.py --lens /path/to/jlens_k2.safetensors --prompt "Fact: ... is" [--position -1] [--every 4]
The lens file fixes the loop count (meta["n_loops"]); the model is run with that many loops.
"""
import argparse, sys, pathlib, torch
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2] / "src"))
from dory.model import Dory, snapshot_path, load_tokenizer
from dory.jlens import JLens, logit_lens, node_key
ap = argparse.ArgumentParser()
ap.add_argument("--lens", required=True); ap.add_argument("--prompt", required=True)
ap.add_argument("--position", type=int, default=-1); ap.add_argument("--every", type=int, default=4)
ap.add_argument("--topk", type=int, default=8); ap.add_argument("--chat", action="store_true")
args = ap.parse_args()
snap = snapshot_path(); tok = load_tokenizer(snap)
model = Dory.from_snapshot(snap, device="cuda", dtype=torch.float32)
lens = JLens.load(args.lens, device="cuda")
text = args.prompt
if args.chat:
text = tok.apply_chat_template([{"role": "user", "content": text}], tokenize=False, add_generation_prompt=True, enable_thinking=False)
ids = tok(text, return_tensors="pt", add_special_tokens=not args.chat).input_ids.cuda()
states = {}
with torch.no_grad():
final = model.forward(ids, n_loops=lens.k, tap=lambda n, h: states.__setitem__(n, h))
dec = lambda lg: [tok.decode([t]) for t in lg.topk(args.topk).indices.tolist()]
print(f"loops {lens.k}; reading token {tok.decode([ids[0, args.position].item()])!r}; model's next-token top-{args.topk}: {dec(model.unembed(final)[0, args.position])}")
for i, node in enumerate(lens.nodes()):
if i % args.every and node[0] not in ("x", "u", "s"):
continue
h = states[node][0, args.position]
print(f"{node_key(node):10s} J-lens {dec(lens.logits(model, h, node))} | logit lens {dec(logit_lens(model, h))[:4]}")
|