Download as_run/readout_example.py from senku21x/dory_implementation: direct link, hf CLI and curl.
- Browser
- Download file 1.95 kB
-
https://huggingface.co/senku21x/dory_implementation/resolve/main/as_run/readout_example.py
- Command line
-
hf download hf://senku21x/dory_implementation/as_run/readout_example.py
-
curl -L -o readout_example.py https://huggingface.co/senku21x/dory_implementation/resolve/main/as_run/readout_example.py
1.95 kB
| """Apply a fitted Dory J-lens to a prompt: top tokens at every source node for one position. | |
| python readout_example.py --lens /path/to/jlens_k2.safetensors --prompt "Fact: ... is" [--position -1] [--every 4] | |
| The lens file fixes the loop count (meta["n_loops"]); the model is run with that many loops. | |
| """ | |
| import argparse, sys, pathlib, torch | |
| sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2] / "src")) | |
| from dory.model import Dory, snapshot_path, load_tokenizer | |
| from dory.jlens import JLens, logit_lens, node_key | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--lens", required=True); ap.add_argument("--prompt", required=True) | |
| ap.add_argument("--position", type=int, default=-1); ap.add_argument("--every", type=int, default=4) | |
| ap.add_argument("--topk", type=int, default=8); ap.add_argument("--chat", action="store_true") | |
| args = ap.parse_args() | |
| snap = snapshot_path(); tok = load_tokenizer(snap) | |
| model = Dory.from_snapshot(snap, device="cuda", dtype=torch.float32) | |
| lens = JLens.load(args.lens, device="cuda") | |
| text = args.prompt | |
| if args.chat: | |
| text = tok.apply_chat_template([{"role": "user", "content": text}], tokenize=False, add_generation_prompt=True, enable_thinking=False) | |
| ids = tok(text, return_tensors="pt", add_special_tokens=not args.chat).input_ids.cuda() | |
| states = {} | |
| with torch.no_grad(): | |
| final = model.forward(ids, n_loops=lens.k, tap=lambda n, h: states.__setitem__(n, h)) | |
| dec = lambda lg: [tok.decode([t]) for t in lg.topk(args.topk).indices.tolist()] | |
| print(f"loops {lens.k}; reading token {tok.decode([ids[0, args.position].item()])!r}; model's next-token top-{args.topk}: {dec(model.unembed(final)[0, args.position])}") | |
| for i, node in enumerate(lens.nodes()): | |
| if i % args.every and node[0] not in ("x", "u", "s"): | |
| continue | |
| h = states[node][0, args.position] | |
| print(f"{node_key(node):10s} J-lens {dec(lens.logits(model, h, node))} | logit lens {dec(logit_lens(model, h))[:4]}") | |