dory_implementation / dory /lens_eval.py
senku21x's picture
Dory forward pass, J-lens / R-lens / J++ code, fit records
498b9c9 verified
Raw History Blame Contribute Delete
2.47 kB
"""The intermediate-variable readout evaluation (Gurnee et al. 2026 prompt sets; scoring as in the J++ post)."""
from __future__ import annotations
import json
from pathlib import Path
import torch
TASKS = ("multihop", "multilingual", "typo", "association", "poetry")
def token_variants(tok, word: str) -> list[int]:
out = []
for s in {" " + word, word, " " + word.capitalize(), word.capitalize(), " " + word.lower(), word.lower()}:
t = tok(s, add_special_tokens=False).input_ids
if len(t) == 1 and t[0] not in out:
out.append(t[0])
return out
def load_items(tok, folder: str | Path, tasks=TASKS) -> list[dict]:
"""Items with: task, name, ids [1, T], pos (readout position), targets (one list of token ids per usable
intermediate), answer (the expected next word, or None)."""
items = []
for task in tasks:
for it in json.loads((Path(folder) / f"lens-eval-{task}.json").read_text())["items"]:
ids = tok(it["prompt"].rstrip(), return_tensors="pt").input_ids
pos = ids.shape[1] - 1
if task == "poetry": # read at the newline that ends line 1
nl = [i for i in range(ids.shape[1]) if "\n" in tok.decode([ids[0, i].item()])]
if not nl:
continue
pos = nl[-1]
targets = [v for v in (token_variants(tok, w) for w in it["intermediates"]) if v]
if targets:
items.append({"task": task, "name": it["name"], "ids": ids, "pos": pos, "targets": targets,
"answer": it.get("target")})
return items
def semantic_mask(tok, vocab_size: int) -> torch.Tensor:
"""True for tokens dropped from the readout: no letter or digit, or a special token."""
special = set(tok.all_special_ids)
out = []
for i in range(vocab_size):
s = tok.decode([i])
out.append(i in special or s.startswith("<|") or s in ("<s>", "</s>", "<unk>", "<think>", "</think>") or not any(c.isalnum() for c in s))
return torch.tensor(out)
def ranks(logits: torch.Tensor, targets: list[list[int]], mask: torch.Tensor | None) -> list[int]:
"""Rank (1 = top) of each intermediate: the best rank over its token variants. logits [V]."""
if mask is not None:
logits = logits.masked_fill(mask, -1e9)
return [min(int((logits > logits[t]).sum()) + 1 for t in variants) for variants in targets]