"""The intermediate-variable readout evaluation (Gurnee et al. 2026 prompt sets; scoring as in the J++ post).""" from __future__ import annotations import json from pathlib import Path import torch TASKS = ("multihop", "multilingual", "typo", "association", "poetry") def token_variants(tok, word: str) -> list[int]: out = [] for s in {" " + word, word, " " + word.capitalize(), word.capitalize(), " " + word.lower(), word.lower()}: t = tok(s, add_special_tokens=False).input_ids if len(t) == 1 and t[0] not in out: out.append(t[0]) return out def load_items(tok, folder: str | Path, tasks=TASKS) -> list[dict]: """Items with: task, name, ids [1, T], pos (readout position), targets (one list of token ids per usable intermediate), answer (the expected next word, or None).""" items = [] for task in tasks: for it in json.loads((Path(folder) / f"lens-eval-{task}.json").read_text())["items"]: ids = tok(it["prompt"].rstrip(), return_tensors="pt").input_ids pos = ids.shape[1] - 1 if task == "poetry": # read at the newline that ends line 1 nl = [i for i in range(ids.shape[1]) if "\n" in tok.decode([ids[0, i].item()])] if not nl: continue pos = nl[-1] targets = [v for v in (token_variants(tok, w) for w in it["intermediates"]) if v] if targets: items.append({"task": task, "name": it["name"], "ids": ids, "pos": pos, "targets": targets, "answer": it.get("target")}) return items def semantic_mask(tok, vocab_size: int) -> torch.Tensor: """True for tokens dropped from the readout: no letter or digit, or a special token.""" special = set(tok.all_special_ids) out = [] for i in range(vocab_size): s = tok.decode([i]) out.append(i in special or s.startswith("<|") or s in ("", "", "", "", "") or not any(c.isalnum() for c in s)) return torch.tensor(out) def ranks(logits: torch.Tensor, targets: list[list[int]], mask: torch.Tensor | None) -> list[int]: """Rank (1 = top) of each intermediate: the best rank over its token variants. logits [V].""" if mask is not None: logits = logits.masked_fill(mask, -1e9) return [min(int((logits > logits[t]).sum()) + 1 for t in variants) for variants in targets]