Download dory/lens_eval.py from senku21x/dory_implementation: direct link, hf CLI and curl.
- Browser
- Download file 2.47 kB
-
https://huggingface.co/senku21x/dory_implementation/resolve/main/dory/lens_eval.py
- Command line
-
hf download hf://senku21x/dory_implementation/dory/lens_eval.py
-
curl -L -o lens_eval.py https://huggingface.co/senku21x/dory_implementation/resolve/main/dory/lens_eval.py
2.47 kB
| """The intermediate-variable readout evaluation (Gurnee et al. 2026 prompt sets; scoring as in the J++ post).""" | |
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| import torch | |
| TASKS = ("multihop", "multilingual", "typo", "association", "poetry") | |
| def token_variants(tok, word: str) -> list[int]: | |
| out = [] | |
| for s in {" " + word, word, " " + word.capitalize(), word.capitalize(), " " + word.lower(), word.lower()}: | |
| t = tok(s, add_special_tokens=False).input_ids | |
| if len(t) == 1 and t[0] not in out: | |
| out.append(t[0]) | |
| return out | |
| def load_items(tok, folder: str | Path, tasks=TASKS) -> list[dict]: | |
| """Items with: task, name, ids [1, T], pos (readout position), targets (one list of token ids per usable | |
| intermediate), answer (the expected next word, or None).""" | |
| items = [] | |
| for task in tasks: | |
| for it in json.loads((Path(folder) / f"lens-eval-{task}.json").read_text())["items"]: | |
| ids = tok(it["prompt"].rstrip(), return_tensors="pt").input_ids | |
| pos = ids.shape[1] - 1 | |
| if task == "poetry": # read at the newline that ends line 1 | |
| nl = [i for i in range(ids.shape[1]) if "\n" in tok.decode([ids[0, i].item()])] | |
| if not nl: | |
| continue | |
| pos = nl[-1] | |
| targets = [v for v in (token_variants(tok, w) for w in it["intermediates"]) if v] | |
| if targets: | |
| items.append({"task": task, "name": it["name"], "ids": ids, "pos": pos, "targets": targets, | |
| "answer": it.get("target")}) | |
| return items | |
| def semantic_mask(tok, vocab_size: int) -> torch.Tensor: | |
| """True for tokens dropped from the readout: no letter or digit, or a special token.""" | |
| special = set(tok.all_special_ids) | |
| out = [] | |
| for i in range(vocab_size): | |
| s = tok.decode([i]) | |
| out.append(i in special or s.startswith("<|") or s in ("<s>", "</s>", "<unk>", "<think>", "</think>") or not any(c.isalnum() for c in s)) | |
| return torch.tensor(out) | |
| def ranks(logits: torch.Tensor, targets: list[list[int]], mask: torch.Tensor | None) -> list[int]: | |
| """Rank (1 = top) of each intermediate: the best rank over its token variants. logits [V].""" | |
| if mask is not None: | |
| logits = logits.masked_fill(mask, -1e9) | |
| return [min(int((logits > logits[t]).sum()) + 1 for t in variants) for variants in targets] | |