Download as_run/bootstrap.py from senku21x/dory_implementation: direct link, hf CLI and curl.
- Browser
- Download file 5.05 kB
-
https://huggingface.co/senku21x/dory_implementation/resolve/main/as_run/bootstrap.py
- Command line
-
hf download hf://senku21x/dory_implementation/as_run/bootstrap.py
-
curl -L -o bootstrap.py https://huggingface.co/senku21x/dory_implementation/resolve/main/as_run/bootstrap.py
5.05 kB
| """Paired bootstrap over test items for the lens comparison of fit_jpp.py, recomputed from the saved lens files. | |
| python bootstrap.py --k 2 --lens-dir <folder with rlens_k2.safetensors and jpp_k2.safetensors> --run k2_n64_e8_s0 | |
| Rebuilds the same dev/test split (same seed and filter), scores each lens on the test half at the same seven | |
| nodes, and resamples test items within each task 2,000 times. The interval covers which test items were drawn; | |
| it does not cover the noise of the fit or the choice of split. | |
| """ | |
| import argparse, json, random, sys, pathlib, torch | |
| HERE = pathlib.Path(__file__).resolve().parent; PROJ = HERE.parents[1] | |
| sys.path.insert(0, str(PROJ / "src")) | |
| from dory.model import Dory, snapshot_path, load_tokenizer, justnorm | |
| from dory.jlens import JLens, source_nodes, node_key | |
| from dory.lens_eval import load_items, semantic_mask, TASKS | |
| ap = argparse.ArgumentParser(); ap.add_argument("--k", type=int, required=True); ap.add_argument("--lens-dir", required=True) | |
| ap.add_argument("--run", required=True); ap.add_argument("--seed", type=int, default=0); args = ap.parse_args() | |
| torch.backends.cuda.matmul.allow_tf32 = True | |
| snap = snapshot_path(); tok = load_tokenizer(snap); model = Dory.from_snapshot(snap, device="cuda", dtype=torch.float32) | |
| K, V = args.k, model.cfg["vocab_size"]; nodes = source_nodes(model, K); keys = [node_key(v) for v in nodes]; N = len(nodes) | |
| items = load_items(tok, PROJ / "evals" / "lens_eval"); mask = semantic_mask(tok, V).cuda() | |
| sub = [v for v in nodes if v[0] in ("in", "x", "rec", "s", "out")] | |
| seven = [nodes.index(sub[i]) for i in sorted({round(f * (len(sub) + 1)) for f in torch.linspace(0.125, 0.875, 7).tolist()})] | |
| H = torch.zeros(len(items), len(seven), model.cfg["hidden_size"], device="cuda"); correct = [] | |
| for i, it in enumerate(items): | |
| st = {} | |
| with torch.no_grad(): | |
| final = model.forward(it["ids"].cuda(), n_loops=K, tap=lambda nm, h: st.__setitem__(nm, h)) | |
| H[i] = torch.stack([st[nodes[j]][0, it["pos"]] for j in seven]) | |
| top = tok.decode([model.unembed(final)[0, -1].argmax().item()]).strip() | |
| correct.append(it["answer"] is None or (len(top) > 0 and it["answer"].lower().startswith(top.lower()))) | |
| keep = [i for i, it in enumerate(items) if it["task"] not in ("multihop", "multilingual") or correct[i]] | |
| rng = random.Random(args.seed); test = [] | |
| for task in TASKS: | |
| idx = [i for i in keep if items[i]["task"] == task]; rng.shuffle(idx); test += idx[len(idx) // 2 :] | |
| D = pathlib.Path(args.lens_dir) | |
| files = {"J-lens": pathlib.Path.home() / f"interp/artifacts/dory_lens/E001_jlens_fit/k{K}_n50_skip16_s0/jlens_k{K}.safetensors", | |
| "R-lens": D / f"rlens_k{K}.safetensors", "J++": D / f"jpp_k{K}.safetensors"} | |
| score = {} # lens -> item -> fraction of intermediates recovered at 10 | |
| def item_scores(J): | |
| out = {} | |
| best = {i: [10**9] * len(items[i]["targets"]) for i in test} | |
| for c, j in enumerate(seven): | |
| h = H[test, c] | |
| lg = model.unembed(justnorm(h if J is None else h @ J[keys[j]].cuda().T)).masked_fill(mask, -1e9) | |
| for r, i in enumerate(test): | |
| for q, tv in enumerate(items[i]["targets"]): | |
| best[i][q] = min(best[i][q], min(int((lg[r] > lg[r, t]).sum()) + 1 for t in tv)) | |
| return {i: sum(b <= 10 for b in best[i]) / len(best[i]) for i in test} | |
| score["logit lens"] = item_scores(None) | |
| for name, f in files.items(): | |
| lens = JLens.load(f, device="cpu"); score[name] = item_scores(lens.J); del lens | |
| by_task = {t: [i for i in test if items[i]["task"] == t] for t in TASKS} | |
| macro = lambda s, pick: sum(sum(s[i] for i in pick[t]) / len(pick[t]) for t in TASKS) / len(TASKS) | |
| point = {n: macro(s, by_task) for n, s in score.items()} | |
| g = random.Random(1); B = 2000; draws = {n: [] for n in score} | |
| for _ in range(B): | |
| pick = {t: [g.choice(by_task[t]) for _ in by_task[t]] for t in TASKS} | |
| for n, s in score.items(): | |
| draws[n].append(macro(s, pick)) | |
| def ci(x): | |
| x = sorted(x); return x[int(0.025 * B)], x[int(0.975 * B)] | |
| res = {"k": K, "n_test_items": len(test), "items_per_task": {t: len(v) for t, v in by_task.items()}, "recall10_seven": {}, "differences": {}} | |
| print(f"k={K}: {len(test)} test items {res['items_per_task']}; recall@10 at any of seven nodes, macro over tasks, 95% bootstrap interval") | |
| for n in score: | |
| lo, hi = ci(draws[n]); res["recall10_seven"][n] = {"point": point[n], "lo": lo, "hi": hi} | |
| print(f" {n:12s} {point[n]:.3f} [{lo:.3f}, {hi:.3f}]") | |
| for a, b in (("J++", "R-lens"), ("R-lens", "J-lens"), ("J++", "J-lens"), ("J-lens", "logit lens")): | |
| dd = [x - y for x, y in zip(draws[a], draws[b])]; lo, hi = ci(dd) | |
| res["differences"][f"{a} minus {b}"] = {"point": point[a] - point[b], "lo": lo, "hi": hi, "share_of_resamples_above_zero": sum(x > 0 for x in dd) / B} | |
| print(f" {a} minus {b}: {point[a] - point[b]:+.3f} [{lo:+.3f}, {hi:+.3f}] above zero in {sum(x > 0 for x in dd) / B:.3f} of resamples") | |
| (HERE / "results" / args.run / "bootstrap.json").write_text(json.dumps(res, indent=1)) | |