"""Paired bootstrap over test items for the lens comparison of fit_jpp.py, recomputed from the saved lens files. python bootstrap.py --k 2 --lens-dir --run k2_n64_e8_s0 Rebuilds the same dev/test split (same seed and filter), scores each lens on the test half at the same seven nodes, and resamples test items within each task 2,000 times. The interval covers which test items were drawn; it does not cover the noise of the fit or the choice of split. """ import argparse, json, random, sys, pathlib, torch HERE = pathlib.Path(__file__).resolve().parent; PROJ = HERE.parents[1] sys.path.insert(0, str(PROJ / "src")) from dory.model import Dory, snapshot_path, load_tokenizer, justnorm from dory.jlens import JLens, source_nodes, node_key from dory.lens_eval import load_items, semantic_mask, TASKS ap = argparse.ArgumentParser(); ap.add_argument("--k", type=int, required=True); ap.add_argument("--lens-dir", required=True) ap.add_argument("--run", required=True); ap.add_argument("--seed", type=int, default=0); args = ap.parse_args() torch.backends.cuda.matmul.allow_tf32 = True snap = snapshot_path(); tok = load_tokenizer(snap); model = Dory.from_snapshot(snap, device="cuda", dtype=torch.float32) K, V = args.k, model.cfg["vocab_size"]; nodes = source_nodes(model, K); keys = [node_key(v) for v in nodes]; N = len(nodes) items = load_items(tok, PROJ / "evals" / "lens_eval"); mask = semantic_mask(tok, V).cuda() sub = [v for v in nodes if v[0] in ("in", "x", "rec", "s", "out")] seven = [nodes.index(sub[i]) for i in sorted({round(f * (len(sub) + 1)) for f in torch.linspace(0.125, 0.875, 7).tolist()})] H = torch.zeros(len(items), len(seven), model.cfg["hidden_size"], device="cuda"); correct = [] for i, it in enumerate(items): st = {} with torch.no_grad(): final = model.forward(it["ids"].cuda(), n_loops=K, tap=lambda nm, h: st.__setitem__(nm, h)) H[i] = torch.stack([st[nodes[j]][0, it["pos"]] for j in seven]) top = tok.decode([model.unembed(final)[0, -1].argmax().item()]).strip() correct.append(it["answer"] is None or (len(top) > 0 and it["answer"].lower().startswith(top.lower()))) keep = [i for i, it in enumerate(items) if it["task"] not in ("multihop", "multilingual") or correct[i]] rng = random.Random(args.seed); test = [] for task in TASKS: idx = [i for i in keep if items[i]["task"] == task]; rng.shuffle(idx); test += idx[len(idx) // 2 :] D = pathlib.Path(args.lens_dir) files = {"J-lens": pathlib.Path.home() / f"interp/artifacts/dory_lens/E001_jlens_fit/k{K}_n50_skip16_s0/jlens_k{K}.safetensors", "R-lens": D / f"rlens_k{K}.safetensors", "J++": D / f"jpp_k{K}.safetensors"} score = {} # lens -> item -> fraction of intermediates recovered at 10 def item_scores(J): out = {} best = {i: [10**9] * len(items[i]["targets"]) for i in test} for c, j in enumerate(seven): h = H[test, c] lg = model.unembed(justnorm(h if J is None else h @ J[keys[j]].cuda().T)).masked_fill(mask, -1e9) for r, i in enumerate(test): for q, tv in enumerate(items[i]["targets"]): best[i][q] = min(best[i][q], min(int((lg[r] > lg[r, t]).sum()) + 1 for t in tv)) return {i: sum(b <= 10 for b in best[i]) / len(best[i]) for i in test} score["logit lens"] = item_scores(None) for name, f in files.items(): lens = JLens.load(f, device="cpu"); score[name] = item_scores(lens.J); del lens by_task = {t: [i for i in test if items[i]["task"] == t] for t in TASKS} macro = lambda s, pick: sum(sum(s[i] for i in pick[t]) / len(pick[t]) for t in TASKS) / len(TASKS) point = {n: macro(s, by_task) for n, s in score.items()} g = random.Random(1); B = 2000; draws = {n: [] for n in score} for _ in range(B): pick = {t: [g.choice(by_task[t]) for _ in by_task[t]] for t in TASKS} for n, s in score.items(): draws[n].append(macro(s, pick)) def ci(x): x = sorted(x); return x[int(0.025 * B)], x[int(0.975 * B)] res = {"k": K, "n_test_items": len(test), "items_per_task": {t: len(v) for t, v in by_task.items()}, "recall10_seven": {}, "differences": {}} print(f"k={K}: {len(test)} test items {res['items_per_task']}; recall@10 at any of seven nodes, macro over tasks, 95% bootstrap interval") for n in score: lo, hi = ci(draws[n]); res["recall10_seven"][n] = {"point": point[n], "lo": lo, "hi": hi} print(f" {n:12s} {point[n]:.3f} [{lo:.3f}, {hi:.3f}]") for a, b in (("J++", "R-lens"), ("R-lens", "J-lens"), ("J++", "J-lens"), ("J-lens", "logit lens")): dd = [x - y for x, y in zip(draws[a], draws[b])]; lo, hi = ci(dd) res["differences"][f"{a} minus {b}"] = {"point": point[a] - point[b], "lo": lo, "hi": hi, "share_of_resamples_above_zero": sum(x > 0 for x in dd) / B} print(f" {a} minus {b}: {point[a] - point[b]:+.3f} [{lo:+.3f}, {hi:+.3f}] above zero in {sum(x > 0 for x in dd) / B:.3f} of resamples") (HERE / "results" / args.run / "bootstrap.json").write_text(json.dumps(res, indent=1))