Download as_run/test_lrp_rules.py from senku21x/dory_implementation: direct link, hf CLI and curl.
- Browser
- Download file 2.36 kB
-
https://huggingface.co/senku21x/dory_implementation/resolve/main/as_run/test_lrp_rules.py
- Command line
-
hf download hf://senku21x/dory_implementation/as_run/test_lrp_rules.py
-
curl -L -o test_lrp_rules.py https://huggingface.co/senku21x/dory_implementation/resolve/main/as_run/test_lrp_rules.py
2.36 kB
| """The LRP rules must not change the forward pass, and must change the gradient. | |
| Run: CUDA_VISIBLE_DEVICES=1 HF_HOME=/workspace/huggingface python tests/test_lrp_rules.py""" | |
| import sys, pathlib, torch | |
| sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "src")) | |
| from dory.model import Dory, snapshot_path, load_tokenizer, lrp_rules, LRP | |
| from dory.jlens import prompt_jacobians | |
| torch.manual_seed(0) | |
| snap = snapshot_path(); tok = load_tokenizer(snap) | |
| m = Dory.from_snapshot(snap, device="cuda", dtype=torch.float32) | |
| ids = tok("The Amazon River in South America is the largest river by discharge volume of water in the world, and the " | |
| "disputed longest river system in the world in comparison to the Nile.", return_tensors="pt").input_ids.cuda() | |
| with torch.no_grad(): | |
| a = m.logits(ids, n_loops=2) | |
| with lrp_rules(): | |
| b = m.logits(ids, n_loops=2) | |
| assert not LRP.on | |
| print("forward: max abs logit difference with the rules on:", (a - b).abs().max().item(), "| logit scale", a.abs().max().item()) | |
| assert (a - b).abs().max().item() < 1e-2 | |
| nodes = [("in", 5), ("x",), ("rec", 1, 30), ("s", 1), ("rec", 2, 30), ("s", 2), ("out", 62)] | |
| rows = torch.randperm(2560, device="cuda")[:96] | |
| bins = ((0, 16), (16, None)) | |
| plain, sq_p, hs = prompt_jacobians(m, ids, 2, nodes, chunk=96, rows=rows, bins=bins) | |
| with lrp_rules(): | |
| lrp, sq_l, _ = prompt_jacobians(m, ids, 2, nodes, chunk=96, rows=rows, bins=bins) | |
| T = ids.shape[1] | |
| print(f"{'node':14s} {'|J| plain':>10s} {'|J| LRP':>9s} {'cosine':>7s} own-state term |J h|/|J| per position: {'plain':>6s} {'LRP':>6s} {'random dir (LRP)':>17s}") | |
| for j, n in enumerate(nodes): | |
| P, L = plain[j, 1], lrp[j, 1] | |
| cos = torch.nn.functional.cosine_similarity(P.flatten(), L.flatten(), dim=0).item() | |
| h = hs[j, 1] / (T - 16); h = h / h.norm() # mean state direction of this prompt | |
| r = torch.randn_like(h); r /= r.norm() | |
| print(f"{str(n):14s} {P.norm().item():10.2f} {L.norm().item():9.2f} {cos:7.3f} {'':38s} {(P @ h).norm().item() / P.norm().item():6.3f} {(L @ h).norm().item() / L.norm().item():6.3f} {(L @ r).norm().item() / L.norm().item():17.3f}") | |
| print("gradient growth toward early nodes, per-position rms norm (plain / LRP):", [f"{sq_p[j, 16:].mean().sqrt().item():.2f}/{sq_l[j, 16:].mean().sqrt().item():.2f}" for j in range(len(nodes))]) | |
| print("OK") | |