dory_implementation / as_run /test_lrp_rules.py
senku21x's picture
Dory forward pass, J-lens / R-lens / J++ code, fit records
498b9c9 verified
Raw History Blame Contribute Delete
2.36 kB
"""The LRP rules must not change the forward pass, and must change the gradient.
Run: CUDA_VISIBLE_DEVICES=1 HF_HOME=/workspace/huggingface python tests/test_lrp_rules.py"""
import sys, pathlib, torch
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "src"))
from dory.model import Dory, snapshot_path, load_tokenizer, lrp_rules, LRP
from dory.jlens import prompt_jacobians
torch.manual_seed(0)
snap = snapshot_path(); tok = load_tokenizer(snap)
m = Dory.from_snapshot(snap, device="cuda", dtype=torch.float32)
ids = tok("The Amazon River in South America is the largest river by discharge volume of water in the world, and the "
"disputed longest river system in the world in comparison to the Nile.", return_tensors="pt").input_ids.cuda()
with torch.no_grad():
a = m.logits(ids, n_loops=2)
with lrp_rules():
b = m.logits(ids, n_loops=2)
assert not LRP.on
print("forward: max abs logit difference with the rules on:", (a - b).abs().max().item(), "| logit scale", a.abs().max().item())
assert (a - b).abs().max().item() < 1e-2
nodes = [("in", 5), ("x",), ("rec", 1, 30), ("s", 1), ("rec", 2, 30), ("s", 2), ("out", 62)]
rows = torch.randperm(2560, device="cuda")[:96]
bins = ((0, 16), (16, None))
plain, sq_p, hs = prompt_jacobians(m, ids, 2, nodes, chunk=96, rows=rows, bins=bins)
with lrp_rules():
lrp, sq_l, _ = prompt_jacobians(m, ids, 2, nodes, chunk=96, rows=rows, bins=bins)
T = ids.shape[1]
print(f"{'node':14s} {'|J| plain':>10s} {'|J| LRP':>9s} {'cosine':>7s} own-state term |J h|/|J| per position: {'plain':>6s} {'LRP':>6s} {'random dir (LRP)':>17s}")
for j, n in enumerate(nodes):
P, L = plain[j, 1], lrp[j, 1]
cos = torch.nn.functional.cosine_similarity(P.flatten(), L.flatten(), dim=0).item()
h = hs[j, 1] / (T - 16); h = h / h.norm() # mean state direction of this prompt
r = torch.randn_like(h); r /= r.norm()
print(f"{str(n):14s} {P.norm().item():10.2f} {L.norm().item():9.2f} {cos:7.3f} {'':38s} {(P @ h).norm().item() / P.norm().item():6.3f} {(L @ h).norm().item() / L.norm().item():6.3f} {(L @ r).norm().item() / L.norm().item():17.3f}")
print("gradient growth toward early nodes, per-position rms norm (plain / LRP):", [f"{sq_p[j, 16:].mean().sqrt().item():.2f}/{sq_l[j, 16:].mean().sqrt().item():.2f}" for j in range(len(nodes))])
print("OK")