"""Pieces of the J++ lens (Ayonrinde & Lindsey 2026) beyond the LRP backward pass: the activation router (k-means on PCA projections), and the learned signed weights that combine the expert Jacobians.""" from __future__ import annotations import torch from .model import justnorm class Router: """Assigns a state to one of E clusters: unit-normalise, centre, project on the top PCA directions, nearest centroid.""" def __init__(self, mean, comps, centroids): self.mean, self.comps, self.centroids = mean, comps, centroids def project(self, h): return (justnorm(h.float()) - self.mean) @ self.comps.T def assign(self, h): return torch.cdist(self.project(h), self.centroids).argmin(-1) def state_dict(self): return {"router/mean": self.mean, "router/comps": self.comps, "router/centroids": self.centroids} def fit_router(acts: torch.Tensor, n_clusters: int = 8, n_pca: int = 64, seed: int = 0, iters: int = 100) -> Router: x = justnorm(acts.float()); mean = x.mean(0); x = x - mean evals, evecs = torch.linalg.eigh((x.T @ x) / len(x)) comps = evecs[:, -n_pca:].T.contiguous() z = x @ comps.T g = torch.Generator(device="cpu").manual_seed(seed) cent = z[torch.randint(len(z), (1,), generator=g)] for _ in range(n_clusters - 1): # k-means++ seeding d2 = torch.cdist(z, cent).min(1).values.pow(2) cent = torch.cat([cent, z[torch.multinomial((d2 / d2.sum()).cpu(), 1, generator=g).to(z.device)]]) for _ in range(iters): a = torch.cdist(z, cent).argmin(1) new = torch.stack([z[a == c].mean(0) if (a == c).any() else cent[c] for c in range(n_clusters)]) if torch.allclose(new, cent, atol=1e-6): break cent = new return Router(mean, comps, cent) def fit_expert_weights(feats: torch.Tensor, targets: list[list[int]], unembed, vocab_mask: torch.Tensor, init: torch.Tensor, steps: int = 200, lr: float = 0.05) -> torch.Tensor: """Signed, unit-norm weights over experts for one node. feats [P, E, d]: each expert's transport of the P labelled states. The readout of weights w is unembed(N(sum_e w_e feats[:, e])). Loss: cross-entropy of the labelled token over the kept vocabulary (a smooth stand-in for recall@10). `targets[p]` lists acceptable token ids.""" v = init.clone().float().requires_grad_(True) opt = torch.optim.Adam([v], lr=lr) P = feats.shape[0] tmask = torch.zeros(P, vocab_mask.shape[0], dtype=torch.bool, device=feats.device) for p, t in enumerate(targets): tmask[p, t] = True with torch.enable_grad(): for _ in range(steps): w = v / v.norm() lg = unembed(justnorm(torch.einsum("e,ped->pd", w, feats))).masked_fill(vocab_mask, -1e9) loss = (torch.logsumexp(lg, -1) - torch.logsumexp(lg.masked_fill(~tmask, -1e9), -1)).mean() opt.zero_grad(); loss.backward(); opt.step() return (v / v.norm()).detach()