bank.py

"""AnchoredBank — the E1-form anchored FFN, born on its own null path.

    Bank(x) = trunk(x) + sum_k w_k(x) * sigmoid(g_k) * expert_k(x)

Trunk: always-on d->ff->d GELU expert. Dispatch: the signed aleph address
over a learned K x d codebook read against the layer input (K=3 fat
experts, the shape validated at parity under encoder pressure). Gates
init -3.0; expert OUTPUT projections zero-init, so at birth the dispatch
contributes exactly zero and the bank is bit-identical to its dense
control (the C6 null path). No balance machinery of any kind —
differentiation is an attractor, pressure stays out of the task gradient.
"""
from __future__ import annotations

import torch
import torch.nn as nn
import torch.nn.functional as F

from .address import AlephAddress


class AnchoredBank(nn.Module):
    def __init__(self, d: int, n_experts: int = 3, ff: int | None = None,
                 tau: float = 0.1, gate_init: float = -3.0):
        super().__init__()
        ff = ff or d
        self.n_experts = n_experts
        self.t_in = nn.Linear(d, ff, bias=False)
        self.t_out = nn.Linear(ff, d, bias=False)
        nn.init.orthogonal_(self.t_in.weight)
        nn.init.orthogonal_(self.t_out.weight)
        self.addr = AlephAddress(n_experts, d, tau)
        w_in = torch.empty(n_experts, d, ff)
        for k in range(n_experts):
            nn.init.orthogonal_(w_in[k])
        self.w_in = nn.Parameter(w_in)
        self.w_out = nn.Parameter(torch.zeros(n_experts, ff, d))  # null path
        self.gates = nn.Parameter(torch.full((n_experts,), gate_init))
        self.last_dispatch = None  # (mean|w| per expert, w sample) for instruments

    def forward(self, x, disable_dispatch: bool = False):
        trunk = self.t_out(F.gelu(self.t_in(x)))
        if disable_dispatch:
            return trunk
        w = self.addr.signed(x)                                   # (B, n, K)
        with torch.no_grad():
            self.last_dispatch = w.detach()
        h = F.gelu(torch.einsum("bnd,kdf->bnkf", x, self.w_in))
        e = torch.einsum("bnkf,kfd->bnkd", h, self.w_out)
        return trunk + torch.einsum("bnk,bnkd->bnd",
                                    w * torch.sigmoid(self.gates), e)