splat_attention v2: CausalSplatHUB (the AR-validated causal form), dtype-aware clamps (fp16 landmine defused), header carries the full causal record — binding surface, corrected supply law, optimizer + precision guidance, the open problem stated plainly

splat_attention.py

@@ -74,6 +74,38 @@
 #   Frozen-everything remains fine for INFERENCE-style play and the
 #   static properties above.
 #
+# CAUSAL / AR RECORD (2026-08-09/10, rank-controlled recall battery,
+# clean protocol — see trainers/ for full replication):
+#   CausalSplatHUB (included below) is the causal prefix-sum form of the
+#   aleph read, measured against matched softmax attention on in-context
+#   key-value binding with dial-able demand (key rank R):
+#     low demand (R=4):      near parity  (.92 vs .96, shared data ceiling)
+#     moderate demand (R=16): .90 vs .99  — army extrapolates toward parity
+#     high demand (R=64):    .84 vs .99  — army-SATURATED below parity.
+#   THE OPEN PROBLEM, stated plainly: at high binding demand the linear
+#   aleph read saturates below softmax parity and count does not close it.
+#   Something else is needed there — address dimensionality, per-depth
+#   allocation, and optimizer geometry are the live suspects.
+#   SUPPLY LAW (corrected): the army-size knee tracks the task's
+#   addressing DEMAND, but the constant is demand-dependent (4R at low
+#   demand, >=16R at moderate, saturating at high). Provision generously
+#   and measure CONSUMED address erank PER LAYER — late layers
+#   under-consume badly (uniform per-layer K wastes most of its supply).
+#   OPTIMIZER (measured, decisive): momentum-geometric optimizers
+#   (Muon-style orthogonalized momentum, plain SGD-momentum) beat Adam by
+#   ~.09 on this mechanism, and the mechanism is ~20x more
+#   optimizer-sensitive than softmax attention. Treat the optimizer as
+#   part of the architecture.
+#   PRECISION: train in fp32 or bf16 (bf16 measured >= fp32); do NOT
+#   train through fp8 (fails); fp8 e4m3 INFERENCE of trained weights is
+#   viable (~5% cost). All normalizer clamps in this file are dtype-aware
+#   because 1e-9-class constants flush to ZERO in fp16 (measured NaN at
+#   ~5% of sharp reads before the fix).
+#   REPLICATOR'S WARNING: evaluation-protocol faults can fake
+#   architecture plateaus (a window-truncation ceiling masqueraded as a
+#   softmax plateau at .936 for two days of this record). The trainers
+#   directory ships the corrected harness; use its clean gauges.
+#
 # Status: research prototype. Trained-at-scale results pending; treat
 # every number above as what it is — a measurement on the stated probe.
 # =========================================================================
@@ -179,7 +211,11 @@ class SplatAttention(nn.Module):
         centers = frac * (L - 1)
         g = torch.exp(-0.5 * ((pos.unsqueeze(0) - centers.unsqueeze(1))
                               / sig) ** 2)
-        g = g / g.sum(dim=0, keepdim=True).clamp(min=1e-9)
+        if g.dtype in (torch.float32, torch.float64):
+            _cl = 1e-9
+        else:                      # half dtypes: 1e-9 flushes to 0 (landmine)
+            _cl = float(torch.finfo(g.dtype).tiny) * 8
+        g = g / g.sum(dim=0, keepdim=True).clamp(min=_cl)
         n_glob = int(round(self.M * self.global_frac))
         if n_glob > 0:
             g[:n_glob] = 1.0
@@ -229,10 +265,64 @@ class SplatAttention(nn.Module):
                                           c0, Mc)
             out = out.add_(part)
             den = den.add_(dpart)
-        out = out / den.unsqueeze(-1).clamp_min(1e-9)
+        if den.dtype in (torch.float32, torch.float64):
+            _cl = 1e-9
+        else:                      # half dtypes: 1e-9 flushes to 0 (landmine)
+            _cl = float(torch.finfo(den.dtype).tiny) * 8
+        out = out / den.unsqueeze(-1).clamp_min(_cl)
         return self.drop(self.w_o(out))
 
 
+
+class CausalSplatHUB(nn.Module):
+    """Causal (autoregressive) aleph linear attention — the AR-validated
+    form from the rank-controlled recall battery. Prefix-sum memories over
+    the 2K oriented halves of the address; no selection event; O(L*K*d).
+
+    Guidance from the measured record (see header): D should match the
+    content's intrinsic dimensionality; K should be provisioned to the
+    task's addressing demand (knee tracks demand, constant is
+    demand-dependent — measure consumed address erank per layer); train
+    with momentum-geometric optimizers; fp32/bf16 only."""
+
+    def __init__(self, d_model, K=64, D=16, tau=0.1):
+        super().__init__()
+        self.K, self.D, self.tau = K, D, tau
+        self.codebook = nn.Parameter(F.normalize(torch.randn(K, D), dim=-1))
+        self.q = nn.Linear(d_model, D, bias=False)
+        self.k = nn.Linear(d_model, D, bias=False)
+        self.v = nn.Linear(d_model, d_model, bias=False)
+        self.o = nn.Linear(d_model, d_model, bias=False)
+        for m in (self.q, self.k, self.v, self.o):
+            nn.init.orthogonal_(m.weight)
+
+    def _oriented(self, x):
+        A = F.normalize(self.codebook, dim=-1)
+        u = (F.normalize(x, dim=-1) @ A.T) / self.tau
+        m = u.abs().amax(dim=-1, keepdim=True)
+        ep, en = torch.exp(u - m), torch.exp(-u - m)
+        Z = (ep + en).sum(dim=-1, keepdim=True)
+        return ep / Z, en / Z
+
+    def forward(self, x):
+        qp, qn = self._oriented(self.q(x))
+        kp, kn = self._oriented(self.k(x))
+        v = self.v(x)
+        Sp = torch.cumsum(torch.einsum("blk,bld->blkd", kp, v), dim=1)
+        Sn = torch.cumsum(torch.einsum("blk,bld->blkd", kn, v), dim=1)
+        zp = torch.cumsum(kp, dim=1)
+        zn = torch.cumsum(kn, dim=1)
+        num = (torch.einsum("blk,blkd->bld", qp, Sp)
+               + torch.einsum("blk,blkd->bld", qn, Sn))
+        den = ((qp * zp).sum(-1, keepdim=True)
+               + (qn * zn).sum(-1, keepdim=True))
+        if den.dtype in (torch.float32, torch.float64):
+            cl = 1e-12
+        else:                      # half dtypes: small constants flush to 0
+            cl = float(torch.finfo(den.dtype).tiny) * 8
+        return self.o(num / den.clamp_min(cl))
+
+
 def _demo():
     torch.manual_seed(0)
     dev = "cuda" if torch.cuda.is_available() else "cpu"