Create constellation_cantor_routing.py

constellation_cantor_routing.py

@@ -0,0 +1,823 @@
+#!/usr/bin/env python3
+"""
+Constellation-Cantor Relay — O(S) Cross-Token Routing
+
+This is likely one of the most powerful routing mechanisms that can exist in current spectrum
+until more formulas are resolved.
+
+=======================================================
+Replaces attention entirely with triangulation-mediated hierarchical routing.
+
+Architecture:
+  per-token:   constellation relay (triangulate → patchwork → gated residual)
+  cross-token: Cantor router (hierarchical scatter/gather through anchor tree)
+
+The triangulation profile IS the routing key. Tokens near the same anchor
+on S^(d-1) share information at level 0. Anchor pairs share at level 1.
+Quads at level 2. Full global at level log2(A).
+
+Total cross-token cost: O(S × n_levels) = O(S × 4) for 16 anchors.
+Total per-token cost:   O(S × tri_dim × pw_hidden).
+No attention anywhere. Fully O(S).
+
+Benchmarks:
+  1. Throughput: cantor-relay vs hybrid vs pure relay vs attention
+  2. Cross-token causal intervention at scale
+  3. Geometric preservation
+  4. Trained task requiring cross-token routing
+"""
+
+import os
+os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+import math
+import time
+import gc
+from collections import OrderedDict
+
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+torch.backends.cuda.matmul.allow_tf32 = True
+torch.backends.cudnn.allow_tf32 = True
+
+
+# ══════════════════════════════════════════════════════════════════
+# ACTIVATIONS
+# ══════════════════════════════════════════════════════════════════
+
+class SquaredReLU(nn.Module):
+    def forward(self, x): return F.relu(x) ** 2
+
+
+# ══════════════════════════════════════════════════════════════════
+# CONSTELLATION RELAY — per-token geometric layer
+# ══════════════════════════════════════════════════════════════════
+
+class ConstellationRelay(nn.Module):
+    """Per-token constellation triangulation + patchwork. O(S)."""
+
+    def __init__(self, dim=256, patch_dim=16, n_anchors=16, n_phases=3):
+        super().__init__()
+        self.dim = dim
+        self.patch_dim = patch_dim
+        self.n_patches = dim // patch_dim
+        self.n_anchors = n_anchors
+        self.n_phases = n_phases
+        P, A, d = self.n_patches, n_anchors, patch_dim
+
+        self.ln = nn.LayerNorm(dim)
+
+        home = torch.empty(P, A, d)
+        nn.init.xavier_normal_(home.view(P * A, d))
+        home = F.normalize(home.view(P, A, d), dim=-1)
+        self.register_buffer('home', home)
+        self.anchors = nn.Parameter(home.clone())
+
+        tri_dim = P * A * n_phases
+        self.tri_dim = tri_dim
+        pw_hidden = tri_dim * 2
+
+        self.patchwork = nn.Sequential(
+            nn.Linear(tri_dim, pw_hidden),
+            SquaredReLU(),
+            nn.LayerNorm(pw_hidden),
+            nn.Linear(pw_hidden, dim),
+        )
+        self.gate = nn.Parameter(torch.tensor(-3.0))
+
+    def drift(self):
+        h = F.normalize(self.home.float(), dim=-1)
+        c = F.normalize(self.anchors.float(), dim=-1)
+        return torch.acos((h * c).sum(-1).clamp(-1 + 1e-6, 1 - 1e-6))
+
+    def at_phase(self, t):
+        h = F.normalize(self.home.float(), dim=-1)
+        c = F.normalize(self.anchors.float(), dim=-1)
+        omega = self.drift().unsqueeze(-1)
+        so = omega.sin().clamp(min=1e-6)
+        return torch.sin((1-t)*omega)/so * h + torch.sin(t*omega)/so * c
+
+    def triangulate(self, patches_n):
+        phases = torch.linspace(0, 1, self.n_phases, device=patches_n.device).tolist()
+        tris = []
+        for t in phases:
+            at = F.normalize(self.at_phase(t), dim=-1).to(patches_n.dtype)
+            tris.append(1.0 - torch.einsum('bpd,pad->bpa', patches_n, at))
+        return torch.cat(tris, dim=-1).reshape(patches_n.shape[0], -1)
+
+    def forward(self, x):
+        """x: (B*S, D) or (B, S, D)"""
+        is_seq = x.dim() == 3
+        if is_seq:
+            B, S, D = x.shape
+            x_flat = x.reshape(B * S, D)
+        else:
+            x_flat = x
+
+        residual = x_flat
+        h = self.ln(x_flat)
+        patches = h.reshape(-1, self.n_patches, self.patch_dim)
+        patches_n = F.normalize(patches, dim=-1)
+        tri = self.triangulate(patches_n)
+        pw_out = self.patchwork(tri)
+        g = self.gate.sigmoid()
+        out = residual + g * pw_out
+
+        if is_seq:
+            return out.reshape(B, S, D), tri.reshape(B, S, -1)
+        return out, tri
+
+    def forward_no_tri(self, x):
+        """Original forward without returning tri — for compatibility."""
+        out, _ = self.forward(x)
+        return out
+
+
+# ══════════════════════════════════════════════════════════════════
+# CANTOR CONSTELLATION ROUTER — hierarchical cross-token, O(S)
+# ══════════════════════════════════════════════════════════════════
+
+class CantorConstellationRouter(nn.Module):
+    """
+    Hierarchical cross-token routing through the constellation anchor tree.
+
+    The triangulation profile assigns each token to a region on S^(d-1).
+    A binary tree over anchors defines the routing hierarchy:
+
+      Level 0: A groups   (per-anchor, local neighbors)
+      Level 1: A/2 groups (anchor pairs, nearby interaction)
+      Level 2: A/4 groups (quads, medium range)
+      ...
+      Level L: 1 group    (global summary)
+
+    At each level:
+      1. Soft-assign tokens to groups via triangulation weights
+      2. Weighted scatter: aggregate token representations per group
+      3. Transform: per-level MLP on group summaries
+      4. Weighted gather: distribute transformed summaries back to tokens
+      5. Gated residual addition
+
+    Cost: O(S × L × D) where L = log2(A) + 1 = 5 for A=16.
+    Memory: O(S × D + A × D) — no S² term anywhere.
+    """
+
+    def __init__(self, dim=256, n_anchors=16, n_patches=16):
+        super().__init__()
+        self.dim = dim
+        self.n_anchors = n_anchors
+        self.n_patches = n_patches
+        self.n_levels = int(math.log2(n_anchors)) + 1  # 5 for A=16
+
+        # Build anchor hierarchy — which anchors merge at each level
+        # Level l: anchors are grouped into bins of size 2^l
+        # The ordering is determined at init from anchor geometry
+
+        # Per-level transforms: group_dim → dim
+        self.level_mlps = nn.ModuleList()
+        self.level_gates = nn.ParameterList()
+        self.level_lns = nn.ModuleList()
+
+        for l in range(self.n_levels):
+            n_groups = max(1, n_anchors // (2 ** l))
+            self.level_mlps.append(nn.Sequential(
+                nn.Linear(dim, dim * 2),
+                SquaredReLU(),
+                nn.Linear(dim * 2, dim),
+            ))
+            self.level_lns.append(nn.LayerNorm(dim))
+            self.level_gates.append(nn.Parameter(torch.tensor(-3.0)))
+
+        # Projection from triangulation distances to routing weights
+        # Input: per-token distances to each anchor (n_patches × n_anchors)
+        self.weight_proj = nn.Linear(n_patches * n_anchors, n_anchors)
+
+    def compute_routing_weights(self, tri, n_anchors):
+        """
+        Extract soft anchor assignment weights from triangulation profile.
+
+        tri: (BS, tri_dim) — full triangulation (n_patches × n_anchors × n_phases)
+        Returns: (BS, n_anchors) — soft assignment weights (sum to 1)
+        """
+        BS = tri.shape[0]
+        # Extract phase-0 distances: first n_patches * n_anchors values
+        # These are 1 - cos(token, anchor) for each patch × anchor
+        phase0 = tri[:, :self.n_patches * n_anchors]
+
+        # Average over patches to get per-anchor proximity
+        # phase0: (BS, n_patches * n_anchors) → reshape → mean over patches
+        dists = phase0.reshape(BS, self.n_patches, n_anchors).mean(dim=1)  # (BS, A)
+
+        # Convert distances to weights: closer = higher weight
+        # dists are in [0, 2] (1 - cos), so proximity = 2 - dists
+        proximity = (2.0 - dists).clamp(min=0)
+        weights = F.softmax(proximity * 5.0, dim=-1)  # temperature-scaled
+        return weights
+
+    def forward(self, x, tri):
+        """
+        x: (B, S, D) token representations
+        tri: (B, S, tri_dim) triangulation profiles from constellation
+
+        Returns: (B, S, D) with cross-token information routed through anchor hierarchy
+        """
+        B, S, D = x.shape
+        x_flat = x.reshape(B * S, D)
+        tri_flat = tri.reshape(B * S, -1)
+
+        # Compute soft routing weights: (BS, A)
+        weights = self.compute_routing_weights(tri_flat, self.n_anchors)
+
+        h = x_flat  # working copy
+
+        for level in range(self.n_levels):
+            group_size = 2 ** level
+            n_groups = max(1, self.n_anchors // group_size)
+
+            # Merge anchor weights into group weights
+            # Reshape weights (BS, A) → (BS, n_groups, group_size) → sum over group
+            if n_groups > 1:
+                group_weights = weights.reshape(B * S, n_groups, group_size).sum(dim=-1)
+            else:
+                group_weights = weights.sum(dim=-1, keepdim=True)  # (BS, 1)
+
+            # Normalize group weights
+            group_weights = group_weights / (group_weights.sum(dim=-1, keepdim=True) + 1e-8)
+
+            # Weighted scatter: aggregate tokens into groups
+            # group_sums[g] = sum_s(group_weights[s, g] * h[s])
+            # Shape: (BS, n_groups, 1) × (BS, 1, D) summed over BS
+            # But we need per-batch grouping. Reshape to (B, S, ...) for batched ops.
+
+            gw = group_weights.reshape(B, S, n_groups)        # (B, S, G)
+            hh = h.reshape(B, S, D)                            # (B, S, D)
+
+            # Weighted sum: (B, G, S) @ (B, S, D) → (B, G, D)
+            group_summary = torch.bmm(gw.transpose(1, 2), hh)  # (B, G, D)
+
+            # Normalize by total weight per group
+            weight_sums = gw.sum(dim=1).unsqueeze(-1).clamp(min=1e-8)  # (B, G, 1)
+            group_summary = group_summary / weight_sums
+
+            # Transform
+            gs_flat = group_summary.reshape(B * n_groups, D)
+            gs_flat = self.level_lns[level](gs_flat)
+            gs_transformed = self.level_mlps[level](gs_flat)
+            gs_transformed = gs_transformed.reshape(B, n_groups, D)
+
+            # Weighted gather: distribute back to tokens
+            # update[s] = sum_g(group_weights[s, g] * gs_transformed[g])
+            # (B, S, G) @ (B, G, D) → (B, S, D)
+            token_update = torch.bmm(gw, gs_transformed).reshape(B * S, D)
+
+            # Gated residual
+            g = self.level_gates[level].sigmoid()
+            h = h + g * token_update
+
+        return h.reshape(B, S, D)
+
+
+# ══════════════════════════════════════════════════════════════════
+# CONSTELLATION-CANTOR RELAY — FULL O(S) TRANSFORMER LAYER
+# ══════════════════════════════════════════════════════════════════
+
+class ConstellationCantorRelay(nn.Module):
+    """
+    Complete O(S) transformer layer. No attention.
+
+    per-token:   ConstellationRelay (triangulate → patchwork → gated residual)
+    cross-token: CantorConstellationRouter (hierarchical scatter/gather through anchors)
+
+    The triangulation from the per-token relay is reused as routing keys
+    for the cross-token path — no redundant computation.
+    """
+
+    def __init__(self, dim=256, patch_dim=16, n_anchors=16, n_phases=3):
+        super().__init__()
+        self.relay = ConstellationRelay(
+            dim=dim, patch_dim=patch_dim, n_anchors=n_anchors, n_phases=n_phases)
+        self.router = CantorConstellationRouter(
+            dim=dim, n_anchors=n_anchors, n_patches=dim // patch_dim)
+        self.gate_relay = nn.Parameter(torch.tensor(-2.0))
+        self.gate_router = nn.Parameter(torch.tensor(-2.0))
+
+    def forward(self, x):
+        """x: (B, S, D)"""
+        B, S, D = x.shape
+
+        # Per-token relay — returns delta + triangulation
+        relay_out, tri = self.relay(x)  # (B, S, D), (B, S, tri_dim)
+        relay_delta = relay_out - x
+
+        # Cross-token routing using triangulation as routing key
+        routed = self.router(x, tri)  # (B, S, D)
+        router_delta = routed - x
+
+        # Gated combination
+        gr = self.gate_relay.sigmoid()
+        gc = self.gate_router.sigmoid()
+        return x + gr * relay_delta + gc * router_delta
+
+
+# ══════════════════════════════════════════════════════════════════
+# COMPARISON ARCHITECTURES
+# ══════════════════════════════════════════════════════════════════
+
+class VanillaAttention(nn.Module):
+    """Standard attention layer for comparison. O(S²)."""
+    def __init__(self, dim=256, n_heads=4):
+        super().__init__()
+        self.n_heads = n_heads
+        self.head_dim = dim // n_heads
+        self.ln = nn.LayerNorm(dim)
+        self.qkv = nn.Linear(dim, 3 * dim)
+        self.proj = nn.Linear(dim, dim)
+
+    def forward(self, x):
+        B, S, D = x.shape
+        h = self.ln(x)
+        qkv = self.qkv(h).reshape(B, S, 3, self.n_heads, self.head_dim)
+        q, k, v = qkv.unbind(2)
+        q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
+        attn = F.scaled_dot_product_attention(q, k, v)
+        return x + self.proj(attn.transpose(1, 2).reshape(B, S, D))
+
+
+class HybridRelay(nn.Module):
+    """Constellation relay + vanilla attention. For comparison."""
+    def __init__(self, dim=256, n_heads=4):
+        super().__init__()
+        self.relay = ConstellationRelay(dim=dim)
+        self.n_heads = n_heads
+        self.head_dim = dim // n_heads
+        self.qkv = nn.Linear(dim, 3 * dim)
+        self.attn_proj = nn.Linear(dim, dim)
+        self.attn_ln = nn.LayerNorm(dim)
+        self.gate_relay = nn.Parameter(torch.tensor(-2.0))
+        self.gate_attn = nn.Parameter(torch.tensor(-2.0))
+
+    def forward(self, x):
+        B, S, D = x.shape
+        relay_out = self.relay.forward_no_tri(x)
+        relay_delta = relay_out - x
+
+        h = self.attn_ln(x)
+        qkv = self.qkv(h).reshape(B, S, 3, self.n_heads, self.head_dim)
+        q, k, v = qkv.unbind(2)
+        q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
+        attn = F.scaled_dot_product_attention(q, k, v)
+        attn_out = self.attn_proj(attn.transpose(1, 2).reshape(B, S, D))
+
+        gr = self.gate_relay.sigmoid()
+        ga = self.gate_attn.sigmoid()
+        return x + gr * relay_delta + ga * attn_out
+
+
+class PureRelayLayer(nn.Module):
+    """Relay-only, no cross-token. For comparison."""
+    def __init__(self, dim=256):
+        super().__init__()
+        self.relay = ConstellationRelay(dim=dim)
+
+    def forward(self, x):
+        return self.relay.forward_no_tri(x)
+
+
+# ══════════════════════════════════════════════════════════════════
+# UTILITIES
+# ══════════════════════════════════════════════════════════════════
+
+def reset_vram():
+    gc.collect()
+    torch.cuda.empty_cache()
+    torch.cuda.reset_peak_memory_stats()
+
+def peak_mb():
+    return torch.cuda.max_memory_allocated() / 1e6
+
+D = 256
+
+print("=" * 80)
+print("CONSTELLATION-CANTOR RELAY — O(S) CROSS-TOKEN ROUTING BENCHMARK")
+print(f"  Device: {torch.cuda.get_device_name()}")
+print(f"  VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB")
+print(f"  Dimension: {D}")
+print("=" * 80)
+
+
+# ══════════════════════════════════════════════════════════════════
+# TEST 1: THROUGHPUT — ALL FOUR ARCHITECTURES
+# ══════════════════════════════════════════════════════════════════
+
+print(f"\n{'━'*80}")
+print("TEST 1: Throughput Scaling — 4 architectures, S=64 to 131K")
+print("  Single layer, B=1, fp16")
+print(f"{'━'*80}")
+
+SEQ_LENGTHS = [64, 256, 1024, 4096, 16384, 32768, 65536, 131072]
+
+print(f"\n  {'S':>8} {'relay':>9} {'cantor':>9} {'hybrid':>9} {'attn':>9} "
+      f"{'c/r':>6} {'c/a':>6} {'c_MB':>7}")
+
+for S in SEQ_LENGTHS:
+    results = {}
+
+    for name, make_layer in [
+        ("relay",  lambda: PureRelayLayer(D)),
+        ("cantor", lambda: ConstellationCantorRelay(D)),
+        ("hybrid", lambda: HybridRelay(D)),
+        ("attn",   lambda: VanillaAttention(D)),
+    ]:
+        try:
+            reset_vram()
+            layer = make_layer().to(DEVICE).half()
+            x = F.normalize(torch.randn(1, S, D, device=DEVICE, dtype=torch.float16), dim=-1)
+
+            # Warmup
+            with torch.no_grad():
+                for _ in range(3):
+                    _ = layer(x)
+            torch.cuda.synchronize()
+
+            t0 = time.perf_counter()
+            with torch.no_grad():
+                for _ in range(10):
+                    _ = layer(x)
+            torch.cuda.synchronize()
+            ms = (time.perf_counter() - t0) / 10 * 1000
+            mb = peak_mb()
+            results[name] = (ms, mb)
+
+            del layer, x
+            reset_vram()
+
+        except (torch.cuda.OutOfMemoryError, RuntimeError):
+            results[name] = (float('inf'), float('inf'))
+            reset_vram()
+
+    r = results.get("relay", (0, 0))[0]
+    c = results.get("cantor", (0, 0))[0]
+    h = results.get("hybrid", (0, 0))[0]
+    a = results.get("attn", (0, 0))[0]
+    c_mb = results.get("cantor", (0, 0))[1]
+
+    def fmt(v):
+        return f"{v:>8.2f}ms" if v < float('inf') else "      OOM"
+
+    cr_ratio = f"{c/r:>5.1f}×" if r > 0 and c < float('inf') else "    -"
+    ca_ratio = f"{c/a:>5.1f}×" if a > 0 and a < float('inf') and c < float('inf') else "    -"
+
+    print(f"  {S:>8} {fmt(r)} {fmt(c)} {fmt(h)} {fmt(a)} "
+          f"{cr_ratio} {ca_ratio} {c_mb:>7.0f}")
+
+
+# ══════════════════════════════════════════════════════════════════
+# TEST 2: CROSS-TOKEN CAUSAL INTERVENTION — CANTOR vs HYBRID
+# ══════════════════════════════════════════════════════════════════
+
+print(f"\n{'━'*80}")
+print("TEST 2: Cross-Token Causal Intervention")
+print("  Modify token 0, measure effect on token S//2")
+print("  4 layers deep. Compare: cantor relay vs hybrid vs pure relay")
+print(f"{'━'*80}")
+
+N_LAYERS = 4
+
+print(f"\n  {'S':>8} {'arch':>10} {'Δ_mid':>10} {'Δ_last':>10} "
+      f"{'cos_orig':>10} {'time_ms':>10}")
+
+for S in [64, 256, 1024, 4096, 16384]:
+    for arch_name, make_stack in [
+        ("cantor", lambda: nn.ModuleList([ConstellationCantorRelay(D) for _ in range(N_LAYERS)])),
+        ("hybrid", lambda: nn.ModuleList([HybridRelay(D) for _ in range(N_LAYERS)])),
+        ("relay",  lambda: nn.ModuleList([PureRelayLayer(D) for _ in range(N_LAYERS)])),
+    ]:
+        try:
+            reset_vram()
+            torch.manual_seed(42)
+            stack = make_stack().to(DEVICE).half()
+
+            x = F.normalize(torch.randn(1, S, D, device=DEVICE, dtype=torch.float16), dim=-1)
+            x_mod = x.clone()
+            x_mod[:, 0] = F.normalize(torch.randn(1, D, device=DEVICE, dtype=torch.float16), dim=-1)
+
+            torch.cuda.synchronize()
+            t0 = time.perf_counter()
+
+            with torch.no_grad():
+                h = x.clone()
+                h_mod = x_mod.clone()
+                for layer in stack:
+                    h = layer(h)
+                    h_mod = layer(h_mod)
+
+            torch.cuda.synchronize()
+            elapsed = (time.perf_counter() - t0) * 1000
+
+            mid = S // 2
+            delta_mid = (h[0, mid].float() - h_mod[0, mid].float()).norm().item()
+            delta_last = (h[0, -1].float() - h_mod[0, -1].float()).norm().item()
+            cos_orig = F.cosine_similarity(
+                x[0, mid:mid+1].float(), h[0, mid:mid+1].float()).item()
+
+            print(f"  {S:>8} {arch_name:>10} {delta_mid:>10.4f} {delta_last:>10.4f} "
+                  f"{cos_orig:>10.4f} {elapsed:>10.1f}")
+
+            del stack, x, x_mod, h, h_mod
+            reset_vram()
+
+        except (torch.cuda.OutOfMemoryError, RuntimeError):
+            print(f"  {S:>8} {arch_name:>10}       OOM")
+            reset_vram()
+
+    print()
+
+
+# ══════════════════════════════════════════════════════════════════
+# TEST 3: GEOMETRIC PRESERVATION WITH CROSS-TOKEN ROUTING
+# ══════════════════════════════════════════════════════════════════
+
+print(f"\n{'━'*80}")
+print("TEST 3: Geometric Preservation — does Cantor routing hurt geometry?")
+print("  8 layers, S=4096. Compare cos_to_orig, CV, eff_dim.")
+print(f"{'━'*80}")
+
+def compute_cv(points, n_samples=500):
+    N = points.shape[0]
+    if N < 5: return float('nan')
+    points = F.normalize(points.float(), dim=-1)
+    vols = []
+    for _ in range(n_samples):
+        idx = torch.randperm(min(N, 2000), device=points.device)[:5]
+        pts = points[idx].unsqueeze(0)
+        gram = torch.bmm(pts, pts.transpose(1, 2))
+        norms = torch.diagonal(gram, dim1=1, dim2=2)
+        d2 = norms.unsqueeze(2) + norms.unsqueeze(1) - 2 * gram
+        d2 = F.relu(d2)
+        cm = torch.zeros(1, 6, 6, device=points.device, dtype=torch.float32)
+        cm[:, 0, 1:] = 1; cm[:, 1:, 0] = 1; cm[:, 1:, 1:] = d2
+        v2 = -torch.linalg.det(cm) / 9216
+        if v2[0].item() > 1e-20:
+            vols.append(v2[0].sqrt().cpu())
+    if len(vols) < 50: return float('nan')
+    vt = torch.stack(vols)
+    return (vt.std() / (vt.mean() + 1e-8)).item()
+
+GEO_DEPTH = 8
+GEO_S = 4096
+
+print(f"\n  {'arch':>10} {'cos_orig':>10} {'norm':>8} {'CV':>8} "
+      f"{'eff_dim':>8} {'self_sim':>10}")
+
+for arch_name, make_stack in [
+    ("relay",  lambda: nn.ModuleList([PureRelayLayer(D) for _ in range(GEO_DEPTH)])),
+    ("cantor", lambda: nn.ModuleList([ConstellationCantorRelay(D) for _ in range(GEO_DEPTH)])),
+    ("hybrid", lambda: nn.ModuleList([HybridRelay(D) for _ in range(GEO_DEPTH)])),
+    ("attn",   lambda: nn.ModuleList([VanillaAttention(D) for _ in range(GEO_DEPTH)])),
+]:
+    try:
+        reset_vram()
+        torch.manual_seed(42)
+        stack = make_stack().to(DEVICE).half()
+
+        x = F.normalize(torch.randn(1, GEO_S, D, device=DEVICE, dtype=torch.float16), dim=-1)
+
+        with torch.no_grad():
+            h = x.clone()
+            for layer in stack:
+                h = layer(h)
+
+        x_s = x[0, :512].float()
+        h_s = h[0, :512].float()
+        cos = F.cosine_similarity(x_s, h_s).mean().item()
+        norm = h_s.norm(dim=-1).mean().item()
+        h_n = F.normalize(h_s, dim=-1)
+        sim = h_n @ h_n.T
+        mask = ~torch.eye(512, device=DEVICE, dtype=torch.bool)
+        self_sim = sim[mask].mean().item()
+        cv = compute_cv(h_n, 500)
+
+        _, S_vals, _ = torch.linalg.svd(h_n[:256], full_matrices=False)
+        p = S_vals / S_vals.sum()
+        ed = p.pow(2).sum().reciprocal().item()
+
+        print(f"  {arch_name:>10} {cos:>10.4f} {norm:>8.4f} {cv:>8.4f} "
+              f"{ed:>8.1f} {self_sim:>10.6f}")
+
+        del stack, x, h
+        reset_vram()
+
+    except (torch.cuda.OutOfMemoryError, RuntimeError):
+        print(f"  {arch_name:>10}       OOM")
+        reset_vram()
+
+
+# ═══��══════════════════════════════════════════════════════════════
+# TEST 4: TRAINED CROSS-TOKEN TASK — ALL ARCHITECTURES
+# ══════════════════════════════════════════════════════════════════
+
+print(f"\n{'━'*80}")
+print("TEST 4: Trained Cross-Token Task")
+print("  Label = (token_0_class + token_1_class) % 10")
+print("  Pure relay CANNOT solve this (zero cross-token info).")
+print("  4 layers, 500 steps, S=8.")
+print(f"{'━'*80}")
+
+S_TASK = 8
+N_CLS = 10
+N_SAMPLES = 4096
+STEPS = 500
+
+torch.manual_seed(777)
+keys_a = F.normalize(torch.randn(N_CLS, D, device=DEVICE), dim=-1)
+keys_b = F.normalize(torch.randn(N_CLS, D, device=DEVICE), dim=-1)
+
+task_x = F.normalize(torch.randn(N_SAMPLES, S_TASK, D, device=DEVICE), dim=-1).clone()
+label_a = torch.randint(0, N_CLS, (N_SAMPLES,), dtype=torch.long, device=DEVICE)
+label_b = torch.randint(0, N_CLS, (N_SAMPLES,), dtype=torch.long, device=DEVICE)
+task_x[:, 0] = keys_a[label_a] + torch.randn(N_SAMPLES, D, device=DEVICE) * 0.2
+task_x[:, 1] = keys_b[label_b] + torch.randn(N_SAMPLES, D, device=DEVICE) * 0.2
+task_x = F.normalize(task_x, dim=-1)
+task_y = ((label_a + label_b) % N_CLS).long()
+
+print(f"\n  {'arch':>10} {'acc':>8} {'loss':>8} {'cross_Δ':>10} {'params':>10}")
+
+for arch_name, make_stack in [
+    ("relay",  lambda: nn.ModuleList([PureRelayLayer(D) for _ in range(4)])),
+    ("cantor", lambda: nn.ModuleList([ConstellationCantorRelay(D) for _ in range(4)])),
+    ("hybrid", lambda: nn.ModuleList([HybridRelay(D) for _ in range(4)])),
+    ("attn",   lambda: nn.ModuleList([VanillaAttention(D) for _ in range(4)])),
+]:
+    torch.manual_seed(42)
+
+    class TaskModel(nn.Module):
+        def __init__(self, stack):
+            super().__init__()
+            self.layers = stack
+            self.pool = nn.Linear(D * S_TASK, D)
+            self.head = nn.Linear(D, N_CLS)
+
+        def forward(self, x):
+            for layer in self.layers:
+                x = layer(x)
+            return self.head(F.gelu(self.pool(x.reshape(x.shape[0], -1))))
+
+    model = TaskModel(make_stack()).to(DEVICE)
+    n_params = sum(p.numel() for p in model.parameters())
+    opt = torch.optim.Adam(model.parameters(), lr=3e-4)
+
+    for step in range(STEPS):
+        idx = torch.randint(0, N_SAMPLES, (128,))
+        logits = model(task_x[idx])
+        loss = F.cross_entropy(logits, task_y[idx])
+        if torch.isnan(loss) or torch.isinf(loss):
+            break
+        opt.zero_grad()
+        loss.backward()
+        nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+        opt.step()
+
+    model.eval()
+    with torch.no_grad():
+        logits = model(task_x[:1024])
+        acc = (logits.argmax(-1) == task_y[:1024]).float().mean().item()
+        final_loss = F.cross_entropy(logits, task_y[:1024]).item()
+
+        # Cross-token intervention
+        h1 = task_x[:64].clone()
+        for layer in model.layers:
+            h1 = layer(h1)
+        h2 = task_x[:64].clone()
+        h2[:, 0] = F.normalize(torch.randn(64, D, device=DEVICE), dim=-1)
+        for layer in model.layers:
+            h2 = layer(h2)
+        cross_delta = (h1[:, 1] - h2[:, 1]).norm(dim=-1).mean().item()
+
+    print(f"  {arch_name:>10} {acc:>8.1%} {final_loss:>8.4f} {cross_delta:>10.4f} {n_params:>10,}")
+
+    del model
+    reset_vram()
+
+
+# ══════════════════════════════════════════════════════════════════
+# TEST 5: THE O(S²) WALL — CANTOR vs ATTENTION at depth 8
+# ══════════════════════════════════════════════════════════════════
+
+print(f"\n{'━'*80}")
+print("TEST 5: The O(S²) Wall — Cantor vs Attention, 8 layers deep")
+print(f"{'━'*80}")
+
+WALL_DEPTH = 8
+
+print(f"\n  {'S':>8} {'cantor_ms':>10} {'attn_ms':>10} {'speedup':>8} "
+      f"{'c_cos':>8} {'a_cos':>8} {'c_MB':>8} {'a_MB':>8}")
+
+for S in [1024, 4096, 8192, 16384, 32768, 65536, 131072]:
+    c_result = None
+    a_result = None
+
+    # Cantor
+    try:
+        reset_vram()
+        torch.manual_seed(42)
+        c_stack = nn.ModuleList([
+            ConstellationCantorRelay(D) for _ in range(WALL_DEPTH)
+        ]).to(DEVICE).half()
+
+        x = F.normalize(torch.randn(1, S, D, device=DEVICE, dtype=torch.float16), dim=-1)
+        with torch.no_grad():
+            h = x.clone()
+            for layer in c_stack:
+                h = layer(h)
+        torch.cuda.synchronize()
+
+        t0 = time.perf_counter()
+        with torch.no_grad():
+            h = x.clone()
+            for layer in c_stack:
+                h = layer(h)
+        torch.cuda.synchronize()
+        c_ms = (time.perf_counter() - t0) * 1000
+        c_mb = peak_mb()
+        c_cos = F.cosine_similarity(x[0, :256].float(), h[0, :256].float()).mean().item()
+        c_result = (c_ms, c_cos, c_mb)
+
+        del x, h, c_stack
+        reset_vram()
+    except (torch.cuda.OutOfMemoryError, RuntimeError):
+        reset_vram()
+
+    # Attention
+    try:
+        reset_vram()
+        torch.manual_seed(42)
+        a_stack = nn.ModuleList([
+            VanillaAttention(D) for _ in range(WALL_DEPTH)
+        ]).to(DEVICE).half()
+
+        x = F.normalize(torch.randn(1, S, D, device=DEVICE, dtype=torch.float16), dim=-1)
+        with torch.no_grad():
+            h = x.clone()
+            for layer in a_stack:
+                h = layer(h)
+        torch.cuda.synchronize()
+
+        t0 = time.perf_counter()
+        with torch.no_grad():
+            h = x.clone()
+            for layer in a_stack:
+                h = layer(h)
+        torch.cuda.synchronize()
+        a_ms = (time.perf_counter() - t0) * 1000
+        a_mb = peak_mb()
+        a_cos = F.cosine_similarity(x[0, :256].float(), h[0, :256].float()).mean().item()
+        a_result = (a_ms, a_cos, a_mb)
+
+        del x, h, a_stack
+        reset_vram()
+    except (torch.cuda.OutOfMemoryError, RuntimeError):
+        reset_vram()
+
+    c_str = f"{c_result[0]:>9.1f}ms" if c_result else "       OOM"
+    a_str = f"{a_result[0]:>9.1f}ms" if a_result else "       OOM"
+    sp = f"{a_result[0]/c_result[0]:>7.1f}×" if c_result and a_result else "      -"
+    cc = f"{c_result[1]:>8.4f}" if c_result else "     ---"
+    ac = f"{a_result[1]:>8.4f}" if a_result else "     ---"
+    cm = f"{c_result[2]:>8.0f}" if c_result else "     OOM"
+    am = f"{a_result[2]:>8.0f}" if a_result else "     OOM"
+
+    print(f"  {S:>8} {c_str} {a_str} {sp} {cc} {ac} {cm} {am}")
+
+    if c_result is None:
+        print(f"  → Cantor OOM at S={S}, stopping")
+        break
+
+
+# ══════════════════════════════════════════════════════════════════
+# SUMMARY
+# ══════════════════════════════════════════════════════════════════
+
+print(f"\n{'='*80}")
+print("CONSTELLATION-CANTOR RELAY — BENCHMARK COMPLETE")
+print(f"{'='*80}")
+print(f"""
+  Architecture:
+    per-token:   constellation relay (triangulate → patchwork → gated residual)
+    cross-token: cantor router (hierarchical scatter/gather through anchor tree)
+    total:       O(S) time, O(S) memory, no attention
+
+  5 tests:
+    T1: Throughput — relay vs cantor vs hybrid vs attention, S to 131K
+    T2: Cross-token causal intervention — who routes strongest?
+    T3: Geometric preservation — does cross-token routing hurt geometry?
+    T4: Trained cross-token task — accuracy on interaction-dependent labels
+    T5: O(S²) wall — cantor vs attention at 8 layers to OOM
+
+  Key questions answered:
+    • Is the cantor router faster than attention at all sequence lengths?
+    • Does it provide meaningful cross-token interaction?
+    • Does the routing hurt per-token geometric preservation?
+    • Can it solve tasks that require cross-token information?
+""")