"""Resynthesis appended science layers. These layers are the trainable reasoning stack mounted after the frozen Resynthesis base hidden states. The recurrent unit is a routed expert inside the MoE layer, not the trunk. Forward paths remain tensor-owned; JSON receipts are boundary artifacts only. The Resynthesis stack composes structural experts, FFN specialists, MILT translation, cosine audit, and additive retention surfaces for: - hidden_size=4096 (integrated Resynthesis graph parent) - Science domain experts (physics, chemistry, biology, math, logic, proof, etc.) - NoNE transfer surfaces for cross-domain knowledge transfer All config fields are INITIAL VALUES, NOT CAPS (uncapped-policy: intentional). """ from __future__ import annotations import contextlib import hashlib from dataclasses import dataclass from typing import TYPE_CHECKING, Any, cast import torch import torch.nn as nn import torch.nn.functional as F from resynthesis.causal_algebra import ( CausalAlgebraConfig, CausalAlgebraWorldGraph, CausalTheoryProofPacket, CausalWorldState, ) from resynthesis.causal_integration_tensor import ( CausalIntegrationOutput, CausalIntegrationTensor, ) from resynthesis.config import GLYPH_DIM, RESYNTHESIS_HIDDEN_SIZE from resynthesis.delta_attn_res import DeltaBlockAttnRes from resynthesis.kda_expert import CRConditionedKDAExpert from resynthesis.sequence_parallel import usp_softmax_boundary from resynthesis.varlen_ring_attention import varlen_ring_softmax_boundary from resynthesis.none_paging import ( NoNEPageForwardPacket, NoNEPagedExpertRuntime, project_resynthesis_expert_weight_int4_qat_boundary, ) from resynthesis.quantile_balancing import ( ANTI_THOMPSON_FAIL_COUNTS_BUFFER_SUFFIX, QuantileBalancingRouter, ) if TYPE_CHECKING: from resynthesis.molecular_geometry import MolecularInputPacket from resynthesis.stacked_single_pass import MuonHeadGeometryPacket GLYPH_INPUT_DIM = GLYPH_DIM SCIENCE_ATTENTION_TILE_TOKENS = 256 SCIENCE_ACTION_DIM = 4 FUNCTIONAL_CAPABILITY_INITIALIZATION_SCHEME = ( "sha256_family_catalog_seeded_xavier_zero_bias_zero_open_growth_v2" ) LANGUAGE_ABILITY_ROUTING_INITIALIZATION_SCHEME = ( "catalog_incidence_zero_route_scale_v1" ) def _language_family_ability_incidence( *, family_ids: tuple[str, ...], capability_dim: int, device: torch.device, ) -> torch.Tensor: """Build the derived catalog incidence on the owning module device.""" from resynthesis.language_catalog import LANGUAGE_ABILITY_AXIS_IDS from resynthesis.language_experts import ( NONE_LANGUAGE_EXPERT_ABILITY_ASSIGNMENTS, ) language_ability_ordinal = { ability_id: ordinal for ordinal, ability_id in enumerate(LANGUAGE_ABILITY_AXIS_IDS) } family_ordinal = { family_id: ordinal for ordinal, family_id in enumerate(family_ids) } incidence = torch.zeros( capability_dim, len(LANGUAGE_ABILITY_AXIS_IDS), device=device, ) incidence_pairs = tuple( (family_index, language_ability_ordinal[ability_id]) for family_id, ability_ids in NONE_LANGUAGE_EXPERT_ABILITY_ASSIGNMENTS if (family_index := family_ordinal.get(family_id)) is not None for ability_id in ability_ids ) if incidence_pairs: incidence_indices = torch.tensor( incidence_pairs, dtype=torch.long, device=device, ) incidence.index_put_( ( incidence_indices[:, 0], incidence_indices[:, 1], ), torch.ones( incidence_indices.shape[0], dtype=incidence.dtype, device=device, ), ) return F.normalize(incidence, dim=-1) def _family_catalog_seeded_xavier_uniform_( tensor: torch.Tensor, *, layer_idx: int, family_ids: tuple[str, ...], ) -> None: """Initialize a functional-family projection independently of global RNG.""" if tensor.device.type == "meta": return family_identity = "\n".join(family_ids) seed = int.from_bytes( hashlib.sha256( ( "resynthesis.functional_capability.v1:" f"{layer_idx}:{family_identity}" ).encode("utf-8") ).digest()[:8], byteorder="little", signed=False, ) generator = torch.Generator(device=tensor.device) generator.manual_seed(seed) nn.init.xavier_uniform_(tensor, generator=generator) class IntentContextPivotAttention(nn.Module): """Exact causal Q/K/V attention with intent/action context ``C`` and relation ``R``. The score for query position ``i`` and causal key position ``j`` is ``Q_i K_j^T + g_ck Q_i C_j^T + g_cq C_i K_j^T + g_ca (Q_i C^a_j^T + C^a_i K_j^T) + g_r R_i^Q (R_j^K)^T``. ``C`` combines hidden context and the model-owned intent glyph. ``C^a`` modulates a model-owned acquisition-action glyph with that intent/context tensor. Its independent score gate can learn directly while legacy C gates remain at compatibility-zero initialization. ``R`` is projected from hidden state plus the NoNE-selected expert-intent mixture, so relational connectivity remains part of the trained graph. Learned scalar gates preserve the existing Q/K/V path when initialized to zero. Two-axis online-softmax tiling retains every causal edge without materializing a sequence-by-sequence matrix. Tile width is an execution seed, never a context or attention cap (uncapped-policy: intentional). """ def __init__( self, hidden_size: int, num_heads: int, *, glyph_dim: int = GLYPH_INPUT_DIM, tile_tokens: int = SCIENCE_ATTENTION_TILE_TOKENS, ) -> None: super().__init__() heads = max(1, int(num_heads)) if hidden_size % heads != 0: raise ValueError("hidden_size must divide evenly across attention heads") if tile_tokens < 1: raise ValueError("attention tile width must be positive") self.hidden_size = int(hidden_size) self.num_heads = heads self.head_dim = self.hidden_size // self.num_heads self.glyph_dim = int(glyph_dim) self.tile_tokens = int(tile_tokens) self.q_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=True) self.k_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=True) self.v_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=True) self.c_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=True) self.intent_c_proj = nn.Linear(self.glyph_dim, self.hidden_size, bias=False) self.action_c_proj = nn.Linear(self.glyph_dim, self.hidden_size, bias=False) self.r_query_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=True) self.r_key_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=True) self.intent_r_query_proj = nn.Linear( self.glyph_dim, self.hidden_size, bias=False, ) self.intent_r_key_proj = nn.Linear( self.glyph_dim, self.hidden_size, bias=False, ) self.out_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=True) self.intent_pivot_scale = nn.Parameter(torch.zeros(())) self.action_pivot_scale = nn.Parameter(torch.zeros(())) self.context_query_pivot_scale = nn.Parameter(torch.zeros(())) self.relation_connectivity_scale = nn.Parameter(torch.zeros(())) self.reset_parameters() def reset_parameters(self) -> None: nn.init.xavier_uniform_(self.q_proj.weight) nn.init.xavier_uniform_(self.k_proj.weight) nn.init.xavier_uniform_(self.v_proj.weight) nn.init.xavier_uniform_(self.c_proj.weight) nn.init.xavier_uniform_(self.intent_c_proj.weight) nn.init.xavier_uniform_(self.action_c_proj.weight) nn.init.xavier_uniform_(self.r_query_proj.weight) nn.init.xavier_uniform_(self.r_key_proj.weight) nn.init.xavier_uniform_(self.intent_r_query_proj.weight) nn.init.xavier_uniform_(self.intent_r_key_proj.weight) nn.init.xavier_uniform_(self.out_proj.weight) nn.init.zeros_(self.q_proj.bias) nn.init.zeros_(self.k_proj.bias) nn.init.zeros_(self.v_proj.bias) nn.init.zeros_(self.c_proj.bias) nn.init.zeros_(self.r_query_proj.bias) nn.init.zeros_(self.r_key_proj.bias) nn.init.zeros_(self.out_proj.bias) with torch.no_grad(): self.intent_pivot_scale.zero_() self.action_pivot_scale.zero_() self.context_query_pivot_scale.zero_() self.relation_connectivity_scale.zero_() def _as_heads(self, tensor: torch.Tensor) -> torch.Tensor: batch, seq, _width = tensor.shape return tensor.reshape(batch, seq, self.num_heads, self.head_dim).transpose(1, 2) def _merge_heads(self, tensor: torch.Tensor) -> torch.Tensor: batch, _heads, seq, head_dim = tensor.shape return tensor.transpose(1, 2).reshape(batch, seq, self.num_heads * head_dim) def _context_intent_action_c_state( self, hidden: torch.Tensor, intent_glyph: torch.Tensor, action_glyph: torch.Tensor, ) -> torch.Tensor: """Compose C from hidden context, intent, and action. The action contribution is gated by the same model-owned action pivot scale used by the direct action score term. At the default zero gate, action-conditioned C is exactly the legacy context+intent C path. """ action_context = self.action_c_proj(action_glyph) action_gate = torch.tanh(self.action_pivot_scale).to( device=hidden.device, dtype=hidden.dtype, ) return cast( torch.Tensor, self.c_proj(hidden) + self.intent_c_proj(intent_glyph) + action_gate * action_context, ) @staticmethod def _apply_attention_mask( scores: torch.Tensor, attn_mask: torch.Tensor | None, query_start: int, query_end: int, key_start: int, key_end: int, ) -> torch.Tensor: if attn_mask is None: return scores if attn_mask.ndim == 2: tile_mask = attn_mask[ query_start:query_end, key_start:key_end, ].view(1, 1, query_end - query_start, key_end - key_start) elif attn_mask.ndim == 4: tile_mask = attn_mask[ ..., query_start:query_end, key_start:key_end, ] else: raise ValueError("attention mask must be [sequence, sequence] or rank four") tile_mask = tile_mask.to(device=scores.device) if tile_mask.dtype == torch.bool: return scores.masked_fill(tile_mask, float("-inf")) return scores + tile_mask.to(dtype=scores.dtype) def _exact_tiled_attention( self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, context: torch.Tensor, action_context: torch.Tensor, relation_query: torch.Tensor, relation_key: torch.Tensor, *, attn_mask: torch.Tensor | None, ) -> torch.Tensor: """Compute exact causal C/R/action attention with bounded score-tile memory.""" sequence = query.shape[-2] positions = torch.arange(sequence, device=query.device) scale = self.head_dim**-0.5 context_key_gate = torch.tanh(self.intent_pivot_scale).to( device=query.device, dtype=query.dtype, ) context_query_gate = torch.tanh(self.context_query_pivot_scale).to( device=query.device, dtype=query.dtype, ) relation_gate = torch.tanh(self.relation_connectivity_scale).to( device=query.device, dtype=query.dtype, ) action_gate = torch.tanh(self.action_pivot_scale).to( device=query.device, dtype=query.dtype, ) output_tiles: tuple[torch.Tensor, ...] = () for query_start in range(0, sequence, self.tile_tokens): query_end = min(sequence, query_start + self.tile_tokens) query_tile = query[..., query_start:query_end, :] context_query_tile = context[..., query_start:query_end, :] action_query_tile = action_context[..., query_start:query_end, :] relation_query_tile = relation_query[..., query_start:query_end, :] running_max = query_tile.new_full(query_tile.shape[:-1], float("-inf")) running_sum = query_tile.new_zeros(query_tile.shape[:-1]) running_value = value.new_zeros( (*query_tile.shape[:-1], value.shape[-1]) ) query_positions = positions[query_start:query_end] for key_start in range(0, query_end, self.tile_tokens): key_end = min(query_end, key_start + self.tile_tokens) key_tile = key[..., key_start:key_end, :] context_key_tile = context[..., key_start:key_end, :] action_key_tile = action_context[..., key_start:key_end, :] relation_key_tile = relation_key[..., key_start:key_end, :] scores = torch.matmul(query_tile, key_tile.transpose(-2, -1)) scores = scores + context_key_gate * torch.matmul( query_tile, context_key_tile.transpose(-2, -1), ) scores = scores + context_query_gate * torch.matmul( context_query_tile, key_tile.transpose(-2, -1), ) scores = scores + action_gate * ( torch.matmul(query_tile, action_key_tile.transpose(-2, -1)) + torch.matmul(action_query_tile, key_tile.transpose(-2, -1)) ) scores = scores + relation_gate * torch.matmul( relation_query_tile, relation_key_tile.transpose(-2, -1), ) scores = scores * scale key_positions = positions[key_start:key_end] causal_mask = key_positions.unsqueeze(0).gt( query_positions.unsqueeze(1) ) scores = scores.masked_fill( causal_mask.view( 1, 1, query_end - query_start, key_end - key_start, ), float("-inf"), ) scores = self._apply_attention_mask( scores, attn_mask, query_start, query_end, key_start, key_end, ) tile_max = scores.amax(dim=-1) next_max = torch.maximum(running_max, tile_max) prior_scale = torch.where( torch.isfinite(running_max), (running_max - next_max).exp(), torch.zeros_like(running_max), ) weights = torch.where( torch.isfinite(scores), (scores - next_max.unsqueeze(-1)).exp(), torch.zeros_like(scores), ) value_tile = value[..., key_start:key_end, :] running_value = ( running_value * prior_scale.unsqueeze(-1) + torch.matmul(weights, value_tile) ) running_sum = ( running_sum * prior_scale + weights.sum(dim=-1) ) running_max = next_max output_tiles += ( running_value / running_sum.clamp_min( torch.finfo(running_sum.dtype).tiny ).unsqueeze(-1), ) return torch.cat(output_tiles, dim=-2) def forward( self, hidden: torch.Tensor, *, intent_glyph_context: torch.Tensor, action_glyph_context: torch.Tensor | None = None, relation_glyph_context: torch.Tensor | None = None, attn_mask: torch.Tensor | None = None, ) -> torch.Tensor: if hidden.ndim != 3: raise ValueError("intent-context attention hidden must be [batch, seq, hidden]") if intent_glyph_context.ndim != 3: raise ValueError( "intent glyph context must be [batch, seq, glyph_dim]" ) if intent_glyph_context.shape[:2] != hidden.shape[:2]: raise ValueError("intent glyph context batch/seq differs from hidden") if intent_glyph_context.shape[-1] != self.glyph_dim: raise ValueError("intent glyph context width differs from glyph_dim") if action_glyph_context is not None: if action_glyph_context.ndim != 3: raise ValueError( "action glyph context must be [batch, seq, glyph_dim]" ) if action_glyph_context.shape != intent_glyph_context.shape: raise ValueError("action glyph context geometry differs from intent") relation_glyph = ( intent_glyph_context if relation_glyph_context is None else relation_glyph_context ) if relation_glyph.shape != intent_glyph_context.shape: raise ValueError("relation glyph context geometry differs from intent") if ( hidden.shape[1] == 1 and attn_mask is None and not torch.is_grad_enabled() ): # With one unmasked causal key, softmax has one element and its # exact weight is one regardless of Q/K/C/action/relation scores. # This path is confined to inference/frozen-parent execution so it # cannot remove trainable projection gradients. return cast(torch.Tensor, self.out_proj(self.v_proj(hidden))) query = self._as_heads(self.q_proj(hidden)) key = self._as_heads(self.k_proj(hidden)) value = self._as_heads(self.v_proj(hidden)) intent_dtype = intent_glyph_context.to(dtype=hidden.dtype) action_glyph = ( intent_dtype if action_glyph_context is None else action_glyph_context.to(dtype=hidden.dtype) ) context = self._context_intent_action_c_state( hidden, intent_dtype, action_glyph, ) action_heads = self._as_heads(self.action_c_proj(action_glyph)) context_heads = self._as_heads(context) relation_glyph = relation_glyph.to(dtype=hidden.dtype) relation_query = self._as_heads( self.r_query_proj(hidden) + self.intent_r_query_proj(relation_glyph) ) relation_key = self._as_heads( self.r_key_proj(hidden) + self.intent_r_key_proj(relation_glyph) ) mixed = self._merge_heads( self._exact_tiled_attention( query, key, value, context_heads, action_heads, relation_query, relation_key, attn_mask=attn_mask, ) ) return cast(torch.Tensor, self.out_proj(mixed)) def expand_context_intent_channel( context_intent: torch.Tensor, reference: torch.Tensor, ) -> torch.Tensor: """Expand context-intent ``C`` to ``[batch, sequence, hidden]``. Accepts ``C`` as ``[batch, sequence]`` (broadcast across hidden) or ``[batch, sequence, hidden]``. There is no host cap on sequence length or hidden width (uncapped-policy: intentional). """ if reference.ndim != 3: raise ValueError("reference hidden must be [batch, sequence, hidden]") batch, sequence, hidden_size = reference.shape if context_intent.ndim == 2: if context_intent.shape != (batch, sequence): raise ValueError( "context-intent [batch, sequence] geometry differs from reference" ) return context_intent.unsqueeze(-1).expand(batch, sequence, hidden_size) if context_intent.ndim == 3: if context_intent.shape != (batch, sequence, hidden_size): raise ValueError( "context-intent [batch, sequence, hidden] geometry differs from reference" ) return context_intent raise ValueError( "context-intent must be [batch, sequence] or [batch, sequence, hidden]" ) def context_intent_action_c_state( context_intent: torch.Tensor, *, context_action: torch.Tensor | None, action_gate: torch.Tensor | float, ) -> torch.Tensor: """Condition an expanded C channel on action without changing zero-gate C.""" if context_action is None: return context_intent if context_action.shape != context_intent.shape: raise ValueError("action channel geometry must match expanded C channel") if isinstance(action_gate, float): if action_gate == 0.0: return context_intent return context_intent + action_gate * torch.tanh(context_action) gate = action_gate.to( device=context_intent.device, dtype=context_intent.dtype, ) while gate.ndim < context_intent.ndim: gate = gate.unsqueeze(-1) return context_intent + gate * torch.tanh(context_action) def compose_long_pool_attention_scores( query: torch.Tensor, keys: torch.Tensor, scale: torch.Tensor | float, context_intent: torch.Tensor | None = None, *, context_action: torch.Tensor | None = None, intent_query_context: torch.Tensor | None = None, action_query_context: torch.Tensor | None = None, intent_additive_gate: torch.Tensor | float = 1.0, action_additive_gate: torch.Tensor | float = 1.0, intent_multiplicative_gate: torch.Tensor | float = 0.0, action_multiplicative_gate: torch.Tensor | float = 0.0, ) -> torch.Tensor: """STACK+COMPOSE long-pool scores: Q·K plus optional intent and action ``C``. Baseline ``Q·K`` always remains (compose, do not replace). When intent and/or action channels are present, scores add ``gate_add * (Q·C)`` for each active channel and optionally ``mult_gate * (Q⊙C_q)(K⊙C)`` on both intent and action. When both channels are absent the result is pure ``Q·K`` (identity). """ content_scores = torch.matmul(query.unsqueeze(1), keys.transpose(1, 2)).squeeze(1) scores = content_scores * scale if context_intent is not None: context = expand_context_intent_channel(context_intent, keys) action_for_c = ( None if context_action is None else expand_context_intent_channel(context_action, keys) ) context = context_intent_action_c_state( context, context_action=action_for_c, action_gate=action_additive_gate, ) intent_scores = ( torch.matmul(query.unsqueeze(1), context.transpose(1, 2)).squeeze(1) * scale ) gate_add = ( float(intent_additive_gate) if isinstance(intent_additive_gate, float) else intent_additive_gate.to(device=query.device, dtype=query.dtype) ) scores = scores + gate_add * intent_scores if context_action is not None: action = expand_context_intent_channel(context_action, keys) action_scores = ( torch.matmul(query.unsqueeze(1), action.transpose(1, 2)).squeeze(1) * scale ) action_gate = ( float(action_additive_gate) if isinstance(action_additive_gate, float) else action_additive_gate.to(device=query.device, dtype=query.dtype) ) scores = scores + action_gate * action_scores if context_intent is not None and not ( isinstance(intent_multiplicative_gate, float) and intent_multiplicative_gate == 0.0 ): context = expand_context_intent_channel(context_intent, keys) action_for_c = ( None if context_action is None else expand_context_intent_channel(context_action, keys) ) context = context_intent_action_c_state( context, context_action=action_for_c, action_gate=action_multiplicative_gate, ) gate_mult = ( float(intent_multiplicative_gate) if isinstance(intent_multiplicative_gate, float) else intent_multiplicative_gate.to(device=query.device, dtype=query.dtype) ) context_query = ( context[:, -1, :] if intent_query_context is None else intent_query_context.to(device=query.device, dtype=query.dtype) ) if context_query.shape != query.shape: raise ValueError("intent query C geometry must match query geometry") query_mod = query * context_query keys_mod = keys * context mult_scores = ( torch.matmul( query_mod.unsqueeze(1), keys_mod.transpose(1, 2) ).squeeze(1) * scale ) scores = scores + gate_mult * mult_scores if context_action is not None and not ( isinstance(action_multiplicative_gate, float) and action_multiplicative_gate == 0.0 ): action = expand_context_intent_channel(context_action, keys) action_gate_mult = ( float(action_multiplicative_gate) if isinstance(action_multiplicative_gate, float) else action_multiplicative_gate.to(device=query.device, dtype=query.dtype) ) action_query = ( action[:, -1, :] if action_query_context is None else action_query_context.to(device=query.device, dtype=query.dtype) ) if action_query.shape != query.shape: raise ValueError("action query C geometry must match query geometry") query_mod = query * action_query keys_mod = keys * action action_mult_scores = ( torch.matmul( query_mod.unsqueeze(1), keys_mod.transpose(1, 2) ).squeeze(1) * scale ) scores = scores + action_gate_mult * action_mult_scores return scores def online_softmax_last_token_pool( hidden: torch.Tensor, *, context_intent: torch.Tensor | None = None, context_action: torch.Tensor | None = None, chunk_tokens: int, intent_additive_gate: torch.Tensor | float = 1.0, action_additive_gate: torch.Tensor | float = 1.0, intent_multiplicative_gate: torch.Tensor | float = 0.0, action_multiplicative_gate: torch.Tensor | float = 0.0, ) -> torch.Tensor: """Exact last-token attention pool with optional intent/action compose and tiling.""" if hidden.ndim != 3 or hidden.shape[1] == 0: raise ValueError("hidden sequence pool requires [batch, sequence, hidden]") batch, sequence, hidden_size = hidden.shape query = hidden[:, -1, :] scale = hidden_size**-0.5 intent_query_context: torch.Tensor | None = None action_query_context: torch.Tensor | None = None if context_intent is not None: intent_query_context = expand_context_intent_channel(context_intent, hidden)[:, -1, :] if context_action is not None: action_query_context = expand_context_intent_channel(context_action, hidden)[:, -1, :] if sequence <= chunk_tokens: scores = compose_long_pool_attention_scores( query, hidden, scale, context_intent, context_action=context_action, intent_query_context=intent_query_context, action_query_context=action_query_context, intent_additive_gate=intent_additive_gate, action_additive_gate=action_additive_gate, intent_multiplicative_gate=intent_multiplicative_gate, action_multiplicative_gate=action_multiplicative_gate, ) weights = varlen_ring_softmax_boundary( usp_softmax_boundary(scores, dim=-1), dim=-1, ) return torch.matmul(weights.unsqueeze(1), hidden).squeeze(1) running_max = hidden.new_full((batch,), float("-inf")) running_sum = hidden.new_zeros((batch,)) running_out = hidden.new_zeros((batch, hidden_size)) tile = chunk_tokens for start in range(0, sequence, tile): end = min(sequence, start + tile) chunk = hidden[:, start:end, :] chunk_context: torch.Tensor | None = None if context_intent is not None: if context_intent.ndim == 2: chunk_context = context_intent[:, start:end] else: chunk_context = context_intent[:, start:end, :] chunk_action: torch.Tensor | None = None if context_action is not None: if context_action.ndim == 2: chunk_action = context_action[:, start:end] else: chunk_action = context_action[:, start:end, :] scores = compose_long_pool_attention_scores( query, chunk, scale, chunk_context, context_action=chunk_action, intent_query_context=intent_query_context, action_query_context=action_query_context, intent_additive_gate=intent_additive_gate, action_additive_gate=action_additive_gate, intent_multiplicative_gate=intent_multiplicative_gate, action_multiplicative_gate=action_multiplicative_gate, ) chunk_max = scores.amax(dim=-1) new_max = torch.maximum(running_max, chunk_max) prior_scale = (running_max - new_max).exp() prior_scale = torch.where( torch.isfinite(running_max), prior_scale, torch.zeros_like(prior_scale), ) weights = (scores - new_max.unsqueeze(-1)).exp() running_out = running_out * prior_scale.unsqueeze(-1) + torch.matmul( weights.unsqueeze(1), chunk ).squeeze(1) running_sum = running_sum * prior_scale + weights.sum(dim=-1) running_max = new_max return running_out / running_sum.clamp_min( torch.finfo(running_sum.dtype).tiny ).unsqueeze(-1) def _deterministic_xavier_tensor( reference: torch.Tensor, shape: tuple[int, ...], name: str, ) -> torch.Tensor: """Create reproducible migration weights without changing global RNG state.""" value = reference.new_empty(shape) if value.device.type == "meta": return value seed = int.from_bytes( hashlib.sha256(name.encode("utf-8")).digest()[:8], byteorder="little", signed=False, ) generator = torch.Generator(device=value.device) generator.manual_seed(seed) nn.init.xavier_uniform_(value, generator=generator) return value def adapt_attention_state_to_context_relation( state: dict[str, torch.Tensor], ) -> tuple[dict[str, torch.Tensor], bool]: """Adapt attention state into the live Q/K/V/C(intent+action)/R geometry. Preserves trained Q/K/V slices from ``in_proj_*`` exactly. Initializes the new C (context/intent/action) and R (relational connectivity) projections deterministically. All newly introduced score gates are zero, so legacy behavior is preserved until training opens the additional pathways. An already trained C gate is retained exactly. """ adapted = dict(state) changed = False prefixes: set[str] = set() for name in state: for marker in ( "attention_expert.in_proj_weight", "attention_expert.q_proj.weight", ): if name.endswith(marker): prefixes.add(name[: -len(marker)] + "attention_expert.") for prefix in sorted(prefixes): in_proj_weight = adapted.pop(f"{prefix}in_proj_weight", None) in_proj_bias = adapted.pop(f"{prefix}in_proj_bias", None) if isinstance(in_proj_weight, torch.Tensor): if in_proj_weight.ndim != 2 or in_proj_weight.shape[0] % 3 != 0: raise RuntimeError( f"legacy attention in_proj_weight geometry differs: {prefix}" ) width = in_proj_weight.shape[0] // 3 q_w = in_proj_weight[:width] k_w = in_proj_weight[width : 2 * width] v_w = in_proj_weight[2 * width : 3 * width] adapted[f"{prefix}q_proj.weight"] = q_w.contiguous().clone() adapted[f"{prefix}k_proj.weight"] = k_w.contiguous().clone() adapted[f"{prefix}v_proj.weight"] = v_w.contiguous().clone() if isinstance(in_proj_bias, torch.Tensor): if in_proj_bias.shape[0] != 3 * width: raise RuntimeError( f"legacy attention in_proj_bias geometry differs: {prefix}" ) q_b = in_proj_bias[:width] k_b = in_proj_bias[width : 2 * width] v_b = in_proj_bias[2 * width : 3 * width] adapted[f"{prefix}q_proj.bias"] = q_b.contiguous().clone() adapted[f"{prefix}k_proj.bias"] = k_b.contiguous().clone() adapted[f"{prefix}v_proj.bias"] = v_b.contiguous().clone() else: zeros = in_proj_weight.new_zeros(width) adapted[f"{prefix}q_proj.bias"] = zeros.clone() adapted[f"{prefix}k_proj.bias"] = zeros.clone() adapted[f"{prefix}v_proj.bias"] = zeros.clone() changed = True q_weight = adapted.get(f"{prefix}q_proj.weight") if not isinstance(q_weight, torch.Tensor) or q_weight.ndim != 2: raise RuntimeError(f"attention Q projection is absent: {prefix}") width = q_weight.shape[0] if f"{prefix}c_proj.weight" not in adapted: adapted[f"{prefix}c_proj.weight"] = _deterministic_xavier_tensor( q_weight, tuple(q_weight.shape), f"{prefix}c_proj.weight", ) adapted[f"{prefix}c_proj.bias"] = q_weight.new_zeros(width) changed = True if f"{prefix}intent_c_proj.weight" not in adapted: adapted[f"{prefix}intent_c_proj.weight"] = ( _deterministic_xavier_tensor( q_weight, (width, GLYPH_INPUT_DIM), f"{prefix}intent_c_proj.weight", ) ) changed = True if f"{prefix}intent_pivot_scale" not in adapted: adapted[f"{prefix}intent_pivot_scale"] = q_weight.new_zeros(()) changed = True if f"{prefix}action_c_proj.weight" not in adapted: adapted[f"{prefix}action_c_proj.weight"] = ( _deterministic_xavier_tensor( q_weight, (width, GLYPH_INPUT_DIM), f"{prefix}action_c_proj.weight", ) ) changed = True layer_prefix = prefix[: -len("attention_expert.")] action_bridge_name = f"{layer_prefix}action_glyph_bridge.weight" if ( "science_stack.science_layer_" in layer_prefix and action_bridge_name not in adapted ): adapted[action_bridge_name] = _deterministic_xavier_tensor( q_weight, (GLYPH_INPUT_DIM, SCIENCE_ACTION_DIM), action_bridge_name, ) changed = True mhc_scale_name = ( f"{layer_prefix}mhc_distinct_hypothesis_scale" ) if ( "science_stack.science_layer_" in layer_prefix and mhc_scale_name not in adapted ): adapted[mhc_scale_name] = q_weight.new_zeros(()) changed = True if f"{prefix}action_pivot_scale" not in adapted: adapted[f"{prefix}action_pivot_scale"] = q_weight.new_zeros(()) changed = True if f"{prefix}context_query_pivot_scale" not in adapted: adapted[f"{prefix}context_query_pivot_scale"] = ( q_weight.new_zeros(()) ) changed = True for projection in ("r_query_proj", "r_key_proj"): weight_name = f"{prefix}{projection}.weight" bias_name = f"{prefix}{projection}.bias" if weight_name not in adapted: adapted[weight_name] = _deterministic_xavier_tensor( q_weight, tuple(q_weight.shape), weight_name, ) adapted[bias_name] = q_weight.new_zeros(width) changed = True for projection in ("intent_r_query_proj", "intent_r_key_proj"): weight_name = f"{prefix}{projection}.weight" if weight_name not in adapted: adapted[weight_name] = _deterministic_xavier_tensor( q_weight, (width, GLYPH_INPUT_DIM), weight_name, ) changed = True if f"{prefix}relation_connectivity_scale" not in adapted: adapted[f"{prefix}relation_connectivity_scale"] = q_weight.new_zeros( () ) changed = True return adapted, changed def adapt_native_attention_expert_state( state: dict[str, torch.Tensor], target_state: dict[str, torch.Tensor], ) -> tuple[dict[str, torch.Tensor], bool]: """Add deterministic native graph tensors to older additive snapshots. KDA/QB/Delta-AttnRes and the causal algebra world graph are explicit growth surfaces. Unknown missing keys remain missing so the strict checkpoint loader still detects unrelated corruption or architecture drift. """ adapted = dict(state) changed = False def is_native_attention_surface(name: str) -> bool: if name.endswith(ANTI_THOMPSON_FAIL_COUNTS_BUFFER_SUFFIX): # Durable routing outcomes are a single atomic family. Their # versioned adapter below must distinguish whole-family absence # from partial/corrupt state; the broad architecture initializer # must not silently pre-seed them one tensor at a time. return False if ".quantile_router.trauma_state." in name: # Trauma is likewise one hard-knowledge authority per router. # Its migration must preserve the complete learned family or seed # the complete constructor family; generic adoption would hide # partial/corrupt state and erase its exact growth receipt. return False science_layer_surface = "science_stack.science_layer_" in name and ( ".quantile_router." in name or ".kda_expert." in name or name.endswith(".quantile_route_scale") or name.endswith(".ffn_up") or name.endswith(".ffn_latent_up") or name.endswith(".situ_glu_scale") or name.endswith(".stable_latent_moe_scale") or name.endswith( ".paged_expert_runtime.executor.situ_glu_scale" ) or name.endswith( ".paged_expert_runtime.executor.latent_rmsnorm_scale" ) ) causal_algebra_surface = ( name.startswith("science_stack.causal_algebra_world_graph.") or name.startswith("causal_algebra_world_graph.") ) return ( science_layer_surface or causal_algebra_surface or name.startswith("science_stack.delta_attn_res.") ) for name, target in target_state.items(): if name in adapted or not is_native_attention_surface(name): continue reference = target.detach().to(device="cpu") causal_algebra_surface = ( name.startswith("science_stack.causal_algebra_world_graph.") or name.startswith("causal_algebra_world_graph.") ) preserve_target = ( causal_algebra_surface or name.endswith(".depth_connection_logits") or name.endswith(".short_conv.weight") or reference.ndim < 2 ) adapted[name] = ( reference.clone() if preserve_target else _deterministic_xavier_tensor( reference, tuple(reference.shape), name, ) ) changed = True return adapted, changed def adapt_mha_state_to_intent_context_c( state: dict[str, torch.Tensor], ) -> tuple[dict[str, torch.Tensor], bool]: """Backward-compatible name for the complete Q/K/V/C/R migration.""" return adapt_attention_state_to_context_relation(state) @dataclass(frozen=True) class ResynthesisScienceLayerConfig: """Science layer config — ALL FIELDS ARE INITIAL VALUES, NOT CAPS. ``num_layers`` and ``num_experts`` define the trained checkpoint geometry. The active loop rotates and recombines those existing pathways. A future geometry migration must be separately trained, retained, and cold-reload verified; this module does not claim unimplemented in-place growth. recursive_steps=0 means the RBO's confidence-based stop gate controls traversal depth, NOT this field. """ # Current vocabulary/projection seam. Successor additive generations may # add wider learned structure around it; this inherited width is not a # model-capacity ceiling. hidden_size: int = RESYNTHESIS_HIDDEN_SIZE # ``None`` follows the accepted generation's full current hidden seam. # An explicit value is an initial materialized transfer rank, never a # maximum: prefix-preserving successor migration may widen it without # replacing already accepted rows/columns. knowledge_transfer_dim: int | None = None num_layers: int = 4 num_experts: int = 8 expert_hidden_size: int = 1024 memory_slots: int = 1 # minimal seed; grows on demand attention_heads: int = 16 mhc_heads: int = 8 recursive_steps: int = 0 # 0 = RBO confidence controls (no cap) residual_init: float = 0.02 logit_residual_init: float = -4.0 kl_anchor_weight: float = 0.1 kl_anchor_warmup_steps: int = 100 glyph_input_dim: int = GLYPH_INPUT_DIM action_input_dim: int = SCIENCE_ACTION_DIM attention_tile_tokens: int = SCIENCE_ATTENTION_TILE_TOKENS enable_molecular_science: bool = True causal_world_size: int = 64 causal_hypothesis_count: int = 4 causal_primitive_count: int = 8 causal_program_steps: int = 4 causal_domain_count: int = 8 causal_operator_rank: int = 16 @dataclass(frozen=True) class ScienceTraversalState: """Session-owned per-example tensor memory for causal NoNE rotation. The leading batch axis is required even for a single example. Keeping traversal pressure independent prevents one prompt in a validation batch from selecting experts or exhausting an arm on behalf of another prompt. """ expert_visits: torch.Tensor expert_selections: torch.Tensor layer_visits: torch.Tensor traversal_index: torch.Tensor @dataclass(frozen=True) class _PagedSparseDeltaBankWorkspace: """Fully overwritten tensor views over one reusable device allocation.""" delta_bank_t: torch.Tensor relation_bank_t: torch.Tensor projected_delta_bank_t: torch.Tensor projected_relation_bank_t: torch.Tensor @dataclass(frozen=True) class ScienceLayerResult: """Tensor-native output of one adaptive science expert layer.""" hidden: torch.Tensor expert_routes: torch.Tensor expert_visit: torch.Tensor @dataclass(frozen=True) class ScienceStackResult: """Tensor-native output of the complete recursive science stack.""" hidden: torch.Tensor expert_routes: torch.Tensor layer_routes: torch.Tensor traversal_state: ScienceTraversalState causal_proof: CausalTheoryProofPacket | None = None def _batched_ffn_expert_mixture( hidden: torch.Tensor, gate_weights: torch.Tensor, gate: torch.Tensor, up: torch.Tensor, down: torch.Tensor, situ_glu_scale: torch.Tensor, latent_up: torch.Tensor, stable_latent_moe_scale: torch.Tensor, ) -> torch.Tensor: """Execute additive legacy and Stable-LatentMoE paths tensor-natively. r152 branch training keeps inherited experts frozen but still needs their exact model-owned mixture as the context supplied to trainable NoNE pages. The zero-initialized blends preserve that historical function. Continued training can independently open bounded SiTU-GLU activations and the normalized shared latent up-projection without silently rewriting accepted expert knowledge. """ if ( hidden.ndim != 3 or gate_weights.ndim != 3 or gate.ndim != 3 or up.ndim != 3 or down.ndim != 3 or gate_weights.shape[:2] != hidden.shape[:2] or gate_weights.shape[-1] != gate.shape[0] or gate.shape != up.shape or gate.shape[0] != down.shape[0] or hidden.shape[-1] != gate.shape[1] or gate.shape[2] != down.shape[1] or down.shape[2] != hidden.shape[-1] or latent_up.shape != (gate.shape[2], hidden.shape[-1]) or situ_glu_scale.numel() != 1 or stable_latent_moe_scale.numel() != 1 ): raise ValueError("batched FFN expert geometry differs") gate_hidden_t = torch.einsum("bsh,ehf->bsef", hidden, gate) up_hidden_t = torch.einsum("bsh,ehf->bsef", hidden, up) legacy_expert_hidden_t = F.silu(gate_hidden_t) # SiTU-GLU uses smooth beta_gate=4 and beta_up=25 caps. Both # branches remain differentiable and approximately linear near the origin # while their multiplicative output cannot grow without bound. situ_gate_t = ( 4.0 * torch.tanh(gate_hidden_t / 4.0) * torch.sigmoid(gate_hidden_t) ) situ_up_t = 25.0 * torch.tanh(up_hidden_t / 25.0) situ_expert_hidden_t = situ_gate_t * situ_up_t situ_blend_t = torch.tanh(situ_glu_scale).to( dtype=legacy_expert_hidden_t.dtype ) expert_hidden_t = legacy_expert_hidden_t + situ_blend_t * ( situ_expert_hidden_t - legacy_expert_hidden_t ) expert_output_t = torch.einsum( "bsef,efh->bseh", expert_hidden_t, down, ) legacy_mixture_t = torch.sum( gate_weights.unsqueeze(-1) * expert_output_t, dim=2, ) routed_latent_t = torch.sum( gate_weights.unsqueeze(-1) * expert_hidden_t, dim=2, ) normalized_latent_t = F.rms_norm( routed_latent_t, (routed_latent_t.shape[-1],), ) stable_latent_output_t = torch.matmul( normalized_latent_t, latent_up, ) stable_blend_t = torch.tanh(stable_latent_moe_scale).to( dtype=legacy_mixture_t.dtype ) return legacy_mixture_t + stable_blend_t * ( stable_latent_output_t - legacy_mixture_t ) class ResynthesisScienceLayer(nn.Module): """One routed science layer with recurrent, attention, memory, glyph, and FFN experts. Per-expert identity system (MILT): - expert_intent_glyphs [E, 168]: semantic identity in glyph space - expert_role_tag [E, 128]: learned role embedding (physics, math, logic, etc.) - expert_specialization [E]: learned scalar — how specialized each expert is - layer_depth_signal [128]: learned embedding — identifies this layer's position """ intent_plane_anchor_mean: torch.Tensor _expert_history_states: torch.Tensor _inherited_dense_frozen_for_paged_training: bool paged_expert_runtime: NoNEPagedExpertRuntime | None last_paged_expert_packet: NoNEPageForwardPacket | None recurrent_expert_id = 0 attention_expert_id = 1 memory_expert_id = 2 glyph_anchor_expert_id = 3 structural_expert_count = 4 ROLE_DIM = 128 def __init__(self, cfg: ResynthesisScienceLayerConfig, layer_idx: int) -> None: super().__init__() self.cfg = cfg self.layer_idx = int(layer_idx) self.num_experts = max(self.structural_expert_count + 1, int(cfg.num_experts)) self.num_ffn_experts = self.num_experts - self.structural_expert_count self.attention_heads = self._valid_heads(cfg.hidden_size, cfg.attention_heads) self.mhc_heads = self._valid_heads(cfg.hidden_size, cfg.mhc_heads) self.norm = nn.LayerNorm(cfg.hidden_size) self.output_norm = nn.LayerNorm(cfg.hidden_size) self.router = nn.Linear(cfg.hidden_size, self.num_experts, bias=False) expert_ids_t = torch.arange(1, self.num_experts + 1, dtype=torch.float32) self.expert_activation_prior = nn.Parameter( 1.0e-3 * torch.sin(expert_ids_t * 0.6180339887) ) self.expert_intent_glyphs = nn.Parameter(torch.empty(self.num_experts, cfg.glyph_input_dim)) self.intent_query_proj = nn.Linear(cfg.hidden_size, cfg.glyph_input_dim, bias=False) self.language_match_scale = nn.Parameter(torch.tensor(0.5)) self.register_buffer("intent_plane_anchor_mean", torch.zeros(cfg.glyph_input_dim), persistent=True) self.expert_role_tag = nn.Parameter(torch.empty(self.num_experts, self.ROLE_DIM)) nn.init.xavier_uniform_(self.expert_role_tag) expert_slots = torch.arange(1, self.num_experts + 1, dtype=torch.float32) self.expert_specialization = nn.Parameter( torch.sin(expert_slots * 0.37) * 0.05 ) self.role_query_proj = nn.Linear(cfg.hidden_size, self.ROLE_DIM, bias=False) nn.init.xavier_uniform_(self.role_query_proj.weight) self.role_match_scale = nn.Parameter(torch.tensor(0.3)) source_slots = expert_slots.unsqueeze(1) target_slots = expert_slots.unsqueeze(0) compatibility = ( torch.sin(source_slots * target_slots * 0.1732050808) + torch.cos(source_slots * 0.6180339887 + target_slots * 0.1178511302) ) * 0.01 compatibility = compatibility + torch.eye(self.num_experts) * 0.02 self.expert_compatibility = nn.Parameter(compatibility) self.expert_transfer_scale = nn.Parameter(torch.tensor(0.10)) self.expert_rotation_pressure = nn.Parameter(torch.tensor(4.0)) from resynthesis.corpus_training import NONE_FUNCTIONAL_EXPERT_FAMILIES family_ids = tuple( family_id for family_id, _description, _claims in NONE_FUNCTIONAL_EXPERT_FAMILIES ) capability_dim = max(8, len(family_ids)) self.expert_capability_proj = nn.Linear(self.ROLE_DIM, capability_dim) self.capability_match_scale = nn.Parameter(torch.zeros(())) self.register_buffer( "language_family_ability_incidence", _language_family_ability_incidence( family_ids=family_ids, capability_dim=capability_dim, device=self.expert_capability_proj.weight.device, ), persistent=False, ) self.language_ability_match_scale = nn.Parameter(torch.zeros(())) self.expert_depth_pref = nn.Parameter( torch.cos(expert_slots * 0.29 + float(layer_idx) * 0.11) * 0.05 ) self.expert_transfer_affinity = nn.Parameter( torch.sin(expert_slots * 0.41 + float(layer_idx) * 0.17) * 0.05 ) self.expert_history_gru = nn.GRUCell(1, self.ROLE_DIM) self.register_buffer( "_expert_history_states", F.normalize(self.expert_role_tag.detach(), dim=-1) * 1.0e-3, persistent=True, ) _family_catalog_seeded_xavier_uniform_( self.expert_capability_proj.weight, layer_idx=self.layer_idx, family_ids=family_ids, ) nn.init.zeros_(self.expert_capability_proj.bias) self.layer_role_head = nn.Linear(self.ROLE_DIM, 5) nn.init.xavier_uniform_(self.layer_role_head.weight) self.layer_depth_signal = nn.Parameter(torch.empty(self.ROLE_DIM)) nn.init.normal_(self.layer_depth_signal, mean=float(layer_idx) * 0.1, std=0.02) self.layer_complexity = nn.Parameter(torch.tensor(0.5)) self.recurrent_expert = nn.GRU( input_size=cfg.hidden_size, hidden_size=cfg.hidden_size, num_layers=1, batch_first=True, ) self.attention_expert = IntentContextPivotAttention( cfg.hidden_size, self.attention_heads, glyph_dim=cfg.glyph_input_dim, tile_tokens=cfg.attention_tile_tokens, ) # Quantile routing is an in-graph additive pathway, not a host feature # switch. Its zero blend preserves the dense route for legacy snapshots # while gradients can open the sparse frontier during continuation. self.quantile_router = QuantileBalancingRouter(self.num_experts) from resynthesis.anti_systems_bridge import TensorAntiThompsonRegistry self._anti_thompson_registry = TensorAntiThompsonRegistry( num_arms=self.num_experts, ) self.quantile_router.bind_anti_thompson_registry_boundary( self._anti_thompson_registry, ) self.quantile_route_scale = nn.Parameter(torch.zeros(())) self.kda_expert = CRConditionedKDAExpert( cfg.hidden_size, self.attention_heads, glyph_dim=cfg.glyph_input_dim, ) self.action_glyph_bridge = nn.Linear( cfg.action_input_dim, cfg.glyph_input_dim, bias=False, ) self.memory_bank = nn.Parameter(torch.empty(max(1, int(cfg.memory_slots)), cfg.hidden_size)) self.memory_query = nn.Linear(cfg.hidden_size, cfg.hidden_size, bias=False) self.memory_out = nn.Linear(cfg.hidden_size, cfg.hidden_size, bias=False) self.glyph_proj = nn.Linear(cfg.hidden_size, cfg.hidden_size, bias=False) self.glyph_gate = nn.Linear(cfg.hidden_size, cfg.hidden_size, bias=False) self.mhc_distinct_hypothesis_scale = nn.Parameter(torch.zeros(())) self.ffn_gate_up = nn.Parameter(torch.empty(self.num_ffn_experts, cfg.hidden_size, cfg.expert_hidden_size)) self.ffn_up = nn.Parameter( torch.empty( self.num_ffn_experts, cfg.hidden_size, cfg.expert_hidden_size, ) ) self.ffn_down = nn.Parameter(torch.empty(self.num_ffn_experts, cfg.expert_hidden_size, cfg.hidden_size)) self.ffn_latent_up = nn.Parameter( torch.empty(cfg.expert_hidden_size, cfg.hidden_size) ) # Both new paths begin as exact additive identities. Their learned # scalar gates can open only through the model's training loss. self.situ_glu_scale = nn.Parameter(torch.zeros(())) self.stable_latent_moe_scale = nn.Parameter(torch.zeros(())) self.residual_scale = nn.Parameter(torch.tensor(float(cfg.residual_init))) self.glyph_translate_proj = nn.Linear(cfg.hidden_size, cfg.glyph_input_dim, bias=False) self.glyph_translate_back = nn.Linear(cfg.glyph_input_dim, cfg.hidden_size, bias=False) self.translate_scale = nn.Parameter(torch.tensor(0.0)) self.audit_scale = nn.Parameter(torch.tensor(0.0)) self._inherited_dense_frozen_for_paged_training = False self.paged_expert_runtime = None self.last_paged_expert_packet = None # These tensors are consumed by the loss belonging to one exact # decode/training arm. They must not retain the completed arm's # autograd graph while the next CUDA wave is materialized. self.last_gate_logits: torch.Tensor | None = None self.last_gate_weights: torch.Tensor | None = None self.reset_parameters() @staticmethod def _valid_heads(width: int, requested: int) -> int: heads = max(1, int(requested)) while heads > 1 and width % heads != 0: heads -= 1 return max(1, heads) def reset_parameters(self) -> None: # A checkpoint-direct construction deliberately creates this module on # ``meta`` and immediately assigns every persistent tensor from an # authority-checked checkpoint. Initializing those tensors would write # tens of GiB only to overwrite them, and the anchor scalar read is not # defined for meta tensors. if self.expert_intent_glyphs.device.type == "meta": return nn.init.xavier_uniform_(self.router.weight) nn.init.xavier_uniform_(self.memory_query.weight) nn.init.xavier_uniform_(self.memory_out.weight) nn.init.xavier_uniform_(self.glyph_proj.weight) nn.init.xavier_uniform_(self.glyph_gate.weight) nn.init.normal_(self.memory_bank, mean=0.0, std=0.02) nn.init.xavier_uniform_(self.ffn_gate_up) nn.init.xavier_uniform_(self.ffn_up) nn.init.xavier_uniform_(self.ffn_down) nn.init.xavier_uniform_(self.ffn_latent_up) nn.init.zeros_(self.situ_glu_scale) nn.init.zeros_(self.stable_latent_moe_scale) nn.init.xavier_uniform_(self.glyph_translate_proj.weight) nn.init.xavier_uniform_(self.glyph_translate_back.weight) nn.init.xavier_uniform_(self.intent_query_proj.weight) nn.init.xavier_uniform_(self.action_glyph_bridge.weight) with torch.no_grad(): orthogonal = torch.randn(self.num_experts, self.cfg.glyph_input_dim) orthogonal = F.normalize(orthogonal, dim=-1) anchor = F.normalize(self.intent_plane_anchor_mean.float(), dim=-1) if anchor.any(): intent = F.normalize(0.7 * orthogonal + 0.3 * anchor.unsqueeze(0), dim=-1) else: intent = orthogonal self.expert_intent_glyphs.copy_(intent) def rebuild_nonpersistent_buffers(self) -> None: """Rebuild catalog-derived state after direct checkpoint assignment.""" from resynthesis.corpus_training import NONE_FUNCTIONAL_EXPERT_FAMILIES # The dataclass registry is runtime plumbing; its fail bank is the # router's persistent checkpoint buffer. Rebind after meta-device # strict assignment without resetting learned outcome history. self.quantile_router.bind_anti_thompson_registry_boundary( self._anti_thompson_registry, ) self.quantile_router.rebuild_nonpersistent_buffers() family_ids = tuple( family_id for family_id, _description, _claims in NONE_FUNCTIONAL_EXPERT_FAMILIES ) self.language_family_ability_incidence = ( _language_family_ability_incidence( family_ids=family_ids, capability_dim=self.expert_capability_proj.out_features, device=self.expert_capability_proj.weight.device, ) ) def intent_anchor_loss(self) -> torch.Tensor: anchor = self.intent_plane_anchor_mean intent = F.normalize(self.expert_intent_glyphs.float(), dim=-1) anchor_n = F.normalize(anchor.float(), dim=-1) loss = ( 1.0 - F.linear(intent, anchor_n.unsqueeze(0)).squeeze(-1) ).mean() anchor_active_t = anchor.ne(0).any().to(dtype=loss.dtype) return loss * anchor_active_t def expert_role_rows(self) -> torch.Tensor: return F.normalize(self.expert_role_tag.float(), dim=-1) def expert_identity_separation_loss(self) -> torch.Tensor: intent = F.normalize(self.expert_intent_glyphs.float(), dim=-1) sim = torch.matmul(intent, intent.t()) eye = torch.eye(sim.shape[0], dtype=sim.dtype, device=sim.device) return ((sim - eye) ** 2).mean() def attach_paged_expert_runtime( self, runtime: NoNEPagedExpertRuntime, ) -> None: """Attach a trained page router/executor without replacing seed experts.""" if runtime.hidden_size != self.cfg.hidden_size: raise ValueError("paged expert hidden geometry differs") if runtime.action_size != self.cfg.action_input_dim: raise ValueError("paged expert action geometry differs") if not torch.equal( runtime.router.layer_id_t.detach().cpu(), torch.tensor(self.layer_idx, dtype=torch.long), ): raise ValueError("paged expert layer identity differs") self.paged_expert_runtime = runtime def muon_head_geometry_boundary( self, ) -> tuple["MuonHeadGeometryPacket", ...]: """Declare exact Q/K/V and KDA output-head optimizer geometry. Parameter names are migration surfaces, not head-layout authority. This construction-time boundary binds the live projection objects to their model-owned head counts so per-head Muon can orthogonalize every head independently without host inference or a routing flag. """ from resynthesis.stacked_single_pass import MuonHeadGeometryPacket attention = self.attention_expert kda = self.kda_expert return ( MuonHeadGeometryPacket( cast(nn.Parameter, attention.q_proj.weight), attention.num_heads, ), MuonHeadGeometryPacket( cast(nn.Parameter, attention.k_proj.weight), attention.num_heads, ), MuonHeadGeometryPacket( cast(nn.Parameter, attention.v_proj.weight), attention.num_heads, ), MuonHeadGeometryPacket( cast(nn.Parameter, kda.q_proj.weight), kda.num_heads, ), MuonHeadGeometryPacket( cast(nn.Parameter, kda.k_proj.weight), kda.num_heads, ), MuonHeadGeometryPacket( cast(nn.Parameter, kda.v_proj.weight), kda.num_heads, ), ) @torch.no_grad() def project_trained_expert_weights_to_int4_qat_boundary( self, ) -> torch.Tensor: """Project only gradient-updated dense FFN experts after optimizer step.""" projected_count_t = self.ffn_gate_up.new_zeros((), dtype=torch.long) for parameter_t in ( self.ffn_gate_up, self.ffn_up, self.ffn_down, ): if parameter_t.grad is None: continue parameter_t.copy_( project_resynthesis_expert_weight_int4_qat_boundary( parameter_t ) ) projected_count_t.add_( torch.ones_like(projected_count_t) ) return projected_count_t def seal_inherited_dense_freeze_for_paged_training_boundary(self) -> None: """Record one verified, launch-lifetime inherited-expert freeze. Fast-release branch training freezes the inherited science stack for the lifetime of that loaded model. Re-walking three complete module parameter trees in every layer forward only rediscovers that immutable fact between CUDA waves. Verify the exact modules and dense FFN parameters once at the freeze boundary, then let the forward use the sealed result without changing routing or the page-owned gradients. """ inherited_parameters = ( *self.recurrent_expert.parameters(), *self.attention_expert.parameters(), *self.kda_expert.parameters(), self.ffn_gate_up, self.ffn_up, self.ffn_down, self.ffn_latent_up, self.situ_glu_scale, self.stable_latent_moe_scale, ) if any(parameter.requires_grad for parameter in inherited_parameters): raise RuntimeError( "cannot seal paged-training dense freeze while inherited " "science parameters remain trainable" ) self._inherited_dense_frozen_for_paged_training = True def _glyph_anchor(self, x: torch.Tensor) -> torch.Tensor: projected: torch.Tensor = self.glyph_proj(x) if self.mhc_heads <= 1: return projected batch, seq, width = projected.shape head_dim = width // self.mhc_heads heads = projected.reshape(batch, seq, self.mhc_heads, head_dim) consensus = heads.mean(dim=2, keepdim=True) compatibility = consensus.expand( -1, -1, self.mhc_heads, -1, ) distinct = compatibility + torch.tanh( self.mhc_distinct_hypothesis_scale ) * (heads - compatibility) return distinct.reshape(batch, seq, width) def _recurrent_trainable(self) -> bool: return any(param.requires_grad for param in self.recurrent_expert.parameters()) @staticmethod def _module_trainable(module: nn.Module) -> bool: return any(param.requires_grad for param in module.parameters()) def none_transfer_gate_weights(self, gate_weights: torch.Tensor) -> torch.Tensor: output_dtype = gate_weights.dtype # Routing probabilities are a control surface. Keep this tiny matrix # operation explicitly in float32: outer BF16 autocast otherwise sees # the parameter's pre-autocast dtype while lowering ``Tensor.to`` and # can emit a float/BF16 GEMM after compilation. with torch.autocast(device_type=gate_weights.device.type, enabled=False): active_gates = gate_weights.float() compatibility = torch.softmax( self.expert_compatibility.float(), dim=-1, ) affinity = torch.sigmoid( self.expert_transfer_affinity.float(), ).view(1, 1, -1) transferred = torch.matmul(active_gates, compatibility) * affinity transfer_scale = torch.sigmoid(self.expert_transfer_scale.float()) combined = active_gates + transfer_scale * transferred normalized = combined / combined.sum( dim=-1, keepdim=True, ).clamp_min(1.0e-9) return normalized.to(dtype=output_dtype) def forward( self, hidden: torch.Tensor, expert_visit: torch.Tensor, expert_selection_count: torch.Tensor, expert_bias: torch.Tensor, action_context: torch.Tensor, ) -> ScienceLayerResult: x = self.norm(hidden) # Trainability cannot change during one module call. Resolve it once # instead of walking the GRU/attention/KDA parameter trees repeatedly # between CUDA kernels. inherited_dense_frozen = ( self._inherited_dense_frozen_for_paged_training ) recurrent_trainable = ( False if inherited_dense_frozen else self._recurrent_trainable() ) attention_trainable = ( False if inherited_dense_frozen else self._module_trainable(self.attention_expert) ) kda_trainable = ( False if inherited_dense_frozen else self._module_trainable(self.kda_expert) ) ffn_trainable = ( False if inherited_dense_frozen else ( self.ffn_gate_up.requires_grad or self.ffn_up.requires_grad or self.ffn_down.requires_grad or self.ffn_latent_up.requires_grad or self.situ_glu_scale.requires_grad or self.stable_latent_moe_scale.requires_grad ) ) # When only paged runtimes remain trainable, dense routing must not # retain an activation tape on the live residual. Pages still receive # ``x`` (with inter-layer gradients); routers/experts use ``dense_x``. paged_sparse_training = ( self.training and self.paged_expert_runtime is not None and not attention_trainable and not recurrent_trainable and not ffn_trainable ) dense_x = x.detach() if paged_sparse_training else x gate_logits = self.router(dense_x) + self.expert_activation_prior.to( dtype=dense_x.dtype, device=dense_x.device, ).view(1, 1, -1) batch_size = hidden.shape[0] if expert_visit.shape != (batch_size, self.num_experts): raise ValueError("expert visit state geometry differs from the science layer") if expert_selection_count.shape != (batch_size, self.num_experts): raise ValueError("expert selection state geometry differs from the science layer") if expert_bias.shape != (batch_size, self.num_experts): raise ValueError("expert bias geometry differs from the science layer") rotation_pressure = F.softplus(self.expert_rotation_pressure).to( dtype=dense_x.dtype, device=dense_x.device, ) gate_logits = gate_logits - rotation_pressure * expert_visit.to( dtype=dense_x.dtype, device=dense_x.device, ).unsqueeze(1) gate_logits = gate_logits - rotation_pressure * expert_selection_count.to( dtype=dense_x.dtype, device=dense_x.device, ).unsqueeze(1) gate_logits = gate_logits + expert_bias.to( dtype=dense_x.dtype, device=dense_x.device, ).unsqueeze(1) input_glyph = F.normalize(self.intent_query_proj(dense_x), dim=-1) intent = F.normalize(self.expert_intent_glyphs.to(dtype=dense_x.dtype), dim=-1) language_match = F.linear(input_glyph, intent) gate_logits = gate_logits + torch.tanh(self.language_match_scale) * language_match input_role = F.normalize(self.role_query_proj(dense_x), dim=-1) role_tags = F.normalize(self.expert_role_tag.to(dtype=dense_x.dtype), dim=-1) role_match = F.linear(input_role, role_tags) gate_logits = gate_logits + torch.tanh(self.role_match_scale) * role_match spec_scores = torch.sigmoid(self.expert_specialization.to(dtype=dense_x.dtype)) gate_logits = gate_logits + spec_scores.view(1, 1, -1) * 0.1 * role_match capability_query = F.normalize( self.expert_capability_proj(input_role), dim=-1, ) expert_capabilities = F.normalize( self.expert_capability_proj(role_tags), dim=-1, ) capability_match = torch.matmul( capability_query, expert_capabilities.t(), ) gate_logits = gate_logits + torch.tanh( self.capability_match_scale ) * capability_match language_incidence = self.language_family_ability_incidence.to( dtype=dense_x.dtype, device=dense_x.device, ) language_ability_query = F.normalize( torch.matmul(capability_query, language_incidence), dim=-1, ) expert_language_abilities = F.normalize( torch.matmul(expert_capabilities, language_incidence), dim=-1, ) language_ability_match = torch.matmul( language_ability_query, expert_language_abilities.t(), ) gate_logits = gate_logits + torch.tanh( self.language_ability_match_scale ) * language_ability_match # Quantile Balancing uses previous-step beta and emits a sparse forward # frontier. The learned additive blend is zero for legacy migration but # remains differentiable, so continued task loss can open it. dense_gates = torch.softmax(gate_logits, dim=-1) from resynthesis.hard_knowledge_router_boundary import ( refresh_hard_knowledge_packet, ) refresh_hard_knowledge_packet(self.quantile_router) sparse_gates = self.quantile_router.route(gate_logits) route_scale = torch.tanh(self.quantile_route_scale) routed_gates = ( dense_gates + route_scale * (sparse_gates - dense_gates) ).clamp_min(torch.finfo(dense_gates.dtype).tiny) routed_gates = routed_gates / routed_gates.sum( dim=-1, keepdim=True, ) gate_weights = self.none_transfer_gate_weights(routed_gates) self.last_gate_logits = gate_logits self.last_gate_weights = gate_weights # Recurrent expert gru_input = dense_x.contiguous() recurrent_context = ( torch.no_grad() if self.training and not recurrent_trainable else contextlib.nullcontext() ) with recurrent_context: recurrent_output, _state = self.recurrent_expert(gru_input) if self.training and not recurrent_trainable: recurrent_output = recurrent_output.detach() # Attention expert — Q/K/V plus C context pivot and NoNE-owned R edges. attention_context = ( torch.no_grad() if self.training and not attention_trainable else contextlib.nullcontext() ) routed_intent = F.normalize( input_glyph + torch.matmul(gate_weights, intent), dim=-1, ) action_glyph = F.normalize( self.action_glyph_bridge(action_context.to(dtype=dense_x.dtype)), dim=-1, ).unsqueeze(1).expand(-1, dense_x.shape[1], -1) with attention_context: attention_output = self.attention_expert( dense_x, intent_glyph_context=input_glyph, action_glyph_context=action_glyph, relation_glyph_context=routed_intent, ) if self.training and not attention_trainable: attention_output = attention_output.detach() # C/R-conditioned KDA is part of the same attention path. Its own # zero-init blend gives exact additive compatibility. kda_context = ( torch.no_grad() if self.training and not kda_trainable else contextlib.nullcontext() ) with kda_context: kda_output = self.kda_expert( dense_x, intent_glyph_context=input_glyph, relation_glyph_context=routed_intent, ) if self.training and not kda_trainable: kda_output = kda_output.detach() attention_output = attention_output + kda_output # Memory expert memory_trainable = ( self.memory_bank.requires_grad or self.memory_query.weight.requires_grad or self.memory_out.weight.requires_grad ) memory_context = torch.no_grad() if self.training and not memory_trainable else contextlib.nullcontext() with memory_context: scale = float(max(1, dense_x.shape[-1])) ** 0.5 memory_scores = torch.matmul(self.memory_query(dense_x), self.memory_bank.t()) / scale memory_output = self.memory_out( torch.matmul( varlen_ring_softmax_boundary( usp_softmax_boundary(memory_scores, dim=-1), dim=-1, ), self.memory_bank, ) ) if self.training and not memory_trainable: memory_output = memory_output.detach() # Glyph anchor expert glyph_trainable = self.glyph_proj.weight.requires_grad or self.glyph_gate.weight.requires_grad glyph_context = torch.no_grad() if self.training and not glyph_trainable else contextlib.nullcontext() with glyph_context: glyph_output = dense_x + torch.sigmoid(self.glyph_gate(dense_x)) * self._glyph_anchor(dense_x) if self.training and not glyph_trainable: glyph_output = glyph_output.detach() # Mix structural experts mixed = ( gate_weights.narrow(-1, self.recurrent_expert_id, 1) * recurrent_output + gate_weights.narrow(-1, self.attention_expert_id, 1) * attention_output + gate_weights.narrow(-1, self.memory_expert_id, 1) * memory_output + gate_weights.narrow(-1, self.glyph_anchor_expert_id, 1) * glyph_output ) # FFN experts. Keep the complete model-owned bank active while # launching its independent projections as batched contractions. ffn_context = torch.no_grad() if self.training and not ffn_trainable else contextlib.nullcontext() with ffn_context: ffn_output = _batched_ffn_expert_mixture( dense_x, gate_weights.narrow( -1, self.structural_expert_count, self.num_ffn_experts, ), self.ffn_gate_up, self.ffn_up, self.ffn_down, self.situ_glu_scale, self.ffn_latent_up, self.stable_latent_moe_scale, ) if self.training and not ffn_trainable: ffn_output = ffn_output.detach() mixed = mixed + ffn_output # Out-of-core native pages are selected by their resident model router. # The storage boundary materializes exactly those IDs; dense seed # experts remain additive compatibility paths and are never replaced. paged_runtime = self.paged_expert_runtime if paged_runtime is not None: # Fast-release / paged-sparse training freezes dense experts and # layer routers. Keep inter-layer gradients for page weights, but # do not ask autograd to store the frozen routing/MILT/audit tape # beside those page residuals on one 96GB accelerator. structural_mixed = mixed.detach() if paged_sparse_training else mixed pathway_t = ( structural_mixed.mean(dim=1) if paged_sparse_training else mixed.mean(dim=1) ) paged_packet = paged_runtime( x, action_context.to(device=x.device, dtype=x.dtype), pathway_t, ) mixed = structural_mixed + paged_packet.output_t if paged_sparse_training: # The model consumes ``paged_packet.output_t`` above, so its # live autograd edge remains in ``mixed`` until backward. # Route evidence is read only after forward and must not keep # the completed wave's graph alive on the layer. Retaining # that graph made the next wave synchronously destroy its # autograd nodes when this attribute was replaced. self.last_paged_expert_packet = NoNEPageForwardPacket( output_t=paged_packet.output_t.detach(), page_ids_t=paged_packet.page_ids_t.detach(), route_probability_t=( paged_packet.route_probability_t.detach() ), route_entropy_t=paged_packet.route_entropy_t.detach(), generation_t=paged_packet.generation_t.detach(), ) else: self.last_paged_expert_packet = paged_packet else: self.last_paged_expert_packet = None if paged_sparse_training: with torch.no_grad(): glyph_translated = self.glyph_translate_back( F.normalize(self.glyph_translate_proj(mixed), dim=-1) ) audit_mixed = mixed + torch.tanh(self.translate_scale) * glyph_translated mixed_glyph = F.normalize(self.intent_query_proj(audit_mixed), dim=-1) audit_match = (gate_weights * F.linear(mixed_glyph, intent)).sum( dim=-1, keepdim=True, ) audit_factor = 1.0 + torch.tanh(self.audit_scale) * audit_match residual_scale = torch.tanh(self.residual_scale) # Page grads flow through ``mixed``; frozen control scales stay # constant; ``hidden`` keeps the inter-layer page residual chain. out = self.output_norm(hidden + residual_scale * audit_factor * mixed) else: # MILT cross-expert translation glyph_translated = self.glyph_translate_back( F.normalize(self.glyph_translate_proj(mixed), dim=-1) ) mixed = mixed + torch.tanh(self.translate_scale) * glyph_translated # Cosine-as-confidence audit mixed_glyph = F.normalize(self.intent_query_proj(mixed), dim=-1) audit_match = (gate_weights * F.linear(mixed_glyph, intent)).sum( dim=-1, keepdim=True, ) audit_factor = 1.0 + torch.tanh(self.audit_scale) * audit_match out = self.output_norm( hidden + torch.tanh(self.residual_scale) * audit_factor * mixed ) return ScienceLayerResult( hidden=out, expert_routes=gate_weights, expert_visit=gate_weights.mean(dim=1), ) class ResynthesisScienceLayerStack(nn.Module): """Recursive NoNE layer graph for appended Resynthesis science capacity. Owns the glyph_projection (glyph dim -> hidden) so its weights appear under ``science_stack.glyph_projection.*``. Bridges the 168-dim learned glyph bank into the 2048-dim hidden space. The live architecture is a Nest of Native Experts: routed expert pressure transfers through learned compatibility edges and each layer receives the recurrent hidden state produced by prior layers. """ _step_count: torch.Tensor _paged_sparse_delta_bank_workspace_t: torch.Tensor depth_index_t: torch.Tensor layer_index_t: torch.Tensor layer_selector_t: torch.Tensor def __init__(self, cfg: ResynthesisScienceLayerConfig) -> None: super().__init__() self.cfg = cfg self.num_layers = max(1, int(cfg.num_layers)) self.num_experts = max(ResynthesisScienceLayer.structural_expert_count + 1, int(cfg.num_experts)) self.recursive_steps = max(1, int(cfg.recursive_steps)) if int(cfg.recursive_steps) > 0 else 1 layer_ids_t = torch.arange(1, self.num_layers + 1, dtype=torch.float32) self.traversal_gate = nn.Parameter(1.0e-3 * torch.sin(layer_ids_t * 0.4142135624)) # Existing rows are exactly one, preserving the pre-growth forward. # Checkpoint migration appends exact-zero rows so added reasoning layers # begin as differentiable no-ops and acquire authority only by training. self.layer_execution_scale = nn.Parameter( torch.ones(self.num_layers, dtype=torch.float32) ) self.layer_rotation_pressure = nn.Parameter(torch.tensor(4.0)) layer_slots = torch.arange(1, self.num_layers + 1, dtype=torch.float32) layer_transfer = ( torch.sin(layer_slots.unsqueeze(1) * layer_slots.unsqueeze(0) * 0.1936491673) + torch.cos(layer_slots.unsqueeze(1) * 0.4142135624 + layer_slots.unsqueeze(0) * 0.2718281828) ) * 0.01 layer_transfer = layer_transfer + torch.eye(self.num_layers) * 0.02 self.layer_transfer_graph = nn.Parameter(layer_transfer) self.layer_transfer_scale = nn.Parameter(torch.tensor(0.10)) self.logit_residual_scale = nn.Parameter(torch.tensor(float(cfg.logit_residual_init))) self.glyph_input_dim = max(1, int(cfg.glyph_input_dim)) self.glyph_projection = nn.Linear(self.glyph_input_dim, int(cfg.hidden_size), bias=False) self.layer_identity_glyphs = nn.Parameter(torch.empty(self.num_layers, self.glyph_input_dim)) self.layer_identity_query_proj = nn.Linear(int(cfg.hidden_size), self.glyph_input_dim, bias=False) self.layer_identity_scale = nn.Parameter(torch.tensor(0.35)) self.long_context_anchor_query = nn.Linear(int(cfg.hidden_size), 1, bias=False) self.long_context_anchor_gain = nn.Parameter(torch.zeros(())) self.register_buffer( "layer_selector_t", torch.eye(self.num_layers, dtype=torch.float32), persistent=False, ) self.register_buffer( "layer_index_t", torch.arange(self.num_layers, dtype=torch.long), persistent=False, ) self.register_buffer( "depth_index_t", torch.arange( self.num_layers * self.recursive_steps, dtype=torch.long, ), persistent=False, ) self.register_buffer( "_paged_sparse_delta_bank_workspace_t", torch.empty(0), persistent=False, ) self.glyph_to_hidden_fn: Any = None self.kl_anchor_weight = float(cfg.kl_anchor_weight) self.kl_anchor_warmup = int(cfg.kl_anchor_warmup_steps) self.register_buffer("_step_count", torch.zeros((), dtype=torch.long), persistent=True) self.last_kl_anchor_loss: Any = None self.last_kl_anchor_loss_live: Any = None self.last_intent_anchor_loss_live: Any = None self.last_layer_identity_contrastive_loss_live: Any = None self.last_causal_algebra_loss_live: torch.Tensor | None = None self.last_causal_algebra_packet: CausalTheoryProofPacket | None = None self.last_capability_integration: CausalIntegrationOutput | None = None for layer_idx in range(self.num_layers): self.add_module(f"science_layer_{layer_idx}", ResynthesisScienceLayer(cfg, layer_idx)) causal_world_size = min( max(2, int(cfg.causal_world_size)), int(cfg.hidden_size), ) self.causal_algebra_world_graph = CausalAlgebraWorldGraph( CausalAlgebraConfig( hidden_size=int(cfg.hidden_size), action_size=int(cfg.action_input_dim), pathway_size=( self.num_layers * self.num_experts + self.num_layers ), world_size=causal_world_size, hypothesis_count=int(cfg.causal_hypothesis_count), primitive_count=int(cfg.causal_primitive_count), program_steps=int(cfg.causal_program_steps), domain_count=int(cfg.causal_domain_count), operator_rank=min( max(1, int(cfg.causal_operator_rank)), causal_world_size, ), ) ) # Capability integration: bridges the causal spine's proof packet to the # exploration/value/intent/calibration tensor modules. Consumes the # proof's disagreement + observation-error tensors and produces shaped # action logits, value estimates, intent composite, and calibrated # confidence — the four signals RBO and learn_loop consume. transfer_dim = ( int(cfg.hidden_size) if cfg.knowledge_transfer_dim is None else int(cfg.knowledge_transfer_dim) ) if transfer_dim < 1: raise ValueError( "knowledge-transfer dimension must be positive when materialized" ) self.capability_integration = CausalIntegrationTensor( hidden_size=int(cfg.hidden_size), transfer_dim=transfer_dim, hypothesis_count=int(cfg.causal_hypothesis_count), ) self.molecular_science: nn.Module | None = None if bool(cfg.enable_molecular_science): from resynthesis.molecular_geometry import ( MolecularGeometryConfig, MolecularScienceBank, ) self.molecular_science = MolecularScienceBank( MolecularGeometryConfig(hidden_size=int(cfg.hidden_size)) ) self.last_molecular_packet: Any = None self.delta_attn_res = DeltaBlockAttnRes( int(cfg.hidden_size), max_blocks=self.num_layers * max(1, self.recursive_steps), glyph_dim=int(cfg.glyph_input_dim), ) nn.init.xavier_uniform_(self.glyph_projection.weight) nn.init.zeros_(self.long_context_anchor_query.weight) self._reset_layer_identities() def _layer(self, layer_idx: int) -> ResynthesisScienceLayer: layer = getattr(self, f"science_layer_{layer_idx}") if not isinstance(layer, ResynthesisScienceLayer): raise RuntimeError("registered science layer has an invalid module type") return layer def rebuild_nonpersistent_buffers(self) -> None: """Rebuild every dense layer's derived, non-checkpoint state.""" device = self.layer_identity_glyphs.device self.capability_integration.rebuild_nonpersistent_buffers() self.layer_selector_t = torch.eye( self.num_layers, dtype=torch.float32, device=device, ) self.layer_index_t = torch.arange( self.num_layers, dtype=torch.long, device=device, ) self.depth_index_t = torch.arange( self.num_layers * self.recursive_steps, dtype=torch.long, device=device, ) self._paged_sparse_delta_bank_workspace_t = ( self.layer_identity_glyphs.new_empty(0) ) for layer_idx in range(self.num_layers): self._layer(layer_idx).rebuild_nonpersistent_buffers() def _paged_sparse_delta_bank_workspace_boundary( self, hidden: torch.Tensor, *, active_depth: int, ) -> _PagedSparseDeltaBankWorkspace: """Return contiguous, fully overwritten sparse-training bank views. Delta AttnRes is a frozen, no-grad control surface in this lane. Every active depth slice is copied before its prefix is read, so initializing four complete banks to zero only adds device writes. A flat high-water allocation also supports changing wave shapes without multiplying unrelated batch and sequence capacity maxima. """ batch_size, sequence_size, hidden_size = hidden.shape hidden_bank_elements = ( 3 * active_depth * batch_size * sequence_size * hidden_size ) relation_bank_elements = ( active_depth * batch_size * sequence_size * self.glyph_input_dim ) required_elements = hidden_bank_elements + relation_bank_elements workspace_t = self._paged_sparse_delta_bank_workspace_t if ( workspace_t.device != hidden.device or workspace_t.dtype != hidden.dtype or workspace_t.numel() < required_elements ): workspace_t = hidden.new_empty(required_elements) self._paged_sparse_delta_bank_workspace_t = workspace_t active_workspace_t = workspace_t.narrow(0, 0, required_elements) hidden_banks_t = active_workspace_t.narrow( 0, 0, hidden_bank_elements, ).view( 3, active_depth, batch_size, sequence_size, hidden_size, ) relation_bank_t = active_workspace_t.narrow( 0, hidden_bank_elements, relation_bank_elements, ).view( active_depth, batch_size, sequence_size, self.glyph_input_dim, ) return _PagedSparseDeltaBankWorkspace( delta_bank_t=hidden_banks_t.select(0, 0), relation_bank_t=relation_bank_t, projected_delta_bank_t=hidden_banks_t.select(0, 1), projected_relation_bank_t=hidden_banks_t.select(0, 2), ) def attach_paged_expert_runtime( self, layer_idx: int, runtime: NoNEPagedExpertRuntime, ) -> None: """Attach one layer-owned page runtime at an explicit load boundary.""" self._layer(layer_idx).attach_paged_expert_runtime(runtime) def _reset_layer_identities(self) -> None: nn.init.xavier_uniform_(self.layer_identity_query_proj.weight) with torch.no_grad(): layer_ids = torch.arange(1, self.num_layers + 1, dtype=torch.float32).unsqueeze(1) dims = torch.arange(1, self.glyph_input_dim + 1, dtype=torch.float32).unsqueeze(0) rows = torch.sin(layer_ids * dims * 0.017) + torch.cos(layer_ids * dims * 0.031) self.layer_identity_glyphs.copy_(F.normalize(rows, dim=-1)) def layer_identity_rows(self) -> torch.Tensor: return F.normalize(self.layer_identity_glyphs.float(), dim=-1) def layer_identity_match(self, hidden: torch.Tensor, layer_idx: int) -> torch.Tensor: query = F.normalize(self.layer_identity_query_proj(hidden), dim=-1) ident = F.normalize( self.layer_identity_glyphs[int(layer_idx)].to(dtype=query.dtype, device=query.device), dim=-1, ) return F.linear(query, ident.view(1, -1)) def layer_identity_logits(self, hidden: torch.Tensor) -> torch.Tensor: query = F.normalize(self.layer_identity_query_proj(hidden), dim=-1) rows = F.normalize(self.layer_identity_glyphs.to(dtype=query.dtype, device=query.device), dim=-1) return F.linear(query, rows) def layer_identity_separation_loss(self) -> torch.Tensor: rows = self.layer_identity_rows() sim = torch.matmul(rows, rows.t()) eye = torch.eye(sim.shape[0], dtype=sim.dtype, device=sim.device) return ((sim - eye) ** 2).mean() def task_intent_logits(self, hidden: torch.Tensor) -> torch.Tensor: """Read five nonexclusive task-intent axes from routed science state. The existing per-layer role heads were checkpointed but previously had no active consumer. They now form a shared model-owned classifier over inherent knowledge, reasoning, agentic action, agentic research, and validation. Labels are consumed only at the loss boundary. """ if hidden.ndim != 3 or hidden.shape[1] < 1: raise ValueError("task-intent hidden must be [batch, sequence, hidden]") pooled = online_softmax_last_token_pool( hidden, chunk_tokens=self.cfg.attention_tile_tokens, ) logits = hidden.new_zeros(hidden.shape[0], 5) for layer_idx in range(self.num_layers): layer = self._layer(layer_idx) role = F.normalize(layer.role_query_proj(pooled), dim=-1) logits = logits + layer.layer_role_head(role) return logits / self.num_layers def logit_residual_alpha(self) -> torch.Tensor: return torch.sigmoid(self.logit_residual_scale) def load_balance_loss(self) -> torch.Tensor: """Expert utilization loss from Quantile Balancing hard assignments. The prior Switch-style loss used ``(avg_gates > 0)`` on dense softmax gates, which is always true and yields a constant ``num_experts`` with ~0 router gradient. QB hard masks provide a real load signal; soft gate importance still carries the differentiable path. """ device = self.layer_execution_scale.device qb_terms = tuple( self._layer(layer_idx).quantile_router.utilization_balance_loss().to( device=device ) for layer_idx in range(self.num_layers) ) page_qb_terms = tuple( runtime.router.quantile_router.utilization_balance_loss().to( device=device ) for layer_idx in range(self.num_layers) if ( runtime := self._layer(layer_idx).paged_expert_runtime ) is not None ) return torch.stack(qb_terms + page_qb_terms).mean() def record_anti_thompson_from_outcome( self, expert_routes_t: torch.Tensor, batch_correctness_t: torch.Tensor, *, floor_t: float = 0.35, ) -> None: """Push anti-Thompson fail counts for routing arms on weak outcomes.""" if expert_routes_t.numel() == 0: return correctness = batch_correctness_t.reshape(-1).detach() failed_mask = correctness.lt(floor_t) success_mask = correctness.ge(floor_t) if expert_routes_t.ndim != 4: raise ValueError("anti-thompson expert route tensor rank differs") if expert_routes_t.shape[1] != correctness.shape[0]: raise ValueError( "anti-thompson expert route batch geometry differs" ) # The caller combines attempt and recursive-depth traversal into the # leading route axis. Reduce only traversal and token axes so each # outcome updates the arm that actually participated for that batch row. arm_ids = ( expert_routes_t.detach() .float() .mean(dim=(0, 2)) .argmax(dim=-1) .to(dtype=torch.long) ) for layer_idx in range(self.num_layers): layer = self._layer(layer_idx) registry = getattr( layer.quantile_router, "_anti_thompson_registry", None, ) if registry is not None: registry = ( layer.quantile_router .bind_anti_thompson_registry_boundary(registry) ) registry.record_outcome_masks( arm_ids, failed_mask, success_mask, ) runtime = layer.paged_expert_runtime if runtime is not None: page_registry = getattr( runtime.router.quantile_router, "_anti_thompson_registry", None, ) if page_registry is not None: page_registry = ( runtime.router.quantile_router .bind_anti_thompson_registry_boundary( page_registry ) ) page_registry.record_outcome_masks( arm_ids, failed_mask, success_mask, ) def begin_decode_arm_boundary(self) -> torch.Tensor: """Fence auxiliary-loss graphs to one decode or CUDA training wave.""" reference_t = self.layer_execution_scale cleared_t = reference_t.new_zeros((), dtype=torch.long) # A bulk page-training wave can leave tens of GiB reachable through # module diagnostics even after backward and deletion of its returned # result. Release only completed-arm references here, immediately # before a new arm is allowed to build a graph. Persistent routing, # working-memory, page-gradient, and optimizer tensors are untouched. self.last_kl_anchor_loss_live = None self.last_layer_identity_contrastive_loss_live = None self.last_intent_anchor_loss_live = None self.last_causal_algebra_loss_live = None self.last_molecular_packet = None for layer_idx in range(self.num_layers): layer = self._layer(layer_idx) layer.last_gate_logits = None layer.last_gate_weights = None layer.last_paged_expert_packet = None proof_t = ( layer.quantile_router.begin_route_arm_boundary() .to(device=cleared_t.device, dtype=torch.long) ) cleared_t = cleared_t + proof_t.reshape(()) return cleared_t def begin_quantile_balancing_step_boundary(self) -> torch.Tensor: """Open one model-wide QB transaction before gradient accumulation.""" proof_t = self.layer_execution_scale.new_zeros((), dtype=torch.long) for layer_idx in range(self.num_layers): layer = self._layer(layer_idx) proof_t = proof_t + ( layer.quantile_router.begin_expert_bias_step_boundary() .to(device=proof_t.device, dtype=torch.long) .reshape(()) ) runtime = layer.paged_expert_runtime if runtime is not None: proof_t = proof_t + ( runtime.router.quantile_router .begin_expert_bias_step_boundary() .to(device=proof_t.device, dtype=torch.long) .reshape(()) ) return proof_t def commit_quantile_balancing_step_boundary(self) -> torch.Tensor: """Commit every pooled QB histogram once after optimizer success.""" proof_t = self.layer_execution_scale.new_zeros((), dtype=torch.long) for layer_idx in range(self.num_layers): layer = self._layer(layer_idx) dense_bias_t = ( layer.quantile_router.commit_expert_bias_step_boundary() ) proof_t = proof_t + torch.isfinite(dense_bias_t).all().to( device=proof_t.device, dtype=torch.long, ) runtime = layer.paged_expert_runtime if runtime is not None: paged_bias_t = ( runtime.router.quantile_router .commit_expert_bias_step_boundary() ) proof_t = proof_t + torch.isfinite(paged_bias_t).all().to( device=proof_t.device, dtype=torch.long, ) return proof_t def project_trained_expert_weights_to_int4_qat_boundary( self, ) -> torch.Tensor: """Apply post-step QAT to trained FFN experts across every layer.""" proof_t = self.layer_execution_scale.new_zeros((), dtype=torch.long) for layer_idx in range(self.num_layers): proof_t = proof_t + ( self._layer(layer_idx) .project_trained_expert_weights_to_int4_qat_boundary() .to(device=proof_t.device, dtype=torch.long) .reshape(()) ) return proof_t def project_glyphs(self, patterns: torch.Tensor) -> torch.Tensor: if getattr(self, "glyph_to_hidden_fn", None) is not None: projected: torch.Tensor = self.glyph_to_hidden_fn(patterns) return projected projected = self.glyph_projection( patterns.to(self.glyph_projection.weight.dtype) ) return projected def initial_traversal_state(self, hidden: torch.Tensor) -> ScienceTraversalState: """Create caller-owned rotation memory on the active tensor device.""" return ScienceTraversalState( expert_visits=hidden.new_zeros( hidden.shape[0], self.num_layers, self.num_experts, ), expert_selections=hidden.new_zeros( hidden.shape[0], self.num_layers, self.num_experts, dtype=torch.long, ), layer_visits=hidden.new_zeros(hidden.shape[0], self.num_layers), traversal_index=hidden.new_zeros(hidden.shape[0], dtype=torch.long), ) def forward( self, hidden: torch.Tensor, traversal_state: ScienceTraversalState | None = None, *, action_context: torch.Tensor, expert_bias: torch.Tensor | None = None, layer_bias: torch.Tensor | None = None, molecular_input: MolecularInputPacket | None = None, causal_world_state: CausalWorldState | None = None, ) -> ScienceStackResult: y = hidden state = traversal_state or self.initial_traversal_state(hidden) batch_size = hidden.shape[0] if state.expert_visits.shape != ( batch_size, self.num_layers, self.num_experts, ): raise ValueError("science expert traversal memory geometry differs") if state.expert_selections.shape != ( batch_size, self.num_layers, self.num_experts, ): raise ValueError("science expert selection memory geometry differs") if state.layer_visits.shape != (batch_size, self.num_layers): raise ValueError("science layer traversal memory geometry differs") if state.traversal_index.shape != (batch_size,): raise ValueError("science traversal index geometry differs") if action_context.shape != ( hidden.shape[0], self.cfg.action_input_dim, ): raise ValueError( "science stack action context geometry differs from the trained policy" ) active_action_context = action_context.to( device=hidden.device, dtype=hidden.dtype, ) active_expert_bias = ( hidden.new_zeros(batch_size, self.num_experts) if expert_bias is None else expert_bias.to(device=hidden.device, dtype=hidden.dtype) ) active_layer_bias = ( hidden.new_zeros(batch_size, self.num_layers) if layer_bias is None else layer_bias.to(device=hidden.device, dtype=hidden.dtype) ) if active_expert_bias.shape == (self.num_experts,): active_expert_bias = active_expert_bias.unsqueeze(0).expand( batch_size, -1, ) if active_layer_bias.shape == (self.num_layers,): active_layer_bias = active_layer_bias.unsqueeze(0).expand( batch_size, -1, ) if active_expert_bias.shape != (batch_size, self.num_experts): raise ValueError("science stack expert bias geometry differs") if active_layer_bias.shape != (batch_size, self.num_layers): raise ValueError("science stack layer bias geometry differs") expert_visits = state.expert_visits.to(device=hidden.device, dtype=hidden.dtype) expert_selections = state.expert_selections.to( device=hidden.device, dtype=torch.long, ) layer_visits = state.layer_visits.to(device=hidden.device, dtype=hidden.dtype) if causal_world_state is not None: active_causal_state = causal_world_state.validated( self.causal_algebra_world_graph.cfg, batch_size, ) counterweight_t = ( active_causal_state.exploration_counterweight_t.to( device=hidden.device, dtype=hidden.dtype, ) .clamp(min=0.0, max=1.0) .unsqueeze(-1) ) causal_action_probability_t = ( active_causal_state.action_policy_t.to( device=hidden.device, dtype=hidden.dtype, ) ) # The learned causal policy, not a host flag or random sampler, # decides how far this phase moves from exploitation toward the # most informative credible action. active_action_context = torch.lerp( active_action_context, causal_action_probability_t, counterweight_t, ) expert_revisit_t = expert_visits.mean(dim=1) expert_revisit_t = expert_revisit_t / expert_revisit_t.mean( dim=-1, keepdim=True, ).clamp_min(torch.finfo(hidden.dtype).eps) layer_revisit_t = layer_visits / layer_visits.mean( dim=-1, keepdim=True, ).clamp_min(torch.finfo(hidden.dtype).eps) active_expert_bias = ( active_expert_bias - counterweight_t * expert_revisit_t ) active_layer_bias = ( active_layer_bias - counterweight_t * layer_revisit_t ) previous_layer_idx: int | None = None active_depth = self.num_layers * self.recursive_steps # Fast-release freezes inherited science weights and trains only paged # runtimes. Delta-AttnRes over the growing depth bank is then a frozen # control surface: keep its live forward values, but do not ask autograd # to retain an 11-layer attention tape beside the page residuals. first_layer = self._layer(0) # Page parameters are materialized from the immutable store only after # routing. They intentionally are not registered on the resident # runtime, so ``runtime.parameters()`` cannot prove whether the current # candidate window will produce page gradients. Match the layer-local # sparse-training contract instead: an attached paged runtime plus # frozen inherited experts means the trainable objects are the # dynamically materialized pages. paged_runtime_attached = any( self._layer(layer_idx).paged_expert_runtime is not None for layer_idx in range(self.num_layers) ) # Fast-release seals this exact launch-lifetime fact only after # verifying every inherited attention/recurrent/KDA/FFN parameter is # frozen. Reuse that proof here instead of walking complete parameter # trees in every science-stack forward between CUDA waves. dense_experts_frozen = ( first_layer._inherited_dense_frozen_for_paged_training ) paged_sparse_training = ( self.training and torch.is_grad_enabled() and paged_runtime_attached and dense_experts_frozen ) invariant_control_ctx = ( torch.no_grad() if paged_sparse_training else contextlib.nullcontext() ) with invariant_control_ctx: layer_identity_scale_t = torch.tanh(self.layer_identity_scale).to( device=hidden.device, dtype=hidden.dtype, ) layer_rotation_pressure_t = F.softplus( self.layer_rotation_pressure ).to( device=hidden.device, dtype=hidden.dtype, ) layer_transfer_scale_t = torch.tanh( self.layer_transfer_scale ).to( device=hidden.device, dtype=hidden.dtype, ) normalized_layer_identity_rows_t = F.normalize( self.layer_identity_glyphs.to( device=hidden.device, dtype=self.layer_identity_query_proj.weight.dtype, ), dim=-1, ) normalized_intent_glyph_rows_t = F.normalize( self.layer_identity_glyphs.to( device=hidden.device, dtype=hidden.dtype, ), dim=-1, ) expert_routes_t = hidden.new_zeros( active_depth, hidden.shape[0], hidden.shape[1], self.num_experts, ) layer_routes_t = hidden.new_zeros(active_depth, batch_size) layer_identity_losses_t = torch.zeros( active_depth, device=hidden.device, dtype=torch.float32, ) if paged_sparse_training: delta_workspace = ( self._paged_sparse_delta_bank_workspace_boundary( hidden, active_depth=active_depth, ) ) delta_bank_t = delta_workspace.delta_bank_t relation_bank_t = delta_workspace.relation_bank_t projected_delta_bank_t = ( delta_workspace.projected_delta_bank_t ) projected_relation_bank_t = ( delta_workspace.projected_relation_bank_t ) else: delta_bank_t = hidden.new_zeros( active_depth, hidden.shape[0], hidden.shape[1], hidden.shape[2], ) relation_bank_t = hidden.new_zeros( active_depth, hidden.shape[0], hidden.shape[1], self.glyph_input_dim, ) projected_delta_bank_t = hidden.new_zeros( active_depth, hidden.shape[0], hidden.shape[1], hidden.shape[2], ) projected_relation_bank_t = hidden.new_zeros( active_depth, hidden.shape[0], hidden.shape[1], hidden.shape[2], ) depth_index = 0 for traversal_step in range(self.recursive_steps): for slot_idx in range(self.num_layers): layer_idx = (slot_idx + traversal_step) % self.num_layers depth_position_t = self.depth_index_t.narrow( 0, depth_index, 1, ) layer_position_t = self.layer_index_t.narrow( 0, layer_idx, 1, ) if self.training and not paged_sparse_training: pooled_identity_hidden = y.mean(dim=1, keepdim=True) identity_query_t = F.normalize( self.layer_identity_query_proj( pooled_identity_hidden ), dim=-1, ) layer_logits = F.linear( identity_query_t, normalized_layer_identity_rows_t, ).reshape(batch_size, -1).float() target = layer_position_t.expand(batch_size) identity_loss_t = F.cross_entropy( layer_logits, target, ).reshape(1) layer_identity_losses_t.select(0, depth_index).copy_( identity_loss_t.detach().reshape(()) ) before_layer = y layer_module = self._layer(layer_idx) visit_t = expert_visits[:, layer_idx, :] selection_t = expert_selections[:, layer_idx, :] # The layer boundary already detaches inherited dense/router # computation and the page executor detaches its input. Keep # the light residual carrier live here so every independently # routed page remains connected to the final loss; detaching # it discarded all but the final layer's page gradients. layer_input = y layer_result = layer_module( layer_input, visit_t, selection_t, active_expert_bias, active_action_context, ) updated = layer_result.hidden gate_weights = layer_result.expert_routes if paged_sparse_training: expert_routes_t.select(0, depth_index).copy_( gate_weights.detach() ) else: expert_routes_t = expert_routes_t.index_copy( 0, depth_position_t, gate_weights.unsqueeze(0), ) gate_ctx = ( torch.no_grad() if paged_sparse_training else contextlib.nullcontext() ) with gate_ctx: layer_match = F.linear( F.normalize( self.layer_identity_query_proj(y), dim=-1, ), normalized_layer_identity_rows_t[layer_idx].view(1, -1), ) gate_logit = ( self.traversal_gate[layer_idx] + active_layer_bias[:, layer_idx] + layer_identity_scale_t * layer_match.mean(dim=1).squeeze(-1) - layer_rotation_pressure_t * layer_visits[:, layer_idx] ) if previous_layer_idx is not None: transfer_edge = self.layer_transfer_graph[ previous_layer_idx, layer_idx, ].to( dtype=gate_logit.dtype, device=gate_logit.device, ) gate_logit = ( gate_logit + layer_transfer_scale_t * transfer_edge ) execution_scale = self.layer_execution_scale[layer_idx].to( dtype=y.dtype, device=y.device, ) gate = ( torch.sigmoid(gate_logit).to(dtype=y.dtype, device=y.device) * execution_scale ).view(batch_size, 1, 1) if paged_sparse_training: gate = gate.detach() execution_scale = execution_scale.detach() # Page grads stay local to this layer's residual. The carrier # into the next layer is the detached input plus this layer's # page delta, so the 11-layer tape cannot accumulate. layer_delta_t = updated - layer_input next_y = layer_input + gate * layer_delta_t else: layer_delta_t = updated - y next_y = y + gate * layer_delta_t if self.training and torch.is_grad_enabled(): # Added reasoning layers begin behind an exact zero output # gate. Keep that migrated forward identity while letting # their physical parameters learn on the first update. next_y = next_y + (1.0 - gate).detach() * ( layer_delta_t - layer_delta_t.detach() ) y = next_y # Delta AttnRes remains in the forward. C is the active layer # identity; R is the per-example routed expert relation. Under # paged-sparse training the control surface is frozen, so its # depth-bank attention must not retain activations. attn_ctx = ( torch.no_grad() if paged_sparse_training else contextlib.nullcontext() ) with attn_ctx: intent_glyph = normalized_intent_glyph_rows_t[layer_idx] layer = self._layer(layer_idx) relation_glyph = F.normalize( torch.matmul( gate_weights.detach().mean(dim=1) if paged_sparse_training else gate_weights.mean(dim=1), layer.expert_intent_glyphs.to(dtype=y.dtype), ), dim=-1, ) relation_sequence_t = relation_glyph.unsqueeze(1).expand( -1, y.shape[1], -1, ) active_delta_t = ( y.detach() - before_layer.detach() if paged_sparse_training else y - before_layer ) projected_active_delta_t = self.delta_attn_res.key_proj( active_delta_t ) projected_relation_t = self.delta_attn_res.relation_k_proj( relation_sequence_t.to(dtype=y.dtype) ) if paged_sparse_training: delta_bank_t.select(0, depth_index).copy_(active_delta_t) relation_bank_t.select(0, depth_index).copy_( relation_sequence_t ) projected_delta_bank_t.select(0, depth_index).copy_( projected_active_delta_t ) projected_relation_bank_t.select(0, depth_index).copy_( projected_relation_t ) else: delta_bank_t = delta_bank_t.index_copy( 0, depth_position_t, active_delta_t.unsqueeze(0), ) relation_bank_t = relation_bank_t.index_copy( 0, depth_position_t, relation_sequence_t.unsqueeze(0), ) projected_delta_bank_t = projected_delta_bank_t.index_copy( 0, depth_position_t, projected_active_delta_t.unsqueeze(0), ) projected_relation_bank_t = ( projected_relation_bank_t.index_copy( 0, depth_position_t, projected_relation_t.unsqueeze(0), ) ) attn_delta_t = execution_scale * self.delta_attn_res( y.detach() if paged_sparse_training else y, delta_bank_t=delta_bank_t.narrow(0, 0, depth_index + 1), intent_glyph_t=intent_glyph, relation_glyph_t=relation_sequence_t, relation_bank_t=relation_bank_t.narrow( 0, 0, depth_index + 1, ), projected_delta_bank_t=projected_delta_bank_t.narrow( 0, 0, depth_index + 1, ), projected_relation_bank_t=projected_relation_bank_t.narrow( 0, 0, depth_index + 1, ), ) y = y + ( attn_delta_t.detach() if paged_sparse_training else attn_delta_t ) depth_index += 1 if paged_sparse_training: layer_routes_t.select(0, depth_index - 1).copy_( gate.reshape(batch_size) ) else: layer_routes_t = layer_routes_t.index_copy( 0, depth_position_t, gate.reshape(1, batch_size), ) selected_expert = gate_weights.mean(dim=1).argmax(dim=-1) selected_expert_delta_t = torch.zeros_like( selection_t, ).scatter( 1, selected_expert.unsqueeze(1), 1, ) expert_selections = expert_selections.index_add( 1, layer_position_t, selected_expert_delta_t.unsqueeze(1), ) layer_visits = layer_visits.index_add( 1, layer_position_t, gate.reshape(batch_size, 1), ) expert_visits = expert_visits.index_add( 1, layer_position_t, ( gate.reshape(batch_size, 1) * ( layer_result.expert_visit.detach() if paged_sparse_training else layer_result.expert_visit ) ).unsqueeze(1), ) previous_layer_idx = layer_idx # KL-anchor if self.training: with torch.no_grad(): self._step_count.add_(1) warmup_frac = ( self._step_count.to(device=hidden.device, dtype=hidden.dtype) / max(1, self.kl_anchor_warmup) ).clamp(max=1.0) effective_weight = self.kl_anchor_weight * warmup_frac if paged_sparse_training: with torch.no_grad(): anchor_signal = self.long_context_anchor_query(hidden).mean() context_multiplier = ( 1.0 + torch.tanh(self.long_context_anchor_gain) * torch.tanh(anchor_signal) ) effective_weight = effective_weight * context_multiplier.clamp_min(0.05) if self.kl_anchor_weight > 0 and self.training: cos_sim = F.cosine_similarity(y.flatten(), hidden.flatten(), dim=0) kl_loss = (1.0 - cos_sim) * effective_weight self.last_kl_anchor_loss = kl_loss.detach() else: self.last_kl_anchor_loss = None self.last_kl_anchor_loss_live = None self.last_layer_identity_contrastive_loss_live = None self.last_intent_anchor_loss_live = None else: anchor_signal = self.long_context_anchor_query(hidden).mean() context_multiplier = 1.0 + torch.tanh(self.long_context_anchor_gain) * torch.tanh(anchor_signal) effective_weight = effective_weight * context_multiplier.clamp_min(0.05) if self.kl_anchor_weight > 0 and self.training: cos_sim = F.cosine_similarity(y.flatten(), hidden.flatten(), dim=0) kl_loss = (1.0 - cos_sim) * effective_weight self.last_kl_anchor_loss = kl_loss.detach() self.last_kl_anchor_loss_live = kl_loss else: self.last_kl_anchor_loss = None self.last_kl_anchor_loss_live = None if self.training: separation_terms = tuple( 0.02 * self._layer(layer_idx).expert_identity_separation_loss() for layer_idx in range(self.num_layers) ) intent_terms = tuple( self._layer(layer_idx).intent_anchor_loss() for layer_idx in range(self.num_layers) ) layer_identity_loss = layer_identity_losses_t.mean() self.last_layer_identity_contrastive_loss_live = layer_identity_loss anchor_terms = ( *separation_terms, *intent_terms, 0.02 * self.layer_identity_separation_loss(), layer_identity_loss, ) self.last_intent_anchor_loss_live = torch.stack(anchor_terms).sum() else: self.last_layer_identity_contrastive_loss_live = None self.last_intent_anchor_loss_live = None output_hidden = y if self.molecular_science is not None: molecular_forward = cast(Any, self.molecular_science) if molecular_input is None: if paged_sparse_training: with torch.no_grad(): output_hidden = molecular_forward.forward_hidden(y.detach()) # Keep the page residual chain live; molecular control is frozen. output_hidden = y + (output_hidden - y).detach() else: output_hidden = molecular_forward.forward_hidden(y) self.last_molecular_packet = None else: if paged_sparse_training: with torch.no_grad(): output_hidden, molecular_packet = molecular_forward( y.detach(), molecular_input=molecular_input, ) # Keep the page residual chain live; molecular control is frozen. output_hidden = y + (output_hidden - y).detach() else: output_hidden, molecular_packet = molecular_forward( y, molecular_input=molecular_input, ) self.last_molecular_packet = molecular_packet else: self.last_molecular_packet = None causal_pathway_context_t = torch.cat( ( expert_visits.reshape(batch_size, -1), layer_visits, ), dim=-1, ) if paged_sparse_training: # Page-only v1 branches keep every causal parameter frozen, so these # detached inputs naturally build no graph and retain the prior # low-memory behavior. A v2 branch may explicitly reopen only the # compact working-memory/exploration heads. The identity term keeps # the dynamically materialized page residual live while the causal # result contributes gradients solely to those isolated heads. causal_result = self.causal_algebra_world_graph( output_hidden.detach(), action_context_t=active_action_context.detach(), pathway_context_t=causal_pathway_context_t.detach(), prior_state=( causal_world_state.detached() if causal_world_state is not None else None ), ) output_hidden = ( output_hidden + causal_result.hidden_t - output_hidden.detach() ) self.last_causal_algebra_loss_live = ( causal_result.auxiliary_loss_t if causal_result.auxiliary_loss_t.requires_grad else None ) self.last_causal_algebra_packet = causal_result.proof.detached() if self.last_causal_algebra_loss_live is not None: if self.last_intent_anchor_loss_live is None: self.last_intent_anchor_loss_live = ( self.last_causal_algebra_loss_live ) else: self.last_intent_anchor_loss_live = ( self.last_intent_anchor_loss_live + self.last_causal_algebra_loss_live ) else: causal_result = self.causal_algebra_world_graph( output_hidden, action_context_t=active_action_context, pathway_context_t=causal_pathway_context_t, prior_state=causal_world_state, ) output_hidden = causal_result.hidden_t self.last_causal_algebra_packet = causal_result.proof self.last_causal_algebra_loss_live = ( causal_result.auxiliary_loss_t if self.training and torch.is_grad_enabled() else None ) if self.last_causal_algebra_loss_live is not None: if self.last_intent_anchor_loss_live is None: self.last_intent_anchor_loss_live = ( self.last_causal_algebra_loss_live ) else: self.last_intent_anchor_loss_live = ( self.last_intent_anchor_loss_live + self.last_causal_algebra_loss_live ) # Bridge the causal spine's proof packet to the capability integration # layer. This consumes the proof's disagreement + observation-error # tensors and produces shaped action logits, value, intent, and # calibrated confidence — stored on self for RBO and learn_loop to read # (same pattern as last_causal_algebra_packet). if causal_result.proof.falsifying_experiment is not None: from resynthesis.additive_training_context_boundary import ( read_additive_training_context_boundary, ) training_ctx = read_additive_training_context_boundary(self) capability_integration = self.capability_integration( action_logits=active_action_context, causal_disagreement=( causal_result.proof.falsifying_experiment.disagreement_t ), observation_error=causal_result.proof.observation_error_t, predicted_outcomes=causal_result.proof.predicted_outcome_t, posterior=causal_result.proof.world_state.posterior_t, student_hidden=output_hidden, prior_additive_logits=( training_ctx.prior_additive_logits if training_ctx else None ), learned_teacher_logits=( training_ctx.learned_teacher_logits if training_ctx else None ), prior_additive_hidden=( training_ctx.prior_additive_hidden if training_ctx else None ), prompt_len=training_ctx.prompt_len if training_ctx else 0, contact_feature_stack=( training_ctx.contact_feature_stack if training_ctx else None ), page_count=training_ctx.page_count if training_ctx else 0, ) self.last_capability_integration = capability_integration causal_capability_loss = ( capability_integration.causal_auxiliary_loss ) if ( causal_capability_loss is not None and causal_capability_loss.requires_grad ): # This target-free loss belongs to the same executable causal # proof that produced ``causal_result``. Keep it on the live # causal loss surface so page-coupled training cannot discard # CCL/MHC gradients between the science stack and RBO loss # boundary. self.last_causal_algebra_loss_live = ( causal_capability_loss if self.last_causal_algebra_loss_live is None else self.last_causal_algebra_loss_live + causal_capability_loss ) # The transfer bank is part of the accepted additive graph. Its # zero-scale initialization is an exact identity, and subsequent # gradients may grow cross-family capability without routing # knowledge back through the frozen parent. if capability_integration.transferred_hidden is not None: output_hidden = capability_integration.transferred_hidden return ScienceStackResult( hidden=output_hidden, expert_routes=expert_routes_t, layer_routes=layer_routes_t, traversal_state=ScienceTraversalState( expert_visits=expert_visits, expert_selections=expert_selections, layer_visits=layer_visits, traversal_index=state.traversal_index.to(device=hidden.device) + active_depth, ), causal_proof=self.last_causal_algebra_packet, ) def build_resynthesis_science_stack( cfg: ResynthesisScienceLayerConfig | None = None, ) -> ResynthesisScienceLayerStack: return ResynthesisScienceLayerStack(cfg or ResynthesisScienceLayerConfig())