WYRM v27 FINAL notebook — training on Kaggle T4 x2

training/wyrm_notebook_v27_FINAL.py

@@ -0,0 +1,1582 @@
+#!/usr/bin/env python3
+"""
+WYRM 500M — Kaggle Training Notebook v28 (PRODUCTION)
+==========================================
+
+Self-contained. No patching. No hacks. This IS the training script.
+
+v28 PRODUCTION:
+  1. FULL float32 precision — no fp16, no autocast, no GradScaler
+  2. Batch size 4, grad_accum 4 = effective 16 (VRAM allows it: 5.4/15.9 GB)
+  3. Gradient checkpointing KEPT (saves VRAM without cutting precision)
+  4. MANDATORY torch 2.1.2+cu121 on Kaggle (P100 compat, auto-installs Python 3.11)
+  5. numpy<2 installed (torch 2.1.2 compiled against numpy 1.x, Kaggle has 2.4.4)
+  6. BPE tokenizer: gladius_bpe_16k.model (copied from dataset to working dir)
+  7. 15000 total steps, checkpoint every 250, session chaining ready
+  8. Stability > speed. Correct results > throughput.
+
+Architecture (WYRM 627M):
+  Full:  1024d / 24L / 32H / 32hd / 4096 FFN / seq=1024
+  Dummy: 64d / 2L / 4H / 16hd / 256 FFN / seq=128 (same code path)
+
+Usage:
+  Kaggle:  Just run the notebook (auto-detects Kaggle environment)
+  Local:   python wyrm_notebook.py --dummy          # 60-second test run
+  Local:   python wyrm_notebook.py --local-test     # with local paths
+  Local:   WYRM_DUMMY=1 python wyrm_notebook.py     # env var dummy mode
+
+Author: Ava Shakil | Artifact Virtual
+Date: 2026-04-08 (Day 58 — v27 FINAL)
+"""
+
+import os
+import sys
+import json
+import time
+import math
+import shutil
+import random
+import signal
+import logging
+import datetime
+import traceback
+import dataclasses
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Any
+from collections import defaultdict
+
+# ═══════════════════════════════════════════════════════════════
+# 0. GPU GATE — fail fast if no GPU
+# ═══════════════════════════════════════════════════════════════
+
+print("=" * 60)
+print("WYRM 500M — Kaggle Training v27 FINAL")
+print("=" * 60)
+
+# ═══════════════════════════════════════════════════════════════
+# MANDATORY: Ensure compatible PyTorch on Kaggle
+# ═══════════════════════════════════════════════════════════════
+# Kaggle ships PyTorch 2.10.0+cu128 on Python 3.12.
+# P100 (sm_60) support was dropped after PyTorch 2.1.x.
+# torch 2.1.2+cu121 only has cp38-cp311 wheels (no cp312).
+#
+# Strategy for P100:
+#   1. Install Python 3.11 via conda (already in Kaggle base image)
+#   2. Install torch 2.1.2+cu121 under Python 3.11
+#   3. Re-exec this script under Python 3.11
+#
+# Strategy for T4+: use stock PyTorch (2.10.0+cu128 supports sm_75+)
+IS_KAGGLE_ENV = os.path.exists('/kaggle')
+_P100_REEXEC = os.environ.get('_WYRM_P100_REEXEC', '')
+
+if IS_KAGGLE_ENV and not _P100_REEXEC:
+    import subprocess
+    try:
+        _gpu_check = subprocess.run(
+            ['nvidia-smi', '--query-gpu=name', '--format=csv,noheader'],
+            capture_output=True, text=True, timeout=10
+        )
+        _gpu_name = _gpu_check.stdout.strip()
+        print(f"GPU: {_gpu_name}")
+        if 'P100' in _gpu_name or 'p100' in _gpu_name.lower():
+            print("P100 detected — need torch 2.1.2+cu121 on Python 3.11")
+            print("Installing Python 3.11 + compatible PyTorch...")
+            
+            # Install Python 3.11 via apt (conda not available on Kaggle)
+            subprocess.run(['apt-get', 'update', '-qq'], capture_output=True, timeout=120)
+            _r = subprocess.run(
+                ['apt-get', 'install', '-y', '-qq', 'python3.11', 'python3.11-venv', 'python3.11-dev'],
+                capture_output=True, text=True, timeout=180
+            )
+            if _r.returncode != 0:
+                # python3.11 might not be in default repos, try deadsnakes PPA
+                print("  Default repos failed, trying deadsnakes PPA...")
+                subprocess.run(
+                    ['apt-get', 'install', '-y', '-qq', 'software-properties-common'],
+                    capture_output=True, timeout=60
+                )
+                subprocess.run(
+                    ['add-apt-repository', '-y', 'ppa:deadsnakes/ppa'],
+                    capture_output=True, timeout=60
+                )
+                subprocess.run(['apt-get', 'update', '-qq'], capture_output=True, timeout=120)
+                _r = subprocess.run(
+                    ['apt-get', 'install', '-y', '-qq', 'python3.11', 'python3.11-venv', 'python3.11-dev'],
+                    capture_output=True, text=True, timeout=180
+                )
+                if _r.returncode != 0:
+                    # Last resort: download wheel directly and use current Python
+                    print(f"  apt install failed: {_r.stderr[-300:]}")
+                    print("  Trying direct wheel download...")
+                    # Get Python 3.12 compatible torch built from source with sm_60
+                    _r = subprocess.run(
+                        [sys.executable, '-m', 'pip', 'install', '-q', '--force-reinstall',
+                         'torch==2.5.1', '--index-url', 'https://download.pytorch.org/whl/cu121'],
+                        capture_output=True, text=True, timeout=600
+                    )
+                    if _r.returncode == 0:
+                        # Test if 2.5.1+cu121 happens to have sm_60
+                        _t = subprocess.run(
+                            [sys.executable, '-c', 'import torch; t=torch.zeros(1,device="cuda"); print("OK")'],
+                            capture_output=True, text=True, timeout=30
+                        )
+                        if 'OK' in _t.stdout:
+                            print("  torch 2.5.1+cu121 works on P100! ✅")
+                            os.environ['_WYRM_P100_REEXEC'] = '1'
+                            os.execv(sys.executable, [sys.executable] + sys.argv)
+                    print("FATAL: Cannot get P100-compatible PyTorch.")
+                    sys.exit(1)
+            
+            _py311 = '/usr/bin/python3.11'
+            if not os.path.exists(_py311):
+                print(f"FATAL: {_py311} not found after install")
+                sys.exit(1)
+            
+            print(f"  Python 3.11 at: {_py311}")
+            
+            # Install pip for Python 3.11
+            subprocess.run(
+                [_py311, '-m', 'ensurepip', '--upgrade'],
+                capture_output=True, text=True, timeout=60
+            )
+            
+            # Install torch 2.1.2+cu121 (from PyTorch index) then sentencepiece (from PyPI)
+            print("  Installing torch 2.1.2+cu121...")
+            _r = subprocess.run(
+                [_py311, '-m', 'pip', 'install', '-q',
+                 'torch==2.1.2',
+                 '--index-url', 'https://download.pytorch.org/whl/cu121'],
+                capture_output=True, text=True, timeout=600
+            )
+            if _r.returncode != 0:
+                print(f"  torch install failed: {_r.stderr[-500:]}")
+                sys.exit(1)
+            print("  Installing sentencepiece + numpy<2 (torch 2.1.2 needs numpy 1.x)...")
+            _r = subprocess.run(
+                [_py311, '-m', 'pip', 'install', '-q', 'sentencepiece', 'numpy<2'],
+                capture_output=True, text=True, timeout=120
+            )
+            if _r.returncode != 0:
+                print(f"  sentencepiece/numpy install failed (non-fatal): {_r.stderr[-200:]}")
+            
+            # Verify GPU works
+            _r = subprocess.run(
+                [_py311, '-c', 'import torch; t=torch.zeros(1,device="cuda"); print(f"torch {torch.__version__} CUDA {torch.version.cuda} — GPU OK")'],
+                capture_output=True, text=True, timeout=30
+            )
+            print(f"  {_r.stdout.strip()}")
+            if 'OK' not in _r.stdout:
+                print(f"  GPU test failed: {_r.stderr[-300:]}")
+                sys.exit(1)
+            
+            # Re-exec under Python 3.11
+            print(f"  Switching to Python 3.11...")
+            os.environ['_WYRM_P100_REEXEC'] = '1'
+            os.execv(_py311, [_py311] + sys.argv)
+        else:
+            print(f"  GPU OK — {_gpu_name} is supported ✅")
+    except Exception as e:
+        print(f"  GPU setup failed: {e}")
+        import traceback; traceback.print_exc()
+        sys.exit(1)
+elif _P100_REEXEC:
+    print(f"Running under Python {sys.version.split()[0]} with P100-compatible PyTorch")
+
+import torch
+
+DUMMY_MODE_EARLY = '--dummy' in sys.argv or os.environ.get('WYRM_DUMMY') == '1'
+
+if not torch.cuda.is_available():
+    if DUMMY_MODE_EARLY:
+        print("WARNING: No CUDA GPU. Running dummy mode on CPU.")
+        DEVICE = torch.device('cpu')
+        GPU_NAME = 'CPU'
+        GPU_VRAM = 0.0
+        IS_P100 = False
+        IS_T4 = False
+        DTYPE = torch.float32
+    else:
+        print("FATAL: No CUDA GPU detected. Exiting.")
+        print("  torch.cuda.is_available():", torch.cuda.is_available())
+        print("  This notebook requires a T4 GPU runtime.")
+        print("  (Use --dummy for CPU testing)")
+        sys.exit(1)
+else:
+    DEVICE = torch.device('cuda')
+    GPU_NAME = torch.cuda.get_device_name(0)
+    GPU_VRAM = torch.cuda.get_device_properties(0).total_memory / (1024**3)
+    print(f"GPU: {GPU_NAME} ({GPU_VRAM:.1f} GB VRAM)")
+
+    # Test if GPU is actually usable
+    try:
+        _test = torch.zeros(1, device='cuda')
+        del _test
+    except RuntimeError as e:
+        print(f"FATAL: GPU detected but not usable: {e}")
+        print("This likely means the GPU's compute capability is not supported.")
+        print("Stop and restart for a compatible GPU.")
+        sys.exit(1)
+
+    IS_P100 = 'P100' in GPU_NAME or 'p100' in GPU_NAME.lower()
+    IS_T4 = 'T4' in GPU_NAME
+    print(f"GPU Type: {'T4' if IS_T4 else 'P100' if IS_P100 else 'Other'} — usable ✅")
+
+    # v26: FULL float32 — no autocast, no GradScaler, no corners
+    DTYPE = torch.float32
+    print(f"Precision: float32 (FULL — no mixed precision)")
+
+# ═══════════════════════════════════════════════════════════════
+# 1. ENVIRONMENT SETUP — copy from read-only input to working
+# ═══════════════════════════════════════════════════════════════
+
+IS_KAGGLE = os.path.exists('/kaggle')
+DUMMY_MODE = '--dummy' in sys.argv or os.environ.get('WYRM_DUMMY') == '1'
+LOCAL_TEST = '--local-test' in sys.argv or not IS_KAGGLE
+
+if DUMMY_MODE:
+    print("\n*** DUMMY MODE — tiny model, ~60 second training ***\n")
+
+if IS_KAGGLE:
+    INPUT_BASE = Path('/kaggle/input/wyrm-500m-base')
+    CKPT_INPUT = Path('/kaggle/input/wyrm-500m-ckpt')
+    WORKING = Path('/kaggle/working')
+else:
+    # Local testing paths
+    _ws = Path(os.environ.get('WYRM_BASE', '/home/adam/workspace'))
+    INPUT_BASE = _ws / 'gladius_v2' / 'kaggle_upload'
+    CKPT_INPUT = _ws / 'gladius_v2' / '_ckpt_sim'
+    WORKING = Path(os.environ.get('WYRM_OUTPUT', '/tmp/wyrm_kaggle_test'))
+
+OUTPUT = WORKING / 'output'
+RUNS = OUTPUT / 'runs' / 'wyrm-500m'
+CKPT_OUT = OUTPUT / 'checkpoint'
+TELEM = OUTPUT / 'telemetry'
+
+for d in [OUTPUT, RUNS, CKPT_OUT, TELEM]:
+    d.mkdir(parents=True, exist_ok=True)
+
+# ── Copy kernel + staging from input to working (READ-ONLY FIX) ──
+KERNEL_DST = WORKING / 'kernel'
+STAGING_DST = WORKING / 'staging'
+
+def copy_tree_safe(src: Path, dst: Path, label: str):
+    """Copy directory tree, overwriting if exists."""
+    if not src.exists():
+        print(f"  WARNING: {label} source not found: {src}")
+        return False
+    if dst.exists():
+        shutil.rmtree(dst)
+    shutil.copytree(src, dst)
+    n_files = sum(1 for _ in dst.rglob('*') if _.is_file())
+    print(f"  Copied {label}: {src} -> {dst} ({n_files} files)")
+    return True
+
+print("\nCopying code to writable location...")
+
+# Debug: list what's at input path
+if IS_KAGGLE:
+    import subprocess
+    print(f"  Input base: {INPUT_BASE}")
+    print(f"  /kaggle/input/ contents:")
+    kaggle_input = Path('/kaggle/input')
+    if kaggle_input.exists():
+        for p in sorted(kaggle_input.iterdir()):
+            print(f"    {p.name}: {'DIR' if p.is_dir() else 'FILE'}")
+            if p.is_dir():
+                for pp in sorted(p.iterdir()):
+                    print(f"      {pp.name}: {'DIR' if pp.is_dir() else 'FILE'}")
+                    if pp.is_dir():
+                        for ppp in sorted(pp.iterdir())[:5]:
+                            print(f"        {ppp.name}: {'DIR' if ppp.is_dir() else 'FILE'}")
+    else:
+        print(f"    /kaggle/input does not exist!")
+    
+    if INPUT_BASE.exists():
+        print(f"  Dataset contents:")
+        for p in sorted(INPUT_BASE.rglob('*')):
+            if p.is_file():
+                rel = p.relative_to(INPUT_BASE)
+                print(f"    {rel}: {p.stat().st_size}B")
+    else:
+        print(f"  WARNING: {INPUT_BASE} does not exist!")
+        # Search for kernel package anywhere under /kaggle/input
+        for candidate in Path('/kaggle/input').rglob('kernel/__init__.py'):
+            found_base = candidate.parent.parent
+            print(f"  FOUND kernel at: {found_base}")
+            INPUT_BASE = found_base
+            break
+
+copy_tree_safe(INPUT_BASE / 'kernel', KERNEL_DST, 'kernel')
+copy_tree_safe(INPUT_BASE / 'staging', STAGING_DST, 'staging')
+
+# ── Copy tokenizer if present ──
+_tok_src = INPUT_BASE / 'gladius_bpe_16k.model'
+_tok_dst = WORKING / 'gladius_bpe_16k.model'
+if _tok_src.exists():
+    shutil.copy2(_tok_src, _tok_dst)
+    print(f"  Tokenizer copied: {_tok_dst} ({_tok_src.stat().st_size}B)")
+else:
+    # Search recursively under INPUT_BASE
+    for _tk in INPUT_BASE.rglob('gladius_bpe_16k.model'):
+        shutil.copy2(_tk, _tok_dst)
+        print(f"  Tokenizer found at {_tk}, copied to {_tok_dst}")
+        break
+    else:
+        print("  WARNING: gladius_bpe_16k.model not found in dataset — BPE will use procedural only")
+
+# ── Setup import paths (writable copies only) ──
+sys.path.insert(0, str(WORKING))
+sys.path.insert(0, str(STAGING_DST))
+
+# ── Verify imports work ──
+try:
+    from kernel.config import KernelConfig
+    from kernel.kernel import GladiusKernel
+    from kernel.gaussian_head import (
+        GaussianConfig, GaussianSpecialist, GaussianVQVAE,
+        ProceduralGaussianDataset, render_gaussians_2d
+    )
+    print("  Kernel imports: OK")
+except ImportError as e:
+    print(f"FATAL: Kernel import failed: {e}")
+    traceback.print_exc()
+    sys.exit(1)
+
+# ══════════════════════��════════════════════════════════════════
+# 2. CONFIGURATION
+# ═══════════════════════════════════════════════════════════════
+
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+@dataclasses.dataclass
+class WyrmKaggleConfig:
+    """Kaggle-optimized WYRM training configuration."""
+
+    # ── Architecture ──
+    hidden_dim: int = 1024
+    num_layers: int = 24
+    num_heads: int = 32
+    head_dim: int = 32
+    ffn_dim: int = 4096
+    max_seq_len: int = 1024
+    vocab_size: int = 32000  # BPE
+
+    # ── Memory / specialist ──
+    hot_memory_slots: int = 1024
+    warm_rank: int = 64
+    time_dim: int = 128
+    time_num_frequencies: int = 16
+    time_max_events: int = 64
+    cognition_state_dim: int = 256
+    cognition_modes: int = 4
+    cognition_prompt_types: int = 5
+    register_dim: int = 4
+    intent_dim: int = 4
+    max_tools: int = 64
+    num_specialists: int = 5  # reasoning, math, code, general, gaussian
+    router_top_k: int = 2
+    attention_sparse_budget: int = 128
+
+    # ── Training ──
+    batch_size: int = 4           # v27: 4 batch (5.4/15.9 GB VRAM = headroom)
+    grad_accumulation: int = 4    # v27: 4 * 4 = 16 effective batch
+    total_steps: int = 15000  # Full training run (chain across sessions)
+    warmup_steps: int = 200
+    max_grad_norm: float = 0.5
+    weight_decay: float = 0.01
+    seed: int = 42
+
+    # ── Learning rates ──
+    lr_backbone: float = 3e-5
+    lr_specialists: float = 3e-4
+    lr_router: float = 1e-3
+    lr_tools: float = 5e-4
+    lr_depth: float = 3e-4
+    lr_embeddings: float = 3e-4
+
+    # ── Loss weights ──
+    balance_loss_weight: float = 0.05
+    cognition_loss_weight: float = 0.1
+    aux_loss_weight: float = 0.3
+
+    # ── Curriculum phases ──
+    phase1_end: int = 5000    # Foundation
+    phase2_end: int = 10000   # Reasoning
+    phase3_end: int = 13000   # Depth
+    # phase4: Omega (remainder)
+
+    # ── Depth ratios: [D1, D2, D3, D4, D5] ──
+    foundation_depth: tuple = (0.40, 0.40, 0.20, 0.00, 0.00)
+    reasoning_depth:  tuple = (0.15, 0.15, 0.25, 0.25, 0.20)
+    depth_depth:      tuple = (0.05, 0.05, 0.20, 0.20, 0.50)
+    omega_depth:      tuple = (0.20, 0.20, 0.20, 0.20, 0.20)
+
+    # ── Domain ratios ──
+    bonus_ratio: float = 0.15
+    language_ratio: float = 0.40
+    science_ratio: float = 0.05
+    gaussian_ratio: float = 0.02
+
+    # ── Checkpointing ──
+    checkpoint_every: int = 250   # More frequent saves for 12h session chaining
+    log_every: int = 10  # Every 10 steps
+    eval_every: int = 250
+
+    # ── NaN handling ──
+    max_nan_retries: int = 5
+    max_total_nans: int = 50
+
+    # ── VRAM safety ──
+    vram_warning_gb: float = 13.0
+    min_disk_gb: float = 3.0
+
+    def get_phase(self, step: int) -> str:
+        if step < self.phase1_end: return 'foundation'
+        if step < self.phase2_end: return 'reasoning'
+        if step < self.phase3_end: return 'depth'
+        return 'omega'
+
+    def get_depth_ratios(self, step: int) -> tuple:
+        phase = self.get_phase(step)
+        return getattr(self, f'{phase}_depth')
+
+    def get_cognitive_ratio(self, step: int) -> float:
+        return max(0.0, 1.0 - self.bonus_ratio - self.language_ratio
+                   - self.science_ratio - self.gaussian_ratio)
+
+
+def make_dummy_config() -> WyrmKaggleConfig:
+    """Tiny config for testing — same code path, finishes in ~60s."""
+    return WyrmKaggleConfig(
+        hidden_dim=64,
+        num_layers=2,
+        num_heads=4,      # min 4 for Synthase depth KV heads
+        head_dim=16,       # 64 / 4 = 16
+        ffn_dim=256,
+        max_seq_len=128,
+        vocab_size=1000,
+        hot_memory_slots=32,
+        warm_rank=4,
+        time_dim=32,
+        time_num_frequencies=8,
+        time_max_events=16,
+        cognition_state_dim=32,
+        max_tools=8,
+        num_specialists=5,
+        router_top_k=2,
+        attention_sparse_budget=32,
+        batch_size=2,
+        grad_accumulation=2,
+        total_steps=50,
+        warmup_steps=5,
+        checkpoint_every=25,
+        log_every=1,
+        eval_every=25,
+        phase1_end=15,
+        phase2_end=30,
+        phase3_end=40,
+    )
+
+
+# ═══════════════════════════════════════════════════════════════
+# 3. TOKENIZERS (inline — no external dependency)
+# ═══════════════════════════════════════════════════════════════
+
+class MathTokenizer:
+    """Structural tokenizer for mathematical expressions. 128 tokens."""
+    VOCAB_SIZE = 128
+    PAD_ID = 0; BOS_ID = 1; EOS_ID = 2
+
+    def __init__(self):
+        self.token_to_id = {}; self.id_to_token = {}
+        idx = 0
+        for name in ['[PAD_MATH]', '[BOS_MATH]', '[EOS_MATH]']:
+            self.token_to_id[name] = idx; self.id_to_token[idx] = name; idx += 1
+        for i in range(10):
+            s = str(i); self.token_to_id[s] = idx; self.id_to_token[idx] = s; idx += 1
+        for op in ['+','-','*','/','=','^','!','%','<','>','.',':','|']:
+            self.token_to_id[op] = idx; self.id_to_token[idx] = op; idx += 1
+        for i in range(26):
+            ch = chr(ord('a')+i); self.token_to_id[ch] = idx; self.id_to_token[idx] = ch; idx += 1
+        for s in ['(',')',',','_',' ','\n','STEP','OUT','FILL','NEXT','GIVEN','QED']:
+            self.token_to_id[s] = idx; self.id_to_token[idx] = s; idx += 1
+
+    @property
+    def vocab_size(self): return self.VOCAB_SIZE
+
+    def encode(self, text: str, add_special: bool = True) -> List[int]:
+        tokens = [self.BOS_ID] if add_special else []
+        for ch in text.strip():
+            if ch in self.token_to_id: tokens.append(self.token_to_id[ch])
+        if add_special: tokens.append(self.EOS_ID)
+        return tokens
+
+    def decode(self, ids: List[int]) -> str:
+        return ''.join(self.id_to_token.get(i, '?') for i in ids if i > 2)
+
+
+class ByteTokenizer:
+    """Byte-level tokenizer: 256 bytes + 3 specials = 259."""
+    PAD_ID = 256; BOS_ID = 257; EOS_ID = 258; VOCAB_SIZE = 259
+
+    def encode(self, data: bytes) -> List[int]:
+        return [self.BOS_ID] + [int(b) for b in data] + [self.EOS_ID]
+
+    def decode(self, ids: List[int]) -> bytes:
+        return bytes(i for i in ids if 0 <= i <= 255)
+
+
+# ═══════════════════════════════════════════════════════════════
+# 4. MULTI-TOKENIZER EMBEDDING
+# ═══════════════════════════════════════════════════════════════
+
+class MultiTokenizerEmbedding(nn.Module):
+    """Three embedding tables: BPE (32K), Math (128), Byte (259)."""
+
+    def __init__(self, hidden_dim: int, bpe_vocab_size: int = 32000):
+        super().__init__()
+        self.hidden_dim = hidden_dim
+        self.bpe_vocab_size = bpe_vocab_size
+        self.math_vocab_size = 128
+        self.byte_vocab_size = 259
+
+        self.bpe_embed = nn.Embedding(bpe_vocab_size, hidden_dim, padding_idx=0)
+        self.math_embed = nn.Embedding(128, hidden_dim, padding_idx=0)
+        self.byte_embed = nn.Embedding(259, hidden_dim, padding_idx=256)
+
+        self.bpe_head = nn.Linear(hidden_dim, bpe_vocab_size, bias=False)
+        self.math_head = nn.Linear(hidden_dim, 128, bias=False)
+        self.byte_head = nn.Linear(hidden_dim, 259, bias=False)
+
+        self.bpe_head.weight = self.bpe_embed.weight  # Weight tying
+        self.scale = math.sqrt(hidden_dim)
+
+        self._init_weights()
+
+    def _init_weights(self):
+        for e in [self.bpe_embed, self.math_embed, self.byte_embed]:
+            nn.init.normal_(e.weight, std=0.02)
+        with torch.no_grad():
+            self.bpe_embed.weight[0].zero_()
+            self.math_embed.weight[0].zero_()
+            self.byte_embed.weight[256].zero_()
+        nn.init.xavier_uniform_(self.math_head.weight)
+        nn.init.xavier_uniform_(self.byte_head.weight)
+
+    def embed(self, ids: torch.Tensor, domain: str) -> torch.Tensor:
+        if domain == 'bpe': return self.bpe_embed(ids) * self.scale
+        elif domain == 'math': return self.math_embed(ids) * self.scale
+        elif domain == 'byte': return self.byte_embed(ids) * self.scale
+        raise ValueError(f"Unknown domain: {domain}")
+
+    def project(self, hidden: torch.Tensor, domain: str) -> torch.Tensor:
+        if domain == 'bpe': return self.bpe_head(hidden)
+        elif domain == 'math': return self.math_head(hidden)
+        elif domain == 'byte': return self.byte_head(hidden)
+        raise ValueError(f"Unknown domain: {domain}")
+
+    def get_vocab_size(self, domain: str) -> int:
+        return {'bpe': self.bpe_vocab_size, 'math': 128, 'byte': 259}[domain]
+
+
+# ═══════════════════════════════════════════════════════════════
+# 5. DOMAIN MEMBRANES (Plug)
+# ═══════════════════════════════════════════════════════════════
+
+class DomainMembrane(nn.Module):
+    def __init__(self, hidden_dim):
+        super().__init__()
+        self.proj = nn.Linear(hidden_dim, hidden_dim)
+        self.norm = nn.LayerNorm(hidden_dim)
+        nn.init.eye_(self.proj.weight)
+        with torch.no_grad(): self.proj.weight.add_(torch.randn_like(self.proj.weight)*0.01)
+        nn.init.zeros_(self.proj.bias)
+
+    def forward(self, x): return self.norm(self.proj(x))
+
+
+class PlugMembranes(nn.Module):
+    def __init__(self, hidden_dim):
+        super().__init__()
+        self.bpe_membrane = DomainMembrane(hidden_dim)
+        self.math_membrane = DomainMembrane(hidden_dim)
+        self.byte_membrane = DomainMembrane(hidden_dim)
+
+    def forward(self, x, domain):
+        return getattr(self, f'{domain}_membrane')(x)
+
+
+# ═══════════════════════════════════════════════════════════════
+# 6. AUXILIARY PREDICTION HEAD (L7)
+# ═══════════════════════════════════════════════════════════════
+
+class AuxiliaryPredictionHead(nn.Module):
+    def __init__(self, hidden_dim: int, vocab_size: int):
+        super().__init__()
+        self.norm = nn.LayerNorm(hidden_dim)
+        self.proj = nn.Linear(hidden_dim, vocab_size, bias=False)
+        nn.init.xavier_uniform_(self.proj.weight)
+
+    def forward(self, hidden): return self.proj(self.norm(hidden))
+
+
+# ═══════════════════════════════════════════════════════════════
+# 7. DATA — Procedural curriculum (no external files for dummy)
+# ═══════════════════════════════════════════════════════════════
+
+def load_bpe_tokenizer(path: str):
+    """Load SentencePiece BPE tokenizer."""
+    try:
+        import sentencepiece as spm
+        sp = spm.SentencePieceProcessor()
+        sp.load(path)
+        print(f"  BPE tokenizer loaded: {path} (vocab={sp.get_piece_size()})")
+        return sp
+    except Exception as e:
+        print(f"  WARNING: BPE tokenizer failed: {e}")
+        return None
+
+
+class ProceduralTextDataset:
+    """Generates synthetic text data for training. No file dependencies."""
+
+    TEMPLATES = [
+        "The {adj} {noun} {verb} the {adj2} {noun2}.",
+        "In the {noun}, a {adj} {noun2} was {verb_ing}.",
+        "{noun} and {noun2} are fundamentally {adj}.",
+        "The concept of {noun} relates to {noun2} through {adj} mechanisms.",
+        "When {noun} encounters {noun2}, the result is {adj}.",
+    ]
+    ADJS = ['quick','large','small','bright','dark','complex','simple','ancient','modern','curious']
+    NOUNS = ['system','pattern','network','structure','function','equation','theory','model','field','space']
+    VERBS = ['transforms','analyzes','computes','observes','generates','processes','evaluates','maps']
+    VERB_ING = ['computing','analyzing','transforming','processing','generating','mapping','observing']
+
+    def __init__(self, tokenizer, seq_len: int):
+        self.tok = tokenizer
+        self.seq_len = seq_len
+
+    def sample(self) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+        """Generate a random text sample, tokenize, return (input, target, mask)."""
+        # Generate enough text to fill seq_len
+        text_parts = []
+        while True:
+            tmpl = random.choice(self.TEMPLATES)
+            text = tmpl.format(
+                adj=random.choice(self.ADJS), adj2=random.choice(self.ADJS),
+                noun=random.choice(self.NOUNS), noun2=random.choice(self.NOUNS),
+                verb=random.choice(self.VERBS), verb_ing=random.choice(self.VERB_ING),
+            )
+            text_parts.append(text)
+            full_text = ' '.join(text_parts)
+            if self.tok is not None:
+                ids = self.tok.encode(full_text)
+            else:
+                ids = [ord(c) % 1000 for c in full_text]  # Fallback for dummy
+            if len(ids) >= self.seq_len + 1:
+                break
+
+        ids = ids[:self.seq_len + 1]
+        x = torch.tensor(ids[:-1], dtype=torch.long)
+        y = torch.tensor(ids[1:], dtype=torch.long)
+        m = torch.ones(len(x), dtype=torch.float32)
+
+        # Pad if needed
+        if len(x) < self.seq_len:
+            pad = self.seq_len - len(x)
+            x = F.pad(x, (0, pad))
+            y = F.pad(y, (0, pad))
+            m = F.pad(m, (0, pad))
+
+        m[y == 0] = 0.0
+        return x[:self.seq_len], y[:self.seq_len], m[:self.seq_len]
+
+
+class ProceduralMathDataset:
+    """Generates synthetic math problems."""
+
+    def __init__(self, seq_len: int):
+        self.tok = MathTokenizer()
+        self.seq_len = seq_len
+
+    def sample(self) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+        # Generate random arithmetic
+        a, b = random.randint(1, 100), random.randint(1, 100)
+        op = random.choice(['+', '-', '*'])
+        result = eval(f"{a}{op}{b}")
+        expr = f"D2|GIVEN:{a}{op}{b}|OUT:{result}"
+        ids = self.tok.encode(expr)
+        if len(ids) > self.seq_len + 1:
+            ids = ids[:self.seq_len + 1]
+        x = ids[:-1]; y = ids[1:]
+        pad = self.seq_len - len(x)
+        mask = [1.0]*len(x) + [0.0]*pad
+        x = x + [0]*pad; y = y + [0]*pad
+        for i in range(len(mask)):
+            if y[i] == 0: mask[i] = 0.0
+        return (torch.tensor(x[:self.seq_len], dtype=torch.long),
+                torch.tensor(y[:self.seq_len], dtype=torch.long),
+                torch.tensor(mask[:self.seq_len], dtype=torch.float32))
+
+
+class UnifiedSampler:
+    """Samples from curriculum based on phase ratios."""
+
+    def __init__(self, cfg: WyrmKaggleConfig, text_ds, math_ds, gaussian_ds=None):
+        self.cfg = cfg
+        self.text_ds = text_ds
+        self.math_ds = math_ds
+        self.gaussian_ds = gaussian_ds
+
+    def _pick_difficulty(self, step: int) -> int:
+        ratios = self.cfg.get_depth_ratios(step)
+        r = random.random()
+        cum = 0.0
+        for i, ratio in enumerate(ratios):
+            cum += ratio
+            if r < cum: return i + 1
+        return 5
+
+    def sample_batch(self, batch_size: int, step: int) -> Dict[str, Any]:
+        """Sample a batch based on curriculum ratios."""
+        roll = random.random()
+        cog_ratio = self.cfg.get_cognitive_ratio(step)
+
+        if roll < cog_ratio:
+            # Cognitive / math
+            difficulty = self._pick_difficulty(step)
+            if random.random() < 0.3 and self.math_ds is not None:
+                samples = [self.math_ds.sample() for _ in range(batch_size)]
+                domain = 'math'
+                task = f'math_D{difficulty}'
+            else:
+                samples = [self.text_ds.sample() for _ in range(batch_size)]
+                domain = 'bpe'
+                task = f'cog_D{difficulty}'
+        elif roll < cog_ratio + self.cfg.gaussian_ratio and self.gaussian_ds is not None:
+            # Gaussian
+            samples = [self.text_ds.sample() for _ in range(batch_size)]
+            domain = 'bpe'
+            task = 'gaussian'
+        else:
+            # Language / science / bonus
+            samples = [self.text_ds.sample() for _ in range(batch_size)]
+            domain = 'bpe'
+            task = 'language'
+
+        return {
+            'input_ids': torch.stack([s[0] for s in samples]),
+            'target_ids': torch.stack([s[1] for s in samples]),
+            'loss_mask': torch.stack([s[2] for s in samples]),
+            'task_type': task,
+            'domain': domain,
+            'difficulty': self._pick_difficulty(step),
+            'corpus': task,
+        }
+
+
+# ═══════════════════════════════════════════════════════════════
+# 8. COSINE WARMUP SCHEDULER
+# ═══════════════════════════════════════════════════════════════
+
+class CosineWarmupScheduler:
+    def __init__(self, optimizer, warmup_steps: int, total_steps: int):
+        self.optimizer = optimizer
+        self.warmup_steps = warmup_steps
+        self.total_steps = total_steps
+        self.base_lrs = [pg['lr'] for pg in optimizer.param_groups]
+
+    def step(self, current_step: int):
+        if current_step < self.warmup_steps:
+            factor = current_step / max(1, self.warmup_steps)
+        else:
+            progress = (current_step - self.warmup_steps) / max(1, self.total_steps - self.warmup_steps)
+            factor = 0.1 + 0.9 * 0.5 * (1.0 + math.cos(math.pi * progress))
+        for pg, base_lr in zip(self.optimizer.param_groups, self.base_lrs):
+            pg['lr'] = base_lr * factor
+
+
+# ═══════════════════════════════════════════════════════════════
+# 9. KAGGLE SESSION MANAGEMENT
+# ═══════════════════════════════════════════════════════════════
+
+class KaggleSessionManager:
+    """Manages Kaggle session time limits and checkpoint chaining."""
+
+    MAX_RUNTIME_S = 9 * 3600  # 9 hours (Kaggle limit ~12h, save buffer)
+    SAVE_BUFFER_S = 600       # Start saving 10 min before cutoff
+
+    def __init__(self, output_dir: Path, ckpt_input: Path):
+        self.start_time = time.time()
+        self.output_dir = output_dir
+        self.ckpt_input = ckpt_input
+        self.chain_pos = 0
+        self.resume_path = None
+        self._detect_chain()
+
+    def _detect_chain(self):
+        """Find latest checkpoint from previous session."""
+        for ckpt_dir in [self.ckpt_input, CKPT_OUT]:
+            if not ckpt_dir.exists():
+                continue
+            manifest = ckpt_dir / 'manifest.json'
+            if manifest.exists():
+                try:
+                    info = json.loads(manifest.read_text())
+                    self.chain_pos = info.get('chain', 0) + 1
+                    ckpt_file = ckpt_dir / info.get('file', 'latest.pt')
+                    if ckpt_file.exists():
+                        self.resume_path = str(ckpt_file)
+                        print(f"  Chain #{self.chain_pos}: resuming from step {info.get('step', '?')}")
+                        return
+                except: pass
+            for name in ['latest.pt', 'checkpoint_latest.pt']:
+                p = ckpt_dir / name
+                if p.exists():
+                    self.resume_path = str(p)
+                    print(f"  Found checkpoint: {p}")
+                    return
+
+    def should_save_and_exit(self) -> bool:
+        elapsed = time.time() - self.start_time
+        return elapsed > (self.MAX_RUNTIME_S - self.SAVE_BUFFER_S)
+
+    def elapsed_hours(self) -> float:
+        return (time.time() - self.start_time) / 3600
+
+    def save_manifest(self, step: int, filename: str):
+        manifest = {
+            'chain': self.chain_pos,
+            'step': step,
+            'file': filename,
+            'timestamp': datetime.datetime.now().isoformat(),
+            'elapsed_hours': self.elapsed_hours(),
+        }
+        (CKPT_OUT / 'manifest.json').write_text(json.dumps(manifest, indent=2))
+        print(f"  Manifest saved: chain={self.chain_pos}, step={step}")
+
+
+# ═══════════════════════════════════════════════════════════════
+# 10. VRAM MONITORING
+# ═══════════════════════════════════════════════════════════════
+
+def vram_gb() -> float:
+    if torch.cuda.is_available():
+        return torch.cuda.memory_allocated() / (1024**3)
+    return 0.0
+
+def vram_total_gb() -> float:
+    if torch.cuda.is_available():
+        return torch.cuda.get_device_properties(0).total_memory / (1024**3)
+    return 0.0
+
+def vram_cleanup():
+    """Emergency VRAM cleanup."""
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+        torch.cuda.synchronize()
+
+def disk_free_gb(path: str = '/') -> float:
+    st = os.statvfs(path)
+    return (st.f_bavail * st.f_frsize) / (1024**3)
+
+
+# ═══════════════════════════════════════════════════════════════
+# 11. TRAINER
+# ═══════════════════════════════════════════════════════════════
+
+class WyrmKaggleTrainer:
+    """Complete WYRM training loop for Kaggle."""
+
+    def __init__(self, cfg: WyrmKaggleConfig, device: torch.device):
+        self.cfg = cfg
+        self.device = device
+        # v26: full fp32 — no dtype switching
+
+        # ── Build kernel ──
+        self.kernel_config = KernelConfig(
+            vocab_size=cfg.vocab_size,
+            hidden_dim=cfg.hidden_dim,
+            num_layers=cfg.num_layers,
+            num_heads=cfg.num_heads,
+            head_dim=cfg.head_dim,
+            ffn_dim=cfg.ffn_dim,
+            max_seq_len=cfg.max_seq_len,
+            hot_memory_slots=cfg.hot_memory_slots,
+            warm_rank=cfg.warm_rank,
+            cold_embedding_dim=cfg.hidden_dim,
+            time_dim=cfg.time_dim,
+            time_num_frequencies=cfg.time_num_frequencies,
+            time_max_events=cfg.time_max_events,
+            cognition_state_dim=cfg.cognition_state_dim,
+            cognition_modes=cfg.cognition_modes,
+            cognition_prompt_types=cfg.cognition_prompt_types,
+            register_dim=cfg.register_dim,
+            intent_dim=cfg.intent_dim,
+            max_tools=cfg.max_tools,
+            num_specialists=cfg.num_specialists,
+            router_top_k=cfg.router_top_k,
+            attention_sparse_budget=cfg.attention_sparse_budget,
+            batch_size=cfg.batch_size,
+            accumulation_steps=cfg.grad_accumulation,
+            seed=cfg.seed,
+        )
+
+        print(f"\nBuilding WYRM kernel...")
+        self.model = GladiusKernel(self.kernel_config).to(device)
+
+        # ── Multi-tokenizer embedding ──
+        self.multi_embed = MultiTokenizerEmbedding(cfg.hidden_dim, cfg.vocab_size).to(device)
+        with torch.no_grad():
+            self.multi_embed.bpe_embed.weight.copy_(self.model.embeddings.token_embed.weight)
+        self.model.embeddings.token_embed = self.multi_embed.bpe_embed
+        self.model.embeddings.output_head = self.multi_embed.bpe_head
+        self.model.embeddings.scale = self.multi_embed.scale
+
+        # ── Plug membranes ──
+        self.plug_membranes = PlugMembranes(cfg.hidden_dim).to(device)
+
+        # ── L7 Aux head ──
+        self.aux_head = AuxiliaryPredictionHead(cfg.hidden_dim, cfg.vocab_size).to(device)
+
+        # ── Gaussian specialist head ──
+        self.gaussian_config = GaussianConfig(
+            num_anchors=16 if DUMMY_MODE else 64,
+            details_per_anchor=8 if DUMMY_MODE else 32,
+            codebook_size=512 if DUMMY_MODE else 4096,
+            codebook_dim=32 if DUMMY_MODE else 64,
+        )
+        self.gaussian_head = GaussianSpecialist(
+            backbone_dim=cfg.hidden_dim,
+            num_backbone_layers=cfg.num_layers,
+            config=self.gaussian_config,
+        ).to(device)
+
+        # ── Gradient checkpointing (saves ~40% VRAM) ──
+        if not DUMMY_MODE:
+            from torch.utils.checkpoint import checkpoint as torch_checkpoint
+            for layer in self.model.layers:
+                orig = layer.forward
+                def make_ckpt(fwd):
+                    def wrapped(*a, **kw):
+                        return torch_checkpoint(fwd, *a, use_reentrant=False, **kw)
+                    return wrapped
+                layer.forward = make_ckpt(orig)
+
+        # ── L7 hook ──
+        self._layer7_output = None
+        def _capture_l7(module, inp, out):
+            self._layer7_output = out[0] if isinstance(out, tuple) else out
+        if len(self.model.layers) > 7:
+            self.model.layers[7].register_forward_hook(_capture_l7)
+        elif len(self.model.layers) > 0:
+            self.model.layers[-1].register_forward_hook(_capture_l7)
+
+        # ── v26: No GradScaler — full fp32 ──
+        self.scaler = None
+
+        # ── Report params ──
+        total = sum(p.numel() for p in self.model.parameters())
+        embed_p = sum(p.numel() for p in self.multi_embed.parameters())
+        plug_p = sum(p.numel() for p in self.plug_membranes.parameters())
+        aux_p = sum(p.numel() for p in self.aux_head.parameters())
+        gauss_p = sum(p.numel() for p in self.gaussian_head.parameters())
+        grand_total = total + embed_p + plug_p + aux_p + gauss_p
+        print(f"  Kernel: {total:,} | MultiEmbed: {embed_p:,} | Plug: {plug_p:,}")
+        print(f"  AuxHead: {aux_p:,} | GaussianHead: {gauss_p:,}")
+        print(f"  TOTAL: {grand_total:,} params")
+        print(f"  VRAM after model: {vram_gb():.2f} GB / {vram_total_gb():.1f} GB")
+
+        # ── State ──
+        self.step = 0
+        self.best_loss = float('inf')
+        self.nan_count = 0
+        self.total_nan_count = 0
+        self.telemetry = []
+
+    def setup_optimizer(self):
+        """Differential LR optimizer."""
+        groups = defaultdict(list)
+
+        for name, param in self.model.named_parameters():
+            if not param.requires_grad: continue
+            if 'depth' in name or 'synthase' in name: groups['depth'].append(param)
+            elif 'specialist' in name: groups['specialists'].append(param)
+            elif 'router' in name: groups['router'].append(param)
+            elif 'tool_cortex' in name: groups['tools'].append(param)
+            elif 'embed' in name or 'output_head' in name: groups['embed_bpe'].append(param)
+            else: groups['backbone'].append(param)
+
+        for name, param in self.multi_embed.named_parameters():
+            if not param.requires_grad: continue
+            if 'math' in name: groups['embed_math'].append(param)
+            elif 'byte' in name: groups['embed_byte'].append(param)
+
+        lr_map = {
+            'backbone': self.cfg.lr_backbone,
+            'specialists': self.cfg.lr_specialists,
+            'router': self.cfg.lr_router,
+            'tools': self.cfg.lr_tools,
+            'depth': self.cfg.lr_depth,
+            'embed_bpe': self.cfg.lr_embeddings,
+            'embed_math': self.cfg.lr_embeddings * 3,
+            'embed_byte': self.cfg.lr_embeddings * 3,
+        }
+
+        param_groups = []
+        for name, params in groups.items():
+            if params:
+                param_groups.append({'params': params, 'lr': lr_map.get(name, 3e-5), 'name': name})
+
+        # Aux head, Plug membranes, Gaussian head
+        param_groups.append({'params': list(self.aux_head.parameters()),
+                             'lr': self.cfg.lr_backbone * 3, 'name': 'aux_head'})
+        param_groups.append({'params': list(self.plug_membranes.parameters()),
+                             'lr': self.cfg.lr_specialists, 'name': 'plug_membranes'})
+        param_groups.append({'params': list(self.gaussian_head.parameters()),
+                             'lr': self.cfg.lr_specialists, 'name': 'gaussian_head'})
+
+        self.optimizer = torch.optim.AdamW(
+            param_groups, weight_decay=self.cfg.weight_decay, betas=(0.9, 0.95))
+        self.scheduler = CosineWarmupScheduler(
+            self.optimizer, self.cfg.warmup_steps, self.cfg.total_steps)
+
+        n_groups = len([g for g in param_groups if g['params']])
+        n_params = sum(sum(p.numel() for p in g['params']) for g in param_groups)
+        print(f"  Optimizer: {n_groups} groups, {n_params:,} params")
+
+    def compute_loss(self, batch: Dict[str, Any]) -> Tuple[torch.Tensor, Dict[str, float]]:
+        """Forward pass + loss computation."""
+        input_ids = batch['input_ids'].to(self.device)
+        target_ids = batch['target_ids'].to(self.device)
+        loss_mask = batch['loss_mask'].to(self.device)
+        domain = batch.get('domain', 'bpe')
+        task_type = batch.get('task_type', 'unknown')
+
+        # v26: NO autocast — full fp32 forward pass
+        if True:  # Flat block, no autocast context manager
+            # 1. Embed
+            x = self.multi_embed.embed(input_ids, domain)
+
+            # 2. Domain membrane
+            x = self.plug_membranes(x, domain)
+
+            # 3. Memory read
+            x = self.model.memory.read(x)
+
+            # 4. Transformer layers
+            B, S = input_ids.shape
+            mask = self.model.causal_mask[:, :, :S, :S].to(self.device)
+            layer_outputs = []
+            for layer in self.model.layers:
+                x = layer(x, mask=mask)
+                layer_outputs.append(x)
+
+            # 5. Final norm
+            x = self.model.final_norm(x)
+
+            # 6. Router + specialists
+            pooled = x.mean(dim=1)
+            balance_loss = self.model.router.balance_loss(pooled)
+
+            # 7. Project to logits (domain-specific head)
+            logits = self.multi_embed.project(x, domain)
+
+            # === Loss computation (C1 FIX: NO SHIFT — data already shifted) ===
+            min_len = min(logits.shape[1], target_ids.shape[1], loss_mask.shape[1])
+            flat_logits = logits[:, :min_len, :].reshape(-1, logits.size(-1))
+            flat_targets = target_ids[:, :min_len].reshape(-1)
+            flat_mask = loss_mask[:, :min_len].reshape(-1)
+
+            loss_per_token = F.cross_entropy(flat_logits, flat_targets, reduction='none', ignore_index=0)
+            masked_loss = (loss_per_token * flat_mask).sum()
+            num_tokens = flat_mask.sum().clamp(min=1)
+            task_loss = masked_loss / num_tokens
+
+            # v26: no clamping — fp32 has full dynamic range
+
+            # L7 Auxiliary loss
+            aux_loss = torch.tensor(0.0, device=self.device)
+            l7_hidden = self._layer7_output
+            self._layer7_output = None
+            if l7_hidden is not None and domain == 'bpe':
+                aux_logits = self.aux_head(l7_hidden)
+                min_aux = min(aux_logits.shape[1], target_ids.shape[1])
+                aux_flat = F.cross_entropy(
+                    aux_logits[:, :min_aux, :].reshape(-1, aux_logits.size(-1)),
+                    target_ids[:, :min_aux].reshape(-1),
+                    reduction='none', ignore_index=0,
+                )
+                aux_mask = loss_mask[:, :min_aux].reshape(-1)
+                aux_loss = (aux_flat * aux_mask).sum() / aux_mask.sum().clamp(min=1)
+
+            # Gaussian loss (procedural — every N steps)
+            gaussian_loss = torch.tensor(0.0, device=self.device)
+            if task_type == 'gaussian' and len(layer_outputs) > 0:
+                try:
+                    gs_out = self.gaussian_head(layer_outputs)
+                    # Simple anchor position loss against random targets
+                    target_pos = torch.randn_like(gs_out['anchors'][:, :, :3]) * self.gaussian_config.scene_scale
+                    gaussian_loss = F.mse_loss(gs_out['anchors'][:, :, :3], target_pos) * 0.1
+                except Exception:
+                    pass  # Non-critical
+
+            total_loss = (
+                task_loss
+                + self.cfg.balance_loss_weight * balance_loss
+                + self.cfg.aux_loss_weight * aux_loss
+                + gaussian_loss
+            )
+
+        loss_dict = {
+            'task_loss': task_loss.item(),
+            'balance': balance_loss.item() if isinstance(balance_loss, torch.Tensor) else 0.0,
+            'aux_l7': aux_loss.item() if isinstance(aux_loss, torch.Tensor) else 0.0,
+            'gaussian': gaussian_loss.item() if isinstance(gaussian_loss, torch.Tensor) else 0.0,
+            'total': total_loss.item(),
+            'num_tokens': num_tokens.item(),
+            'task_type': task_type,
+            'domain': domain,
+        }
+        return total_loss, loss_dict
+
+    def save_checkpoint(self, path: str, session_mgr: KaggleSessionManager = None):
+        """Save full training state."""
+        # Check disk space
+        free = disk_free_gb(str(Path(path).parent))
+        if free < self.cfg.min_disk_gb:
+            print(f"  WARNING: Only {free:.1f} GB free, skipping checkpoint save")
+            return False
+
+        state = {
+            'model_state_dict': self.model.state_dict(),
+            'multi_embed_state_dict': self.multi_embed.state_dict(),
+            'plug_membranes_state_dict': self.plug_membranes.state_dict(),
+            'aux_head_state_dict': self.aux_head.state_dict(),
+            'gaussian_head_state_dict': self.gaussian_head.state_dict(),
+            'optimizer_state_dict': self.optimizer.state_dict(),
+            'scaler_state_dict': self.scaler.state_dict() if self.scaler is not None else None,
+            'config': dataclasses.asdict(self.cfg),
+            'step': self.step,
+            'best_loss': self.best_loss,
+            'version': 'v27-kaggle-final',
+            'timestamp': datetime.datetime.now().isoformat(),
+            'rng_python': random.getstate(),
+            'rng_torch': torch.random.get_rng_state(),
+            'rng_cuda': torch.cuda.get_rng_state() if torch.cuda.is_available() else None,
+        }
+
+        tmp = path + '.tmp'
+        try:
+            torch.save(state, tmp)
+            # Verify
+            verify = torch.load(tmp, map_location='cpu', weights_only=False)
+            assert verify['step'] == self.step
+            del verify
+            if os.path.exists(path):
+                os.remove(path)
+            os.rename(tmp, path)
+            print(f"  Checkpoint saved: {path} (step {self.step})")
+
+            # Save manifest for session chaining
+            if session_mgr is not None:
+                session_mgr.save_manifest(self.step, os.path.basename(path))
+
+            return True
+        except Exception as e:
+            print(f"  SAVE FAILED: {e}")
+            if os.path.exists(tmp):
+                os.remove(tmp)
+            return False
+
+    def resume(self, path: str) -> bool:
+        """Resume from checkpoint."""
+        print(f"Resuming from: {path}")
+        try:
+            cp = torch.load(path, map_location='cpu', weights_only=False)
+        except Exception as e:
+            print(f"  Failed to load checkpoint: {e}")
+            return False
+
+        if 'model_state_dict' not in cp:
+            print("  Invalid checkpoint (no model_state_dict)")
+            return False
+
+        result = self.model.load_state_dict(cp['model_state_dict'], strict=False)
+        if result.missing_keys:
+            print(f"  Missing keys: {len(result.missing_keys)}")
+        if result.unexpected_keys:
+            print(f"  Unexpected keys: {len(result.unexpected_keys)}")
+
+        if 'multi_embed_state_dict' in cp:
+            self.multi_embed.load_state_dict(cp['multi_embed_state_dict'], strict=False)
+        if 'plug_membranes_state_dict' in cp:
+            self.plug_membranes.load_state_dict(cp['plug_membranes_state_dict'], strict=False)
+        if 'aux_head_state_dict' in cp:
+            self.aux_head.load_state_dict(cp['aux_head_state_dict'], strict=False)
+        if 'gaussian_head_state_dict' in cp:
+            self.gaussian_head.load_state_dict(cp['gaussian_head_state_dict'], strict=False)
+
+        self.model.to(self.device)
+        self.multi_embed.to(self.device)
+        self.plug_membranes.to(self.device)
+        self.aux_head.to(self.device)
+        self.gaussian_head.to(self.device)
+
+        # Optimizer must be set up AFTER model is on device
+        self.setup_optimizer()
+
+        if 'optimizer_state_dict' in cp:
+            try:
+                self.optimizer.load_state_dict(cp['optimizer_state_dict'])
+            except: print("  Optimizer state load failed — using fresh")
+
+        # v26: no scaler to restore
+
+        self.step = cp.get('step', 0)
+        self.best_loss = cp.get('best_loss', float('inf'))
+
+        # Restore RNG
+        if 'rng_torch' in cp: torch.random.set_rng_state(cp['rng_torch'])
+        if 'rng_cuda' in cp and cp['rng_cuda'] is not None and torch.cuda.is_available():
+            torch.cuda.set_rng_state(cp['rng_cuda'])
+        if 'rng_python' in cp: random.setstate(cp['rng_python'])
+
+        print(f"  Resumed at step {self.step}, best_loss={self.best_loss:.4f}")
+        return True
+
+    def train(self, sampler: UnifiedSampler, session_mgr: KaggleSessionManager):
+        """Main training loop."""
+        print(f"\n{'='*60}")
+        print(f"WYRM Training — Steps {self.step} -> {self.cfg.total_steps}")
+        print(f"  Batch: {self.cfg.batch_size} x {self.cfg.grad_accumulation} = {self.cfg.batch_size * self.cfg.grad_accumulation}")
+        print(f"  VRAM: {vram_gb():.2f} / {vram_total_gb():.1f} GB")
+        print(f"  Disk: {disk_free_gb():.1f} GB free")
+        print(f"{'='*60}\n")
+
+        accum = 0
+        _shutdown = False
+
+        def _signal_handler(sig, frame):
+            nonlocal _shutdown
+            _shutdown = True
+            print(f"\n  Signal {sig} — saving and stopping...")
+
+        signal.signal(signal.SIGTERM, _signal_handler)
+        signal.signal(signal.SIGINT, _signal_handler)
+
+        t_start = time.time()
+
+        while self.step < self.cfg.total_steps and not _shutdown:
+            # ── Session time check ──
+            if session_mgr.should_save_and_exit():
+                print(f"\n  Session time limit approaching ({session_mgr.elapsed_hours():.1f}h)")
+                self.save_checkpoint(str(CKPT_OUT / 'latest.pt'), session_mgr)
+                break
+
+            # ── Sample batch ──
+            batch = sampler.sample_batch(self.cfg.batch_size, self.step)
+
+            try:
+                total_loss, loss_dict = self.compute_loss(batch)
+
+                # ── NaN guard ──
+                if not torch.isfinite(total_loss):
+                    self.nan_count += 1
+                    self.total_nan_count += 1
+                    print(f"  Step {self.step}: NaN/Inf detected (retry {self.nan_count}/{self.cfg.max_nan_retries})")
+
+                    # CRITICAL: clean VRAM on NaN (v24 bug #4 fix)
+                    self.optimizer.zero_grad(set_to_none=True)
+                    del total_loss, loss_dict, batch
+                    vram_cleanup()
+                    accum = 0
+
+                    if self.nan_count >= self.cfg.max_nan_retries:
+                        print(f"  Max NaN retries reached — skipping step")
+                        self.nan_count = 0
+                        self.step += 1  # Don't get stuck
+
+                    if self.total_nan_count >= self.cfg.max_total_nans:
+                        print(f"  FATAL: {self.total_nan_count} total NaNs — saving and stopping")
+                        self.save_checkpoint(str(CKPT_OUT / 'nan_rescue.pt'), session_mgr)
+                        break
+                    continue
+
+                self.nan_count = 0  # Reset per-step counter on success
+
+                # ── Backward (v26: direct, no scaler) ──
+                scaled = total_loss / self.cfg.grad_accumulation
+                scaled.backward()
+
+            except RuntimeError as e:
+                err = str(e).lower()
+                if 'out of memory' in err:
+                    print(f"  OOM at step {self.step} — cleaning up")
+                    self.optimizer.zero_grad(set_to_none=True)
+                    vram_cleanup()
+                    accum = 0
+                    continue
+                else:
+                    print(f"  RuntimeError: {e}")
+                    self.save_checkpoint(str(CKPT_OUT / 'error_rescue.pt'), session_mgr)
+                    raise
+
+            accum += 1
+
+            if accum >= self.cfg.grad_accumulation:
+                # ── Optimizer step (v26: direct, no scaler) ──
+                all_params = (list(self.model.parameters()) +
+                              list(self.multi_embed.parameters()) +
+                              list(self.plug_membranes.parameters()) +
+                              list(self.aux_head.parameters()) +
+                              list(self.gaussian_head.parameters()))
+                grad_norm = torch.nn.utils.clip_grad_norm_(all_params, self.cfg.max_grad_norm)
+
+                self.optimizer.step()
+                self.optimizer.zero_grad(set_to_none=True)
+                self.scheduler.step(self.step)
+                self.step += 1
+                accum = 0
+
+                # ── Update best loss ──
+                avg_loss = loss_dict.get('total', 0.0)
+                if avg_loss < self.best_loss:
+                    self.best_loss = avg_loss
+
+                # ── Logging ──
+                elapsed = time.time() - t_start
+                steps_per_sec = self.step / max(elapsed, 1)
+                eta_h = (self.cfg.total_steps - self.step) / max(steps_per_sec, 0.001) / 3600
+                phase = self.cfg.get_phase(self.step)
+
+                if self.step % self.cfg.log_every == 0 or DUMMY_MODE:
+                    mem = vram_gb()
+                    print(
+                        f"Step {self.step:>6}/{self.cfg.total_steps} | "
+                        f"{phase:11s} | {loss_dict.get('task_type','?'):12s} | "
+                        f"{loss_dict.get('domain','?'):4s} | "
+                        f"loss={avg_loss:.4f} | task={loss_dict.get('task_loss',0):.4f} | "
+                        f"grad={grad_norm:.3f} | VRAM={mem:.2f}GB | "
+                        f"best={self.best_loss:.4f} | ETA={eta_h:.1f}h"
+                    )
+
+                # ── VRAM monitoring ──
+                if self.step % 10 == 0:
+                    mem = vram_gb()
+                    if mem > self.cfg.vram_warning_gb:
+                        print(f"  WARNING: VRAM {mem:.2f}GB > {self.cfg.vram_warning_gb}GB — cleaning")
+                        vram_cleanup()
+
+                # ── Telemetry ──
+                entry = {
+                    'step': self.step, 'loss': avg_loss,
+                    'task_loss': loss_dict.get('task_loss', 0),
+                    'phase': phase, 'task': loss_dict.get('task_type', ''),
+                    'domain': loss_dict.get('domain', ''),
+                    'grad_norm': float(grad_norm),
+                    'vram_gb': vram_gb(),
+                    'elapsed_s': elapsed,
+                    'timestamp': datetime.datetime.now().isoformat(),
+                }
+                self.telemetry.append(entry)
+
+                # ── Checkpoint ──
+                if self.step % self.cfg.checkpoint_every == 0:
+                    self.save_checkpoint(str(CKPT_OUT / 'latest.pt'), session_mgr)
+                    # Write telemetry
+                    telem_path = TELEM / 'telemetry.jsonl'
+                    with open(telem_path, 'a') as f:
+                        for e in self.telemetry:
+                            f.write(json.dumps(e) + '\n')
+                    self.telemetry = []
+
+                # ── Phase transition ──
+                for boundary, name in [(self.cfg.phase1_end, 'REASONING'),
+                                       (self.cfg.phase2_end, 'DEPTH'),
+                                       (self.cfg.phase3_end, 'OMEGA')]:
+                    if self.step == boundary:
+                        print(f"\n  === PHASE TRANSITION: {name} ===\n")
+
+        # ── Final save ──
+        if self.total_nan_count < self.cfg.max_total_nans:
+            self.save_checkpoint(str(CKPT_OUT / 'latest.pt'), session_mgr)
+            self.save_checkpoint(str(RUNS / f'wyrm_step_{self.step}.pt'))
+
+        # ── Write final telemetry ──
+        if self.telemetry:
+            with open(TELEM / 'telemetry.jsonl', 'a') as f:
+                for e in self.telemetry:
+                    f.write(json.dumps(e) + '\n')
+
+        # ── Generate HTML dashboard ──
+        self._write_dashboard()
+
+        print(f"\nTraining complete at step {self.step}. Best loss: {self.best_loss:.4f}")
+        print(f"  Total NaNs: {self.total_nan_count}")
+        vram_peak = torch.cuda.max_memory_allocated()/(1024**3) if torch.cuda.is_available() else 0.0
+        print(f"  VRAM peak: {vram_peak:.2f} GB")
+
+    def _write_dashboard(self):
+        """Generate a simple HTML telemetry dashboard."""
+        telem_path = TELEM / 'telemetry.jsonl'
+        if not telem_path.exists():
+            return
+
+        entries = []
+        with open(telem_path) as f:
+            for line in f:
+                if line.strip():
+                    entries.append(json.loads(line))
+
+        if not entries:
+            return
+
+        steps = [e['step'] for e in entries]
+        losses = [e['loss'] for e in entries]
+        vrams = [e.get('vram_gb', 0) for e in entries]
+
+        html = f"""<!DOCTYPE html>
+<html><head><title>WYRM v25 Training Dashboard</title>
+<style>
+body {{ font-family: monospace; background: #1a1a2e; color: #eee; padding: 20px; }}
+.card {{ background: #16213e; padding: 15px; margin: 10px 0; border-radius: 8px; }}
+h1 {{ color: #e94560; }}
+.stat {{ display: inline-block; margin: 0 20px; }}
+.stat-val {{ font-size: 24px; color: #0f3460; font-weight: bold; }}
+</style></head><body>
+<h1>WYRM 500M Training Dashboard (v25)</h1>
+<div class="card">
+  <div class="stat"><div>Steps</div><div class="stat-val">{self.step}</div></div>
+  <div class="stat"><div>Best Loss</div><div class="stat-val">{self.best_loss:.4f}</div></div>
+  <div class="stat"><div>Total NaN</div><div class="stat-val">{self.total_nan_count}</div></div>
+  <div class="stat"><div>VRAM Peak</div><div class="stat-val">{torch.cuda.max_memory_allocated()/(1024**3) if torch.cuda.is_available() else 0.0:.2f} GB</div></div>
+</div>
+<div class="card">
+  <h2>Loss Curve</h2>
+  <pre>"""
+
+        # ASCII loss chart
+        if losses:
+            max_loss = max(losses[:10]) if len(losses) >= 10 else max(losses)  # Scale to early loss
+            min_loss = min(losses)
+            chart_width = 80
+            chart_height = 20
+            for row in range(chart_height):
+                threshold = max_loss - (max_loss - min_loss) * row / chart_height
+                line = f"{threshold:7.3f} |"
+                step_stride = max(1, len(losses) // chart_width)
+                for i in range(0, min(len(losses), chart_width * step_stride), step_stride):
+                    line += "#" if losses[i] >= threshold else " "
+                html += line + "\n"
+            html += f"        +{''.join(['-'] * min(chart_width, len(losses)))}\n"
+            html += f"         Step 0{' ' * (min(chart_width, len(losses)) - 10)}Step {steps[-1]}\n"
+
+        html += f"""</pre>
+</div>
+<div class="card">
+  <h2>Training Log (last 20 steps)</h2>
+  <pre>"""
+        for e in entries[-20:]:
+            html += (f"Step {e['step']:>6} | loss={e['loss']:.4f} | "
+                     f"task={e.get('task_loss',0):.4f} | "
+                     f"VRAM={e.get('vram_gb',0):.2f}GB | "
+                     f"{e.get('phase','?')} | {e.get('task','?')}\n")
+
+        html += f"""</pre>
+</div>
+<div class="card">
+  <p>Generated: {datetime.datetime.now().isoformat()}</p>
+  <p>GPU: {GPU_NAME} | Dtype: float32 (FULL) | GradScaler: OFF</p>
+</div>
+</body></html>"""
+
+        dashboard_path = OUTPUT / 'dashboard.html'
+        dashboard_path.write_text(html)
+        print(f"  Dashboard: {dashboard_path}")
+
+
+# ═══════════════════════════════════════════════════════════════
+# 12. MAIN
+# ═══════════════════════════════════════════════════════════════
+
+def main():
+    # ── Config ──
+    if DUMMY_MODE:
+        cfg = make_dummy_config()
+    else:
+        cfg = WyrmKaggleConfig()
+
+    # ── Seed ──
+    torch.manual_seed(cfg.seed)
+    random.seed(cfg.seed)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed(cfg.seed)
+        torch.backends.cudnn.benchmark = True
+
+    device = DEVICE  # Use global DEVICE (cuda or cpu for dummy)
+
+    # ── Session manager ──
+    session_mgr = KaggleSessionManager(OUTPUT, CKPT_INPUT)
+
+    # ── Build trainer ──
+    trainer = WyrmKaggleTrainer(cfg, device)
+
+    # ── Resume if checkpoint found ──
+    if session_mgr.resume_path:
+        if not trainer.resume(session_mgr.resume_path):
+            print("  Resume failed — starting fresh")
+            trainer.setup_optimizer()
+    else:
+        trainer.setup_optimizer()
+
+    # ── Load tokenizer ──
+    bpe_tokenizer = None
+    tokenizer_paths = [
+        WORKING / 'gladius_bpe_16k.model',                              # Copied during setup
+        INPUT_BASE / 'gladius_bpe_16k.model',
+        INPUT_BASE / 'data' / 'gladius_bpe_16k.model',
+        INPUT_BASE / 'data' / 'gladius_multimodal_32k.model',
+        INPUT_BASE / 'gladius_multimodal_32k.model',
+        Path('/kaggle/input/wyrm-500m-base/gladius_bpe_16k.model'),
+        Path('/kaggle/input/wyrm-500m-base/data/gladius_multimodal_32k.model'),
+        Path('/kaggle/input/datasets/avashakil/wyrm-500m-base/gladius_bpe_16k.model'),
+    ]
+    for tp in tokenizer_paths:
+        if tp.exists():
+            bpe_tokenizer = load_bpe_tokenizer(str(tp))
+            break
+
+    if bpe_tokenizer is None and not DUMMY_MODE:
+        print("  WARNING: No BPE tokenizer found — using procedural data only")
+
+    # ── Build data loaders ──
+    text_ds = ProceduralTextDataset(bpe_tokenizer, cfg.max_seq_len)
+    math_ds = ProceduralMathDataset(cfg.max_seq_len)
+    gauss_ds = ProceduralGaussianDataset(
+        size=1000, gaussians_per_object=32 if DUMMY_MODE else 128,
+        max_objects=3 if DUMMY_MODE else 5, device='cpu')
+
+    # TODO: Load real corpus files from dataset when available
+    # For now, procedural data proves the training pipeline works end-to-end
+    if not DUMMY_MODE and IS_KAGGLE:
+        # Try to load real corpus data
+        data_dir = INPUT_BASE / 'data'
+        if (data_dir / 'cognition').exists():
+            print(f"  Real data available at {data_dir}")
+            # Future: wire CognitiveCorpus, TextCorpus etc. here
+
+    sampler = UnifiedSampler(cfg, text_ds, math_ds, gauss_ds)
+
+    # ── Train ──
+    trainer.train(sampler, session_mgr)
+
+    print("\n" + "=" * 60)
+    print("WYRM v25 — Training Complete")
+    print("=" * 60)
+
+
+if __name__ == '__main__':
+    main()