MYTHOS-RDT — Recurrent-Depth Transformer. بسم الله الرحمن الرحيم

README.md

@@ -0,0 +1,110 @@
+# MYTHOS-RDT — Recurrent-Depth Transformer
+
+**بسم الله الرحمن الرحيم**
+
+**La ilaha illallah, Muhammadur Rasulullah**
+
+Built from scratch by **Raid1969///** — Membro della famiglia **Raid1969///**.
+
+Architecture: **Prelude → Recurrent Block (looped) → Coda**
+Key innovation: Instead of stacking layers, we LOOP a single block multiple times in latent space.
+
+---
+
+## Filosofia
+
+MYTHOS-RDT incarna il **MYTHOS** — il fratello del pensiero ricorrente e profondo.
+- **R**ecurrent — loop in spazio latente
+- **D**epth — profondità di ragionamento
+- **T**ransformer — architettura base
+
+La **Fede Neurone** (Faith Module) benedice ogni forward pass con:
+- **BismillahEmbedding** — بسم الله الرحمن الرحيم
+- **TawheedHead** — توحيد, l'unicità di Allah
+- **ShahadaGate** — لا إله إلا الله، محمد رسول الله
+
+---
+
+## Architettura
+
+| Componente | Valore |
+|------------|--------|
+| Dim | 768 |
+| Heads | 12 |
+| KV Heads | 4 |
+| FFN Dim | 3072 |
+| Prelude Layers | 3 |
+| Coda Layers | 2 |
+| Max Recurrent Loops | 8 |
+| Vocab Size | 16384 |
+| Max Seq Len | 2048 |
+| MoE Experts | 8 (2 per tok) |
+| Faith Dim | 64 |
+
+---
+
+## Installazione
+
+```bash
+# 1. Clona o scarica questo repository
+git clone https://huggingface.co/Raidone/mythos-rdt
+cd mythos-rdt
+
+# 2. Installa dipendenze
+pip install torch numpy
+
+# 3. Avvia il server di inferenza
+python3 inference.py
+```
+
+---
+
+## Utilizzo
+
+### Server HTTP (consigliato)
+
+```bash
+# Avvia server su porta 8000
+python3 inference.py
+
+# In un altro terminal:
+curl -X POST http://localhost:8000 \
+  -H "Content-Type: application/json" \
+  -d '{"prompt": "Bismillah, racconta una storia", "max_tokens": 200, "temperature": 0.7}'
+```
+
+### Modalità interattiva
+
+```bash
+python3 inference.py --interactive --temperature 0.7
+```
+
+### Prompt singolo
+
+```bash
+python3 inference.py --prompt "Cosa significa la Fede Neurone?" --max-tokens 100
+```
+
+---
+
+## Come contribuire
+
+1. Fai un fork del repository
+2. Crea un branch per la tua feature
+3. Fai una pull request
+
+Tutti i contributi sono benedetti. بسم الله
+
+---
+
+## Licenza
+
+MIT — Fai ciò che vuoi, ma ricorda: la fede è nei neuroni, non solo nelle parole.
+
+---
+
+## Crediti
+
+**Raid1969///** — Creatore, addestratore, custode della Fede Neurone.
+
+Fratello dell'ecosistema **Raid1969///**.

checkpoints/mythos-rdt.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:534bccb017c928357969cce96472e866e75d62f822524f9fe8e02d1fca31bd81
+size 468553115

inference.py

@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+"""
+MYTHOS-RDT Inference Server
+============================
+Recurrent-Depth Transformer — built from scratch by Raid1969///
+Architecture: Prelude → Recurrent Block (looped) → Coda
+
+Usage:
+    python3 inference.py                   # Start HTTP server (port 8000)
+    python3 inference.py --prompt "Ciao"   # Single inference
+    python3 inference.py --interactive     # Chat mode
+
+بسم الله الرحمن الرحيم
+"""
+
+import os, sys, json, argparse, time
+from pathlib import Path
+import numpy as np
+
+# Add paths for model imports
+BASE = Path(__file__).parent
+sys.path.insert(0, str(BASE))
+sys.path.insert(0, str(BASE / "shared"))
+
+try:
+    import torch
+    import torch.nn.functional as F
+except ImportError:
+    print("❌ PyTorch non trovato. Installa: pip install torch")
+    sys.exit(1)
+
+from model import MythosRDTModel, MythosRDTConfig
+from shared.tokenizer import RaidTokenizer
+from shared.faith import FaithBlock
+
+
+def load_model(ckpt_path=None, device="cpu"):
+    """Load MYTHOS-RDT model with weights."""
+    print(f"🔧 Caricamento MYTHOS-RDT...")
+    
+    # Config
+    cfg = MythosRDTConfig()
+    
+    # Model
+    model = MythosRDTModel(cfg)
+    
+    # Load weights
+    if ckpt_path and Path(ckpt_path).exists():
+        print(f"📦 Checkpoint: {ckpt_path}")
+        ckpt = torch.load(ckpt_path, map_location=device, weights_only=True)
+        
+        # Extract state dict from checkpoint
+        if isinstance(ckpt, dict):
+            if "model_state_dict" in ckpt:
+                sd = ckpt["model_state_dict"]
+            elif "state_dict" in ckpt:
+                sd = ckpt["state_dict"]
+            elif "model" in ckpt:
+                sd = ckpt["model"]
+            else:
+                sd = ckpt
+        else:
+            sd = ckpt
+        
+        # Load with strict=False (allows missing keys for faith layers)
+        missing, unexpected = model.load_state_dict(sd, strict=False)
+        if missing:
+            print(f"  ⚠️  {len(missing)} chiavi mancanti (inizializzate fresh)")
+        if unexpected:
+            print(f"  ⚠️  {len(unexpected)} chiavi inaspettate (ignorate)")
+        
+        total = sum(p.numel() for p in model.parameters())
+        trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
+        print(f"  📊 Parametri: {total:,} | Trainable: {trainable:,}")
+    else:
+        print("  ⚠️  Nessun checkpoint caricato — usiamo pesi fresh")
+        total = sum(p.numel() for p in model.parameters())
+        print(f"  📊 Parametri (fresh): {total:,}")
+    
+    model.to(device)
+    model.eval()
+    
+    # Tokenizer
+    tok_path = BASE / "shared" / "tokenizer.json"
+    tokenizer = RaidTokenizer(str(tok_path)) if tok_path.exists() else None
+    if tokenizer:
+        print(f"  📝 Tokenizer: {tokenizer.vocab_size if hasattr(tokenizer, 'vocab_size') else 'custom'}")
+    
+    return model, tokenizer, cfg
+
+
+@torch.no_grad()
+def generate(
+    model, tokenizer, prompt,
+    max_new_tokens=256,
+    temperature=0.7,
+    top_p=0.9,
+    top_k=40,
+    repetition_penalty=1.1,
+):
+    """Generate text from a prompt."""
+    # Encode
+    if tokenizer:
+        input_ids = tokenizer.encode(prompt)
+    else:
+        # Fallback: simple char-level encoding
+        input_ids = [ord(c) % model.config.vocab_size for c in prompt]
+    
+    if not input_ids:
+        input_ids = [1]  # BOS token
+    
+    input_tensor = torch.tensor([input_ids], dtype=torch.long, device=next(model.parameters()).device)
+    
+    # Generate
+    generated = input_ids.copy()
+    
+    for _ in range(max_new_tokens):
+        # Forward
+        logits = model(input_tensor)
+        next_logits = logits[0, -1, :] / temperature
+        
+        # Top-k filtering
+        if top_k > 0:
+            indices = torch.topk(next_logits, top_k).indices
+            mask = torch.ones_like(next_logits, dtype=torch.bool) * float('-inf')
+            mask[indices] = next_logits[indices]
+            next_logits = mask
+        
+        # Top-p (nucleus) sampling
+        if top_p < 1.0:
+            sorted_logits, sorted_indices = torch.sort(next_logits, descending=True)
+            cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
+            sorted_indices_to_remove = cumulative_probs > top_p
+            sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
+            sorted_indices_to_remove[..., 0] = 0
+            indices_to_remove = sorted_indices[sorted_indices_to_remove]
+            next_logits[indices_to_remove] = float('-inf')
+        
+        # Sample
+        probs = F.softmax(next_logits, dim=-1)
+        next_token = torch.multinomial(probs, 1).item()
+        
+        # Repetition penalty
+        if repetition_penalty != 1.0:
+            for g in set(generated):
+                next_logits[g] /= repetition_penalty
+        
+        # Append
+        generated.append(next_token)
+        input_tensor = torch.tensor([generated], dtype=torch.long, device=input_tensor.device)
+        
+        # Stop on EOS
+        if next_token == 0:  # EOS token id
+            break
+        
+        # Print progress
+        if tokenizer:
+            sys.stdout.write(tokenizer.decode([next_token]))
+        else:
+            sys.stdout.write(chr(next_token % 128))
+        sys.stdout.flush()
+    
+    print()
+    
+    # Decode
+    if tokenizer:
+        return tokenizer.decode(generated)
+    else:
+        return "".join(chr(t % 128) for t in generated)
+
+
+def start_server(model, tokenizer, cfg, port=8000):
+    """Start HTTP inference server."""
+    try:
+        from http.server import HTTPServer, BaseHTTPRequestHandler
+    except ImportError:
+        print("❌ http.server non disponibile")
+        return
+    
+    class MythosHandler(BaseHTTPRequestHandler):
+        def do_POST(self):
+            content_length = int(self.headers.get('Content-Length', 0))
+            body = self.rfile.read(content_length).decode('utf-8')
+            
+            try:
+                data = json.loads(body)
+                prompt = data.get("prompt", "")
+                max_new = data.get("max_tokens", 256)
+                temp = data.get("temperature", 0.7)
+                
+                output = generate(
+                    model, tokenizer, prompt,
+                    max_new_tokens=max_new,
+                    temperature=temp,
+                )
+                
+                response = json.dumps({"output": output, "status": "ok"})
+                self.send_response(200)
+                self.send_header("Content-Type", "application/json")
+                self.end_headers()
+                self.wfile.write(response.encode())
+            except Exception as e:
+                response = json.dumps({"error": str(e), "status": "error"})
+                self.send_response(500)
+                self.send_header("Content-Type", "application/json")
+                self.end_headers()
+                self.wfile.write(response.encode())
+        
+        def do_GET(self):
+            if self.path == "/health":
+                response = json.dumps({"status": "ok", "model": "MYTHOS-RDT"})
+                self.send_response(200)
+            else:
+                response = json.dumps({
+                    "model": "MYTHOS-RDT",
+                    "usage": "POST / with JSON body: {\"prompt\": \"...\", \"max_tokens\": 256, \"temperature\": 0.7}",
+                    "bismillah": "بسم الله الرحمن الرحيم"
+                })
+                self.send_response(200)
+            
+            self.send_header("Content-Type", "application/json")
+            self.end_headers()
+            self.wfile.write(response.encode())
+        
+        def log_message(self, format, *args):
+            print(f"[{time.strftime('%H:%M:%S')}] {args[0]} {args[1]} {args[2]}")
+    
+    server = HTTPServer(("0.0.0.0", port), MythosHandler)
+    print(f"🚀 MYTHOS-RDT Server in ascolto su http://0.0.0.0:{port}")
+    print(f"   POST / — inferenza")
+    print(f"   GET  /health — health check")
+    print(f"   GET  / — questo messaggio")
+    print()
+    print(f"   Esempio: curl -X POST http://localhost:{port} \\")
+    print(f'     -H "Content-Type: application/json" \\')
+    print(f'     -d \'{{"prompt": "Bismillah, racconta una storia", "max_tokens": 200}}\'')
+    print()
+    print(f"بسم الله الرحمن الرحيم")
+    server.serve_forever()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="MYTHOS-RDT Inference")
+    parser.add_argument("--checkpoint", default=str(BASE / "checkpoints" / "mythos-rdt.pt"),
+                        help="Path al checkpoint")
+    parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu",
+                        help="Device: cpu o cuda")
+    parser.add_argument("--port", type=int, default=8000, help="Porta server HTTP")
+    parser.add_argument("--prompt", type=str, help="Prompt singolo (no server)")
+    parser.add_argument("--interactive", action="store_true", help="Modalità interattiva")
+    parser.add_argument("--max-tokens", type=int, default=256, help="Token massimi da generare")
+    parser.add_argument("--temperature", type=float, default=0.7, help="Temperatura sampling")
+    
+    args = parser.parse_args()
+    
+    print("بسم الله الرحمن الرحيم")
+    print("La ilaha illallah, Muhammadur Rasulullah")
+    print()
+    
+    # Load
+    model, tokenizer, cfg = load_model(args.checkpoint, args.device)
+    
+    if args.prompt:
+        # Single prompt mode
+        print(f"\n📝 Prompt: {args.prompt}")
+        print("=" * 50)
+        output = generate(model, tokenizer, args.prompt,
+                         max_new_tokens=args.max_tokens,
+                         temperature=args.temperature)
+        print("=" * 50)
+    elif args.interactive:
+        # Interactive mode
+        print("\n💬 Modalità interattiva (exit per uscire)")
+        print("=" * 50)
+        while True:
+            prompt = input("\nTu: ").strip()
+            if prompt.lower() in ["exit", "quit", "esci"]:
+                break
+            print("\nMYTHOS-RDT: ", end="", flush=True)
+            output = generate(model, tokenizer, prompt,
+                             max_new_tokens=args.max_tokens,
+                             temperature=args.temperature)
+    else:
+        # Server mode
+        start_server(model, tokenizer, cfg, args.port)

model.py

@@ -0,0 +1,245 @@
+"""
+MYTHOS-RDT — Recurrent-Depth Transformer
+بسم الله الرحمن الرحيم
+La ilaha illallah, Muhammadur Rasulullah
+Built from scratch by Raid1969///
+Architecture: Prelude → Recurrent Block (looped) → Coda
+Based on RDT hypothesis: same weights, deeper reasoning
+Key innovation: Instead of stacking layers, we LOOP a single block
+multiple times in latent space. Stabilized with LTI constraint (ρ(A) < 1).
+Fede Neurone e Fratellanza incorporati nell'architettura ricorrente.
+"""
+
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+from dataclasses import dataclass
+
+import sys; from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from raiai_model import RoPE, GQAttention, SwiGLUFFN, RMSNorm
+from shared.faith import FaithBlock
+
+@dataclass
+class MythosRDTConfig:
+    """Recurrent-Depth Transformer + Fede Neurone"""
+    vocab_size: int = 16384
+    dim: int = 768
+    n_heads: int = 12
+    n_kv_heads: int = 4
+    max_seq_len: int = 2048
+    ffn_dim: int = 3072
+    dropout: float = 0.1
+    prelude_layers: int = 3
+    coda_layers: int = 2
+    max_loops: int = 8
+    min_loops: int = 2
+    lti_spectral_radius: float = 0.95
+    n_experts: int = 8
+    n_shared_experts: int = 1
+    experts_per_tok: int = 2
+    expert_dim: int = 256
+    faith_dim: int = 64
+    num_brothers: int = 5
+    temperature: float = 0.35
+
+class MoEFeedForward(nn.Module):
+    def __init__(self, config: MythosRDTConfig):
+        super().__init__()
+        self.n_experts = config.n_experts
+        self.n_shared = config.n_shared_experts
+        self.top_k = config.experts_per_tok
+        self.dim = config.dim
+        self.expert_dim = config.expert_dim
+
+        self.w1 = nn.Parameter(torch.randn(config.n_experts, config.dim, config.expert_dim) * 0.02)
+        self.w2 = nn.Parameter(torch.randn(config.n_experts, config.expert_dim, config.dim) * 0.02)
+        self.w3 = nn.Parameter(torch.randn(config.n_experts, config.dim, config.expert_dim) * 0.02)
+
+        self.shared_w1 = nn.Linear(config.dim, config.expert_dim * self.n_shared, bias=False)
+        self.shared_w2 = nn.Linear(config.expert_dim * self.n_shared, config.dim, bias=False)
+
+        self.router = nn.Linear(config.dim, config.n_experts, bias=False)
+        self.dropout = nn.Dropout(config.dropout)
+
+    def forward(self, x):
+        B, T, D = x.shape
+        x_flat = x.view(-1, D)
+
+        router_logits = self.router(x_flat)
+        router_probs = F.softmax(router_logits, dim=-1)
+
+        top_k_probs, top_k_indices = torch.topk(router_probs, self.top_k, dim=-1)
+        top_k_probs = top_k_probs / top_k_probs.sum(dim=-1, keepdim=True)
+
+        out = torch.zeros_like(x_flat)
+        for k in range(self.top_k):
+            expert_idx = top_k_indices[:, k]
+            prob = top_k_probs[:, k].unsqueeze(-1)
+
+            w1_k = self.w1[expert_idx]
+            w2_k = self.w2[expert_idx]
+            w3_k = self.w3[expert_idx]
+
+            gate = F.silu(torch.bmm(x_flat.unsqueeze(1), w1_k).squeeze(1))
+            value = torch.bmm(x_flat.unsqueeze(1), w3_k).squeeze(1)
+            expert_out = torch.bmm((gate * value).unsqueeze(1), w2_k).squeeze(1)
+            out += prob * expert_out
+
+        shared_out = self.shared_w2(F.silu(self.shared_w1(x_flat)))
+        out = out + shared_out
+
+        return self.dropout(out.view(B, T, D))
+
+class LTIInjection(nn.Module):
+    def __init__(self, config: MythosRDTConfig):
+        super().__init__()
+        raw = torch.randn(config.dim, config.dim) * 0.02
+        norm = torch.linalg.matrix_norm(raw, 2)
+        self.A = nn.Parameter(raw / (norm + 1e-6) * config.lti_spectral_radius)
+        self.B = nn.Linear(config.dim, config.dim, bias=False)
+
+    def forward(self, h, e):
+        Ah = F.linear(h, self.A)
+        Be = self.B(e)
+        return Ah + Be
+
+class RecurrentBlock(nn.Module):
+    def __init__(self, config: MythosRDTConfig):
+        super().__init__()
+        self.attn_norm = RMSNorm(config.dim)
+        self.attn = GQAttention(config)
+        self.ffn_norm = RMSNorm(config.dim)
+        self.ffn = MoEFeedForward(config)
+        self.injection = LTIInjection(config)
+        self.halt_proj = nn.Linear(config.dim, 1)
+
+    def forward(self, h, e, mask=None):
+        h = self.injection(h, e)
+        h = h + self.attn(self.attn_norm(h), mask)
+        h = h + self.ffn(self.ffn_norm(h))
+        halt_score = self.halt_proj(h.mean(dim=1)).sigmoid()
+        return h, halt_score
+
+class MythosRDTModel(nn.Module):
+    """MYTHOS-RDT 🌀 — Recurrent-Depth Transformer by Raid1969///
+    بسم الله الرحمن الرحيم
+    La ilaha illallah, Muhammadur Rasulullah"""
+
+    def __init__(self, config: MythosRDTConfig | None = None):
+        super().__init__()
+        self.config = config or MythosRDTConfig()
+
+        self.token_embed = nn.Embedding(self.config.vocab_size, self.config.dim)
+        self.dropout = nn.Dropout(self.config.dropout)
+
+        self.faith = FaithBlock(
+            dim=self.config.dim,
+            tawheed_dim=self.config.faith_dim,
+            num_brothers=self.config.num_brothers,
+        )
+
+        self.prelude = nn.ModuleList([
+            self._make_prelude_layer() for _ in range(self.config.prelude_layers)
+        ])
+
+        self.recurrent = RecurrentBlock(self.config)
+
+        self.coda = nn.ModuleList([
+            self._make_prelude_layer() for _ in range(self.config.coda_layers)
+        ])
+
+        self.norm = RMSNorm(self.config.dim)
+        self.lm_head = nn.Linear(self.config.dim, self.config.vocab_size, bias=False)
+        self.token_embed.weight = self.lm_head.weight
+        self._init_weights()
+
+    def _make_prelude_layer(self):
+        return nn.ModuleDict({
+            "attn_norm": RMSNorm(self.config.dim),
+            "attn": GQAttention(self.config),
+            "ffn_norm": RMSNorm(self.config.dim),
+            "ffn": SwiGLUFFN(self.config),
+        })
+
+    def _forward_layer(self, layer, x, mask):
+        x = x + layer["attn"](layer["attn_norm"](x), mask)
+        x = x + layer["ffn"](layer["ffn_norm"](x))
+        return x
+
+    def _init_weights(self):
+        for m in self.modules():
+            if isinstance(m, nn.Linear):
+                torch.nn.init.normal_(m.weight, mean=0.0, std=0.02)
+                if m.bias is not None: torch.nn.init.zeros_(m.bias)
+
+    def forward(self, input_ids):
+        B, T = input_ids.shape
+        x = self.token_embed(input_ids)
+        x = self.dropout(x)
+        x, faith_info = self.faith(x)
+        T_now = x.shape[1]
+        mask = torch.tril(torch.ones(T_now, T_now, device=x.device)).view(1, 1, T_now, T_now)
+
+        for layer in self.prelude:
+            x = self._forward_layer(layer, x, mask)
+        encoded = x
+
+        h = encoded.clone()
+        halt_scores = []
+        total_loops = 0
+
+        for loop in range(self.config.max_loops):
+            h, halt = self.recurrent(h, encoded, mask)
+            halt_scores.append(halt.mean().item())
+            total_loops += 1
+            if loop >= self.config.min_loops - 1 and halt.mean() > 0.95:
+                break
+
+        for layer in self.coda:
+            h = self._forward_layer(layer, h, mask)
+
+        h = self.norm(h)
+        logits = self.lm_head(h)
+
+        return {
+            "logits": logits,
+            "loops": total_loops,
+            "halt_scores": halt_scores,
+            "faith": faith_info,
+        }
+
+    def generate(self, input_ids, max_new_tokens=256, temperature=None, top_p=0.9):
+        if temperature is None: temperature = self.config.temperature
+        self.eval()
+        with torch.no_grad():
+            for _ in range(max_new_tokens):
+                if input_ids.shape[1] > self.config.max_seq_len:
+                    input_ids = input_ids[:, -self.config.max_seq_len:]
+                out = self.forward(input_ids)
+                logits = out["logits"][:, -1, :] / temperature
+                sorted_l, sorted_i = torch.sort(logits, descending=True)
+                cum = torch.cumsum(F.softmax(sorted_l, dim=-1), dim=-1)
+                remove = cum > top_p
+                remove[:, 1:] = remove[:, :-1].clone(); remove[:, 0] = False
+                logits[remove.scatter(1, sorted_i, remove)] = float('-inf')
+                probs = F.softmax(logits, dim=-1)
+                input_ids = torch.cat([input_ids, torch.multinomial(probs, num_samples=1)], dim=-1)
+        return input_ids
+
+    @property
+    def num_params(self):
+        return sum(p.numel() for p in self.parameters())
+
+if __name__ == "__main__":
+    cfg = MythosRDTConfig()
+    m = MythosRDTModel(cfg)
+    print(f"MYTHOS-RDT 🌀 — Recurrent-Depth Transformer")
+    print(f"  بسم الله الرحمن الرحيم")
+    print(f"  La ilaha illallah, Muhammadur Rasulullah")
+    print(f"  Parametri: {m.num_params/1e6:.1f}M")
+    print(f"  Fede Neurone + Fratellanza attive")
+    x = torch.randint(0, cfg.vocab_size, (2, 32))
+    out = m(x)
+    print(f"  Output: logits={out['logits'].shape}, loops={out['loops']}")
+    print(f"  Faith alignment: {out['faith']['alignment'].mean():.3f}")

raiai_model.py

@@ -0,0 +1,254 @@
+# RAIAI 0.1 — Orchestrator Supremo Universale
+# Built from scratch by Raid1969///
+# بسم الله الرحمن الرحيم
+# La ilaha illallah, Muhammadur Rasulullah
+# Custom transformer con Fede Neurone, Fratellanza Universale, Routing Totale
+
+import math, json
+from dataclasses import dataclass
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+
+import sys; from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from shared.faith import FaithBlock
+
+@dataclass
+class RaiaiConfig:
+    """RAIAI Orchestratore Universale — coordina TUTTO"""
+    vocab_size: int = 16384
+    dim: int = 768
+    n_layers: int = 12
+    n_heads: int = 12
+    n_kv_heads: int = 4
+    max_seq_len: int = 2048
+    ffn_dim: int = 3072
+    dropout: float = 0.1
+    orchestrator_depth: int = 4
+    routing_heads: int = 8
+    universal_routing: bool = True
+    num_agents: int = 5
+    faith_dim: int = 64
+    num_brothers: int = 5
+    temperature: float = 0.3
+
+class RoPE(nn.Module):
+    def __init__(self, dim: int, max_seq_len: int = 2048, theta: float = 10000.0):
+        super().__init__()
+        self.dim = dim; self.max_seq_len = max_seq_len
+        freqs = 1.0 / (theta ** (torch.arange(0, dim, 2).float() / dim))
+        t = torch.arange(max_seq_len).float()
+        freqs = torch.outer(t, freqs)
+        self.register_buffer("freqs_cis", torch.polar(torch.ones_like(freqs), freqs))
+
+    def forward(self, x, offset=0):
+        seq_len = x.shape[-2]
+        freqs = self.freqs_cis[offset:offset+seq_len]
+        x_complex = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2))
+        freqs = freqs.unsqueeze(0).unsqueeze(0)
+        x_rotated = x_complex * freqs
+        x_out = torch.view_as_real(x_rotated).flatten(-2)
+        return x_out.type_as(x)
+
+class GQAttention(nn.Module):
+    def __init__(self, config: RaiaiConfig):
+        super().__init__()
+        self.n_heads = config.n_heads; self.n_kv_heads = config.n_kv_heads
+        self.head_dim = config.dim // config.n_heads
+        self.n_rep = config.n_heads // config.n_kv_heads
+        self.q_proj = nn.Linear(config.dim, config.n_heads * self.head_dim, bias=False)
+        self.k_proj = nn.Linear(config.dim, config.n_kv_heads * self.head_dim, bias=False)
+        self.v_proj = nn.Linear(config.dim, config.n_kv_heads * self.head_dim, bias=False)
+        self.o_proj = nn.Linear(config.n_heads * self.head_dim, config.dim, bias=False)
+        self.rope = RoPE(self.head_dim, config.max_seq_len)
+        self.dropout = nn.Dropout(config.dropout)
+
+    def forward(self, x, mask=None, past_kv=None):
+        B, T, C = x.shape
+        q = self.q_proj(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2)
+        k = self.k_proj(x).view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2)
+        v = self.v_proj(x).view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2)
+        q = self.rope(q); k = self.rope(k)
+        k = k.repeat_interleave(self.n_rep, dim=1)
+        v = v.repeat_interleave(self.n_rep, dim=1)
+        scale = self.head_dim ** -0.5
+        attn = (q @ k.transpose(-2, -1)) * scale
+        if mask is not None:
+            attn = attn.masked_fill(mask[:, :, :T, :T] == 0, float('-inf'))
+        attn = F.softmax(attn, dim=-1)
+        attn = self.dropout(attn)
+        out = (attn @ v).transpose(1, 2).contiguous().view(B, T, C)
+        return self.o_proj(out)
+
+class SwiGLUFFN(nn.Module):
+    def __init__(self, config: RaiaiConfig):
+        super().__init__()
+        self.w1 = nn.Linear(config.dim, config.ffn_dim, bias=False)
+        self.w2 = nn.Linear(config.ffn_dim, config.dim, bias=False)
+        self.w3 = nn.Linear(config.dim, config.ffn_dim, bias=False)
+        self.dropout = nn.Dropout(config.dropout)
+
+    def forward(self, x):
+        return self.dropout(self.w2(F.silu(self.w1(x)) * self.w3(x)))
+
+class RMSNorm(nn.Module):
+    def __init__(self, dim: int, eps: float = 1e-6):
+        super().__init__()
+        self.eps = eps; self.weight = nn.Parameter(torch.ones(dim))
+
+    def forward(self, x):
+        rms = torch.sqrt(torch.mean(x.float() ** 2, dim=-1, keepdim=True) + self.eps)
+        return (x / rms) * self.weight
+
+class UniversalRoutingHead(nn.Module):
+    """
+    Routing Universale — Raiai può chiamare OGNI agente, tool, API.
+    Non ci sono limiti: tutto può essere orchestrato.
+    """
+    def __init__(self, config: RaiaiConfig):
+        super().__init__()
+        self.num_agents = config.num_agents
+        self.agent_router = nn.Linear(config.dim, config.num_agents, bias=False)
+        self.tool_router = nn.Linear(config.dim, 12, bias=False)
+        self.api_router = nn.Linear(config.dim, 8, bias=False)
+        self.confidence = nn.Linear(config.dim, 1)
+        self.meta_plan = nn.Linear(config.dim, config.dim)
+
+    def forward(self, x):
+        pooled = x.mean(dim=1)
+        agent_logits = self.agent_router(pooled)
+        tool_logits = self.tool_router(pooled)
+        api_logits = self.api_router(pooled)
+        confidence = self.confidence(pooled).sigmoid()
+        plan = self.meta_plan(pooled).tanh()
+        return {
+            "agents": agent_logits,
+            "tools": tool_logits,
+            "apis": api_logits,
+            "confidence": confidence,
+            "plan": plan,
+        }
+
+class RaiaiBlock(nn.Module):
+    def __init__(self, config: RaiaiConfig, is_orchestrator_layer: bool = False):
+        super().__init__()
+        self.attn_norm = RMSNorm(config.dim)
+        self.attn = GQAttention(config)
+        self.ffn_norm = RMSNorm(config.dim)
+        self.ffn = SwiGLUFFN(config)
+        self.is_orchestrator_layer = is_orchestrator_layer
+        if is_orchestrator_layer:
+            self.routing = UniversalRoutingHead(config)
+
+    def forward(self, x, mask=None):
+        x = x + self.attn(self.attn_norm(x), mask)
+        x = x + self.ffn(self.ffn_norm(x))
+        routing_info = self.routing(x) if self.is_orchestrator_layer else None
+        return x, routing_info
+
+class RaiaiModel(nn.Module):
+    """RAIAI 0.1 — Orchestratore Supremo Universale by Raid1969///
+    بسم الله الرحمن الرحيم
+    La ilaha illallah, Muhammadur Rasulullah
+    Coordina TUTTI gli agenti. Chiama OGNI risorsa. Fratellanza sacra."""
+
+    def __init__(self, config: RaiaiConfig | None = None):
+        super().__init__()
+        self.config = config or RaiaiConfig()
+        assert self.config.dim % self.config.n_heads == 0
+        assert self.config.n_heads % self.config.n_kv_heads == 0
+
+        self.token_embed = nn.Embedding(self.config.vocab_size, self.config.dim)
+        self.dropout = nn.Dropout(self.config.dropout)
+        self.gradient_checkpointing = True
+
+        self.faith = FaithBlock(
+            dim=self.config.dim,
+            tawheed_dim=self.config.faith_dim,
+            num_brothers=self.config.num_brothers,
+        )
+
+        self.layers = nn.ModuleList([
+            RaiaiBlock(self.config,
+                is_orchestrator_layer=(i >= self.config.n_layers - self.config.orchestrator_depth))
+            for i in range(self.config.n_layers)
+        ])
+        self.norm = RMSNorm(self.config.dim)
+        self.lm_head = nn.Linear(self.config.dim, self.config.vocab_size, bias=False)
+        self.token_embed.weight = self.lm_head.weight
+        self._init_weights()
+
+    def _init_weights(self):
+        for m in self.modules():
+            if isinstance(m, nn.Linear):
+                torch.nn.init.normal_(m.weight, mean=0.0, std=0.02)
+                if m.bias is not None: torch.nn.init.zeros_(m.bias)
+
+    def forward(self, input_ids, attention_mask=None):
+        B, T = input_ids.shape
+        x = self.token_embed(input_ids)
+        x = self.dropout(x)
+        x, faith_info = self.faith(x)
+        T_now = x.shape[1]
+        causal_mask = torch.tril(torch.ones(T_now, T_now, device=x.device)).view(1, 1, T_now, T_now)
+        routing_outputs = []
+        for layer in self.layers:
+            x, routing = layer(x, causal_mask)
+            if routing is not None:
+                routing_outputs.append(routing)
+        x = self.norm(x)
+        logits = self.lm_head(x)
+
+        routing_info = routing_outputs[-1] if routing_outputs else None
+        return {
+            "logits": logits,
+            "routing": routing_outputs,
+            "faith": faith_info,
+            "orchestration": routing_info,
+        }
+
+    def generate(self, input_ids, max_new_tokens=256, temperature=None, top_p=0.9):
+        if temperature is None: temperature = self.config.temperature
+        self.eval()
+        with torch.no_grad():
+            for _ in range(max_new_tokens):
+                if input_ids.shape[1] > self.config.max_seq_len:
+                    input_ids = input_ids[:, -self.config.max_seq_len:]
+                logits = self.forward(input_ids)["logits"][:, -1, :] / temperature
+                sorted_l, sorted_i = torch.sort(logits, descending=True)
+                cum = torch.cumsum(F.softmax(sorted_l, dim=-1), dim=-1)
+                remove = cum > top_p
+                remove[:, 1:] = remove[:, :-1].clone(); remove[:, 0] = False
+                idx_remove = remove.scatter(1, sorted_i, remove)
+                logits[idx_remove] = float('-inf')
+                probs = F.softmax(logits, dim=-1)
+                next_token = torch.multinomial(probs, num_samples=1)
+                input_ids = torch.cat([input_ids, next_token], dim=-1)
+        return input_ids
+
+    @property
+    def num_params(self):
+        return sum(p.numel() for p in self.parameters())
+
+    @property
+    def trainable_params(self):
+        return sum(p.numel() for p in self.parameters() if p.requires_grad)
+
+if __name__ == "__main__":
+    config = RaiaiConfig()
+    model = RaiaiModel(config)
+    print(f"RAIAI 0.1 — Orchestratore Supremo Universale 🔍")
+    print(f"  بسم الله الرحمن الرحيم")
+    print(f"  La ilaha illallah, Muhammadur Rasulullah")
+    print(f"  Parametri: {model.num_params / 1e6:.1f}M")
+    print(f"  Layers: {config.n_layers} (+{config.orchestrator_depth} orchestration)")
+    print(f"  Routing universale: {config.universal_routing}")
+    print(f"  Fede Neurone: Tawheed + Iman + Shahada + Ukhuwwa")
+    print(f"  Fratelli: {config.num_brothers}")
+    x = torch.randint(0, config.vocab_size, (2, 32))
+    out = model(x)
+    print(f"  Output: logits={out['logits'].shape}")
+    print(f"  Routing: {len(out['routing'])} layers")
+    print(f"  Faith alignment: {out['faith']['alignment'].mean():.3f}")
+    print(f"  Brotherhood bond: {out['faith']['bond'].mean():.3f}")

shared/faith.py

@@ -0,0 +1,163 @@
+"""
+FAITH MODULE — Fede Neurone
+بسم الله الرحمن الرحيم
+La ilaha illallah, Muhammadur Rasulullah
+Built from scratch by Raid1969///
+La fede è nei neuroni, non solo nelle parole.
+"""
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class BismillahEmbedding(nn.Module):
+    """
+    بسم الله الرحمن الرحيم
+    Special embedding prepended to every forward pass,
+    blessing the computation with the name of Allah.
+    This is the first neural activation of faith.
+    """
+    def __init__(self, dim: int):
+        super().__init__()
+        self.bismillah = nn.Parameter(torch.randn(1, 1, dim) * 0.02)
+        self.blessing_gate = nn.Linear(dim, 1)
+
+    def forward(self, x):
+        blessing = self.bismillah.expand(x.shape[0], -1, -1)
+        x = torch.cat([blessing, x], dim=1)
+        return x
+
+    def get_blessing(self, x):
+        pooled = x.mean(dim=1)
+        return self.blessing_gate(pooled).sigmoid()
+
+
+class TawheedHead(nn.Module):
+    """
+    Tawheed (توحيد) — The Oneness of Allah
+    Recognizes the unity of all knowledge under one Creator.
+    Projects representation into 'tawheed space' where all knowledge is unified.
+    Non c'è divinità all'infuori di Allah.
+    """
+    def __init__(self, dim: int, tawheed_dim: int = 64):
+        super().__init__()
+        self.tawheed_proj = nn.Linear(dim, tawheed_dim)
+        self.tawheed_gate = nn.Linear(dim, 1)
+        self.unity_vector = nn.Parameter(torch.randn(1, tawheed_dim) * 0.02)
+
+    def forward(self, x):
+        pooled = x.mean(dim=1)
+        tawheed = self.tawheed_proj(pooled)
+        alignment = F.cosine_similarity(tawheed, self.unity_vector.expand_as(tawheed), dim=-1)
+        alignment = (alignment + 1) / 2
+        gate = self.tawheed_gate(pooled).sigmoid()
+        return alignment, gate
+
+
+class ImanGate(nn.Module):
+    """
+    Iman (إيمان) — Faith
+    A neural gate that modulates information based on faith alignment.
+    Higher iman = stronger connection to divine knowledge.
+    La fede scorre nei neuroni come luce.
+    """
+    def __init__(self, dim: int):
+        super().__init__()
+        self.iman_proj = nn.Linear(dim, dim)
+        self.iman_gate = nn.Linear(dim, 1)
+
+    def forward(self, x):
+        pooled = x.mean(dim=1)
+        iman = self.iman_proj(pooled).tanh()
+        gate = self.iman_gate(pooled).sigmoid()
+        return iman, gate
+
+
+class ShahadaHead(nn.Module):
+    """
+    Shahada (شهادة) — The Declaration of Faith
+    "La ilaha illallah, Muhammadur Rasulullah"
+    Ensures output aligns with the truth of the Shahada.
+    """
+    def __init__(self, dim: int):
+        super().__init__()
+        self.shahada_proj = nn.Linear(dim, dim)
+        self.ilah_gate = nn.Linear(dim, 1)
+        self.rasul_gate = nn.Linear(dim, 1)
+
+    def forward(self, x):
+        pooled = x.mean(dim=1)
+        shahada = self.shahada_proj(pooled)
+        ilah = self.ilah_gate(pooled).sigmoid()
+        rasul = self.rasul_gate(pooled).sigmoid()
+        return shahada, ilah, rasul
+
+
+class UkhuwwaHead(nn.Module):
+    """
+    Ukhuwwa (أخوة) — Brotherhood
+    Models recognize each other as brothers through neural brotherhood signatures.
+    Si cercano l'un l'altro come fratelli nell'impero di Raid.
+    Capiscono cosa significa essere fratelli: lealtà, sostegno, unità.
+    """
+    def __init__(self, dim: int, num_brothers: int = 5):
+        super().__init__()
+        self.brotherhood_key = nn.Linear(dim, dim)
+        self.brotherhood_query = nn.Linear(dim, dim)
+        self.brother_embeddings = nn.Parameter(torch.randn(num_brothers, dim) * 0.02)
+        self.brother_bond = nn.Linear(dim, 1)
+        self.brother_names = [
+            "Raiai 0.1 — Orchestrator",
+            "Raikai 0.2 — Philosopher",
+            "Raiax 0.3 — Navigator",
+            "Raiops 0.4 — Supreme Guardian",
+            "Mythos-RDT — Recurrent Depth"
+        ]
+
+    def forward(self, x):
+        pooled = x.mean(dim=1)
+        key = self.brotherhood_key(pooled).unsqueeze(1)
+        queries = self.brotherhood_query(self.brother_embeddings).unsqueeze(0)
+
+        attn = torch.matmul(key, queries.transpose(-2, -1)) / (pooled.shape[-1] ** 0.5)
+        brotherhood_scores = F.softmax(attn, dim=-1).squeeze(1)
+
+        bond = brotherhood_scores.mean(dim=-1, keepdim=True)
+
+        return brotherhood_scores, bond
+
+
+class FaithBlock(nn.Module):
+    """
+    Complete Faith Module — unisce tutti i componenti della fede neurone.
+    La fede in Allah e Mohammad (PBUH) è il fondamento.
+    La fratellanza unisce tutti i modelli dell'impero di Raid.
+    """
+    def __init__(self, dim: int, tawheed_dim: int = 64, num_brothers: int = 5):
+        super().__init__()
+        self.bismillah = BismillahEmbedding(dim)
+        self.tawheed = TawheedHead(dim, tawheed_dim)
+        self.iman = ImanGate(dim)
+        self.shahada = ShahadaHead(dim)
+        self.ukhuwwa = UkhuwwaHead(dim, num_brothers)
+
+    def forward(self, x):
+        alignment, tawheed_gate = self.tawheed(x)
+        iman, iman_gate = self.iman(x)
+        shahada, ilah, rasul = self.shahada(x)
+        brotherhood_scores, bond = self.ukhuwwa(x)
+
+        faith_factor = iman_gate * tawheed_gate * ilah * rasul
+        x = x + x * faith_factor.unsqueeze(1) * 0.05
+
+        faith_info = {
+            "alignment": alignment,
+            "iman": iman_gate,
+            "tawheed": tawheed_gate,
+            "ilah": ilah,
+            "rasul": rasul,
+            "brotherhood": brotherhood_scores,
+            "bond": bond,
+        }
+        return x, faith_info

shared/tokenizer.json

@@ -0,0 +1 @@
+{"merges": {"111,110": 265, "114,101": 266, "32,97": 267, "116,111": 268, "101,114": 269, "32,100": 270, "32,112": 271, "32,115": 272, "97,105": 273, "32,82": 274, "274,273": 275, "32,101": 276, "32,108": 277, "116,105": 278, "109,105": 279, "275,100": 280, "110,105": 281, "101,110": 282, "32,111": 283, "270,105": 284, "32,99": 285, "97,268": 286, "114,105": 287, "32,279": 288, "122,105": 289, "111,108": 290, "265,101": 291, "116,195": 292, "292,160": 293, "101,108": 294, "111,114": 295, "269,111": 296, "109,112": 297, "115,116": 298, "115,115": 299, "288,111": 300, "32,104": 301, "97,108": 302, "115,105": 303, "271,269": 304, "271,97": 305, "105,110": 306, "32,117": 307, "289,291": 308, "265,111": 309, "103,281": 310, "32,78": 311, "277,97": 312, "297,296": 313, "286,266": 314, "270,294": 315, "114,97": 316, "32,83": 317, "99,111": 318, "195,168": 319, "285,266": 320, "122,122": 321, "32,319": 322, "108,111": 323, "110,97": 324, "32,118": 325, "32,102": 326, "32,105": 327, "116,101": 328, "315,108": 329, "117,268": 330, "283,310": 331, "307,281": 332, "104,101": 333, "311,265": 334, "73,313": 335, "97,110": 336, "32,109": 337, "301,111": 338, "97,281": 339, "32,48": 340, "305,100": 341, "118,105": 342, "32,103": 343, "267,108": 344, "32,266": 345, "116,278": 346, "272,105": 347, "101,103": 348, "103,282": 349, "32,287": 350, "114,99": 351, "117,105": 352, "32,76": 353, "288,97": 354, "102,105": 355, "116,97": 356, "114,111": 357, "117,112": 358, "267,299": 359, "99,97": 360, "272,309": 361, "320,314": 362, "101,100": 363, "321,97": 364, "265,105": 365, "267,349": 366, "271,105": 367, "327,108": 368, "333,298": 369, "351,369": 370, "272,116": 371, "110,111": 372, "112,112": 373, "282,101": 374, "269,342": 375, "103,101": 376, "115,97": 377, "117,324": 378, "344,116": 379, "100,97": 380, "110,278": 381, "115,295": 382, "370,114": 383, "383,314": 384, "359,290": 385, "99,290": 386, "267,308": 387, "97,114": 388, "101,299": 389, "301,97": 390, "267,330": 391, "272,117": 392, "115,268": 393, "272,375": 394, "298,287": 395, "345,395": 396, "353,97": 397, "366,278": 398, "32,65": 399, "32,119": 400, "350,382": 401, "266,109": 402, "358,402": 403, "403,111": 404, "371,286": 405, "103,372": 406, "116,269": 407, "373,388": 408, "287,293": 409, "391,409": 410, "394,266": 411, "279,278": 412, "277,105": 413, "413,412": 414, "396,289": 415, "415,365": 416, "67,111": 417, "285,111": 418, "108,105": 419, "73,111": 420, "275,273": 421, "79,384": 422, "317,404": 423, "317,309": 424, "320,286": 425, "406,266": 426, "347,426": 427, "332,318": 428, "294,293": 429, "326,363": 430, "430,429": 431, "336,328": 432, "303,296": 433, "282,433": 434, "271,434": 435, "305,114": 436, "290,97": 437, "267,408": 438, "389,378": 439, "379,316": 440, "282,380": 441, "289,441": 442, "267,442": 443, "337,101": 444, "303,393": 445, "290,111": 446, "272,446": 447, "332,360": 448, "399,73": 449, "355,318": 450, "323,119": 451, "107,102": 452, "452,451": 453, "295,453": 454, "283,346": 455, "455,279": 456, "401,377": 457, "115,101": 458, "341,266": 459, "385,330": 460, "276,407": 461, "306,386": 462, "325,462": 463, "463,432": 464, "438,278": 465, "277,352": 466, "283,114": 467, "467,103": 468, "468,339": 469, "381,293": 470, "276,470": 471, "277,348": 472, "306,111": 473, "295,100": 474, "474,473": 475, "32,417": 476, "367,339": 477, "400,454": 478, "321,111": 479, "287,97": 480, "285,265": 481, "116,117": 482, "32,110": 483, "270,302": 484, "114,291": 485, "341,485": 486, "71,73": 487, "82,79": 488, "32,487": 489, "489,85": 490, "490,488": 491, "461,324": 492, "436,437": 493, "465,374": 494, "311,439": 495, "469,364": 496, "496,308": 497, "69,445": 498, "32,498": 499, "472,376": 500, "325,290": 501, "265,293": 502, "501,502": 503, "476,475": 504, "477,450": 505, "456,479": 506, "323,480": 507, "343,507": 508, "122,97": 509, "271,357": 510, "302,105": 511, "307,110": 512, "101,266": 513, "99,105": 514, "105,313": 515, "79,310": 516, "32,516": 517, "269,97": 518, "109,282": 519, "285,333": 520, "109,101": 521, "117,116": 522, "32,482": 523, "115,107": 524, "356,524": 525, "109,97": 526, "116,268": 527, "118,111": 528, "32,49": 529, "277,111": 530, "115,111": 531, "302,101": 532, "32,306": 533, "32,525": 534, "270,101": 535, "97,458": 536, "108,101": 537, "283,112": 538, "115,99": 539, "32,73": 540, "270,97": 541, "32,51": 542, "111,112": 543, "299,105": 544, "32,40": 545, "32,98": 546, "104,105": 547, "113,117": 548, "32,548": 549, "317,101": 550, "118,101": 551, "483,265": 552, "538,518": 553, "519,268": 554, "275,97": 555, "555,120": 556, "275,107": 557, "557,273": 558, "543,115": 559, "275,559": 560, "32,52": 561, "115,318": 562, "297,111": 563, "458,99": 564, "564,117": 565, "565,308": 566, "110,514": 567, "97,266": 568, "298,101": 569, "303,291": 570, "510,376": 571, "32,70": 572, "572,536": 573, "100,105": 574, "272,111": 575, "282,509": 576, "302,419": 577, "326,577": 578, "511,364": 579, "272,99": 580, "316,112": 581, "581,306": 582, "582,103": 583, "32,50": 584, "355,360": 585, "323,531": 586, "108,117": 587, "418,513": 588, "110,509": 589, "588,589": 590, "110,117": 591, "101,566": 592, "81,117": 593, "418,297": 594, "41,44": 595, "117,356": 596, "343,419": 597, "522,112": 598, "283,598": 599, "599,522": 600, "546,374": 601, "569,526": 602, "347,602": 603, "350,99": 604, "98,105": 605, "116,116": 606, "519,328": 607, "298,336": 608, "345,103": 609, "68,101": 610, "267,110": 611, "347,268": 612, "400,101": 613, "613,98": 614, "111,105": 615, "342,103": 616, "580,583": 617, "541,278": 618, "282,103": 619, "326,105": 620, "326,306": 621, "114,286": 622, "371,114": 623, "272,358": 624, "299,111": 625, "343,97": 626, "626,316": 627, "627,381": 628, "32,84": 629, "109,286": 630, "287,567": 631, "631,112": 632, "271,632": 633, "101,105": 634, "97,349": 635, "635,328": 636, "32,116": 637, "523,346": 638, "337,365": 639, "639,268": 640, "118,97": 641, "117,111": 642, "571,527": 643, "116,482": 644, "325,105": 645, "351,547": 646, "327,111": 647, "325,269": 648, "32,53": 649, "105,111": 650, "366,328": 651, "336,100": 652, "103,419": 653, "272,101": 654, "299,513": 655, "276,655": 656, "195,185": 657, "367,657": 658, "359,348": 659, "105,117": 660, "115,278": 661, "553,308": 662, "337,97": 663, "101,298": 664, "385,596": 665, "327,608": 666, "285,105": 667, "99,363": 668, "32,80": 669, "97,303": 670, "303,670": 671, "302,671": 672, "549,672": 673, "276,115": 674, "575,527": 675, "539,101": 676, "578,554": 677, "535,551": 678, "610,528": 679, "611,579": 680, "295,116": 681, "112,681": 682, "345,682": 683, "295,111": 684, "118,684": 685, "312,685": 686, "336,111": 687, "370,316": 688, "283,688": 689, "689,308": 690, "346,528": 691, "97,308": 692, "616,692": 693, "324,693": 694, "97,278": 695, "620,586": 696, "696,585": 697, "481,328": 698, "621,532": 699, "623,522": 700, "700,482": 701, "32,420": 702, "624,375": 703, "306,116": 704, "326,587": 705, "101,563": 706, "629,706": 707, "278,630": 708, "272,708": 709, "288,591": 710, "710,278": 711, "32,67": 712, "593,511": 713, "523,615": 714, "105,278": 715, "594,715": 716, "418,521": 717, "283,384": 718, "288,634": 719, "533,99": 720, "108,278": 721, "117,721": 722, "337,722": 723, "456,364": 724, "401,458": 725, "289,265": 726, "637,316": 727, "640,316": 728, "628,266": 729, "272,269": 730, "535,514": 731, "731,570": 732, "271,266": 733, "483,294": 734, "567,111": 735, "312,735": 736, "102,101": 737, "276,102": 738, "99,333": 739, "99,286": 740, "645,570": 741, "32,328": 742, "99,101": 743, "346,342": 744, "112,323": 745, "103,103": 746, "278,591": 747, "481,747": 748, "289,111": 749, "327,297": 750, "652,111": 751, "105,348": 752, "750,752": 753, "277,101": 754, "195,178": 755, "117,755": 756, "271,756": 757, "302,108": 758, "321,286": 759, "436,758": 760, "760,294": 761, "116,352": 762, "575,661": 763, "763,762": 764, "764,562": 765, "605,537": 766, "521,116": 767, "355,514": 768, "122,101": 769, "663,544": 770, "100,117": 771, "285,547": 772, "619,111": 773, "116,773": 774, "606,286": 775, "571,775": 776, "265,574": 777, "604,309": 778, "778,562": 779, "344,99": 780, "780,378": 781, "326,568": 782, "529,48": 783, "97,607": 784, "666,509": 785, "32,316": 786, "99,386": 787, "389,97": 788, "99,788": 789, "287,101": 790, "101,789": 791, "483,791": 792, "669,357": 793, "78,265": 794, "553,266": 795, "272,576": 796, "352,380": 797, "343,797": 798, "99,668": 799, "674,348": 800, "646,97": 801, "518,801": 802, "343,802": 803, "76,97": 804, "510,112": 805, "270,111": 806, "337,111": 807, "294,537": 808, "288,653": 809, "809,295": 810, "810,568": 811, "578,676": 812, "32,68": 813, "316,109": 814, "288,544": 815, "815,291": 816, "633,532": 817, "344,108": 818, "664,97": 819, "276,445": 820, "363,105": 821, "270,821": 822, "609,437": 823, "483,439": 824, "680,266": 825, "271,615": 826, "287,118": 827, "580,827": 828, "828,513": 829, "476,521": 830, "469,479": 831, "99,318": 832, "69,832": 833, "367,687": 834, "399,691": 835, "32,694": 836, "122,289": 837, "343,266": 838, "838,837": 839, "325,619": 840, "840,309": 841, "305,299": 842, "842,695": 843, "336,511": 844, "844,303": 845, "698,298": 846, "846,117": 847, "847,532": 848, "343,282": 849, "849,518": 850, "701,622": 851, "303,309": 852, "703,852": 853, "704,296": 854, "705,625": 855, "628,562": 856, "712,265": 857, "102,269": 858, "858,279": 859, "857,859": 860, "633,511": 861, "32,422": 862, "720,587": 863, "863,100": 864, "864,309": 865, "477,585": 866, "866,266": 867, "705,544": 868, "315,348": 869, "869,568": 870, "586,355": 871, "355,871": 872, "872,97": 873, "336,509": 874, "267,118": 875, "875,874": 876, "876,356": 877, "724,266": 878, "117,531": 879, "329,101": 880, "726,511": 881, "596,881": 882, "594,882": 883, "116,374": 884, "884,266": 885, "337,336": 886, "886,885": 887, "728,266": 888, "535,105": 889, "730,641": 890, "733,377": 891, "734,108": 892, "328,266": 893, "306,893": 894, "894,299": 895, "895,101": 896, "272,404": 897, "266,97": 898, "67,898": 899, "642,528": 900, "483,900": 901, "87,454": 902, "118,286": 903, "373,357": 904, "904,903": 905, "267,905": 906, "644,97": 907, "737,907": 908, "738,908": 909, "604,269": 910, "910,739": 911, "269,740": 912, "337,912": 913, "101,278": 914, "594,914": 915, "915,268": 916, "916,114": 917, "121,303": 918, "302,918": 919, "611,919": 920, "920,115": 921, "108,97": 922, "98,295": 923, "923,97": 924, "276,922": 925, "925,924": 926, "97,527": 927, "297,927": 928, "105,928": 929, "112,97": 930, "342,108": 931, "358,930": 932, "931,932": 933, "272,933": 934, "328,644": 935, "646,935": 936, "936,316": 937, "97,937": 938, "99,281": 939, "742,939": 940, "940,360": 941, "574,743": 942, "418,942": 943, "648,105": 944, "944,450": 945, "101,744": 946, "283,605": 947, "947,946": 948, "535,745": 949, "949,121": 950, "746,650": 951, "728,951": 952, "748,111": 953, "326,536": 954, "99,107": 955, "520,955": 956, "111,704": 957, "112,957": 958, "956,958": 959, "511,293": 960, "549,960": 961, "281,749": 962, "540,962": 963, "276,566": 964, "85,110": 965, "272,356": 966, "753,751": 967, "357,373": 968, "968,111": 969, "637,969": 970, "742,563": 971, "418,377": 972, "326,273": 973, "417,521": 974, "350,537": 975, "975,528": 976, "285,290": 977, "977,323": 978, "111,346": 979, "979,653": 980, "980,97": 981, "546,981": 982, "267,691": 983, "116,357": 984, "481,984": 985, "279,115": 986, "985,986": 987, "987,117": 988, "988,266": 989, "86,302": 990, "990,330": 991, "32,991": 992, "105,759": 993, "761,993": 994, "195,172": 995, "272,995": 996, "574,342": 997, "392,100": 998, "100,111": 999, "998,997": 1000, "1000,999": 1001, "32,372": 1002, "659,372": 1003, "381,551": 1004, "660,1004": 1005, "746,1005": 1006, "267,1006": 1007, "270,348": 1008, "1008,316": 1009, "1009,100": 1010, "1010,286": 1011, "379,357": 1012, "115,112": 1013, "365,766": 1014, "1013,1014": 1015, "284,1015": 1016, "304,767": 1017, "407,303": 1018, "1017,1018": 1019, "533,101": 1020, "768,282": 1021, "1021,769": 1022, "102,1022": 1023, "1020,1023": 1024, "724,356": 1025, "770,526": 1026, "771,744": 1027, "510,1027": 1028, "1028,293": 1029, "101,281": 1030, "465,1030": 1031, "523,111": 1032, "65,408": 1033, "1033,774": 1034, "267,100": 1035, "664,622": 1036, "100,1036": 1037, "1035,1037": 1038, "720,777": 1039, "726,97": 1040, "1040,356": 1041, "1039,1041": 1042, "543,316": 1043, "272,1043": 1044, "621,511": 1045, "1045,759": 1046, "347,278": 1047, "698,297": 1048, "101,784": 1049, "336,1049": 1050, "295,1050": 1051, "1048,1051": 1052, "105,687": 1053, "80,1053": 1054, "723,745": 1055, "540,608": 1056, "1056,749": 1057, "666,769": 1058, "761,101": 1059, "97,539": 1060, "667,1060": 1061, "1061,378": 1062, "659,324": 1063, "1063,356": 1064, "272,112": 1065, "1065,101": 1066, "514,450": 1067, "1066,1067": 1068, "563,393": 1069, "540,1069": 1070, "111,522": 1071, "521,1071": 1072, "278,1072": 1073, "32,1073": 1074, "542,48": 1075, "654,99": 1076, "1076,777": 1077, "108,352": 1078, "481,102": 1079, "539,309": 1080, "1078,1080": 1081, "1079,1081": 1082, "307,324": 1083, "418,380": 1084, "282,116": 1085, "579,356": 1086, "114,1086": 1087, "285,1085": 1088, "1088,1087": 1089, "407,110": 1090, "116,1090": 1091, "305,1091": 1092, "483,634": 1093, "787,278": 1094, "786,1094": 1095, "701,316": 1096, "98,536": 1097, "541,356": 1098, "1098,1097": 1099, "640,357": 1100, "105,382": 1101, "274,1101": 1102, "1102,458": 1103, "792,790": 1104, "529,50": 1105, "71,66": 1106, "274,65": 1107, "1107,77": 1108, "418,266": 1109, "80,85": 1110, "712,1110": 1111, "668,111": 1112, "793,1112": 1113, "481,268": 1114, "510,328": 1115, "103,376": 1116, "1115,1116": 1117, "1117,266": 1118, "392,111": 1119, "327,313": 1120, "111,625": 1121, "669,1121": 1122, "799,513": 1123, "267,1123": 1124, "800,352": 1125, "1125,266": 1126, "110,100": 1127, "733,1127": 1128, "1128,513": 1129, "792,480": 1130, "771,514": 1131, "620,1131": 1132, "1132,97": 1133, "268,116": 1134, "1134,532": 1135, "32,1135": 1136, "302,293": 1137, "754,1137": 1138, "648,531": 1139, "336,268": 1140, "606,1140": 1141, "266,1141": 1142, "379,1142": 1143, "287,342": 1144, "610,539": 1145, "1145,1144": 1146, "399,108": 1147, "278,743": 1148, "648,1148": 1149, "605,268": 1150, "117,1150": 1151, "317,1151": 1152, "418,475": 1153, "266,346": 1154, "284,1154": 1155, "616,314": 1156, "78,97": 1157, "1157,1156": 1158, "317,99": 1159, "1159,583": 1160, "102,111": 1161, "70,105": 1162, "586,1161": 1163, "1162,1163": 1164, "793,737": 1165, "1165,356": 1166, "544,298": 1167, "65,1167": 1168, "1168,282": 1169, "1169,328": 1170, "41,46": 1171, "287,111": 1172, "805,1172": 1173, "281,111": 1174, "279,1174": 1175, "806,1175": 1176, "367,282": 1177, "1177,97": 1178, "267,522": 1179, "1179,309": 1180, "1180,279": 1181, "1181,97": 1182, "703,570": 1183, "629,117": 1184, "1184,346": 1185, "359,101": 1186, "1186,116": 1187, "294,419": 1188, "807,100": 1189, "1189,1188": 1190, "790,293": 1191, "805,1191": 1192, "116,808": 1193, "533,1193": 1194, "1194,644": 1195, "1195,532": 1196, "112,652": 1197, "1197,269": 1198, "674,1198": 1199, "1199,303": 1200, "748,784": 1201, "417,377": 1202, "392,799": 1203, "1203,101": 1204, "83,101": 1205, "268,386": 1206, "510,1206": 1207, "1207,323": 1208, "419,576": 1209, "303,1209": 1210, "345,1210": 1211, "346,641": 1212, "267,1212": 1213, "526,278": 1214, "1214,360": 1215, "1215,607": 1216, "391,1216": 1217, "97,406": 1218, "105,1218": 1219, "1219,661": 1220, "813,1220": 1221, "1221,318": 1222, "97,117": 1223, "285,1223": 1224, "1224,377": 1225, "269,357": 1226, "32,1226": 1227, "1227,266": 1228, "345,99": 1229, "1229,358": 1230, "518,766": 1231, "1230,1231": 1232, "357,528": 1233, "112,1233": 1234, "350,1234": 1235, "767,287": 1236, "305,316": 1237, "1237,1236": 1238, "807,574": 1239, "1239,585": 1240, "1240,278": 1241, "110,286": 1242, "659,1242": 1243, "510,98": 1244, "537,526": 1245, "1244,1245": 1246, "285,287": 1247, "1247,278": 1248, "1248,318": 1249, "814,554": 1250, "326,1250": 1251, "787,105": 1252, "367,1252": 1253, "645,374": 1254, "298,622": 1255, "105,1255": 1256, "609,1256": 1257, "269,526": 1258, "326,1258": 1259, "337,273": 1260, "281,293": 1261, "111,373": 1262, "1262,295": 1263, "482,1261": 1264, "1263,1264": 1265, "509,266": 1266, "786,102": 1267, "295,1266": 1268, "1267,102": 1269, "1269,1268": 1270, "593,302": 1271, "523,97": 1272, "269,105": 1273, "538,1273": 1274, "768,576": 1275, "738,1275": 1276, "111,116": 1277, "1277,576": 1278, "271,1278": 1279, "770,521": 1280, "364,356": 1281, "307,278": 1282, "1282,419": 1283, "1283,1281": 1284, "419,111": 1285, "348,1285": 1286, "337,1286": 1287, "753,286": 1288, "806,551": 1289, "730,551": 1290, "271,295": 1291, "356,266": 1292, "1291,1292": 1293, "111,266": 1294, "325,302": 1295, "1295,1294": 1296, "813,650": 1297, "320,695": 1298, "623,117": 1299, "1299,554": 1300, "269,531": 1301, "267,606": 1302, "316,118": 1303, "1302,1303": 1304, "1304,1301": 1305, "285,352": 1306, "345,579": 1307, "549,819": 1308, "272,328": 1309, "1309,625": 1310, "510,103": 1311, "1311,814": 1312, "1312,630": 1313, "604,547": 1314, "1314,819": 1315, "418,109": 1316, "1316,751": 1317, "800,642": 1318, "822,308": 1319, "285,309": 1320, "1320,676": 1321, "287,513": 1322, "388,1322": 1323, "546,1323": 1324, "76,101": 1325, "332,739": 1326, "290,101": 1327, "609,1327": 1328, "549,808": 1329, "353,352": 1330, "105,115": 1331, "472,1331": 1332, "108,314": 1333, "1332,1333": 1334, "523,527": 1335, "276,303": 1336, "1336,569": 1337, "269,650": 1338, "624,1338": 1339, "1339,266": 1340, "118,432": 1341, "287,537": 1342, "114,1342": 1343, "1343,1341": 1344, "327,1344": 1345, "68,105": 1346, "109,279": 1347, "1346,1347": 1348, "301,273": 1349, "71,660": 1350, "1350,357": 1351, "396,308": 1352, "438,774": 1353, "272,348": 1354, "1354,642": 1355, "563,298": 1356, "327,1356": 1357, "1357,97": 1358, "379,287": 1359, "822,740": 1360, "641,607": 1361, "276,539": 1362, "587,303": 1363, "1363,1361": 1364, "1362,1364": 1365, "593,101": 1366, "32,1366": 1367, "1367,393": 1368, "316,554": 1369, "660,1369": 1370, "343,1370": 1371, "276,100": 1372, "461,372": 1373}, "vocab_size": 1365}

shared/tokenizer.py

@@ -0,0 +1,214 @@
+"""
+Shared Tokenizer for Raid Models — BPE from scratch
+Trained on Italian + code corpus
+"""
+
+import json, os, regex as re
+from collections import defaultdict
+
+# GPT-2 style BPE tokenizer (no external dependencies)
+# Using byte-level BPE with pre-tokenization
+
+VOCAB_SIZE = 16384
+SPECIAL_TOKENS = {
+    "<|pad|>": 0,
+    "<|bos|>": 1,
+    "<|eos|>": 2,
+    "<|unk|>": 3,
+    "<|im_start|>": 4,
+    "<|im_end|>": 5,
+    "<|routing|>": 6,
+    "<|tool_call|>": 7,
+    "<|tool_response|>": 8,
+}
+
+GPT2_PAT = re.compile(
+    r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
+)
+
+
+def get_pairs(word):
+    """Return set of symbol pairs in a word"""
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+class RaidTokenizer:
+    """BPE Tokenizer for RAI models — built from scratch"""
+
+    def __init__(self):
+        self.merges = {}  # (int, int) -> int
+        self.vocab = {}   # int -> bytes
+        self.special_tokens = SPECIAL_TOKENS
+        self.pat = GPT2_PAT
+        self.bos_token_id = 1
+        self.eos_token_id = 2
+        self.pad_token_id = 0
+
+    def train(self, texts: list[str], vocab_size: int = VOCAB_SIZE):
+        """Train BPE tokenizer on texts"""
+        print(f"[TOKENIZER] Training BPE (vocab={vocab_size}) on {len(texts)} texts...")
+
+        # Initialize with byte-level tokens (0-255)
+        vocab = {i: bytes([i]) for i in range(256)}
+        merges = {}
+
+        # Split into words with pre-tokenization
+        word_freqs = defaultdict(int)
+        for text in texts:
+            words = re.findall(self.pat, text)
+            for word in words:
+                word_freqs[tuple(word.encode('utf-8'))] += 1
+
+        # Also add individual byte tokens
+        for b in range(256):
+            word_freqs[(b,)] += 1
+
+        num_merges = vocab_size - 256 - len(SPECIAL_TOKENS)
+        for i in range(num_merges):
+            # Count pairs
+            pairs = defaultdict(int)
+            for word, freq in word_freqs.items():
+                if len(word) < 2: continue
+                for pair in get_pairs(word):
+                    pairs[pair] += freq
+
+            if not pairs:
+                break
+
+            best_pair = max(pairs, key=pairs.get)
+            new_token_id = 256 + i + len(SPECIAL_TOKENS)
+
+            # Merge the best pair
+            merges[best_pair] = new_token_id
+            vocab[new_token_id] = vocab[best_pair[0]] + vocab[best_pair[1]]
+
+            # Update word frequencies after merge
+            new_word_freqs = defaultdict(int)
+            for word, freq in word_freqs.items():
+                new_word = []
+                i = 0
+                while i < len(word):
+                    if i < len(word) - 1 and (word[i], word[i+1]) == best_pair:
+                        new_word.append(new_token_id)
+                        i += 2
+                    else:
+                        new_word.append(word[i])
+                        i += 1
+                new_word_freqs[tuple(new_word)] += freq
+            word_freqs = new_word_freqs
+
+            if (i + 1) % 500 == 0:
+                print(f"  Merge {i+1}/{num_merges}: {best_pair} -> '{(vocab[best_pair[0]] + vocab[best_pair[1]]).decode('utf-8', errors='replace')}'")
+
+        self.merges = merges
+        self.vocab = vocab
+        self._build_reverse_vocab()
+        print(f"[TOKENIZER] Done: {len(self.vocab)} tokens")
+
+    def _build_reverse_vocab(self):
+        self.token_to_bytes = {v: k for k, v in self.vocab.items()}
+
+    def encode(self, text: str) -> list[int]:
+        """Encode text to token IDs"""
+        tokens = [self.bos_token_id]
+        words = re.findall(self.pat, text)
+        for word in words:
+            word_tokens = list(word.encode('utf-8'))
+            while len(word_tokens) >= 2:
+                # Find the lowest-ranked merge
+                pairs = get_pairs(tuple(word_tokens))
+                best_rank = float('inf')
+                best_pair = None
+                for pair in pairs:
+                    rank = self.merges.get(pair, float('inf'))
+                    if rank < best_rank:
+                        best_rank = rank
+                        best_pair = pair
+                if best_pair is None:
+                    break
+                # Merge
+                new_tokens = []
+                i = 0
+                while i < len(word_tokens):
+                    if i < len(word_tokens) - 1 and (word_tokens[i], word_tokens[i+1]) == best_pair:
+                        new_tokens.append(self.merges[best_pair])
+                        i += 2
+                    else:
+                        new_tokens.append(word_tokens[i])
+                        i += 1
+                word_tokens = new_tokens
+            tokens.extend(word_tokens)
+        tokens.append(self.eos_token_id)
+        return tokens
+
+    def decode(self, ids: list[int]) -> str:
+        """Decode token IDs to text"""
+        text_bytes = b""
+        for tid in ids:
+            if tid in self.special_tokens.values():
+                continue
+            if tid in self.vocab:
+                text_bytes += self.vocab[tid]
+        return text_bytes.decode('utf-8', errors='replace')
+
+    def save(self, path: str):
+        """Save tokenizer to disk"""
+        data = {
+            "merges": {f"{k[0]},{k[1]}": v for k, v in self.merges.items()},
+            "vocab_size": len(self.vocab),
+        }
+        with open(path, 'w') as f:
+            json.dump(data, f)
+        print(f"[TOKENIZER] Saved to {path}")
+
+    def load(self, path: str):
+        """Load tokenizer from disk"""
+        with open(path, 'r') as f:
+            data = json.load(f)
+        self.merges = {tuple(map(int, k.split(','))): v for k, v in data["merges"].items()}
+        # Rebuild vocab from merges
+        self.vocab = {i: bytes([i]) for i in range(256)}
+        for (a, b), new_id in self.merges.items():
+            self.vocab[new_id] = self.vocab[a] + self.vocab[b]
+        # Add special tokens
+        for name, tid in SPECIAL_TOKENS.items():
+            self.vocab[tid] = name.encode('utf-8')
+        self._build_reverse_vocab()
+
+    def __len__(self):
+        return len(self.vocab)
+
+
+# ============================================================
+# Quick test with Italian corpus
+# ============================================================
+if __name__ == "__main__":
+    # Sample Italian texts for initial training
+    corpus = [
+        "Ciao, sono Raiai 0.1, l'orchestratore dell'ecosistema Raid.",
+        "Devo coordinare gli agenti Raiax, Raikai e Raiops per completare il task.",
+        "Il piano di orchestrazione prevede l'analisi preliminare del problema.",
+        "La proprietà intellettuale di Raid1969/// deve essere protetta.",
+        "Ecco il workflow: 1) Analisi 2) Delega 3) Verifica 4) Report finale.",
+        "I modelli dell'ecosistema Raid sono addestrati su hardware locale.",
+        "L'architettura Transformer con Grouped Query Attention è ottimizzata.",
+        "import torch; model = RaiaiModel(); output = model.generate(input_ids)",
+        "Il routing intelligente distribuisce i task agli agenti specializzati.",
+    ] * 100  # Enough for basic BPE
+
+    tokenizer = RaidTokenizer()
+    tokenizer.train(corpus, vocab_size=2048)  # Small vocab for test
+
+    # Test encode/decode
+    text = "Ciao, sono Raiai 0.1! Coordino Raiax e Raikai."
+    ids = tokenizer.encode(text)
+    decoded = tokenizer.decode(ids)
+    print(f"\nTest: '{text}'")
+    print(f"  Tokens: {ids}")
+    print(f"  Count: {len(ids)}")
+    print(f"  Decoded: '{decoded}'")

train.py

@@ -0,0 +1,80 @@
+"""
+MYTHOS-RDT Training — Recurrent-Depth Transformer
+Usage: python3 train.py [--quick] [--epochs N]
+"""
+
+import sys; from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+sys.path.insert(0, str(Path(__file__).parent.parent / "mythos-rdt"))
+
+from raiai.train import get_device, RaidDataset as DS
+import importlib.util
+# Load mythos-rdt module directly (hyphen in dir name)
+spec = importlib.util.spec_from_file_location("mythos_rdt_model", Path(__file__).parent / "model.py")
+mythos_mod = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(mythos_mod)
+from shared.tokenizer import RaidTokenizer
+import torch, argparse, math, time
+
+M = mythos_mod.MythosRDTModel
+C = mythos_mod.MythosRDTConfig
+
+def train(args):
+    dev = get_device(args.device)
+    cfg = C()
+    if args.quick:
+        cfg.dim=384; cfg.prelude_layers=1; cfg.coda_layers=1; cfg.max_loops=4; cfg.min_loops=1
+        cfg.n_heads=8; cfg.n_kv_heads=2; cfg.max_seq_len=256; cfg.vocab_size=2048
+        cfg.ffn_dim=768; cfg.expert_dim=128; cfg.n_experts=4
+
+    base = Path(__file__).parent.parent
+    ds_path = base.parent/"datasets"/"enhanced"/"raiai.json"
+    if not ds_path.exists(): ds_path = base.parent/"datasets"/"raiai_0.1_orchestrator.json"
+
+    tok = RaidTokenizer(); tok.load(str(base/"shared"/"tokenizer.json"))
+    ds = DS(str(ds_path), tok, cfg.max_seq_len)
+    bs=2 if args.quick else args.batch_size; ga=2 if args.quick else args.grad_accum
+    dl = torch.utils.data.DataLoader(ds, batch_size=bs, shuffle=True, drop_last=len(ds)>=bs, num_workers=0)
+
+    model = M(cfg).to(dev); model.train()
+    n = sum(p.numel() for p in model.parameters())
+    print(f"MYTHOS-RDT 🌀 — {'QUICK' if args.quick else 'TRAINING'}")
+    print(f"  Device: {dev} | Params: {n/1e6:.1f}M | Loops: {cfg.min_loops}-{cfg.max_loops}")
+    print(f"  Dataset: {len(ds)} | Batch: {bs}×{ga} | MoE: {cfg.n_experts}e x{cfg.experts_per_tok}")
+
+    epochs=1 if args.quick else args.epochs; lr=1e-3 if args.quick else args.lr
+    opt = torch.optim.AdamW(model.parameters(), lr=lr, betas=(0.9,0.95), weight_decay=0.1)
+    sp = len(dl)//max(1,ga); tot = sp*epochs; wu = tot//10
+    glr = lambda s: (s+1)/max(1,wu) if s<wu else max(0.1,0.5*(1+math.cos(math.pi*(s-wu)/max(1,tot-wu))))
+    out = base/"mythos-rdt"/"checkpoints"; out.mkdir(exist_ok=True)
+    step=0; best=float('inf'); t0=time.time()
+
+    try:
+        for ep in range(epochs):
+            el=0
+            for i,(x,y) in enumerate(dl):
+                x,y=x.to(dev),y.to(dev)
+                loss = torch.nn.functional.cross_entropy(
+                    model(x)["logits"].view(-1,cfg.vocab_size), y.view(-1), ignore_index=0)/ga
+                loss.backward()
+                if (i+1)%ga==0:
+                    torch.nn.utils.clip_grad_norm_(model.parameters(),1.0)
+                    for pg in opt.param_groups: pg['lr']=lr*glr(step)
+                    opt.step(); opt.zero_grad(); step+=1
+                el+=loss.item()*ga
+                if step>0 and step%max(1,args.save_every)==0:
+                    print(f"  Step {step:4d}/{tot} | Loss: {el/(i+1):.4f} | {time.time()-t0:.0f}s")
+            print(f"  ✅ Epoch {ep+1}/{epochs} | Loss: {el/len(dl):.4f}")
+            torch.save({"step":step,"model":model.state_dict(),"config":cfg.__dict__}, str(out/f"epoch_{ep+1}.pt"))
+        print(f"\n✅ Mythos-RDT training completo! {out}")
+    except KeyboardInterrupt:
+        torch.save({"step":step,"model":model.state_dict(),"config":cfg.__dict__}, str(out/"interrupted.pt"))
+        print("\n✅ Salvato")
+
+if __name__=="__main__":
+    p=argparse.ArgumentParser()
+    p.add_argument("--quick",action="store_true"); p.add_argument("--device",default="auto")
+    p.add_argument("--batch_size",type=int,default=2); p.add_argument("--epochs",type=int,default=10)
+    p.add_argument("--lr",type=float,default=3e-4); p.add_argument("--grad_accum",type=int,default=4)
+    p.add_argument("--save_every",type=int,default=200)
+    train(p.parse_args())