min-spark-preview — Meiosis looped hybrid demo (ZeroGPU)

.gitignore

@@ -0,0 +1,3 @@
+# Binaries live in the HF Space repo, not git (CLAUDE.md: no binaries).
+assets/meiosis.safetensors
+assets/tokenizer.json

README.md

@@ -1,13 +1,54 @@
 ---
-title: Min Spark Preview
-emoji: 🌍
-colorFrom: indigo
-colorTo: green
+title: min-spark · Meiosis preview
+emoji: 🧬
+colorFrom: yellow
+colorTo: indigo
 sdk: gradio
-sdk_version: 6.20.0
-python_version: '3.12'
+sdk_version: "5.50.0"
 app_file: app.py
+python_version: "3.12"
 pinned: false
+license: apache-2.0
+tags:
+  - text-generation
+  - language-model
+  - research
+short_description: A 5.76M looped-hybrid LM — pick the effort and prompt it.
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# min-spark · Meiosis preview
+
+A research demo for **Meiosis** — PICO release 01, a sub-10M-parameter
+decoder-only **looped-hybrid** language model trained from scratch on ~10B
+tokens of filtered fineweb-edu + finemath.
+
+The one mechanic: a single weight-shared body block runs **K times per token**.
+This Space lets you set that effort (K = 2, 3, or 4) and prompt the model on a
+free CPU — ~20 tokens/sec, sized for showing the mechanic, not for throughput.
+
+## K-split
+
+The loop-count trade-off is real (measured on the lm-eval harness):
+
+- **K = 2** — favors commonsense tasks (PIQA, HellaSwag)
+- **K = 3** — favors grammar (BLiMP), the default
+- **K = 4** — deeper grammar passes; diminishing returns on CPU
+
+## What's in here
+
+- `assets/meiosis.safetensors` — the decay-p09 release candidate (23 MB, fp32)
+- `assets/tokenizer.json` — byte-level BPE, vocab 4096 (ADR-0010)
+- `assets/meiosis.py` — the model definition (torch-only, vendored)
+- `loader.py` — model + tokenizer load, generation (mirrors PICO's `infer.py`)
+- `app.py` — the Gradio interface
+
+## Provenance
+
+The preview squeeze gate (2026-07-27) tested whether weight-space averaging of
+late trunk pins (SWA / EMA / Model Stock over p05–p09 + decay-p09) beat the
+final decay-p09 checkpoint on reserved-val NLL. It did **not** — averaging across
+mixed WSD phases regressed val perplexity — so the Space ships decay-p09 as-is.
+See `PICO/results/post_train/preview_winner.json`.
+
+PICO is a monthly series of cheap, fully-trained-and-evaluated small models.
+Source: [eclipse-senpai/PICO](https://github.com/eclipse-senpai/PICO).

app.py

@@ -0,0 +1,226 @@
+"""min-spark-preview — a research demo for Meiosis, PICO release 01.
+
+A 5.76M-parameter looped-hybrid language model. The signature mechanic: one
+weight-shared body block runs `K` times per token (the "effort"). More passes
+sharpen grammar; fewer favor commonsense. The Space makes the loop visible.
+
+Runs on ZeroGPU (free for the creator; visitors consume their own quota). The
+model is tiny so cold-start weight streaming is near-instant.
+"""
+from __future__ import annotations
+
+import spaces  # MUST precede any torch / CUDA-touching import (ZeroGPU hijack)
+import torch
+import gradio as gr
+
+from loader import load_model, load_tokenizer, generate
+
+# ── load at module scope, .to("cuda") eagerly so the hijack packs weights ────
+# "cuda" as a STRING (never an int device id) — ZeroGPU re-allocs device ids.
+print("Loading Meiosis on ZeroGPU ...")
+DEVICE = "cuda"
+MODEL = load_model(DEVICE)
+TOKENIZER = load_tokenizer()
+_PARAMS = sum(p.numel() for p in MODEL.parameters())
+print(f"  {round(_PARAMS/1e6, 2)}M params ready on {DEVICE}")
+
+EFFORTS = [
+    {"id": "low", "k": 2, "name": "Low", "tag": "commonsense"},
+    {"id": "med", "k": 3, "name": "Medium", "tag": "grammar", "default": True},
+    {"id": "high", "k": 4, "name": "High", "tag": "deeper grammar"},
+]
+EFFORT_K = {e["id"]: e["k"] for e in EFFORTS}
+_MAX_K = max(e["k"] for e in EFFORTS)
+
+
+# ── the effort rail (signature) ─────────────────────────────────────────────
+def effort_html(selected_id: str = "med") -> str:
+    cards = []
+    for e in EFFORTS:
+        sel = "selected" if e["id"] == selected_id else ""
+        dots = "".join(
+            f'<span class="dot {"lit" if i < e["k"] else ""}"></span>'
+            for i in range(_MAX_K)
+        )
+        cards.append(f"""
+        <button class="effort-card {sel}" data-id="{e['id']}" type="button">
+          <span class="eff-label">{e['name']}</span>
+          <span class="eff-k">K={e['k']}</span>
+          <span class="eff-dots">{dots}</span>
+          <span class="eff-tag">{e['tag']}</span>
+        </button>""")
+    return f'<div id="effort-rail" max-k="{_MAX_K}">{"".join(cards)}</div>'
+
+
+# On load: attach click listeners to the cards; track selection in window.__effort.
+ATTACH_JS = """
+() => {
+  window.__effort = "med";
+  const rail = document.getElementById('effort-rail');
+  if (!rail) return;
+  rail.querySelectorAll('.effort-card').forEach(btn => {
+    btn.addEventListener('click', () => {
+      rail.querySelectorAll('.effort-card').forEach(b => b.classList.remove('selected'));
+      btn.classList.add('selected');
+      window.__effort = btn.dataset.id;
+    });
+  });
+}
+"""
+
+# Generate-click preamble: override the effort slot with the JS-tracked selection.
+READ_EFFORT_JS = """
+(prompt, eff, mx, t, k) => [prompt, window.__effort || eff || "med", mx, t, k]
+"""
+
+
+def _estimate_duration(prompt, effort, max_new, temperature, top_k):
+    # Tiny model on an RTX PRO 6000: cold-start weight stream (~1-2s) + ~0.5s
+    # per 128 tokens. Declare the realistic worst case; cap polite.
+    return min(60, 8 + int(max_new) * 0.1)
+
+
+@spaces.GPU(duration=_estimate_duration)
+def run_generate(prompt: str, effort: str, max_new: int,
+                 temperature: float, top_k: int):
+    """Generate text from a prompt. `effort` sets the loop count K (2/3/4):
+    more passes sharpen grammar, fewer favor commonsense. Streams token-by-token."""
+    picked = EFFORT_K.get(effort or "med", 3)
+    if not str(prompt).strip():
+        yield ("<div class='outbox' id='specimen'></div>"
+               "<div class='status'><span class='empty'>Type a prompt to start.</span></div>")
+        return
+    outs = []
+    last_count = 0
+    last_tps = 0.0
+    for text, count, tps in generate(
+            MODEL, TOKENIZER, prompt, loops=picked, max_new=int(max_new),
+            temperature=float(temperature), top_k=int(top_k), device=DEVICE):
+        outs.append(text)
+        last_tps, last_count = tps, count
+        status = (f"<div class='status'>"
+                  f"<span class='k-chip'>K={picked}</span>"
+                  f"<span class='stat'>{last_count} tok</span>"
+                  f"<span class='stat'>{last_tps:.1f} tok/s</span>"
+                  f"<span class='stat'>ZeroGPU</span></div>")
+        yield f"<div class='outbox' id='specimen'>{''.join(outs)}</div>", status
+    status = (f"<div class='status'>"
+              f"<span class='k-chip'>K={picked}</span>"
+              f"<span class='stat'>{last_count} tok</span>"
+              f"<span class='stat'>{last_tps:.1f} tok/s</span>"
+              f"<span class='done'>done</span></div>")
+    yield f"<div class='outbox' id='specimen'>{''.join(outs)}</div>", status
+
+
+CSS = """
+@import url('https://fonts.googleapis.com/css2?family=Fraunces:opsz,wght@9..144,300;9..144,500&family=Inter:wght@400;500;600&family=IBM+Plex+Mono:wght@400;500&display=swap');
+
+:root{
+  --bg:#13131A; --surface:#1C1C26; --surface2:#22222E;
+  --hair:#2A2A38; --text:#E8E6DF; --muted:#8A8A98;
+  --amber:#E8A547; --amber-dim:#8a6a2e; --teal:#4FB3A0;
+  --r:10px;
+}
+*{box-sizing:border-box;}
+body, .gradio-container, .gradio-container * { background:var(--bg) !important; color:var(--text) !important; font-family:'Inter',system-ui,sans-serif !important; }
+.gradio-container{ max-width:980px !important; padding:28px 20px 48px !important; }
+
+.head{ display:flex; align-items:baseline; gap:16px; padding-bottom:6px; flex-wrap:wrap; }
+.breed{ font-family:'Fraunces',serif; font-weight:300; font-size:30px; line-height:1; letter-spacing:-0.01em; color:var(--text); }
+.breed em{ font-style:italic; font-weight:500; color:var(--amber); }
+.kicker{ font-family:'IBM Plex Mono',monospace; font-size:11px; letter-spacing:0.22em; text-transform:uppercase; color:var(--muted); border-bottom:1px solid var(--hair); padding-bottom:8px; width:100%; margin-top:4px; }
+.lede{ font-family:'Fraunces',serif; font-weight:300; font-size:15.5px; line-height:1.5; color:#b7b6ad; max-width:62ch; margin:14px 0 4px; }
+.lede b{ font-weight:500; color:var(--teal); }
+
+.panel-h{ font-family:'IBM Plex Mono',monospace; font-size:10.5px; letter-spacing:0.18em; text-transform:uppercase; color:var(--muted); margin:22px 0 10px; }
+#effort-rail{ display:grid; grid-template-columns:repeat(3,1fr); gap:10px; }
+.effort-card{ background:var(--surface); border:1px solid var(--hair); border-radius:var(--r); padding:12px 14px 12px; cursor:pointer; display:grid; grid-template-columns:auto 1fr auto; grid-template-rows:auto auto; align-items:center; gap:2px 10px; transition:border-color .15s, background .15s; text-align:left; }
+.effort-card:hover{ border-color:#3a3a4a; }
+.effort-card.selected{ background:var(--surface2); border-color:var(--amber); box-shadow:0 0 0 1px var(--amber) inset; }
+.eff-label{ font-family:'Inter'; font-weight:600; font-size:14px; color:var(--text); }
+.effort-card.selected .eff-label{ color:var(--amber); }
+.eff-k{ grid-column:2; font-family:'IBM Plex Mono',monospace; font-size:12px; color:var(--muted); }
+.effort-card.selected .eff-k{ color:#c9a05a; }
+.eff-dots{ grid-column:1 / span 3; grid-row:2; display:flex; gap:4px; margin-top:6px; }
+.dot{ width:7px; height:7px; border-radius:50%; background:#33333f; transition:background .15s, box-shadow .15s; }
+.effort-card .dot.lit{ background:#4a4a3a; }
+.effort-card.selected .dot.lit{ background:var(--amber); box-shadow:0 0 5px rgba(232,165,71,.55); }
+.eff-tag{ grid-column:3; grid-row:1; font-family:'Inter'; font-size:10.5px; color:#6f6f7d; text-transform:lowercase; }
+.effort-card.selected .eff-tag{ color:var(--amber-dim); }
+
+textarea#prompt, .gradio-container textarea{ background:var(--surface) !important; border:1px solid var(--hair) !important; border-radius:var(--r) !important; color:var(--text) !important; font-family:'Fraunces',serif !important; font-weight:300 !important; font-size:16px !important; line-height:1.5 !important; }
+textarea#prompt:focus{ border-color:var(--amber) !important; }
+.label-wrap label, .gradio-container .form label{ color:#b7b6ad !important; font-size:11.5px !important; letter-spacing:0.04em !important; }
+
+button#gen, button.primary{ background:var(--amber) !important; color:#1a1409 !important; border:none !important; border-radius:var(--r) !important; font-weight:600 !important; font-size:14px !important; }
+button#gen:hover{ filter:brightness(1.08); }
+
+.outbox{ background:var(--surface); border:1px solid var(--hair); border-radius:var(--r); padding:18px 20px; min-height:140px; font-family:'Fraunces',serif; font-weight:300; font-size:17px; line-height:1.55; color:var(--text); white-space:pre-wrap; word-break:break-word; }
+.empty{ color:#55556a; font-style:italic; font-family:'Fraunces',serif; }
+
+.status{ font-family:'IBM Plex Mono',monospace; font-size:11.5px; display:flex; gap:12px; align-items:center; margin-top:10px; flex-wrap:wrap; }
+.stat{ color:var(--muted); }
+.k-chip{ background:rgba(232,165,71,.12); color:var(--amber); border:1px solid rgba(232,165,71,.35); border-radius:5px; padding:1px 7px; font-weight:500; }
+.done{ color:var(--teal); }
+
+input[type=range]{ accent-color:var(--amber); }
+
+.foot{ margin-top:28px; border-top:1px solid var(--hair); padding-top:12px; font-family:'IBM Plex Mono',monospace; font-size:10.5px; color:#5d5d6c; display:flex; justify-content:space-between; flex-wrap:wrap; gap:8px; }
+.foot a{ color:var(--muted); text-decoration:none; border-bottom:1px dotted var(--hair); }
+.foot a:hover{ color:var(--text); }
+"""
+
+HEADER_HTML = """
+<div class="head">
+  <div class="breed">min&nbsp;·spark <em>preview</em></div>
+  <div class="kicker">PICO release 01 · Meiosis · looped hybrid · 5.76M params · ZeroGPU</div>
+</div>
+<div class="lede">
+  A research demo of a small language model trained from scratch on ten billion tokens.
+  Its one mechanic: a single weight-shared block runs <b>K times per token</b> — that is the
+  &ldquo;effort.&rdquo; More passes sharpen grammar; fewer favor commonsense. Pick a pass count
+  and prompt it.
+</div>
+"""
+
+FOOTER_HTML = """
+<div class="foot">
+  <span>ZeroGPU · generations stop early on &lt;eos&gt; · visitors use their own quota</span>
+  <span><a href="https://github.com/eclipse-senpai/PICO" target="_blank" rel="noopener">PICO on GitHub</a></span>
+</div>
+"""
+
+with gr.Blocks(css=CSS, title="min-spark · Meiosis preview") as demo:
+    gr.HTML(HEADER_HTML)
+
+    gr.HTML('<div class="panel-h">Effort — loops per token</div>')
+    effort_view = gr.HTML(effort_html("med"))
+
+    gr.HTML('<div class="panel-h">Prompt</div>')
+    prompt = gr.Textbox(value="", placeholder="Once upon a time, the very small model",
+                        elem_id="prompt", lines=2, show_label=False)
+
+    with gr.Row():
+        max_new = gr.Slider(8, 256, value=128, step=8, label="Max tokens")
+        temperature = gr.Slider(0.1, 1.6, value=0.8, step=0.05, label="Temperature")
+        top_k = gr.Slider(0, 200, value=50, step=5, label="Top-k  (0 = off)")
+
+    gen_btn = gr.Button("Generate", elem_id="gen", variant="primary")
+
+    out = gr.HTML(value='<div class="outbox" id="specimen"></div>')
+    status = gr.HTML(value='<div class="status"></div>')
+
+    gr.HTML(FOOTER_HTML)
+
+    effort_state = gr.State("med")  # default; overridden by READ_EFFORT_JS preamble
+
+    demo.load(fn=None, js=ATTACH_JS)
+    gen_btn.click(
+        fn=run_generate,
+        js=READ_EFFORT_JS,
+        inputs=[prompt, effort_state, max_new, temperature, top_k],
+        outputs=[out, status],
+    )
+
+if __name__ == "__main__":
+    demo.queue().launch(mcp_server=True)

assets/meiosis.py

@@ -0,0 +1,335 @@
+"""Meiosis: PICO release 1 (2026-07). Tied-embedding looped decoder-only LM.
+
+Spec: research/2026-07-first-release/final-spec.md (approved 2026-07-02).
+embed -> prelude x1 -> [body of `body_blocks` distinct blocks xK loops,
+per-loop LoRA + loop embed, Deep Delta vdim1 residuals] -> coda x1
+-> RMSNorm -> tied unembed. Attention is MHA by default, GQA when
+`n_kv_heads` < `n_heads` (2026-07-05 overhaul knobs, ADR-0009).
+"""
+
+import math
+from dataclasses import dataclass
+
+import torch
+from torch import Tensor, nn
+from torch.nn import functional
+
+EMBED_STD = 0.02
+LOOP_EMBED_STD = 0.02
+
+
+@dataclass
+class MeiosisConfig:
+    # defaults = release shape per ADR-0009 (B'-GQA overhaul, 2026-07-05):
+    # 3-block GQA body x3 loops, vocab 4096, ~5.76M total under the <6M cap
+    vocab_size: int = 4096
+    dim: int = 288
+    n_heads: int = 6
+    n_kv_heads: int | None = 2  # None -> MHA (= n_heads)
+    ffn_hidden: int = 768
+    prelude_layers: int = 1
+    coda_layers: int = 1
+    body_blocks: int = 3  # distinct blocks in the loop body
+    max_loops: int = 4
+    train_loops: int = 3
+    lora_rank: int = 16
+    rope_base: float = 10000.0
+    max_seq_len: int = 512
+    ddl_beta_init: float = 1.0
+    # rms_norm backward amplifies grads by 1/sqrt(eps_rms) when k_in ~ 0 — which is
+    # exactly the zero-init state. 1e-5 gave a 1.7e6x amplifier (1e5-magnitude grad
+    # spikes; > fp16 max at ANY loss scale — the 2026-07-06 fp16 divergence, ADR-0012).
+    # 1e-2 caps it at 1.7e3: fp16-safe, and identical bf16 training curves.
+    ddl_k_eps: float = 1e-2
+    ddl_v_sigmoid_scale: float = 4.0
+    # intra-document attention (ADR-0019): tokens attend only within their own
+    # EOS-delimited document. None = plain causal (pre-mask checkpoints).
+    doc_mask_eos: int | None = 2
+
+    @property
+    def head_dim(self) -> int:
+        return self.dim // self.n_heads
+
+    @property
+    def kv_heads(self) -> int:
+        return self.n_kv_heads if self.n_kv_heads is not None else self.n_heads
+
+    @property
+    def qkv_dim(self) -> int:
+        return self.dim + 2 * self.kv_heads * self.head_dim
+
+
+def build_rope_cache(config: MeiosisConfig, length: int) -> tuple[Tensor, Tensor]:
+    positions = torch.arange(length, dtype=torch.float32)
+    inv_freq = 1.0 / (
+        config.rope_base
+        ** (torch.arange(0, config.head_dim, 2, dtype=torch.float32) / config.head_dim)
+    )
+    angles = torch.outer(positions, inv_freq)
+    return torch.cos(angles), torch.sin(angles)
+
+
+def apply_rope(x: Tensor, cos: Tensor, sin: Tensor) -> Tensor:
+    x_even, x_odd = x[..., 0::2], x[..., 1::2]
+    rotated_even = x_even * cos - x_odd * sin
+    rotated_odd = x_even * sin + x_odd * cos
+    return torch.stack((rotated_even, rotated_odd), dim=-1).flatten(-2)
+
+
+def build_doc_mask(tokens: Tensor, eos_id: int) -> Tensor:
+    """(B,T) tokens -> (B,1,T,T) bool, True where attention is allowed:
+    causal AND same document. Exclusive EOS scan, so an EOS token is the
+    last token of its document (FSX-1 convention)."""
+    is_eos = tokens == eos_id
+    doc_id = torch.cumsum(is_eos, dim=1) - is_eos.to(torch.long)
+    same = doc_id.unsqueeze(2) == doc_id.unsqueeze(1)
+    causal = torch.ones(
+        tokens.shape[1], tokens.shape[1], dtype=torch.bool, device=tokens.device
+    ).tril()
+    return (same & causal).unsqueeze(1)
+
+
+class SwiGlu(nn.Module):
+    def __init__(self, dim: int, hidden: int) -> None:
+        super().__init__()
+        self.gate_up = nn.Linear(dim, 2 * hidden, bias=False)
+        self.down = nn.Linear(hidden, dim, bias=False)
+
+    def forward(self, x: Tensor) -> Tensor:
+        gate, up = self.gate_up(x).chunk(2, dim=-1)
+        return self.down(functional.silu(gate) * up)
+
+
+class Attention(nn.Module):
+    """MHA, or GQA when kv_heads < n_heads (KV repeated to full head count)."""
+
+    def __init__(self, config: MeiosisConfig) -> None:
+        super().__init__()
+        self.n_heads = config.n_heads
+        self.kv_heads = config.kv_heads
+        self.head_dim = config.head_dim
+        self.qkv = nn.Linear(config.dim, config.qkv_dim, bias=False)
+        self.out = nn.Linear(config.dim, config.dim, bias=False)
+
+    def forward(
+        self,
+        x: Tensor,
+        cos: Tensor,
+        sin: Tensor,
+        qkv_delta: Tensor | None = None,
+        attn_mask: Tensor | None = None,
+    ) -> Tensor:
+        batch, seq_len, dim = x.shape
+        kv_dim = self.kv_heads * self.head_dim
+        qkv = self.qkv(x)
+        if qkv_delta is not None:
+            qkv = qkv + qkv_delta
+        q, k, v = qkv.split([dim, kv_dim, kv_dim], dim=-1)
+        q = q.view(batch, seq_len, self.n_heads, self.head_dim).transpose(1, 2)
+        k = k.view(batch, seq_len, self.kv_heads, self.head_dim).transpose(1, 2)
+        v = v.view(batch, seq_len, self.kv_heads, self.head_dim).transpose(1, 2)
+        q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin)
+        if self.kv_heads != self.n_heads:
+            k = k.repeat_interleave(self.n_heads // self.kv_heads, dim=1)
+            v = v.repeat_interleave(self.n_heads // self.kv_heads, dim=1)
+        if attn_mask is not None:
+            attended = functional.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask)
+        else:
+            attended = functional.scaled_dot_product_attention(q, k, v, is_causal=True)
+        return self.out(attended.transpose(1, 2).reshape(batch, seq_len, dim))
+
+
+class DeepDeltaResidual(nn.Module):
+    """DDL vdim1 (arXiv 2601.00417): x <- x + beta * (v - k^T x) * k.
+
+    k is the sublayer output (rms-normed), beta in [0,2] gates between
+    identity / projection / reflection, v is a learned scalar target.
+    Replaces the plain additive residual in the looped block only.
+    """
+
+    def __init__(self, config: MeiosisConfig) -> None:
+        super().__init__()
+        self.k_eps = config.ddl_k_eps
+        self.v_sigmoid_scale = config.ddl_v_sigmoid_scale
+        self.beta = nn.Linear(config.dim, 1, bias=True)
+        self.v_proj = nn.Linear(config.dim, 1, bias=True)
+        beta_p = min(max(config.ddl_beta_init, 0.0), 2.0) / 2.0
+        beta_p = min(max(beta_p, 1e-6), 1.0 - 1e-6)
+        with torch.no_grad():
+            self.beta.bias.fill_(math.log(beta_p) - math.log(1.0 - beta_p))
+
+    def forward(self, x: Tensor, *, k_in: Tensor, context: Tensor) -> Tensor:
+        k_dim = k_in.size(-1)
+        eps_rms = (self.k_eps * self.k_eps) / k_dim
+        k_rms = functional.rms_norm(k_in, [k_dim], eps=eps_rms)
+        k_scale = 1.0 / math.sqrt(k_dim)
+        beta = 2.0 * torch.sigmoid(self.beta(context).float())
+        proj = torch.sum(k_rms * x, dim=-1, keepdim=True, dtype=torch.float32) * k_scale
+        v = torch.sigmoid(self.v_proj(x).float()) * self.v_sigmoid_scale
+        delta = ((beta * (v - proj)) * k_scale).to(dtype=x.dtype)
+        return x + delta * k_rms
+
+
+class Block(nn.Module):
+    """Pre-norm block with plain additive residuals (prelude/coda)."""
+
+    def __init__(self, config: MeiosisConfig) -> None:
+        super().__init__()
+        self.attn_norm = nn.RMSNorm(config.dim)
+        self.attn = Attention(config)
+        self.ffn_norm = nn.RMSNorm(config.dim)
+        self.ffn = SwiGlu(config.dim, config.ffn_hidden)
+
+    def forward(
+        self, x: Tensor, cos: Tensor, sin: Tensor, attn_mask: Tensor | None = None
+    ) -> Tensor:
+        x = x + self.attn(self.attn_norm(x), cos, sin, attn_mask=attn_mask)
+        return x + self.ffn(self.ffn_norm(x))
+
+
+class LoopedBlock(nn.Module):
+    """Shared block with Deep Delta residuals; run K times with per-loop LoRA."""
+
+    def __init__(self, config: MeiosisConfig) -> None:
+        super().__init__()
+        self.attn_norm = nn.RMSNorm(config.dim)
+        self.attn = Attention(config)
+        self.ddl_attn = DeepDeltaResidual(config)
+        self.ffn_norm = nn.RMSNorm(config.dim)
+        self.ffn = SwiGlu(config.dim, config.ffn_hidden)
+        self.ddl_ffn = DeepDeltaResidual(config)
+
+    def forward(
+        self,
+        x: Tensor,
+        cos: Tensor,
+        sin: Tensor,
+        qkv_delta: Tensor,
+        loop_emb: Tensor,
+        attn_mask: Tensor | None = None,
+    ) -> Tensor:
+        # loop_emb conditions the sublayer inputs only — it is not carried in
+        # the residual stream, so the block stays exactly identity at init
+        # (zero-init out-projections -> k=0 -> DDL no-op) for any loop count.
+        x_norm = self.attn_norm(x + loop_emb)
+        x = self.ddl_attn(
+            x,
+            k_in=self.attn(x_norm, cos, sin, qkv_delta, attn_mask=attn_mask),
+            context=x_norm,
+        )
+        x_norm = self.ffn_norm(x + loop_emb)
+        return self.ddl_ffn(x, k_in=self.ffn(x_norm), context=x_norm)
+
+
+class LoopLora(nn.Module):
+    def __init__(self, config: MeiosisConfig) -> None:
+        super().__init__()
+        self.down = nn.ModuleList(
+            nn.Linear(config.dim, config.lora_rank, bias=False)
+            for _ in range(config.max_loops)
+        )
+        self.up = nn.ModuleList(
+            nn.Linear(config.lora_rank, config.qkv_dim, bias=False)
+            for _ in range(config.max_loops)
+        )
+        for up in self.up:
+            nn.init.zeros_(up.weight)
+
+    def forward(self, x: Tensor, loop_index: int) -> Tensor:
+        clamped = min(loop_index, len(self.down) - 1)
+        return self.up[clamped](self.down[clamped](x))
+
+
+class Meiosis(nn.Module):
+    def __init__(self, config: MeiosisConfig) -> None:
+        super().__init__()
+        self.config = config
+        self.embed = nn.Embedding(config.vocab_size, config.dim)
+        self.prelude = nn.ModuleList(Block(config) for _ in range(config.prelude_layers))
+        self.body = nn.ModuleList(LoopedBlock(config) for _ in range(config.body_blocks))
+        self.loop_lora = nn.ModuleList(LoopLora(config) for _ in range(config.body_blocks))
+        self.loop_embed = nn.Embedding(config.max_loops, config.dim)
+        self.coda = nn.ModuleList(Block(config) for _ in range(config.coda_layers))
+        self.final_norm = nn.RMSNorm(config.dim)
+        cos, sin = build_rope_cache(config, config.max_seq_len)
+        self.register_buffer("rope_cos", cos, persistent=False)
+        self.register_buffer("rope_sin", sin, persistent=False)
+        self.register_buffer("last_loop_rms", torch.zeros(config.max_loops), persistent=False)
+
+    def forward(
+        self,
+        tokens: Tensor,
+        loops: int | None = None,
+        return_hidden: bool = False,
+        collect_loop_rms: bool = False,
+        attn_mask: Tensor | None = None,
+    ) -> Tensor | tuple[Tensor, Tensor]:
+        loop_count = loops if loops is not None else self.config.train_loops
+        seq_len = tokens.shape[1]
+        if seq_len > self.config.max_seq_len:
+            raise ValueError(f"seq_len {seq_len} > max {self.config.max_seq_len}")
+        x = self.embed(tokens)
+        if attn_mask is None and self.config.doc_mask_eos is not None:
+            attn_mask = build_doc_mask(tokens, self.config.doc_mask_eos)
+        device_type = tokens.device.type
+        compute_dtype = (
+            torch.get_autocast_dtype(device_type)
+            if torch.is_autocast_enabled(device_type)
+            else x.dtype
+        )
+        cos = self.rope_cos[:seq_len].to(compute_dtype)
+        sin = self.rope_sin[:seq_len].to(compute_dtype)
+        for block in self.prelude:
+            x = block(x, cos, sin, attn_mask=attn_mask)
+        rms_per_loop = []
+        for i in range(loop_count):
+            clamped = min(i, self.config.max_loops - 1)
+            loop_emb = self.loop_embed.weight[clamped]
+            for block, lora in zip(self.body, self.loop_lora):
+                x = block(x, cos, sin, lora(x + loop_emb, i), loop_emb, attn_mask=attn_mask)
+            rms = x.float().pow(2).mean().sqrt()
+            self.last_loop_rms[clamped] = rms.detach()
+            if collect_loop_rms:
+                rms_per_loop.append(rms)
+        for block in self.coda:
+            x = block(x, cos, sin, attn_mask=attn_mask)
+        x = self.final_norm(x)
+        out = x if return_hidden else functional.linear(x, self.embed.weight)
+        if collect_loop_rms:
+            return out, torch.stack(rms_per_loop)
+        return out
+
+
+def init_meiosis(model: Meiosis) -> None:
+    """Mandatory MythosMini-validated init. Never mu-center the tied embedding."""
+    with torch.no_grad():
+        model.embed.weight.normal_(mean=0.0, std=EMBED_STD)
+        model.loop_embed.weight.normal_(mean=0.0, std=LOOP_EMBED_STD)
+        for block in [*model.prelude, *model.body, *model.coda]:
+            nn.init.zeros_(block.attn.out.weight)
+            nn.init.zeros_(block.ffn.down.weight)
+
+
+def count_parameters(model: nn.Module) -> int:
+    return sum(p.numel() for p in model.parameters())
+
+
+def muon_param_split(model: Meiosis) -> tuple[list[nn.Parameter], list[nn.Parameter]]:
+    """Explicit Muon/aux split (ADR-0005). Muon gets the block and LoRA
+    matrices; the tied embedding, loop embeddings, norm gains, and 1-row DDL
+    heads stay on NAdamW. Listed explicitly - no shape heuristics, so a
+    rank-8 pilot LoRA cannot silently fall out of the Muon group.
+    """
+    muon: list[nn.Parameter] = []
+    for block in [*model.prelude, *model.body, *model.coda]:
+        muon += [
+            block.attn.qkv.weight,
+            block.attn.out.weight,
+            block.ffn.gate_up.weight,
+            block.ffn.down.weight,
+        ]
+    for lora in model.loop_lora:
+        muon += [linear.weight for linear in [*lora.down, *lora.up]]
+    muon_ids = {id(p) for p in muon}
+    aux = [p for p in model.parameters() if id(p) not in muon_ids]
+    return muon, aux

loader.py

@@ -0,0 +1,74 @@
+"""Meiosis model loader + byte-BPE tokenizer for the min-spark-preview Space.
+
+ZeroGPU: model loads at module scope with .to("cuda") (string, never an int) so
+the `spaces` hijack packs weights to disk for the GPU worker. Generation runs
+inside @spaces.GPU (decorated in app.py). Vendored paths so the Space has zero
+dependency on the PICO repo layout.
+"""
+from __future__ import annotations
+from pathlib import Path
+import torch
+
+_ASSETS = Path(__file__).parent / "assets"
+_TOK = _ASSETS / "tokenizer.json"
+_CKPT = _ASSETS / "meiosis.safetensors"
+
+# Keep `meiosis.py` importable: it is torch-only and self-contained here.
+import sys
+if str(_ASSETS) not in sys.path:
+    sys.path.insert(0, str(_ASSETS))
+
+from meiosis import Meiosis, MeiosisConfig
+
+# tokenizer.json is a raw HF `tokenizers` artifact (no PreTrainedTokenizerFast
+# wrapper, per ADR-0010) — load it with the `tokenizers` library directly.
+from tokenizers import Tokenizer
+
+EOS_ID = 2  # PICO specials: <pad>=0, <bos>=1, <eos>=2 (ADR-0010)
+
+
+def load_tokenizer():
+    return Tokenizer.from_file(str(_TOK))
+
+
+def load_model(device: str = "cpu") -> Meiosis:
+    from safetensors.torch import load_file
+    model = Meiosis(MeiosisConfig())
+    state = load_file(str(_CKPT))
+    # strict=False: safetensors may lack non-persistent buffers (rope, loop_rms)
+    model.load_state_dict(state, strict=False)
+    model.to(device).eval()
+    return model
+
+
+@torch.no_grad()
+def generate(model, tokenizer, prompt: str, *, loops: int, max_new: int,
+             temperature: float, top_k: int, device: str):
+    """Token-by-token sampling. Yields (text_so_far, token_count, tok_per_s).
+    Mirrors infer.py: EOS prefix, top-k + temperature, stop on EOS. Runs on the
+    GPU worker when invoked under @spaces.GPU; returns only CPU-safe Python
+    objects (str/int/float) so nothing CUDA crosses the pickle boundary."""
+    import time
+    ids = [EOS_ID] + tokenizer.encode(prompt).ids
+    out_text = ""
+    t0 = None
+    count = 0
+    for _ in range(max_new):
+        ctx = ids[-model.config.max_seq_len:]
+        x = torch.tensor([ctx], device=device)
+        logits = model(x, loops=loops)
+        if t0 is None:
+            t0 = time.perf_counter()
+        next_logits = logits[0, -1] / max(temperature, 1e-6)
+        if top_k > 0:
+            topk_vals, _ = torch.topk(next_logits, min(top_k, next_logits.shape[-1]))
+            next_logits[next_logits < topk_vals[-1]] = float("-inf")
+        probs = torch.softmax(next_logits, dim=-1)
+        next_id = int(torch.multinomial(probs, 1).item())
+        if next_id == EOS_ID:
+            break
+        ids.append(next_id)
+        out_text += tokenizer.decode([next_id])
+        count += 1
+        elapsed = time.perf_counter() - t0
+        yield out_text, count, (count / elapsed if elapsed > 0 else 0.0)

requirements.txt

@@ -0,0 +1,5 @@
+# gradio, spaces, huggingface_hub are preinstalled by the Gradio SDK base image —
+# do NOT list them (pinning breaks the runtime). torch is also preinstalled on
+# ZeroGPU; leave it unpinned (the runtime pins one of 2.8/2.9.1/2.10/2.11).
+safetensors>=0.4.1
+tokenizers>=0.21