Add sd-c

sd-c/README.md

@@ -0,0 +1,30 @@
+# SD-C (self-drafting, token router + SSM)
+
+Part of the [Self-Drafting Latent Memory](../README.md) collection.
+
+SD-B plus a 256-channel non-selective diagonal SSM over (detached) embeddings. Its state feeds both the router and the draft query (181 routes). This is the best drafter in the study.
+
+- **Architecture:** 24-layer Llama-style decoder, d = 1024, 24 query / 4 KV heads, SwiGLU, RoPE, tied embeddings, vocabulary 32,768, context 2048. Training config: `w3_sdc_1b` in the repository.
+- **Validation loss:** 2.7111 (1B tokens).
+- **Zero-shot (lm-eval 0.4.13):**
+
+  | HellaSwag | MMLU-cloze | SciQ | WinoGrande | PIQA | ARC-Easy |
+  |---|---|---|---|---|---|
+  | 28.6 | 24.3 | 60.3 | 51.6 | 58.3 | 40.0 |
+- Draft head: top-1 agreement 0.449 (held-out). Accepted tokens per step at gamma=4: 1.04 (head alone; beats a 300M-token n-gram drafter at 0.86), 2.52 (head + online counts, open text). Hybrid with prompt lookup: +0.13 over prompt lookup (open). Implied speedup with the PyTorch drafter: 2.86x (open, gamma=4).
+
+## Files
+
+- `model.pt`: `{"model": state_dict, "config": ..., "step": ..., "params": ...}`, weights only (no optimizer state).
+- `config.json`: the resolved training/model config.
+- `params.json`: parameter counts by component.
+
+## Load
+
+```python
+from ffnmem.infer import load_model   # code from the accompanying repository
+model, cfg = load_model("sd-c/model.pt", device="cuda")
+draft_logits = model.draft_logits(idx, b_idx, t_idx)   # token ids only; no backbone forward
+```
+
+Research checkpoint: not instruction-tuned, not safety-tuned, not for deployment.

sd-c/config.json

@@ -0,0 +1,161 @@
+{
+ "model": {
+  "d_model": 1024,
+  "n_layers": 24,
+  "n_heads": 24,
+  "n_kv_heads": 4,
+  "head_dim": 64,
+  "vocab": 32768,
+  "seq_len": 2048,
+  "d_ff": [
+   2816,
+   2816,
+   1536,
+   1536,
+   1536,
+   1536,
+   2816,
+   2816,
+   2816,
+   2816,
+   2816,
+   2816,
+   1536,
+   1536,
+   1536,
+   1536,
+   2816,
+   2816,
+   2816,
+   2816,
+   2816,
+   2816,
+   2816,
+   2816
+  ],
+  "memory_layers": [
+   1
+  ],
+  "memory": {
+   "routes": 181,
+   "mem_dim": 128,
+   "kv_heads": 4,
+   "q_heads": 16,
+   "head_dim": 64,
+   "execution_mode": "full",
+   "router_input": "embed2_ssm"
+  },
+  "norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "init_std": 0.02,
+  "draft": {
+   "enabled": true,
+   "q_heads": 4,
+   "head_dim": 64,
+   "share_kv_proj": true,
+   "detach_embeddings": true,
+   "loss_weight": 0.1,
+   "loss_position_frac": 0.25,
+   "loss_chunk": 512
+  },
+  "ssm": {
+   "enabled": true,
+   "state_dim": 256,
+   "chunk": 128,
+   "feeds": [
+    "router",
+    "draft_query"
+   ]
+  },
+  "ffn_cut_layers": [
+   2,
+   3,
+   4,
+   5,
+   12,
+   13,
+   14,
+   15
+  ],
+  "ffn_cut_delta": 1280,
+  "layers": [
+   0,
+   1,
+   2,
+   3,
+   4,
+   5,
+   6,
+   7,
+   8,
+   9,
+   10,
+   11,
+   12,
+   13,
+   14,
+   15,
+   16,
+   17,
+   18,
+   19,
+   20,
+   21,
+   22,
+   23
+  ],
+  "memory_overrides": {}
+ },
+ "train": {
+  "seed": 1234,
+  "data_seed": 1234,
+  "val_tokens": 20000000,
+  "global_batch": 256,
+  "micro_batch": 2,
+  "tokens_total": 999817216,
+  "lr": 0.0006,
+  "min_lr_ratio": 0.1,
+  "warmup_frac": 0.02,
+  "warmup_min_steps": 100,
+  "betas": [
+   0.9,
+   0.95
+  ],
+  "eps": 1e-08,
+  "weight_decay": 0.1,
+  "grad_clip": 1.0,
+  "table_lr_mult": 4.0,
+  "precision": "fp16",
+  "act_ckpt": true,
+  "attn_backend": "sdpa",
+  "zero1": false,
+  "ckpt_every": 200,
+  "keep_last": 3,
+  "keep_every": 1000,
+  "eval_every": 200,
+  "eval_max_batches": null,
+  "diag_batches": 4,
+  "log_every": 10,
+  "scaler_min": 1.0,
+  "scaler_patience": 100,
+  "nan_patience": 20,
+  "save_steps": [
+   500,
+   1000,
+   1907
+  ],
+  "strip_old_optimizer": true,
+  "seq_len": 2048
+ },
+ "name": "w3_sdc_1b",
+ "params": {
+  "reference": false,
+  "expected_non_embedding": null,
+  "match_config": "w3_sb_1b.yaml",
+  "match_tolerance": 0.001,
+  "match_target": 369777920
+ },
+ "wandb": {
+  "project": "morbius-wave3"
+ }
+}

sd-c/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f21fd33b12d84c2a9c074daba31a89420a0631db6804ca9658455793467b96b
+size 1613808887

sd-c/params.json

@@ -0,0 +1,13 @@
+{
+ "embedding": 33554432,
+ "attention": 88080384,
+ "ffn": 176160768,
+ "norms": 50176,
+ "memory_tables": 100827136,
+ "memory_readout": 3905792,
+ "draft": 591872,
+ "ssm": 263424,
+ "total": 403433984,
+ "non_embedding": 369879552,
+ "active_non_embedding_per_token": 269098752
+}