Add sb-run2

sb-run2/README.md

@@ -0,0 +1,28 @@
+# SB (run 2)
+
+Part of the [Self-Drafting Latent Memory](../README.md) collection.
+
+Second training run of the SB recipe: same initialisation seed, data and data order. It's the replicate used as the noise floor in the paper. Its codes agree with run 1 on only 37% of positions at equal loss.
+
+- **Architecture:** 24-layer Llama-style decoder, d = 1024, 24 query / 4 KV heads, SwiGLU, RoPE, tied embeddings, vocabulary 32,768, context 2048. Training config: `w3_sb_1b` in the repository.
+- **Validation loss:** 2.7107 (1B tokens).
+- **Zero-shot (lm-eval 0.4.13):**
+
+  | HellaSwag | MMLU-cloze | SciQ | WinoGrande | PIQA | ARC-Easy |
+  |---|---|---|---|---|---|
+  | 29.3 | 24.6 | 60.4 | 51.8 | 60.0 | 38.0 |
+
+## Files
+
+- `model.pt`: `{"model": state_dict, "config": ..., "step": ..., "params": ...}`, weights only (no optimizer state).
+- `config.json`: the resolved training/model config.
+- `params.json`: parameter counts by component.
+
+## Load
+
+```python
+from ffnmem.infer import load_model   # code from the accompanying repository
+model, cfg = load_model("sb-run2/model.pt", device="cuda")
+```
+
+Research checkpoint: not instruction-tuned, not safety-tuned, not for deployment.

sb-run2/config.json

@@ -0,0 +1,140 @@
+{
+ "model": {
+  "d_model": 1024,
+  "n_layers": 24,
+  "n_heads": 24,
+  "n_kv_heads": 4,
+  "head_dim": 64,
+  "vocab": 32768,
+  "seq_len": 2048,
+  "d_ff": [
+   2816,
+   2816,
+   1536,
+   1536,
+   1536,
+   1536,
+   2816,
+   2816,
+   2816,
+   2816,
+   2816,
+   2816,
+   1536,
+   1536,
+   1536,
+   1536,
+   2816,
+   2816,
+   2816,
+   2816,
+   2816,
+   2816,
+   2816,
+   2816
+  ],
+  "memory_layers": [
+   1
+  ],
+  "memory": {
+   "routes": 184,
+   "mem_dim": 128,
+   "kv_heads": 4,
+   "q_heads": 16,
+   "head_dim": 64,
+   "execution_mode": "full"
+  },
+  "norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "init_std": 0.02,
+  "ffn_cut_layers": [
+   2,
+   3,
+   4,
+   5,
+   12,
+   13,
+   14,
+   15
+  ],
+  "ffn_cut_delta": 1280,
+  "layers": [
+   0,
+   1,
+   2,
+   3,
+   4,
+   5,
+   6,
+   7,
+   8,
+   9,
+   10,
+   11,
+   12,
+   13,
+   14,
+   15,
+   16,
+   17,
+   18,
+   19,
+   20,
+   21,
+   22,
+   23
+  ],
+  "memory_overrides": {},
+  "draft": null,
+  "ssm": null
+ },
+ "train": {
+  "seed": 1234,
+  "data_seed": 1234,
+  "val_tokens": 20000000,
+  "global_batch": 256,
+  "micro_batch": 4,
+  "tokens_total": 999817216,
+  "lr": 0.0006,
+  "min_lr_ratio": 0.1,
+  "warmup_frac": 0.02,
+  "warmup_min_steps": 100,
+  "betas": [
+   0.9,
+   0.95
+  ],
+  "eps": 1e-08,
+  "weight_decay": 0.1,
+  "grad_clip": 1.0,
+  "table_lr_mult": 4.0,
+  "precision": "fp16",
+  "act_ckpt": true,
+  "attn_backend": "sdpa",
+  "zero1": false,
+  "ckpt_every": 200,
+  "keep_last": 3,
+  "keep_every": 1000,
+  "eval_every": 200,
+  "eval_max_batches": null,
+  "diag_batches": 4,
+  "log_every": 10,
+  "scaler_min": 1.0,
+  "scaler_patience": 100,
+  "nan_patience": 20,
+  "save_steps": [
+   500,
+   1000,
+   1907
+  ],
+  "strip_old_optimizer": true,
+  "seq_len": 2048
+ },
+ "name": "w3_sb_1b",
+ "params": {
+  "reference": true,
+  "expected_non_embedding": 369777920
+ },
+ "wandb": {
+  "project": "morbius-wave3"
+ }
+}

sb-run2/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:264adb3e641c520f48f3b6410fc80e4e1a7a54f501f305b8da476499e6f65208
+size 1613398819

sb-run2/params.json

@@ -0,0 +1,13 @@
+{
+ "embedding": 33554432,
+ "attention": 88080384,
+ "ffn": 176160768,
+ "norms": 50176,
+ "memory_tables": 102498304,
+ "memory_readout": 2988288,
+ "draft": 0,
+ "ssm": 0,
+ "total": 403332352,
+ "non_embedding": 369777920,
+ "active_non_embedding_per_token": 267326720
+}