Add performance benchmarks: 6.2 s CPU compile, speed, held-out accuracy

README.md

@@ -49,6 +49,61 @@ Full construction details are in [docs/how-it-works.md](docs/how-it-works.md).
 The last rows show its main weakness. It sees only 4 tokens of context, so longer
 completions can drift between topics.
 
+## Performance: why compile a model instead of training one
+
+Measured with `python scripts/benchmark.py` on a 10-core Apple Silicon laptop (32 GB RAM).
+CPU only, no GPU. Raw numbers are in [`docs/benchmarks.json`](docs/benchmarks.json).
+
+**Building it**
+
+| | |
+|---|---|
+| Compile all 380,290,320 weights from text | **6.2 s** (tokenize 0.5 s, build tensors 2.7 s, write 1.5 GB) |
+| Text in, model out, from scratch | about 10 s to download the data + 6 s to compile |
+| Peak memory while compiling | 2.3 GB |
+| GPU hours / compute cost | **0 / $0** |
+| Reproducible | two rebuilds give a byte-identical `model.safetensors` (sha256 `93c23040…`) |
+| Exact | matches its reference n-gram to within 2.5e-5 nats |
+
+For scale: the sibling experiment [tiny-research](https://tiny-research.vercel.app/)
+trained a 34M model with a 5-minute budget on an RTX 4090. Compiling this 380M model takes
+6 seconds on a laptop CPU.
+
+**Using it**
+
+| | |
+|---|---|
+| PyTorch (CPU, 8 threads) | 70 ms per next word; 16 tokens/s greedy generation; 0.3 s to load |
+| Plain TypeScript (Bun, no ML library) | 0.44 s per token; 0.4 s to load |
+
+**How good is it on text it has never seen?** The benchmark compiles the model without a
+fixed 10% of the coding sentences (2,040 lines, 35,039 tokens), then scores each next token:
+
+| | |
+|---|---|
+| Top-1 next-token accuracy | **25.8%** |
+| Top-3 next-token accuracy | **39.6%** |
+| Next token outside the 3,000-word vocabulary | 8.8% (always counted as a miss) |
+
+On the source phrases themselves it completes almost every line exactly.
+
+**Why build a model like this**
+
+- **Iteration in seconds:** edit a `.txt` file, rebuild, and test in under 10 s. There is no
+  training run, no hyperparameter tuning, and no GPU.
+- **Auditable:** every MLP neuron is one context from the source text, so you can trace any
+  prediction back to the phrases that caused it.
+- **Editable:** to remove a behavior, delete the line and recompile. Nothing has to be
+  fine-tuned away.
+- **Deterministic and exact:** the same text always gives the same bytes, and the network
+  computes a known formula.
+- **Standard format:** it's a plain `LlamaForCausalLM`, so it works with `transformers`,
+  `pipeline`, and any runtime that loads Llama safetensors.
+
+**What it costs you:** it doesn't generalize like a trained model, and only 7% of the weights
+are non-zero (26.8M of 380M), so the dense file is mostly empty space. A sparse runtime could
+be 10× smaller and faster.
+
 ## Use it with transformers
 
 ```python
@@ -161,6 +216,7 @@ pip install -r requirements.txt
 python scripts/fetch_coding_prompts.py   # optional: re-download data/coding_prompts.txt
 python build_weights.py                  # writes config, weights and tokenizer
 python test_model.py                     # HF loading, n-gram exactness, pipeline, TypeScript parity
+python scripts/benchmark.py              # compile time, speed, held-out accuracy -> docs/benchmarks.json
 ```
 
 Tested with transformers 4.52.4 and 5.22.1, Bun 1.3 and Node 24.
@@ -180,6 +236,7 @@ excerpts from OpenAssistant oasst2 (Apache-2.0), MBPP (CC-BY-4.0) and CoNaLa (MI
 | `build_weights.py` | the compiler that writes every file above |
 | `phrases/*.txt`, `data/coding_prompts.txt` | source text |
 | `scripts/fetch_coding_prompts.py`, `data/SOURCES.md` | data download and attribution |
+| `scripts/benchmark.py`, `docs/benchmarks.json` | performance measurements |
 | `test_model.py` | verification suite |
 | `model.ts`, `autocomplete.ts`, `ui.ts`, `ui.html` | zero-dependency TypeScript runtime, CLI and web UI |
 | `docs/how-it-works.md` | full construction details |

docs/benchmarks.json

@@ -0,0 +1,47 @@
+{
+  "hardware": {
+    "cpu": "Darwin arm64",
+    "cores": 10,
+    "ram_gb": 32,
+    "gpu": "none used",
+    "python": "3.13.4",
+    "torch": "2.15.0"
+  },
+  "compile": {
+    "compile_seconds": 6.2,
+    "compile_runs": [
+      6.4,
+      6.2
+    ],
+    "peak_memory_gb": 2.3,
+    "byte_identical_rebuild": true,
+    "model_sha256": "93c23040ccf6393710b0ee1d2fff98d57dc4193a27c4911cac7374e2c28f50f6",
+    "file_gb": 1.52,
+    "tokenize_seconds": 0.49,
+    "build_tensors_seconds": 2.74,
+    "source_lines": 21106,
+    "params": 380290320,
+    "nonzero_params": 26792230,
+    "contexts_seen": 434284,
+    "neurons": 24774
+  },
+  "pytorch": {
+    "load_seconds": 0.3,
+    "next_word_ms": 70.3,
+    "generate_tokens_per_second": 16.4,
+    "threads": 8
+  },
+  "typescript": {
+    "runtime": "bun, plain TypeScript, no ML library",
+    "load_seconds": 0.4,
+    "seconds_per_token": 0.44
+  },
+  "max_logprob_error_vs_ngram": 2.4948840973593178e-05,
+  "heldout": {
+    "heldout_lines": 2040,
+    "heldout_tokens": 35039,
+    "next_token_top1": 0.258,
+    "next_token_top3": 0.396,
+    "oov_rate": 0.088
+  }
+}

scripts/benchmark.py

@@ -0,0 +1,193 @@
+"""Measure compile cost, inference speed, exactness and held-out accuracy.
+
+Writes docs/benchmarks.json. Run from the repo root after build_weights.py:
+    python scripts/benchmark.py
+"""
+
+import hashlib
+import json
+import os
+import platform
+import random
+import resource
+import shutil
+import subprocess
+import sys
+import time
+import zlib
+from pathlib import Path
+
+import torch
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+import build_weights as bw  # noqa: E402
+
+PROMPTS = [
+    "Once upon a", "I hope this email", "The quick brown", "Thank you for", "Looking forward to",
+    "In Python,", "How to convert a", "Write a python function to", "You can use the", "Here is an example of",
+]
+
+
+def sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 24), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def hardware() -> dict:
+    cpu = ""
+    if sys.platform == "darwin":
+        cpu = subprocess.run(["sysctl", "-n", "machdep.cpu.brand_string"], capture_output=True, text=True).stdout.strip()
+    cpu = cpu or f"{platform.system()} {platform.machine()}"
+    try:
+        ram = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
+    except (ValueError, OSError):
+        ram = 0
+    return {"cpu": cpu, "cores": os.cpu_count(), "ram_gb": round(ram / 2**30), "gpu": "none used",
+            "python": platform.python_version(), "torch": torch.__version__}
+
+
+def compile_runs(n: int = 2) -> dict:
+    """Full `python build_weights.py` runs: wall time, peak memory, determinism."""
+    times, hashes = [], []
+    for _ in range(n):
+        t0 = time.perf_counter()
+        subprocess.run([sys.executable, "build_weights.py"], cwd=ROOT, check=True, capture_output=True)
+        times.append(time.perf_counter() - t0)
+        hashes.append(sha256(ROOT / "model.safetensors"))
+    peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
+    peak_gb = peak / 2**30 if sys.platform == "darwin" else peak / 2**20
+    return {
+        "compile_seconds": round(min(times), 1),
+        "compile_runs": [round(t, 1) for t in times],
+        "peak_memory_gb": round(peak_gb, 1),
+        "byte_identical_rebuild": len(set(hashes)) == 1,
+        "model_sha256": hashes[0],
+        "file_gb": round((ROOT / "model.safetensors").stat().st_size / 1e9, 2),
+    }
+
+
+def compile_stages() -> dict:
+    t0 = time.perf_counter()
+    phrases, data = bw.load_phrases(), bw.load_data()
+    t1 = time.perf_counter()
+    _, ngram, state, _, info = bw.build(phrases, data)
+    t2 = time.perf_counter()
+    nonzero = sum(int((t != 0).sum()) for t in state.values())
+    return {
+        "tokenize_seconds": round(t1 - t0, 2),
+        "build_tensors_seconds": round(t2 - t1, 2),
+        "source_lines": len(phrases) + len(data),
+        "params": info["params"],
+        "nonzero_params": nonzero,
+        "contexts_seen": info["contexts_seen"],
+        "neurons": info["neurons"],
+    }
+
+
+def torch_inference() -> dict:
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+
+    t0 = time.perf_counter()
+    tok = AutoTokenizer.from_pretrained(ROOT)
+    model = AutoModelForCausalLM.from_pretrained(ROOT).eval()
+    load = time.perf_counter() - t0
+
+    with torch.no_grad():
+        ids = tok("Write a python function to", return_tensors="pt")
+        model(**ids)  # warm-up
+        t0 = time.perf_counter()
+        for _ in range(20):
+            model(**ids)
+        forward_ms = (time.perf_counter() - t0) / 20 * 1000
+
+        new_tokens, t0 = 0, time.perf_counter()
+        for p in PROMPTS:
+            ids = tok(p, return_tensors="pt")
+            out = model.generate(**ids, max_new_tokens=16)
+            new_tokens += out.shape[1] - ids["input_ids"].shape[1]
+        gen = time.perf_counter() - t0
+    return {
+        "load_seconds": round(load, 1),
+        "next_word_ms": round(forward_ms, 1),
+        "generate_tokens_per_second": round(new_tokens / gen, 1),
+        "threads": torch.get_num_threads(),
+    }
+
+
+def ts_inference() -> dict | None:
+    bun = shutil.which("bun")
+    if not bun:
+        return None
+    t0 = time.perf_counter()
+    subprocess.run([bun, "autocomplete.ts", "--json"], cwd=ROOT, check=True, capture_output=True)
+    load = time.perf_counter() - t0
+    t0 = time.perf_counter()
+    out = subprocess.run([bun, "autocomplete.ts", "--json", *PROMPTS], cwd=ROOT, check=True, capture_output=True, text=True)
+    total = time.perf_counter() - t0 - load
+    words = sum(len(bw.pieces(r["completion"])) + 1 for r in json.loads(out.stdout))  # +1: the final EOS step
+    return {"runtime": "bun, plain TypeScript, no ML library", "load_seconds": round(load, 1), "seconds_per_token": round(total / words, 2)}
+
+
+def exactness() -> float:
+    from transformers import AutoModelForCausalLM
+
+    _, vocab, ngram, _, _, _ = bw.compile_model()
+    model = AutoModelForCausalLM.from_pretrained(ROOT).eval()
+    rng, worst = random.Random(0), 0.0
+    with torch.no_grad():
+        for _ in range(50):
+            ids = [0] + [rng.randrange(2, len(vocab)) for _ in range(rng.randint(1, 80))]
+            got = model(torch.tensor([ids])).logits[0, -1].double().log_softmax(-1)
+            worst = max(worst, (got - ngram.reference_logits(ids).log_softmax(-1)).abs().max().item())
+    return worst
+
+
+def heldout(frac: float = 0.1) -> dict:
+    """Compile without 10% of the data lines, then score next-word prediction on them."""
+    phrases, data = bw.load_phrases(), bw.load_data()
+    test = [t for t in data if zlib.crc32(" ".join(t).encode()) % 100 < frac * 100]
+    train = [t for t in data if zlib.crc32(" ".join(t).encode()) % 100 >= frac * 100]
+    vocab, ngram, _, _, _ = bw.build(phrases, train)
+    index = {p: i for i, p in enumerate(vocab)}
+    top1 = top3 = total = oov = 0
+    for toks in test:
+        ids = [0] + [index.get(p, 1) for p in toks] + [0]
+        for t in range(len(ids) - 1):
+            target = ids[t + 1]
+            total += 1
+            if target == 1:
+                oov += 1
+                continue
+            ranked = ngram.reference_logits(ids[: t + 1]).topk(3).indices.tolist()
+            top1 += ranked[0] == target
+            top3 += target in ranked
+    return {
+        "heldout_lines": len(test),
+        "heldout_tokens": total,
+        "next_token_top1": round(top1 / total, 3),
+        "next_token_top3": round(top3 / total, 3),
+        "oov_rate": round(oov / total, 3),
+    }
+
+
+def main() -> None:
+    results = {"hardware": hardware()}
+    print("compiling (2 full runs)...", flush=True)
+    results["compile"] = {**compile_runs(), **compile_stages()}
+    print("inference...", flush=True)
+    results["pytorch"] = torch_inference()
+    results["typescript"] = ts_inference()
+    results["max_logprob_error_vs_ngram"] = exactness()
+    print("held-out accuracy...", flush=True)
+    results["heldout"] = heldout()
+    out = ROOT / "docs" / "benchmarks.json"
+    out.write_text(json.dumps(results, indent=2) + "\n")
+    print(json.dumps(results, indent=2))
+
+
+if __name__ == "__main__":
+    main()