Confidence-gated ghost text for coding prompts: Kneser-Ney compile, 4.5 MB browser export + zero-dep runtime, held-out precision/keystrokes-saved benchmarks
4113449 verified | { | |
| "hardware": { | |
| "cpu": "Darwin arm64", | |
| "cores": 10, | |
| "ram_gb": 32, | |
| "gpu": "none used", | |
| "python": "3.13.4", | |
| "torch": "2.15.0" | |
| }, | |
| "compile_dense": { | |
| "seconds": 6.9, | |
| "runs": [ | |
| 6.9, | |
| 7.0 | |
| ], | |
| "byte_identical_rebuild": true, | |
| "sha256": "ae857db3e321c35782192c198a7778987964e9cc147d2c67428fea9bf5777100", | |
| "mb": 1521.16, | |
| "peak_memory_gb": 2.4, | |
| "tokenize_seconds": 0.48, | |
| "count_seconds": 2.37, | |
| "build_tensors_seconds": 1.07, | |
| "source_lines": 21106, | |
| "params": 380290320, | |
| "nonzero_params": 26792230, | |
| "vocab": 3000, | |
| "min_context_count": 8.0, | |
| "contexts_seen": 434284, | |
| "contexts_kept": 24773 | |
| }, | |
| "compile_web": { | |
| "seconds": 6.6, | |
| "runs": [ | |
| 7.2, | |
| 6.6 | |
| ], | |
| "byte_identical_rebuild": true, | |
| "sha256": "d770bf4ec6644ccab5646e2b1244b58b185378b371c52b1c263b5a0ebf3f8499", | |
| "mb": 4.49, | |
| "contexts": 107864, | |
| "vocab": 6612 | |
| }, | |
| "pytorch": { | |
| "load_seconds": 0.1, | |
| "next_word_ms": 57.9, | |
| "generate_tokens_per_second": 18.0, | |
| "threads": 8 | |
| }, | |
| "web_keystroke_latency": { | |
| "runtime": "web/autocomplete.ts on Bun", | |
| "median_ms": 0.5, | |
| "p95_ms": 0.92, | |
| "max_ms": 3.48 | |
| }, | |
| "max_logprob_error_vs_ngram": 1.6793374655321713e-05, | |
| "heldout_typing_web": { | |
| "policy": { | |
| "tau": 0.72, | |
| "min_support": 0, | |
| "max_words": 8 | |
| }, | |
| "heldout_prompts": 711, | |
| "characters": 42185, | |
| "tau=0.72": { | |
| "precision": 0.9029, | |
| "coverage": 0.3866, | |
| "keystrokes_saved": 0.2296, | |
| "shown": 16034 | |
| }, | |
| "tau=0.8": { | |
| "precision": 0.9351, | |
| "coverage": 0.3412, | |
| "keystrokes_saved": 0.2066, | |
| "shown": 14153 | |
| }, | |
| "tau=0.9": { | |
| "precision": 0.9644, | |
| "coverage": 0.2692, | |
| "keystrokes_saved": 0.1642, | |
| "shown": 11165 | |
| }, | |
| "tau=0.95": { | |
| "precision": 0.9773, | |
| "coverage": 0.2049, | |
| "keystrokes_saved": 0.1176, | |
| "shown": 8497 | |
| } | |
| }, | |
| "heldout_typing_dense_tables": { | |
| "policy": { | |
| "tau": 0.72, | |
| "min_support": 0, | |
| "max_words": 8 | |
| }, | |
| "heldout_prompts": 711, | |
| "characters": 42185, | |
| "tau=0.72": { | |
| "precision": 0.9243, | |
| "coverage": 0.3245, | |
| "keystrokes_saved": 0.1902, | |
| "shown": 13458 | |
| }, | |
| "tau=0.8": { | |
| "precision": 0.9516, | |
| "coverage": 0.2698, | |
| "keystrokes_saved": 0.1574, | |
| "shown": 11191 | |
| }, | |
| "tau=0.9": { | |
| "precision": 0.9736, | |
| "coverage": 0.1921, | |
| "keystrokes_saved": 0.1057, | |
| "shown": 7966 | |
| }, | |
| "tau=0.95": { | |
| "precision": 0.9843, | |
| "coverage": 0.1322, | |
| "keystrokes_saved": 0.0698, | |
| "shown": 5481 | |
| } | |
| }, | |
| "heldout_next_token_dense": { | |
| "heldout_lines": 711, | |
| "heldout_tokens": 9466, | |
| "next_token_top1": 0.267, | |
| "next_token_top3": 0.406, | |
| "oov_rate": 0.07 | |
| } | |
| } | |