ghost-typer / docs /benchmarks.json
tmickleydoyle's picture
v6: modified Kneser-Ney + per-word extension rule (held-out keystrokes saved 22.2% -> 23.0%, precision 90.2% -> 90.8%)
3c53a76 verified
Raw History Blame Contribute Delete
5.49 kB
{
"hardware": {
"cpu": "Darwin arm64",
"cores": 10,
"ram_gb": 32,
"gpu": "none used",
"python": "3.13.4",
"torch": "2.15.0"
},
"compile_dense": {
"seconds": 13.5,
"runs": [
13.5,
14.0
],
"byte_identical_rebuild": true,
"sha256": "60ae8a557cdc9a2c2e8d1fffce45187eb079ab46d0929dead9b23fcebad0d031",
"mb": 1599.67,
"peak_memory_gb": 3.6,
"tokenize_seconds": 2.74,
"count_seconds": 5.36,
"build_tensors_seconds": 2.05,
"source_lines": 126347,
"params": 399916440,
"nonzero_params": 13553535,
"vocab": 6000,
"min_context_count": 37.0,
"contexts_seen": 2127286,
"contexts_kept": 12607
},
"compile_web": {
"seconds": 21.2,
"runs": [
21.4,
21.2
],
"byte_identical_rebuild": true,
"sha256": "b49b81f934b345a8a27348ac8660747b12f7dc457ec58586b0966a49a5644a80",
"mb": 10.69,
"contexts": 512533,
"vocab": 30893,
"deltas": "u8"
},
"pytorch": {
"load_seconds": 0.1,
"next_word_ms": 58.2,
"generate_tokens_per_second": 18.8,
"threads": 8
},
"web_keystroke_latency": {
"runtime": "web/autocomplete.ts on Bun",
"median_ms": 2.47,
"p95_ms": 4.25,
"max_ms": 5.49
},
"max_logprob_error_vs_ngram": 1.4116566347865955e-05,
"heldout_typing_web": {
"policy": {
"tau": 0.62,
"min_support": 0,
"max_words": 8,
"ext": 0.8
},
"heldout": {
"prompts": 711,
"wildchat": 1000
},
"tau=0.62": {
"all": {
"precision": 0.9079,
"coverage": 0.3919,
"keystrokes_saved": 0.23,
"shown": 42183
},
"prompts": {
"precision": 0.9188,
"coverage": 0.4153,
"keystrokes_saved": 0.243,
"shown": 17225
},
"wildchat": {
"precision": 0.9004,
"coverage": 0.3772,
"keystrokes_saved": 0.2219,
"shown": 24958
}
},
"tau=0.8": {
"all": {
"precision": 0.9489,
"coverage": 0.3046,
"keystrokes_saved": 0.1834,
"shown": 32790
},
"prompts": {
"precision": 0.9551,
"coverage": 0.3245,
"keystrokes_saved": 0.1919,
"shown": 13457
},
"wildchat": {
"precision": 0.9446,
"coverage": 0.2922,
"keystrokes_saved": 0.178,
"shown": 19333
}
},
"tau=0.9": {
"all": {
"precision": 0.9669,
"coverage": 0.2444,
"keystrokes_saved": 0.1509,
"shown": 26305
},
"prompts": {
"precision": 0.9706,
"coverage": 0.262,
"keystrokes_saved": 0.1574,
"shown": 10868
},
"wildchat": {
"precision": 0.9642,
"coverage": 0.2333,
"keystrokes_saved": 0.1468,
"shown": 15437
}
},
"tau=0.95": {
"all": {
"precision": 0.9739,
"coverage": 0.1904,
"keystrokes_saved": 0.1208,
"shown": 20498
},
"prompts": {
"precision": 0.9782,
"coverage": 0.2023,
"keystrokes_saved": 0.1239,
"shown": 8389
},
"wildchat": {
"precision": 0.971,
"coverage": 0.183,
"keystrokes_saved": 0.1188,
"shown": 12109
}
}
},
"heldout_typing_dense_tables": {
"policy": {
"tau": 0.62,
"min_support": 0,
"max_words": 8,
"ext": 0.8
},
"heldout": {
"prompts": 711,
"wildchat": 1000
},
"tau=0.62": {
"all": {
"precision": 0.9171,
"coverage": 0.3126,
"keystrokes_saved": 0.1759,
"shown": 33650
},
"prompts": {
"precision": 0.9314,
"coverage": 0.3498,
"keystrokes_saved": 0.2003,
"shown": 14507
},
"wildchat": {
"precision": 0.9062,
"coverage": 0.2893,
"keystrokes_saved": 0.1606,
"shown": 19143
}
},
"tau=0.8": {
"all": {
"precision": 0.958,
"coverage": 0.2188,
"keystrokes_saved": 0.125,
"shown": 23547
},
"prompts": {
"precision": 0.965,
"coverage": 0.2492,
"keystrokes_saved": 0.1428,
"shown": 10334
},
"wildchat": {
"precision": 0.9525,
"coverage": 0.1997,
"keystrokes_saved": 0.1139,
"shown": 13213
}
},
"tau=0.9": {
"all": {
"precision": 0.9764,
"coverage": 0.1481,
"keystrokes_saved": 0.0855,
"shown": 15942
},
"prompts": {
"precision": 0.9813,
"coverage": 0.1738,
"keystrokes_saved": 0.0997,
"shown": 7208
},
"wildchat": {
"precision": 0.9723,
"coverage": 0.132,
"keystrokes_saved": 0.0767,
"shown": 8734
}
},
"tau=0.95": {
"all": {
"precision": 0.9846,
"coverage": 0.0994,
"keystrokes_saved": 0.06,
"shown": 10704
},
"prompts": {
"precision": 0.99,
"coverage": 0.1182,
"keystrokes_saved": 0.0687,
"shown": 4901
},
"wildchat": {
"precision": 0.98,
"coverage": 0.0877,
"keystrokes_saved": 0.0545,
"shown": 5803
}
}
},
"heldout_next_token_dense": {
"heldout_lines": 711,
"heldout_tokens": 9466,
"next_token_top1": 0.253,
"next_token_top3": 0.411,
"oov_rate": 0.04
}
}