| { |
| "regression": { |
| "overall": { |
| "n": 664, |
| "correct": 617, |
| "accuracy": 0.9292168674698795, |
| "macro_f1": 0.8939250255772296, |
| "tp": 117, |
| "fn": 33, |
| "fp": 14, |
| "tn": 500, |
| "deep_recall": 0.78, |
| "false_deep_rate": 0.027237354085603113, |
| "deep_recall_wilson95": [ |
| 0.7071754568460243, |
| 0.8388407052323733 |
| ], |
| "false_deep_wilson95": [ |
| 0.016292663069040196, |
| 0.045196410037485035 |
| ], |
| "brier": 0.05831906802495684, |
| "nll": 0.19566529778461716 |
| }, |
| "by_language": { |
| "en": { |
| "n": 303, |
| "correct": 274, |
| "accuracy": 0.9042904290429042, |
| "macro_f1": 0.8721053780656429, |
| "tp": 61, |
| "fn": 24, |
| "fp": 5, |
| "tn": 213, |
| "deep_recall": 0.7176470588235294, |
| "false_deep_rate": 0.022935779816513763, |
| "deep_recall_wilson95": [ |
| 0.6141587579262482, |
| 0.8023128049451995 |
| ], |
| "false_deep_wilson95": [ |
| 0.009835515458069499, |
| 0.05255855310256116 |
| ], |
| "brier": 0.07394316489740126, |
| "nll": 0.25100498741147925 |
| }, |
| "zh": { |
| "n": 361, |
| "correct": 343, |
| "accuracy": 0.9501385041551247, |
| "macro_f1": 0.9155665280665282, |
| "tp": 56, |
| "fn": 9, |
| "fp": 9, |
| "tn": 287, |
| "deep_recall": 0.8615384615384616, |
| "false_deep_rate": 0.030405405405405407, |
| "deep_recall_wilson95": [ |
| 0.7573167852534054, |
| 0.9254099381231576 |
| ], |
| "false_deep_wilson95": [ |
| 0.016077105119947862, |
| 0.056766689737069984 |
| ], |
| "brier": 0.045205213863320645, |
| "nll": 0.14921674942744484 |
| } |
| }, |
| "by_category": { |
| "completion": { |
| "n": 10, |
| "correct": 10, |
| "accuracy": 1.0, |
| "macro_f1": 1.0, |
| "tp": 6, |
| "fn": 0, |
| "fp": 0, |
| "tn": 4, |
| "deep_recall": 1.0, |
| "false_deep_rate": 0.0, |
| "deep_recall_wilson95": [ |
| 0.6096569663469354, |
| 0.9999999999999999 |
| ], |
| "false_deep_wilson95": [ |
| 0, |
| 0.48990002040399916 |
| ], |
| "brier": 0.0010905407833151288, |
| "nll": 0.019759182865158858 |
| }, |
| "context_completed": { |
| "n": 26, |
| "correct": 26, |
| "accuracy": 1.0, |
| "macro_f1": 0.5, |
| "tp": 0, |
| "fn": 0, |
| "fp": 0, |
| "tn": 26, |
| "deep_recall": null, |
| "false_deep_rate": 0.0, |
| "deep_recall_wilson95": null, |
| "false_deep_wilson95": [ |
| 0, |
| 0.12873304380462175 |
| ], |
| "brier": 0.00024423200617112355, |
| "nll": 0.011072157124723887 |
| }, |
| "context_unfinished": { |
| "n": 28, |
| "correct": 16, |
| "accuracy": 0.5714285714285714, |
| "macro_f1": 0.5087719298245614, |
| "tp": 13, |
| "fn": 9, |
| "fp": 3, |
| "tn": 3, |
| "deep_recall": 0.5909090909090909, |
| "false_deep_rate": 0.5, |
| "deep_recall_wilson95": [ |
| 0.38734477382893806, |
| 0.7674444033117963 |
| ], |
| "false_deep_wilson95": [ |
| 0.18761280689940868, |
| 0.8123871931005913 |
| ], |
| "brier": 0.31489699811230015, |
| "nll": 0.9439517192864099 |
| }, |
| "emotion": { |
| "n": 10, |
| "correct": 10, |
| "accuracy": 1.0, |
| "macro_f1": 1.0, |
| "tp": 4, |
| "fn": 0, |
| "fp": 0, |
| "tn": 6, |
| "deep_recall": 1.0, |
| "false_deep_rate": 0.0, |
| "deep_recall_wilson95": [ |
| 0.5100999795960008, |
| 1 |
| ], |
| "false_deep_wilson95": [ |
| 0, |
| 0.3903430336530645 |
| ], |
| "brier": 3.401839728294804e-05, |
| "nll": 0.0028612694791609336 |
| }, |
| "factual": { |
| "n": 10, |
| "correct": 10, |
| "accuracy": 1.0, |
| "macro_f1": 1.0, |
| "tp": 6, |
| "fn": 0, |
| "fp": 0, |
| "tn": 4, |
| "deep_recall": 1.0, |
| "false_deep_rate": 0.0, |
| "deep_recall_wilson95": [ |
| 0.6096569663469354, |
| 0.9999999999999999 |
| ], |
| "false_deep_wilson95": [ |
| 0, |
| 0.48990002040399916 |
| ], |
| "brier": 0.0026418461177944662, |
| "nll": 0.04135682687840026 |
| }, |
| "memory_fact": { |
| "n": 10, |
| "correct": 9, |
| "accuracy": 0.9, |
| "macro_f1": 0.8901098901098901, |
| "tp": 3, |
| "fn": 1, |
| "fp": 0, |
| "tn": 6, |
| "deep_recall": 0.75, |
| "false_deep_rate": 0.0, |
| "deep_recall_wilson95": [ |
| 0.3006360524426366, |
| 0.9544139373553637 |
| ], |
| "false_deep_wilson95": [ |
| 0, |
| 0.3903430336530645 |
| ], |
| "brier": 0.03388762151034709, |
| "nll": 0.1221164888785166 |
| }, |
| "new_constraint_after_completion": { |
| "n": 30, |
| "correct": 30, |
| "accuracy": 1.0, |
| "macro_f1": 0.5, |
| "tp": 30, |
| "fn": 0, |
| "fp": 0, |
| "tn": 0, |
| "deep_recall": 1.0, |
| "false_deep_rate": null, |
| "deep_recall_wilson95": [ |
| 0.8864829086095221, |
| 1 |
| ], |
| "false_deep_wilson95": null, |
| "brier": 0.0024941061534353476, |
| "nll": 0.03609175294310134 |
| }, |
| "public_conversation": { |
| "n": 362, |
| "correct": 333, |
| "accuracy": 0.919889502762431, |
| "macro_f1": 0.8358738645779591, |
| "tp": 37, |
| "fn": 20, |
| "fp": 9, |
| "tn": 296, |
| "deep_recall": 0.6491228070175439, |
| "false_deep_rate": 0.029508196721311476, |
| "deep_recall_wilson95": [ |
| 0.519416545881409, |
| 0.7599975392182597 |
| ], |
| "false_deep_wilson95": [ |
| 0.01560036886817662, |
| 0.05512067389290219 |
| ], |
| "brier": 0.07042387677218072, |
| "nll": 0.24027162211281072 |
| }, |
| "public_single_turn": { |
| "n": 100, |
| "correct": 95, |
| "accuracy": 0.95, |
| "macro_f1": 0.8669859005054535, |
| "tp": 8, |
| "fn": 3, |
| "fp": 2, |
| "tn": 87, |
| "deep_recall": 0.7272727272727273, |
| "false_deep_rate": 0.02247191011235955, |
| "deep_recall_wilson95": [ |
| 0.4343497069897537, |
| 0.9025411942607853 |
| ], |
| "false_deep_wilson95": [ |
| 0.006184269063094168, |
| 0.07827786859933303 |
| ], |
| "brier": 0.03411899684260777, |
| "nll": 0.11113511759895704 |
| }, |
| "topic_shift": { |
| "n": 30, |
| "correct": 30, |
| "accuracy": 1.0, |
| "macro_f1": 0.5, |
| "tp": 0, |
| "fn": 0, |
| "fp": 0, |
| "tn": 30, |
| "deep_recall": null, |
| "false_deep_rate": 0.0, |
| "deep_recall_wilson95": null, |
| "false_deep_wilson95": [ |
| 0, |
| 0.113517091390478 |
| ], |
| "brier": 1.7556884895978534e-09, |
| "nll": 3.98404888866351e-05 |
| }, |
| "user_pause": { |
| "n": 36, |
| "correct": 36, |
| "accuracy": 1.0, |
| "macro_f1": 1.0, |
| "tp": 4, |
| "fn": 0, |
| "fp": 0, |
| "tn": 32, |
| "deep_recall": 1.0, |
| "false_deep_rate": 0.0, |
| "deep_recall_wilson95": [ |
| 0.5100999795960008, |
| 1 |
| ], |
| "false_deep_wilson95": [ |
| 0, |
| 0.1071827150573635 |
| ], |
| "brier": 9.071688464884205e-06, |
| "nll": 0.00122522356652892 |
| }, |
| "writing": { |
| "n": 12, |
| "correct": 12, |
| "accuracy": 1.0, |
| "macro_f1": 1.0, |
| "tp": 6, |
| "fn": 0, |
| "fp": 0, |
| "tn": 6, |
| "deep_recall": 1.0, |
| "false_deep_rate": 0.0, |
| "deep_recall_wilson95": [ |
| 0.6096569663469354, |
| 0.9999999999999999 |
| ], |
| "false_deep_wilson95": [ |
| 0, |
| 0.3903430336530645 |
| ], |
| "brier": 0.045280168944203315, |
| "nll": 0.17686675055194026 |
| } |
| } |
| }, |
| "regression_scope": "Reused machine-supervised regression; not independent human speech.", |
| "latency_common_128_inputs_twice": { |
| "gpu": { |
| "n": 256, |
| "p50_ms": 10.602879337966442, |
| "p95_ms": 11.419005924835801, |
| "threads": null |
| }, |
| "cpu": { |
| "n": 256, |
| "p50_ms": 47.47643414884806, |
| "p95_ms": 121.94657069630921, |
| "threads": 2 |
| } |
| }, |
| "latency_compiled_full_664_inputs_twice": { |
| "n": 1328, |
| "p50_ms": 2.2120927460491657, |
| "p95_ms": 3.2203245908021927 |
| }, |
| "gpu": "NVIDIA L20", |
| "tokenization_included": true, |
| "result_cache_disabled": true, |
| "compile_and_warm_seconds": 61.08646277990192, |
| "compiled_decisions_checked": 1222, |
| "onnx_decisions_checked": 1222, |
| "tokenizer_parity_rows": 7224, |
| "native_macos_tested": false, |
| "native_windows_tested": false, |
| "live_asr_tts_contention_measured": false, |
| "model_id": "Sorbett/VoiceMem-Jev" |
| } |
|
|
