| { |
| "status": "passed", |
| "conversations": [ |
| "20", |
| "21" |
| ], |
| "positive_events": 148, |
| "negative_spans": 94, |
| "plan": { |
| "protocol": "turn-export-pilot-20261030-v1", |
| "conversations": [ |
| "20", |
| "21" |
| ], |
| "dataset": "mundo-ai/turn-benchmark-dev", |
| "dataset_revision": "8fa18a24be51528a45397b35cbcaecd84202062b", |
| "scorer_revision": "38a6f874322430cb3ca71d8a52aa1e636e88bad8", |
| "gold_sha256": "89f7a4cec6b22a8fdc7703ad1b93a603ee752cef7d096a72cd50ff5e3f919b38", |
| "audio_manifest_sha256": "dada2f249539f67c43879e07298d6211aeabfe0a307814ca3c4cad0f7f57b2f9", |
| "p99": { |
| "input": "independent mono channels", |
| "step_samples": 2560, |
| "gate": "saved Silero VAD onset 0.5 / offset 0.35", |
| "minimum_silence_seconds": 0.3, |
| "maximum_silence_seconds": 2.0, |
| "threshold": 0.5, |
| "readout": "latest output available at a completed 160 ms step", |
| "end_of_stream_flush": false |
| }, |
| "vap": { |
| "input": "stereo channel order speaker_1, speaker_2", |
| "window_samples": 320000, |
| "readout_samples": 8000, |
| "readout": "last frame only, timestamped at input window end", |
| "short_context": "leading zeros", |
| "eot_threshold": 0.9161, |
| "refractory_seconds": 2.0, |
| "onnx_threads": 2 |
| }, |
| "threshold_tuning_on_pilot": false, |
| "graphs": { |
| "p99_coreml": "7b7d37a333e098671ddc428d5f5055acc6fbdf1d92383198d815692031201b67", |
| "vap_coreml": "3d856c039e8aff382b71280546ffeaee5e886ddd8804fe1d75be378968e1fd29", |
| "vap_onnx": "8dc413028f9181115ef46eb3260cd2be01aa61cf123711843071e45db5b7133a" |
| } |
| }, |
| "metrics": { |
| "p99-coreml": { |
| "recall": 0.9054054054054054, |
| "fpr": 0.1595744680851064, |
| "tp": 134, |
| "fn": 14, |
| "fp": 15, |
| "tn": 79, |
| "p10_ms": 300.99999999999784, |
| "p50_ms": 1506.0000000000002, |
| "p90_ms": 2058.9999999999914, |
| "runtime": { |
| "requests": 18958, |
| "model_compute_seconds": 65.37459301610943, |
| "compute_rtf_per_conversation": 0.043102675508419036, |
| "request_ms_p50": 3.0113745015114546, |
| "request_ms_p95": 5.207720579346642 |
| }, |
| "predictions_sha256": "3ea95d48d1710faf1de6e47ac05652009348da0b065de68e5890f4e1321769e3" |
| }, |
| "p99-reference": { |
| "recall": 0.9054054054054054, |
| "fpr": 0.1595744680851064, |
| "tp": 134, |
| "fn": 14, |
| "fp": 15, |
| "tn": 79, |
| "p10_ms": 300.99999999999784, |
| "p50_ms": 1506.0000000000002, |
| "p90_ms": 2058.9999999999914, |
| "runtime": { |
| "requests": 18958, |
| "model_compute_seconds": 46.38250518415589, |
| "compute_rtf_per_conversation": 0.03058084154692321, |
| "request_ms_p50": 1.9732710206881166, |
| "request_ms_p95": 4.183964408002792 |
| }, |
| "predictions_sha256": "3ea95d48d1710faf1de6e47ac05652009348da0b065de68e5890f4e1321769e3" |
| }, |
| "vap-coreml": { |
| "recall": 0.8378378378378378, |
| "fpr": 0.02127659574468085, |
| "tp": 124, |
| "fn": 24, |
| "fp": 2, |
| "tn": 92, |
| "p10_ms": 388.9000000000072, |
| "p50_ms": 945.4999999999956, |
| "p90_ms": 2032.1999999999894, |
| "runtime": { |
| "requests": 3032, |
| "model_compute_seconds": 371.5537947830744, |
| "compute_rtf_per_conversation": 0.24497227304359984, |
| "request_ms_p50": 113.04749950068071, |
| "request_ms_p95": 216.1974496615583 |
| }, |
| "predictions_sha256": "7374a1c9409eb97f6d9ced9ee376ae4654931fe5bd4aaae8e11b67c684780e18" |
| }, |
| "vap-onnx": { |
| "recall": 0.8378378378378378, |
| "fpr": 0.02127659574468085, |
| "tp": 124, |
| "fn": 24, |
| "fp": 2, |
| "tn": 92, |
| "p10_ms": 388.9000000000072, |
| "p50_ms": 945.4999999999956, |
| "p90_ms": 2032.1999999999894, |
| "runtime": { |
| "requests": 3032, |
| "model_compute_seconds": 1158.3884125971235, |
| "compute_rtf_per_conversation": 0.7637468557331274, |
| "request_ms_p50": 377.92920850915834, |
| "request_ms_p95": 416.1279473511968 |
| }, |
| "predictions_sha256": "7374a1c9409eb97f6d9ced9ee376ae4654931fe5bd4aaae8e11b67c684780e18" |
| }, |
| "vap-reference": { |
| "recall": 0.8378378378378378, |
| "fpr": 0.02127659574468085, |
| "tp": 124, |
| "fn": 24, |
| "fp": 2, |
| "tn": 92, |
| "p10_ms": 388.9000000000072, |
| "p50_ms": 945.4999999999956, |
| "p90_ms": 2032.1999999999894, |
| "runtime": { |
| "requests": 3032, |
| "model_compute_seconds": 1423.4309210996144, |
| "compute_rtf_per_conversation": 0.9384942723190351, |
| "request_ms_p50": 409.97106197755784, |
| "request_ms_p95": 703.7348371639378 |
| }, |
| "predictions_sha256": "7374a1c9409eb97f6d9ced9ee376ae4654931fe5bd4aaae8e11b67c684780e18" |
| } |
| }, |
| "maximum_probability_errors": { |
| "p99_coreml": 5.270540714263916e-05, |
| "vap_coreml": 1.1324882507324219e-05, |
| "vap_onnx": 9.59634780883789e-06 |
| }, |
| "event_predictions_identical_to_reference": { |
| "p99-coreml": true, |
| "vap-coreml": true, |
| "vap-onnx": true |
| }, |
| "latency_scope": "Offline audio-timeline replay. Request computation is measured separately and excluded from event timestamps.", |
| "hardware": "Apple M5 Pro, 48 GiB; CPU; ONNX Runtime 2 threads", |
| "validation_versions": { |
| "torch": "2.11.0", |
| "torchaudio": "2.11.0", |
| "numpy": "2.2.6", |
| "coremltools": "9.0", |
| "onnxruntime": "1.31.0" |
| }, |
| "saved_Silero_trace_sha256": "6cdf85f35d4e5d0384af85f725c0a896fe1ffd7c4d9b250883560bb9dc5ff216" |
| } |
|
|
