{
  "status": "passed",
  "conversations": [
    "20",
    "21"
  ],
  "positive_events": 148,
  "negative_spans": 94,
  "plan": {
    "protocol": "turn-export-pilot-20261030-v1",
    "conversations": [
      "20",
      "21"
    ],
    "dataset": "mundo-ai/turn-benchmark-dev",
    "dataset_revision": "8fa18a24be51528a45397b35cbcaecd84202062b",
    "scorer_revision": "38a6f874322430cb3ca71d8a52aa1e636e88bad8",
    "gold_sha256": "89f7a4cec6b22a8fdc7703ad1b93a603ee752cef7d096a72cd50ff5e3f919b38",
    "audio_manifest_sha256": "dada2f249539f67c43879e07298d6211aeabfe0a307814ca3c4cad0f7f57b2f9",
    "p99": {
      "input": "independent mono channels",
      "step_samples": 2560,
      "gate": "saved Silero VAD onset 0.5 / offset 0.35",
      "minimum_silence_seconds": 0.3,
      "maximum_silence_seconds": 2.0,
      "threshold": 0.5,
      "readout": "latest output available at a completed 160 ms step",
      "end_of_stream_flush": false
    },
    "vap": {
      "input": "stereo channel order speaker_1, speaker_2",
      "window_samples": 320000,
      "readout_samples": 8000,
      "readout": "last frame only, timestamped at input window end",
      "short_context": "leading zeros",
      "eot_threshold": 0.9161,
      "refractory_seconds": 2.0,
      "onnx_threads": 2
    },
    "threshold_tuning_on_pilot": false,
    "graphs": {
      "p99_coreml": "7b7d37a333e098671ddc428d5f5055acc6fbdf1d92383198d815692031201b67",
      "vap_coreml": "3d856c039e8aff382b71280546ffeaee5e886ddd8804fe1d75be378968e1fd29",
      "vap_onnx": "8dc413028f9181115ef46eb3260cd2be01aa61cf123711843071e45db5b7133a"
    }
  },
  "metrics": {
    "p99-coreml": {
      "recall": 0.9054054054054054,
      "fpr": 0.1595744680851064,
      "tp": 134,
      "fn": 14,
      "fp": 15,
      "tn": 79,
      "p10_ms": 300.99999999999784,
      "p50_ms": 1506.0000000000002,
      "p90_ms": 2058.9999999999914,
      "runtime": {
        "requests": 18958,
        "model_compute_seconds": 65.37459301610943,
        "compute_rtf_per_conversation": 0.043102675508419036,
        "request_ms_p50": 3.0113745015114546,
        "request_ms_p95": 5.207720579346642
      },
      "predictions_sha256": "3ea95d48d1710faf1de6e47ac05652009348da0b065de68e5890f4e1321769e3"
    },
    "p99-reference": {
      "recall": 0.9054054054054054,
      "fpr": 0.1595744680851064,
      "tp": 134,
      "fn": 14,
      "fp": 15,
      "tn": 79,
      "p10_ms": 300.99999999999784,
      "p50_ms": 1506.0000000000002,
      "p90_ms": 2058.9999999999914,
      "runtime": {
        "requests": 18958,
        "model_compute_seconds": 46.38250518415589,
        "compute_rtf_per_conversation": 0.03058084154692321,
        "request_ms_p50": 1.9732710206881166,
        "request_ms_p95": 4.183964408002792
      },
      "predictions_sha256": "3ea95d48d1710faf1de6e47ac05652009348da0b065de68e5890f4e1321769e3"
    },
    "vap-coreml": {
      "recall": 0.8378378378378378,
      "fpr": 0.02127659574468085,
      "tp": 124,
      "fn": 24,
      "fp": 2,
      "tn": 92,
      "p10_ms": 388.9000000000072,
      "p50_ms": 945.4999999999956,
      "p90_ms": 2032.1999999999894,
      "runtime": {
        "requests": 3032,
        "model_compute_seconds": 371.5537947830744,
        "compute_rtf_per_conversation": 0.24497227304359984,
        "request_ms_p50": 113.04749950068071,
        "request_ms_p95": 216.1974496615583
      },
      "predictions_sha256": "7374a1c9409eb97f6d9ced9ee376ae4654931fe5bd4aaae8e11b67c684780e18"
    },
    "vap-onnx": {
      "recall": 0.8378378378378378,
      "fpr": 0.02127659574468085,
      "tp": 124,
      "fn": 24,
      "fp": 2,
      "tn": 92,
      "p10_ms": 388.9000000000072,
      "p50_ms": 945.4999999999956,
      "p90_ms": 2032.1999999999894,
      "runtime": {
        "requests": 3032,
        "model_compute_seconds": 1158.3884125971235,
        "compute_rtf_per_conversation": 0.7637468557331274,
        "request_ms_p50": 377.92920850915834,
        "request_ms_p95": 416.1279473511968
      },
      "predictions_sha256": "7374a1c9409eb97f6d9ced9ee376ae4654931fe5bd4aaae8e11b67c684780e18"
    },
    "vap-reference": {
      "recall": 0.8378378378378378,
      "fpr": 0.02127659574468085,
      "tp": 124,
      "fn": 24,
      "fp": 2,
      "tn": 92,
      "p10_ms": 388.9000000000072,
      "p50_ms": 945.4999999999956,
      "p90_ms": 2032.1999999999894,
      "runtime": {
        "requests": 3032,
        "model_compute_seconds": 1423.4309210996144,
        "compute_rtf_per_conversation": 0.9384942723190351,
        "request_ms_p50": 409.97106197755784,
        "request_ms_p95": 703.7348371639378
      },
      "predictions_sha256": "7374a1c9409eb97f6d9ced9ee376ae4654931fe5bd4aaae8e11b67c684780e18"
    }
  },
  "maximum_probability_errors": {
    "p99_coreml": 5.270540714263916e-05,
    "vap_coreml": 1.1324882507324219e-05,
    "vap_onnx": 9.59634780883789e-06
  },
  "event_predictions_identical_to_reference": {
    "p99-coreml": true,
    "vap-coreml": true,
    "vap-onnx": true
  },
  "latency_scope": "Offline audio-timeline replay. Request computation is measured separately and excluded from event timestamps.",
  "hardware": "Apple M5 Pro, 48 GiB; CPU; ONNX Runtime 2 threads",
  "validation_versions": {
    "torch": "2.11.0",
    "torchaudio": "2.11.0",
    "numpy": "2.2.6",
    "coremltools": "9.0",
    "onnxruntime": "1.31.0"
  },
  "saved_Silero_trace_sha256": "6cdf85f35d4e5d0384af85f725c0a896fe1ffd7c4d9b250883560bb9dc5ff216"
}