Pyannote-Community-1-CoreML / benchmark.json
aufklarer's picture
Publish Pyannote Community-1 CoreML bundle
d026707 verified
Raw
History Blame Contribute Delete
10.4 kB
{
"amiSingleMeetingDiagnostic": {
"audioSeconds": 1049.3546875,
"endToEnd": {
"peakRssMB": 944.375,
"throughputXRT": 23.988282918384776
},
"exactMatchToUpstream": true,
"exclusive": {
"collar025": {
"derPercent": 22.58107201156437,
"jerPercent": 24.12583550793453
},
"strict": {
"derPercent": 34.977370101045615,
"jerPercent": 34.678204358087
}
},
"oracleSpeakerCount": 4,
"recording": "ami-ES2004a",
"regular": {
"collar025": {
"derPercent": 23.00216585375451,
"jerPercent": 23.619490786739856
},
"strict": {
"derPercent": 35.22383954393369,
"jerPercent": 33.206537162384095
}
}
},
"method": {
"collarsSeconds": [
0.0,
0.25
],
"hostProcessing": "official pyannote.audio Community-1 powerset, speaker counting, VBx, and reconstruction",
"networkDuringInference": false,
"neuralStages": "compiled Core ML",
"overlapScored": true,
"telemetry": false
},
"runtime": {
"computeUnits": "cpu-and-neural-engine",
"coremltoolsVersion": "8.3.0",
"cpu": "Apple M5 Pro",
"machine": "arm64",
"platform": "macOS-26.5.2-arm64-arm-64bit",
"pyannoteAudioVersion": "4.0.7"
},
"schemaVersion": 1,
"sourceModel": "pyannote/speaker-diarization-community-1",
"sourceRevision": "3533c8cf8e369892e6b79ff1bf80f7b0286a54ee",
"voxconverseFiveFileSubset": {
"createdAt": "2026-07-14T14:50:36.914392+00:00",
"endToEnd": {
"peakRssMB": 863.421875,
"throughputXRT": 25.001608024686572
},
"exactMatchToUpstream": true,
"exclusive": {
"collar025": {
"derPercent": 7.8330999608667895,
"jerPercent": 21.825168113918547
},
"strict": {
"derPercent": 10.309137316831276,
"jerPercent": 23.61927726234164
}
},
"files": [
{
"audioSeconds": 148.32,
"detectedSpeakers": 3,
"exclusive": {
"collar025": {
"confusionSeconds": 0.6615937500000002,
"derPercent": 3.2672644439454674,
"falseAlarmSeconds": 0.7983437499999628,
"jerPercent": 3.5324457444260835,
"missSeconds": 2.9057812499999702,
"referenceSpeakerSeconds": 133.62
},
"strict": {
"confusionSeconds": 0.7865937500000002,
"derPercent": 4.908588280338341,
"falseAlarmSeconds": 2.1820312499999526,
"jerPercent": 5.841663331740058,
"missSeconds": 3.76203124999998,
"referenceSpeakerSeconds": 137.12
}
},
"id": "vox-ampme",
"referenceSpeakers": 3,
"regular": {
"collar025": {
"confusionSeconds": 0.6615937500000002,
"derPercent": 4.163930362221178,
"falseAlarmSeconds": 1.9964687499999674,
"jerPercent": 9.504901132865212,
"missSeconds": 2.9057812499999702,
"referenceSpeakerSeconds": 133.62
},
"strict": {
"confusionSeconds": 0.7865937500000002,
"derPercent": 5.782366722578717,
"falseAlarmSeconds": 3.380156249999957,
"jerPercent": 11.412048028876494,
"missSeconds": 3.76203124999998,
"referenceSpeakerSeconds": 137.12
}
}
},
{
"audioSeconds": 194.136,
"detectedSpeakers": 3,
"exclusive": {
"collar025": {
"confusionSeconds": 2.09715625000001,
"derPercent": 13.524134267851926,
"falseAlarmSeconds": 0.0,
"jerPercent": 33.47446402551495,
"missSeconds": 25.592156250000023,
"referenceSpeakerSeconds": 204.74000000000004
},
"strict": {
"confusionSeconds": 3.131187500000001,
"derPercent": 15.926859919571058,
"falseAlarmSeconds": 0.23187500000000938,
"jerPercent": 35.194641726815576,
"missSeconds": 32.281250000000014,
"referenceSpeakerSeconds": 223.8
}
},
"id": "vox-azisu",
"referenceSpeakers": 4,
"regular": {
"collar025": {
"confusionSeconds": 5.680687500000005,
"derPercent": 7.149021319722563,
"falseAlarmSeconds": 1.4464687499999922,
"jerPercent": 30.93591229987478,
"missSeconds": 7.509749999999983,
"referenceSpeakerSeconds": 204.74000000000004
},
"strict": {
"confusionSeconds": 6.850281249999995,
"derPercent": 9.529923480786389,
"falseAlarmSeconds": 3.7775937499999817,
"jerPercent": 32.515982955223386,
"missSeconds": 10.700093749999965,
"referenceSpeakerSeconds": 223.80000000000004
}
}
},
{
"audioSeconds": 259.160875,
"detectedSpeakers": 2,
"exclusive": {
"collar025": {
"confusionSeconds": 0.14215624999999932,
"derPercent": 0.7351801529790701,
"falseAlarmSeconds": 0.4240312500000061,
"jerPercent": 0.9684476301170961,
"missSeconds": 1.2600000000000051,
"referenceSpeakerSeconds": 248.40000000000003
},
"strict": {
"confusionSeconds": 0.6398749999999751,
"derPercent": 2.156736760124583,
"falseAlarmSeconds": 1.8436874999999742,
"jerPercent": 2.7212320527959633,
"missSeconds": 3.05493749999998,
"referenceSpeakerSeconds": 256.8
}
},
"id": "vox-blwmj",
"referenceSpeakers": 2,
"regular": {
"collar025": {
"confusionSeconds": 0.14215624999999932,
"derPercent": 0.472473832528178,
"falseAlarmSeconds": 0.9199374999999963,
"jerPercent": 0.6409231193258367,
"missSeconds": 0.11153124999999875,
"referenceSpeakerSeconds": 248.40000000000003
},
"strict": {
"confusionSeconds": 0.5524374999999964,
"derPercent": 1.8084355529594591,
"falseAlarmSeconds": 2.9842499999999186,
"jerPercent": 2.2384443742178157,
"missSeconds": 1.1073749999999762,
"referenceSpeakerSeconds": 256.8
}
}
},
{
"audioSeconds": 300.264,
"detectedSpeakers": 2,
"exclusive": {
"collar025": {
"confusionSeconds": 2.691531249999919,
"derPercent": 10.18159170067545,
"falseAlarmSeconds": 1.601812500000004,
"jerPercent": 10.631003936851133,
"missSeconds": 24.646812499999974,
"referenceSpeakerSeconds": 284.24
},
"strict": {
"confusionSeconds": 5.173843749999847,
"derPercent": 13.722947492313498,
"falseAlarmSeconds": 5.943906249999916,
"jerPercent": 14.60187647291351,
"missSeconds": 31.730781249999907,
"referenceSpeakerSeconds": 312.24
}
},
"id": "vox-crixb",
"referenceSpeakers": 2,
"regular": {
"collar025": {
"confusionSeconds": 2.5665937499999885,
"derPercent": 3.1307829650998653,
"falseAlarmSeconds": 2.35656249999992,
"jerPercent": 4.0015791666816805,
"missSeconds": 3.9757812499999474,
"referenceSpeakerSeconds": 284.24
},
"strict": {
"confusionSeconds": 4.162781249999981,
"derPercent": 6.574139684217199,
"falseAlarmSeconds": 8.856218749999943,
"jerPercent": 7.647274233385495,
"missSeconds": 7.508093749999858,
"referenceSpeakerSeconds": 312.24
}
}
},
{
"audioSeconds": 155.6114375,
"detectedSpeakers": 3,
"exclusive": {
"collar025": {
"confusionSeconds": 6.980000000000001,
"derPercent": 11.408968739662592,
"falseAlarmSeconds": 0.25465625000000386,
"jerPercent": 39.92085630987593,
"missSeconds": 10.010000000000005,
"referenceSpeakerSeconds": 151.15
},
"strict": {
"confusionSeconds": 7.859874999999977,
"derPercent": 13.51821608040199,
"falseAlarmSeconds": 0.9405624999999989,
"jerPercent": 40.3348462453058,
"missSeconds": 12.720562499999994,
"referenceSpeakerSeconds": 159.20000000000002
}
},
"id": "vox-cwryz",
"referenceSpeakers": 4,
"regular": {
"collar025": {
"confusionSeconds": 6.980000000000001,
"derPercent": 11.408968739662592,
"falseAlarmSeconds": 0.25465625000000386,
"jerPercent": 39.92085630987593,
"missSeconds": 10.010000000000005,
"referenceSpeakerSeconds": 151.15
},
"strict": {
"confusionSeconds": 7.859874999999977,
"derPercent": 13.613614949748722,
"falseAlarmSeconds": 1.0924374999999957,
"jerPercent": 40.50417077688739,
"missSeconds": 12.720562499999994,
"referenceSpeakerSeconds": 159.20000000000002
}
}
}
],
"regular": {
"collar025": {
"derPercent": 4.648825392554887,
"jerPercent": 21.415118827307566
},
"strict": {
"derPercent": 6.987107610452046,
"jerPercent": 23.072546415351944
}
},
"speakerCount": {
"accuracy": 0.6,
"exactFiles": 3,
"scoredFiles": 5
},
"stageTiming": {
"embedding": {
"calls": 1016,
"medianMsPerWindow": 33.23672899932717,
"p95MsPerWindow": 35.55754149965651,
"throughputXRT": 297.19853527611525,
"totalSeconds": 34.18590199497703,
"windows": 1016
},
"segmentation": {
"calls": 1016,
"medianMsPerWindow": 7.43545799923595,
"p95MsPerWindow": 7.705229250859702,
"throughputXRT": 1338.3739525810397,
"totalSeconds": 7.59130135520536,
"windows": 1016
}
},
"totalAudioSeconds": 1057.4923125
}
}