| { | |
| "format": "inflect_v2_modern400_three_asr_v1", | |
| "prompt_count_per_system": 400, | |
| "total_audio_clips": 3200, | |
| "asr_models": { | |
| "qwen3_asr": "Qwen/Qwen3-ASR-1.7B", | |
| "nemotron35_asr": "nvidia/nemotron-3.5-asr-streaming-0.6b", | |
| "whisper_large_v3": "openai/whisper-large-v3" | |
| }, | |
| "macro_average": { | |
| "definition": "unweighted mean of the three corpus-level semantic WERs", | |
| "headline": false, | |
| "bootstrap": "matched-prompt resampling within each system", | |
| "samples": 10000 | |
| }, | |
| "systems": { | |
| "inflect-micro-v2": { | |
| "label": "Inflect-Micro-v2", | |
| "qwen3_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 159, | |
| "words": 6794, | |
| "wer": 0.023403002649396525 | |
| }, | |
| "raw_95ci": [ | |
| 0.017408416590977898, | |
| 0.030176415970287838 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 172, | |
| "words": 6823, | |
| "wer": 0.02520885241096292 | |
| }, | |
| "semantic_95ci": [ | |
| 0.018783412801618264, | |
| 0.03235815602836879 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 0, | |
| "words": 296, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 0, | |
| "words": 309, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 0, | |
| "words": 217, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 0, | |
| "words": 226, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 314, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 329, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 114, | |
| "words": 4250, | |
| "wer": 0.026823529411764704 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 148, | |
| "words": 4215, | |
| "wer": 0.035112692763938316 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 70, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 70, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 412, | |
| "wer": 0.0024271844660194173 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 432, | |
| "wer": 0.0023148148148148147 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 57, | |
| "wer": 0.10526315789473684 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 57, | |
| "wer": 0.10526315789473684 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 342, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 352, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 21, | |
| "words": 81, | |
| "wer": 0.25925925925925924 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 64, | |
| "wer": 0.0625 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 98, | |
| "wer": 0.030612244897959183 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 105, | |
| "wer": 0.02857142857142857 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 11, | |
| "words": 251, | |
| "wer": 0.043824701195219126 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 7, | |
| "words": 256, | |
| "wer": 0.02734375 | |
| } | |
| } | |
| } | |
| }, | |
| "nemotron35_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 429, | |
| "words": 6794, | |
| "wer": 0.06314395054459818 | |
| }, | |
| "raw_95ci": [ | |
| 0.052087944518223406, | |
| 0.07540394973070018 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 372, | |
| "words": 6823, | |
| "wer": 0.05452147149347794 | |
| }, | |
| "semantic_95ci": [ | |
| 0.04496430198997418, | |
| 0.0649688901750832 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 3, | |
| "words": 296, | |
| "wer": 0.010135135135135136 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 3, | |
| "words": 309, | |
| "wer": 0.009708737864077669 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 2, | |
| "words": 217, | |
| "wer": 0.009216589861751152 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 2, | |
| "words": 226, | |
| "wer": 0.008849557522123894 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 314, | |
| "wer": 0.009554140127388535 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 329, | |
| "wer": 0.00303951367781155 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 359, | |
| "words": 4250, | |
| "wer": 0.08447058823529412 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 306, | |
| "words": 4215, | |
| "wer": 0.07259786476868327 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 70, | |
| "wer": 0.07142857142857142 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 70, | |
| "wer": 0.07142857142857142 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 7, | |
| "words": 412, | |
| "wer": 0.01699029126213592 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 6, | |
| "words": 432, | |
| "wer": 0.013888888888888888 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 121, | |
| "wer": 0.01652892561983471 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 121, | |
| "wer": 0.01652892561983471 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 13, | |
| "words": 57, | |
| "wer": 0.22807017543859648 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 13, | |
| "words": 57, | |
| "wer": 0.22807017543859648 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 4, | |
| "words": 342, | |
| "wer": 0.011695906432748537 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 4, | |
| "words": 352, | |
| "wer": 0.011363636363636364 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 81, | |
| "wer": 0.06172839506172839 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 64, | |
| "wer": 0.0625 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 57, | |
| "wer": 0.10526315789473684 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 57, | |
| "wer": 0.10526315789473684 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 6, | |
| "words": 98, | |
| "wer": 0.061224489795918366 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 6, | |
| "words": 105, | |
| "wer": 0.05714285714285714 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 14, | |
| "words": 251, | |
| "wer": 0.055776892430278883 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 14, | |
| "words": 256, | |
| "wer": 0.0546875 | |
| } | |
| } | |
| } | |
| }, | |
| "whisper_large_v3": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 189, | |
| "words": 6794, | |
| "wer": 0.027818663526641155 | |
| }, | |
| "raw_95ci": [ | |
| 0.02115697588400651, | |
| 0.034870148818208345 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 186, | |
| "words": 6823, | |
| "wer": 0.02726073574673897 | |
| }, | |
| "semantic_95ci": [ | |
| 0.020453534904401955, | |
| 0.03492018244013683 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 1, | |
| "words": 296, | |
| "wer": 0.0033783783783783786 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 0, | |
| "words": 309, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 2, | |
| "words": 217, | |
| "wer": 0.009216589861751152 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 0, | |
| "words": 226, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 314, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 329, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 124, | |
| "words": 4250, | |
| "wer": 0.029176470588235293 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 154, | |
| "words": 4215, | |
| "wer": 0.0365361803084223 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 70, | |
| "wer": 0.014285714285714285 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 70, | |
| "wer": 0.014285714285714285 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 412, | |
| "wer": 0.0024271844660194173 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 432, | |
| "wer": 0.0023148148148148147 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 2, | |
| "words": 342, | |
| "wer": 0.005847953216374269 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 352, | |
| "wer": 0.002840909090909091 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 35, | |
| "words": 81, | |
| "wer": 0.43209876543209874 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 10, | |
| "words": 64, | |
| "wer": 0.15625 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 4, | |
| "words": 98, | |
| "wer": 0.04081632653061224 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 4, | |
| "words": 105, | |
| "wer": 0.0380952380952381 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 12, | |
| "words": 251, | |
| "wer": 0.04780876494023904 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 8, | |
| "words": 256, | |
| "wer": 0.03125 | |
| } | |
| } | |
| } | |
| }, | |
| "asr_macro_average": { | |
| "wer": 0.03566368655039328, | |
| "wer_95ci": [ | |
| 0.02878832692497289, | |
| 0.04329524954900782 | |
| ] | |
| } | |
| }, | |
| "inflect-nano-v2": { | |
| "label": "Inflect-Nano-v2", | |
| "qwen3_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 179, | |
| "words": 6794, | |
| "wer": 0.026346776567559613 | |
| }, | |
| "raw_95ci": [ | |
| 0.020206362854686157, | |
| 0.033170442766178 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 190, | |
| "words": 6823, | |
| "wer": 0.02784698812838927 | |
| }, | |
| "semantic_95ci": [ | |
| 0.021251648834823392, | |
| 0.0351635682374982 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 3, | |
| "words": 296, | |
| "wer": 0.010135135135135136 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 1, | |
| "words": 309, | |
| "wer": 0.003236245954692557 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 217, | |
| "wer": 0.004608294930875576 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 226, | |
| "wer": 0.004424778761061947 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 314, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 329, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 123, | |
| "words": 4250, | |
| "wer": 0.028941176470588234 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 159, | |
| "words": 4215, | |
| "wer": 0.03772241992882562 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 70, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 70, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 2, | |
| "words": 412, | |
| "wer": 0.0048543689320388345 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 2, | |
| "words": 432, | |
| "wer": 0.004629629629629629 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 10, | |
| "words": 57, | |
| "wer": 0.17543859649122806 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 8, | |
| "words": 57, | |
| "wer": 0.14035087719298245 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 342, | |
| "wer": 0.0029239766081871343 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 352, | |
| "wer": 0.002840909090909091 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 20, | |
| "words": 81, | |
| "wer": 0.24691358024691357 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 64, | |
| "wer": 0.046875 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 6, | |
| "words": 98, | |
| "wer": 0.061224489795918366 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 6, | |
| "words": 105, | |
| "wer": 0.05714285714285714 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 10, | |
| "words": 251, | |
| "wer": 0.0398406374501992 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 6, | |
| "words": 256, | |
| "wer": 0.0234375 | |
| } | |
| } | |
| } | |
| }, | |
| "nemotron35_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 461, | |
| "words": 6794, | |
| "wer": 0.06785398881365912 | |
| }, | |
| "raw_95ci": [ | |
| 0.05698005698005698, | |
| 0.07978646659933632 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 384, | |
| "words": 6823, | |
| "wer": 0.056280228638428846 | |
| }, | |
| "semantic_95ci": [ | |
| 0.04723173515981735, | |
| 0.06624651912648395 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 8, | |
| "words": 296, | |
| "wer": 0.02702702702702703 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 6, | |
| "words": 309, | |
| "wer": 0.019417475728155338 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 3, | |
| "words": 217, | |
| "wer": 0.013824884792626729 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 3, | |
| "words": 226, | |
| "wer": 0.01327433628318584 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 2, | |
| "words": 314, | |
| "wer": 0.006369426751592357 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 2, | |
| "words": 329, | |
| "wer": 0.0060790273556231 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 377, | |
| "words": 4250, | |
| "wer": 0.08870588235294118 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 303, | |
| "words": 4215, | |
| "wer": 0.07188612099644127 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 8, | |
| "words": 70, | |
| "wer": 0.11428571428571428 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 8, | |
| "words": 70, | |
| "wer": 0.11428571428571428 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 3, | |
| "words": 412, | |
| "wer": 0.007281553398058253 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 3, | |
| "words": 432, | |
| "wer": 0.006944444444444444 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 121, | |
| "wer": 0.024793388429752067 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 121, | |
| "wer": 0.024793388429752067 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 12, | |
| "words": 57, | |
| "wer": 0.21052631578947367 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 12, | |
| "words": 57, | |
| "wer": 0.21052631578947367 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 10, | |
| "words": 342, | |
| "wer": 0.029239766081871343 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 10, | |
| "words": 352, | |
| "wer": 0.028409090909090908 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 81, | |
| "wer": 0.04938271604938271 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 64, | |
| "wer": 0.046875 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 65, | |
| "wer": 0.03076923076923077 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 65, | |
| "wer": 0.03076923076923077 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 57, | |
| "wer": 0.10526315789473684 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 57, | |
| "wer": 0.10526315789473684 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 13, | |
| "words": 98, | |
| "wer": 0.1326530612244898 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 13, | |
| "words": 105, | |
| "wer": 0.12380952380952381 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 10, | |
| "words": 251, | |
| "wer": 0.0398406374501992 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 10, | |
| "words": 256, | |
| "wer": 0.0390625 | |
| } | |
| } | |
| } | |
| }, | |
| "whisper_large_v3": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 192, | |
| "words": 6794, | |
| "wer": 0.028260229614365615 | |
| }, | |
| "raw_95ci": [ | |
| 0.022173274596182086, | |
| 0.03481012658227848 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 181, | |
| "words": 6823, | |
| "wer": 0.026527920269676094 | |
| }, | |
| "semantic_95ci": [ | |
| 0.020260492040520984, | |
| 0.03347280334728033 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 4, | |
| "words": 296, | |
| "wer": 0.013513513513513514 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 1, | |
| "words": 309, | |
| "wer": 0.003236245954692557 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 217, | |
| "wer": 0.004608294930875576 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 0, | |
| "words": 226, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 314, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 329, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 118, | |
| "words": 4250, | |
| "wer": 0.02776470588235294 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 142, | |
| "words": 4215, | |
| "wer": 0.03368920521945433 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 70, | |
| "wer": 0.05714285714285714 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 70, | |
| "wer": 0.05714285714285714 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 2, | |
| "words": 412, | |
| "wer": 0.0048543689320388345 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 2, | |
| "words": 432, | |
| "wer": 0.004629629629629629 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 121, | |
| "wer": 0.008264462809917356 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 121, | |
| "wer": 0.008264462809917356 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 57, | |
| "wer": 0.08771929824561403 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 57, | |
| "wer": 0.08771929824561403 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 4, | |
| "words": 342, | |
| "wer": 0.011695906432748537 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 352, | |
| "wer": 0.008522727272727272 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 34, | |
| "words": 81, | |
| "wer": 0.41975308641975306 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 8, | |
| "words": 64, | |
| "wer": 0.125 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 57, | |
| "wer": 0.03508771929824561 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 57, | |
| "wer": 0.03508771929824561 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 5, | |
| "words": 98, | |
| "wer": 0.05102040816326531 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 5, | |
| "words": 105, | |
| "wer": 0.047619047619047616 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 12, | |
| "words": 251, | |
| "wer": 0.04780876494023904 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 8, | |
| "words": 256, | |
| "wer": 0.03125 | |
| } | |
| } | |
| } | |
| }, | |
| "asr_macro_average": { | |
| "wer": 0.036885045678831406, | |
| "wer_95ci": [ | |
| 0.030208333333333334, | |
| 0.04434228798544271 | |
| ] | |
| } | |
| }, | |
| "kitten-nano-bruno": { | |
| "label": "KittenTTS Nano - Bruno", | |
| "qwen3_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 139, | |
| "words": 6794, | |
| "wer": 0.02045922873123344 | |
| }, | |
| "raw_95ci": [ | |
| 0.014941302027748132, | |
| 0.026771187694128087 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 147, | |
| "words": 6823, | |
| "wer": 0.02154477502564854 | |
| }, | |
| "semantic_95ci": [ | |
| 0.015435835351089588, | |
| 0.0284862043251305 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 1, | |
| "words": 296, | |
| "wer": 0.0033783783783783786 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 1, | |
| "words": 309, | |
| "wer": 0.003236245954692557 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 0, | |
| "words": 217, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 0, | |
| "words": 226, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 314, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 329, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 94, | |
| "words": 4250, | |
| "wer": 0.02211764705882353 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 125, | |
| "words": 4215, | |
| "wer": 0.02965599051008304 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 70, | |
| "wer": 0.014285714285714285 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 70, | |
| "wer": 0.014285714285714285 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 412, | |
| "wer": 0.0024271844660194173 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 432, | |
| "wer": 0.0023148148148148147 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 8, | |
| "words": 57, | |
| "wer": 0.14035087719298245 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 57, | |
| "wer": 0.10526315789473684 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 2, | |
| "words": 342, | |
| "wer": 0.005847953216374269 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 2, | |
| "words": 352, | |
| "wer": 0.005681818181818182 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 19, | |
| "words": 81, | |
| "wer": 0.2345679012345679 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 64, | |
| "wer": 0.03125 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 98, | |
| "wer": 0.01020408163265306 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 105, | |
| "wer": 0.009523809523809525 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 9, | |
| "words": 251, | |
| "wer": 0.035856573705179286 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 5, | |
| "words": 256, | |
| "wer": 0.01953125 | |
| } | |
| } | |
| } | |
| }, | |
| "nemotron35_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 300, | |
| "words": 6794, | |
| "wer": 0.044156608772446274 | |
| }, | |
| "raw_95ci": [ | |
| 0.03619243784022363, | |
| 0.052726998035957096 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 270, | |
| "words": 6823, | |
| "wer": 0.03957203576139528 | |
| }, | |
| "semantic_95ci": [ | |
| 0.03191024333381903, | |
| 0.04806156526789604 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 2, | |
| "words": 296, | |
| "wer": 0.006756756756756757 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 2, | |
| "words": 309, | |
| "wer": 0.006472491909385114 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 217, | |
| "wer": 0.004608294930875576 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 226, | |
| "wer": 0.004424778761061947 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 7, | |
| "words": 314, | |
| "wer": 0.022292993630573247 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 7, | |
| "words": 329, | |
| "wer": 0.02127659574468085 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 243, | |
| "words": 4250, | |
| "wer": 0.057176470588235294 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 214, | |
| "words": 4215, | |
| "wer": 0.050771055753262156 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 70, | |
| "wer": 0.04285714285714286 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 70, | |
| "wer": 0.04285714285714286 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 2, | |
| "words": 412, | |
| "wer": 0.0048543689320388345 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 2, | |
| "words": 432, | |
| "wer": 0.004629629629629629 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 121, | |
| "wer": 0.008264462809917356 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 121, | |
| "wer": 0.008264462809917356 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 7, | |
| "words": 57, | |
| "wer": 0.12280701754385964 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 7, | |
| "words": 57, | |
| "wer": 0.12280701754385964 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 6, | |
| "words": 342, | |
| "wer": 0.017543859649122806 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 6, | |
| "words": 352, | |
| "wer": 0.017045454545454544 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 81, | |
| "wer": 0.04938271604938271 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 64, | |
| "wer": 0.078125 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 65, | |
| "wer": 0.03076923076923077 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 65, | |
| "wer": 0.03076923076923077 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 10, | |
| "words": 98, | |
| "wer": 0.10204081632653061 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 10, | |
| "words": 105, | |
| "wer": 0.09523809523809523 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 8, | |
| "words": 251, | |
| "wer": 0.03187250996015936 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 6, | |
| "words": 256, | |
| "wer": 0.0234375 | |
| } | |
| } | |
| } | |
| }, | |
| "whisper_large_v3": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 146, | |
| "words": 6794, | |
| "wer": 0.021489549602590523 | |
| }, | |
| "raw_95ci": [ | |
| 0.015989811801330127, | |
| 0.02743585940975827 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 148, | |
| "words": 6823, | |
| "wer": 0.021691338121061117 | |
| }, | |
| "semantic_95ci": [ | |
| 0.01564083997103548, | |
| 0.02847497431381183 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 1, | |
| "words": 296, | |
| "wer": 0.0033783783783783786 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 1, | |
| "words": 309, | |
| "wer": 0.003236245954692557 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 217, | |
| "wer": 0.004608294930875576 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 0, | |
| "words": 226, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 314, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 329, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 89, | |
| "words": 4250, | |
| "wer": 0.020941176470588234 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 123, | |
| "words": 4215, | |
| "wer": 0.029181494661921707 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 70, | |
| "wer": 0.014285714285714285 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 70, | |
| "wer": 0.014285714285714285 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 412, | |
| "wer": 0.0024271844660194173 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 432, | |
| "wer": 0.0023148148148148147 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 5, | |
| "words": 342, | |
| "wer": 0.014619883040935672 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 4, | |
| "words": 352, | |
| "wer": 0.011363636363636364 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 32, | |
| "words": 81, | |
| "wer": 0.3950617283950617 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 64, | |
| "wer": 0.09375 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 57, | |
| "wer": 0.03508771929824561 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 57, | |
| "wer": 0.03508771929824561 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 2, | |
| "words": 98, | |
| "wer": 0.02040816326530612 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 2, | |
| "words": 105, | |
| "wer": 0.01904761904761905 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 9, | |
| "words": 251, | |
| "wer": 0.035856573705179286 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 5, | |
| "words": 256, | |
| "wer": 0.01953125 | |
| } | |
| } | |
| } | |
| }, | |
| "asr_macro_average": { | |
| "wer": 0.027602716302701644, | |
| "wer_95ci": [ | |
| 0.021376971841368126, | |
| 0.03446097015650821 | |
| ] | |
| } | |
| }, | |
| "kitten-nano-hugo": { | |
| "label": "KittenTTS Nano - Hugo", | |
| "qwen3_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 150, | |
| "words": 6794, | |
| "wer": 0.022078304386223137 | |
| }, | |
| "raw_95ci": [ | |
| 0.016605437178545186, | |
| 0.028407418310273772 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 163, | |
| "words": 6823, | |
| "wer": 0.023889784552249743 | |
| }, | |
| "semantic_95ci": [ | |
| 0.0173235166738848, | |
| 0.03107674350744091 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 2, | |
| "words": 296, | |
| "wer": 0.006756756756756757 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 2, | |
| "words": 309, | |
| "wer": 0.006472491909385114 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 3, | |
| "words": 217, | |
| "wer": 0.013824884792626729 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 2, | |
| "words": 226, | |
| "wer": 0.008849557522123894 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 314, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 329, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 100, | |
| "words": 4250, | |
| "wer": 0.023529411764705882 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 133, | |
| "words": 4215, | |
| "wer": 0.03155397390272835 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 70, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 70, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 412, | |
| "wer": 0.0024271844660194173 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 432, | |
| "wer": 0.0023148148148148147 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 7, | |
| "words": 57, | |
| "wer": 0.12280701754385964 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 57, | |
| "wer": 0.08771929824561403 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 4, | |
| "words": 342, | |
| "wer": 0.011695906432748537 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 4, | |
| "words": 352, | |
| "wer": 0.011363636363636364 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 20, | |
| "words": 81, | |
| "wer": 0.24691358024691357 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 7, | |
| "words": 64, | |
| "wer": 0.109375 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 98, | |
| "wer": 0.01020408163265306 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 105, | |
| "wer": 0.009523809523809525 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 9, | |
| "words": 251, | |
| "wer": 0.035856573705179286 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 5, | |
| "words": 256, | |
| "wer": 0.01953125 | |
| } | |
| } | |
| } | |
| }, | |
| "nemotron35_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 287, | |
| "words": 6794, | |
| "wer": 0.04224315572564027 | |
| }, | |
| "raw_95ci": [ | |
| 0.034229828850855744, | |
| 0.05104311543810849 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 259, | |
| "words": 6823, | |
| "wer": 0.037959841711856956 | |
| }, | |
| "semantic_95ci": [ | |
| 0.030281077948420748, | |
| 0.04660587639311044 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 3, | |
| "words": 296, | |
| "wer": 0.010135135135135136 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 3, | |
| "words": 309, | |
| "wer": 0.009708737864077669 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 3, | |
| "words": 217, | |
| "wer": 0.013824884792626729 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 3, | |
| "words": 226, | |
| "wer": 0.01327433628318584 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 314, | |
| "wer": 0.0031847133757961785 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 329, | |
| "wer": 0.00303951367781155 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 237, | |
| "words": 4250, | |
| "wer": 0.05576470588235294 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 208, | |
| "words": 4215, | |
| "wer": 0.04934756820877817 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 70, | |
| "wer": 0.05714285714285714 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 70, | |
| "wer": 0.05714285714285714 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 3, | |
| "words": 412, | |
| "wer": 0.007281553398058253 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 3, | |
| "words": 432, | |
| "wer": 0.006944444444444444 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 121, | |
| "wer": 0.01652892561983471 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 121, | |
| "wer": 0.01652892561983471 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 7, | |
| "words": 57, | |
| "wer": 0.12280701754385964 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 7, | |
| "words": 57, | |
| "wer": 0.12280701754385964 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 5, | |
| "words": 342, | |
| "wer": 0.014619883040935672 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 5, | |
| "words": 352, | |
| "wer": 0.014204545454545454 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 81, | |
| "wer": 0.04938271604938271 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 64, | |
| "wer": 0.09375 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 6, | |
| "words": 98, | |
| "wer": 0.061224489795918366 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 6, | |
| "words": 105, | |
| "wer": 0.05714285714285714 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 8, | |
| "words": 251, | |
| "wer": 0.03187250996015936 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 7, | |
| "words": 256, | |
| "wer": 0.02734375 | |
| } | |
| } | |
| } | |
| }, | |
| "whisper_large_v3": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 148, | |
| "words": 6794, | |
| "wer": 0.02178392699440683 | |
| }, | |
| "raw_95ci": [ | |
| 0.01613385121003884, | |
| 0.02785923753665689 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 144, | |
| "words": 6823, | |
| "wer": 0.021105085739410816 | |
| }, | |
| "semantic_95ci": [ | |
| 0.015114082255486121, | |
| 0.027789966359514407 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 1, | |
| "words": 296, | |
| "wer": 0.0033783783783783786 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 1, | |
| "words": 309, | |
| "wer": 0.003236245954692557 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 2, | |
| "words": 217, | |
| "wer": 0.009216589861751152 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 226, | |
| "wer": 0.004424778761061947 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 314, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 329, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 89, | |
| "words": 4250, | |
| "wer": 0.020941176470588234 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 116, | |
| "words": 4215, | |
| "wer": 0.02752075919335706 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 70, | |
| "wer": 0.014285714285714285 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 70, | |
| "wer": 0.014285714285714285 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 412, | |
| "wer": 0.0024271844660194173 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 432, | |
| "wer": 0.0023148148148148147 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 57, | |
| "wer": 0.03508771929824561 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 4, | |
| "words": 342, | |
| "wer": 0.011695906432748537 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 352, | |
| "wer": 0.008522727272727272 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 32, | |
| "words": 81, | |
| "wer": 0.3950617283950617 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 8, | |
| "words": 64, | |
| "wer": 0.125 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 65, | |
| "wer": 0.015384615384615385 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 65, | |
| "wer": 0.015384615384615385 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 57, | |
| "wer": 0.03508771929824561 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 57, | |
| "wer": 0.03508771929824561 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 2, | |
| "words": 98, | |
| "wer": 0.02040816326530612 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 2, | |
| "words": 105, | |
| "wer": 0.01904761904761905 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 10, | |
| "words": 251, | |
| "wer": 0.0398406374501992 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 6, | |
| "words": 256, | |
| "wer": 0.0234375 | |
| } | |
| } | |
| } | |
| }, | |
| "asr_macro_average": { | |
| "wer": 0.027651570667839173, | |
| "wer_95ci": [ | |
| 0.021452384403536747, | |
| 0.0344283318536056 | |
| ] | |
| } | |
| }, | |
| "piper-ryan-low": { | |
| "label": "Piper - Ryan Low", | |
| "qwen3_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 181, | |
| "words": 6794, | |
| "wer": 0.02664115395937592 | |
| }, | |
| "raw_95ci": [ | |
| 0.02094850160023276, | |
| 0.03278688524590164 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 192, | |
| "words": 6823, | |
| "wer": 0.028140114319214423 | |
| }, | |
| "semantic_95ci": [ | |
| 0.021386536529352664, | |
| 0.03574504737295435 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 2, | |
| "words": 296, | |
| "wer": 0.006756756756756757 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 2, | |
| "words": 309, | |
| "wer": 0.006472491909385114 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 0, | |
| "words": 217, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 0, | |
| "words": 226, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 314, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 329, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 130, | |
| "words": 4250, | |
| "wer": 0.03058823529411765 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 153, | |
| "words": 4215, | |
| "wer": 0.036298932384341634 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 70, | |
| "wer": 0.04285714285714286 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 70, | |
| "wer": 0.04285714285714286 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 3, | |
| "words": 412, | |
| "wer": 0.007281553398058253 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 3, | |
| "words": 432, | |
| "wer": 0.006944444444444444 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 121, | |
| "wer": 0.008264462809917356 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 121, | |
| "wer": 0.008264462809917356 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 14, | |
| "words": 57, | |
| "wer": 0.24561403508771928 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 14, | |
| "words": 57, | |
| "wer": 0.24561403508771928 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 342, | |
| "wer": 0.0029239766081871343 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 352, | |
| "wer": 0.002840909090909091 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 11, | |
| "words": 81, | |
| "wer": 0.13580246913580246 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 64, | |
| "wer": 0.046875 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 98, | |
| "wer": 0.030612244897959183 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 105, | |
| "wer": 0.02857142857142857 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 9, | |
| "words": 251, | |
| "wer": 0.035856573705179286 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 5, | |
| "words": 256, | |
| "wer": 0.01953125 | |
| } | |
| } | |
| } | |
| }, | |
| "nemotron35_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 442, | |
| "words": 6794, | |
| "wer": 0.06505740359140418 | |
| }, | |
| "raw_95ci": [ | |
| 0.054938545831482305, | |
| 0.07616320687186828 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 376, | |
| "words": 6823, | |
| "wer": 0.055107723875128245 | |
| }, | |
| "semantic_95ci": [ | |
| 0.04650152107779226, | |
| 0.06445341707791237 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 10, | |
| "words": 296, | |
| "wer": 0.033783783783783786 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 8, | |
| "words": 309, | |
| "wer": 0.025889967637540454 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 7, | |
| "words": 217, | |
| "wer": 0.03225806451612903 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 7, | |
| "words": 226, | |
| "wer": 0.030973451327433628 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 314, | |
| "wer": 0.0031847133757961785 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 329, | |
| "wer": 0.00303951367781155 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 347, | |
| "words": 4250, | |
| "wer": 0.0816470588235294 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 284, | |
| "words": 4215, | |
| "wer": 0.06737841043890866 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 7, | |
| "words": 70, | |
| "wer": 0.1 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 7, | |
| "words": 70, | |
| "wer": 0.1 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 9, | |
| "words": 412, | |
| "wer": 0.021844660194174758 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 9, | |
| "words": 432, | |
| "wer": 0.020833333333333332 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 121, | |
| "wer": 0.008264462809917356 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 121, | |
| "wer": 0.008264462809917356 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 24, | |
| "words": 57, | |
| "wer": 0.42105263157894735 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 24, | |
| "words": 57, | |
| "wer": 0.42105263157894735 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 4, | |
| "words": 342, | |
| "wer": 0.011695906432748537 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 4, | |
| "words": 352, | |
| "wer": 0.011363636363636364 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 81, | |
| "wer": 0.06172839506172839 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 64, | |
| "wer": 0.078125 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 65, | |
| "wer": 0.015384615384615385 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 65, | |
| "wer": 0.015384615384615385 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 57, | |
| "wer": 0.10526315789473684 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 57, | |
| "wer": 0.10526315789473684 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 10, | |
| "words": 98, | |
| "wer": 0.10204081632653061 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 10, | |
| "words": 105, | |
| "wer": 0.09523809523809523 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 10, | |
| "words": 251, | |
| "wer": 0.0398406374501992 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 9, | |
| "words": 256, | |
| "wer": 0.03515625 | |
| } | |
| } | |
| } | |
| }, | |
| "whisper_large_v3": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 192, | |
| "words": 6794, | |
| "wer": 0.028260229614365615 | |
| }, | |
| "raw_95ci": [ | |
| 0.02239140170174653, | |
| 0.034622911335240106 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 196, | |
| "words": 6823, | |
| "wer": 0.028726366700864724 | |
| }, | |
| "semantic_95ci": [ | |
| 0.0221175799086758, | |
| 0.03601981089599279 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 8, | |
| "words": 296, | |
| "wer": 0.02702702702702703 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 5, | |
| "words": 309, | |
| "wer": 0.016181229773462782 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 3, | |
| "words": 217, | |
| "wer": 0.013824884792626729 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 226, | |
| "wer": 0.004424778761061947 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 314, | |
| "wer": 0.0031847133757961785 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 329, | |
| "wer": 0.00303951367781155 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 118, | |
| "words": 4250, | |
| "wer": 0.02776470588235294 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 157, | |
| "words": 4215, | |
| "wer": 0.037247924080664296 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 70, | |
| "wer": 0.014285714285714285 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 70, | |
| "wer": 0.014285714285714285 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 3, | |
| "words": 412, | |
| "wer": 0.007281553398058253 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 3, | |
| "words": 432, | |
| "wer": 0.006944444444444444 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 121, | |
| "wer": 0.008264462809917356 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 121, | |
| "wer": 0.008264462809917356 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 57, | |
| "wer": 0.03508771929824561 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 57, | |
| "wer": 0.03508771929824561 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 342, | |
| "wer": 0.0029239766081871343 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 352, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 32, | |
| "words": 81, | |
| "wer": 0.3950617283950617 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 64, | |
| "wer": 0.09375 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 65, | |
| "wer": 0.015384615384615385 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 65, | |
| "wer": 0.015384615384615385 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 7, | |
| "words": 98, | |
| "wer": 0.07142857142857142 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 8, | |
| "words": 105, | |
| "wer": 0.0761904761904762 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 10, | |
| "words": 251, | |
| "wer": 0.0398406374501992 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 6, | |
| "words": 256, | |
| "wer": 0.0234375 | |
| } | |
| } | |
| } | |
| }, | |
| "asr_macro_average": { | |
| "wer": 0.03732473496506913, | |
| "wer_95ci": [ | |
| 0.030795019157088124, | |
| 0.04458376872169976 | |
| ] | |
| } | |
| }, | |
| "piper-danny-low": { | |
| "label": "Piper - Danny Low", | |
| "qwen3_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 175, | |
| "words": 6794, | |
| "wer": 0.025758021783926995 | |
| }, | |
| "raw_95ci": [ | |
| 0.01956489998539933, | |
| 0.03292759051186017 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 179, | |
| "words": 6823, | |
| "wer": 0.026234794078850945 | |
| }, | |
| "semantic_95ci": [ | |
| 0.019979876383498633, | |
| 0.03340659340659341 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 2, | |
| "words": 296, | |
| "wer": 0.006756756756756757 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 2, | |
| "words": 309, | |
| "wer": 0.006472491909385114 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 2, | |
| "words": 217, | |
| "wer": 0.009216589861751152 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 2, | |
| "words": 226, | |
| "wer": 0.008849557522123894 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 314, | |
| "wer": 0.0031847133757961785 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 329, | |
| "wer": 0.00303951367781155 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 131, | |
| "words": 4250, | |
| "wer": 0.030823529411764704 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 150, | |
| "words": 4215, | |
| "wer": 0.03558718861209965 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 70, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 70, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 5, | |
| "words": 412, | |
| "wer": 0.012135922330097087 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 5, | |
| "words": 432, | |
| "wer": 0.011574074074074073 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 57, | |
| "wer": 0.10526315789473684 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 57, | |
| "wer": 0.10526315789473684 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 342, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 352, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 13, | |
| "words": 81, | |
| "wer": 0.16049382716049382 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 64, | |
| "wer": 0.03125 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 65, | |
| "wer": 0.015384615384615385 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 65, | |
| "wer": 0.015384615384615385 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 2, | |
| "words": 98, | |
| "wer": 0.02040816326530612 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 2, | |
| "words": 105, | |
| "wer": 0.01904761904761905 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 9, | |
| "words": 251, | |
| "wer": 0.035856573705179286 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 5, | |
| "words": 256, | |
| "wer": 0.01953125 | |
| } | |
| } | |
| } | |
| }, | |
| "nemotron35_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 438, | |
| "words": 6794, | |
| "wer": 0.06446864880777156 | |
| }, | |
| "raw_95ci": [ | |
| 0.05425912670007158, | |
| 0.07542874487782668 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 382, | |
| "words": 6823, | |
| "wer": 0.055987102447603694 | |
| }, | |
| "semantic_95ci": [ | |
| 0.04722822174226062, | |
| 0.06521079769487413 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 4, | |
| "words": 296, | |
| "wer": 0.013513513513513514 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 4, | |
| "words": 309, | |
| "wer": 0.012944983818770227 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 4, | |
| "words": 217, | |
| "wer": 0.018433179723502304 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 4, | |
| "words": 226, | |
| "wer": 0.017699115044247787 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 314, | |
| "wer": 0.009554140127388535 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 329, | |
| "wer": 0.00911854103343465 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 349, | |
| "words": 4250, | |
| "wer": 0.08211764705882353 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 295, | |
| "words": 4215, | |
| "wer": 0.06998813760379596 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 70, | |
| "wer": 0.04285714285714286 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 70, | |
| "wer": 0.04285714285714286 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 19, | |
| "words": 412, | |
| "wer": 0.04611650485436893 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 19, | |
| "words": 432, | |
| "wer": 0.04398148148148148 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 121, | |
| "wer": 0.03305785123966942 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 121, | |
| "wer": 0.03305785123966942 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 13, | |
| "words": 57, | |
| "wer": 0.22807017543859648 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 13, | |
| "words": 57, | |
| "wer": 0.22807017543859648 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 342, | |
| "wer": 0.0029239766081871343 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 352, | |
| "wer": 0.002840909090909091 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 81, | |
| "wer": 0.06172839506172839 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 64, | |
| "wer": 0.0625 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 65, | |
| "wer": 0.015384615384615385 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 65, | |
| "wer": 0.015384615384615385 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 163, | |
| "wer": 0.006134969325153374 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 165, | |
| "wer": 0.006060606060606061 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 98, | |
| "wer": 0.030612244897959183 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 105, | |
| "wer": 0.02857142857142857 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 24, | |
| "words": 251, | |
| "wer": 0.09561752988047809 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 23, | |
| "words": 256, | |
| "wer": 0.08984375 | |
| } | |
| } | |
| } | |
| }, | |
| "whisper_large_v3": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 166, | |
| "words": 6794, | |
| "wer": 0.024433323520753607 | |
| }, | |
| "raw_95ci": [ | |
| 0.018651148910772904, | |
| 0.030638170677028845 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 174, | |
| "words": 6823, | |
| "wer": 0.02550197860178807 | |
| }, | |
| "semantic_95ci": [ | |
| 0.019211324570273004, | |
| 0.03259413603214764 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 2, | |
| "words": 296, | |
| "wer": 0.006756756756756757 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 2, | |
| "words": 309, | |
| "wer": 0.006472491909385114 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 2, | |
| "words": 217, | |
| "wer": 0.009216589861751152 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 226, | |
| "wer": 0.004424778761061947 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 314, | |
| "wer": 0.0031847133757961785 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 329, | |
| "wer": 0.00303951367781155 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 100, | |
| "words": 4250, | |
| "wer": 0.023529411764705882 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 140, | |
| "words": 4215, | |
| "wer": 0.033214709371293 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 70, | |
| "wer": 0.014285714285714285 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 70, | |
| "wer": 0.014285714285714285 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 7, | |
| "words": 412, | |
| "wer": 0.01699029126213592 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 7, | |
| "words": 432, | |
| "wer": 0.016203703703703703 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 342, | |
| "wer": 0.0029239766081871343 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 352, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 33, | |
| "words": 81, | |
| "wer": 0.4074074074074074 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 7, | |
| "words": 64, | |
| "wer": 0.109375 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 2, | |
| "words": 163, | |
| "wer": 0.012269938650306749 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 2, | |
| "words": 165, | |
| "wer": 0.012121212121212121 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 57, | |
| "wer": 0.03508771929824561 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 57, | |
| "wer": 0.03508771929824561 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 98, | |
| "wer": 0.030612244897959183 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 105, | |
| "wer": 0.02857142857142857 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 9, | |
| "words": 251, | |
| "wer": 0.035856573705179286 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 5, | |
| "words": 256, | |
| "wer": 0.01953125 | |
| } | |
| } | |
| } | |
| }, | |
| "asr_macro_average": { | |
| "wer": 0.0359079583760809, | |
| "wer_95ci": [ | |
| 0.029313597445943983, | |
| 0.04306619524651345 | |
| ] | |
| } | |
| }, | |
| "supertonic3-m2-3step": { | |
| "label": "Supertonic 3 - M2 - 3-step", | |
| "qwen3_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 200, | |
| "words": 6794, | |
| "wer": 0.02943773918163085 | |
| }, | |
| "raw_95ci": [ | |
| 0.023245740687265377, | |
| 0.03628800917957545 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 207, | |
| "words": 6823, | |
| "wer": 0.03033856075040305 | |
| }, | |
| "semantic_95ci": [ | |
| 0.02375875723423698, | |
| 0.03767123287671233 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 0, | |
| "words": 296, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 0, | |
| "words": 309, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 217, | |
| "wer": 0.004608294930875576 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 226, | |
| "wer": 0.004424778761061947 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 314, | |
| "wer": 0.0031847133757961785 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 329, | |
| "wer": 0.00303951367781155 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 120, | |
| "words": 4250, | |
| "wer": 0.02823529411764706 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 153, | |
| "words": 4215, | |
| "wer": 0.036298932384341634 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 70, | |
| "wer": 0.05714285714285714 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 70, | |
| "wer": 0.05714285714285714 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 6, | |
| "words": 412, | |
| "wer": 0.014563106796116505 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 6, | |
| "words": 432, | |
| "wer": 0.013888888888888888 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 121, | |
| "wer": 0.008264462809917356 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 121, | |
| "wer": 0.008264462809917356 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 10, | |
| "words": 57, | |
| "wer": 0.17543859649122806 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 10, | |
| "words": 57, | |
| "wer": 0.17543859649122806 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 342, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 352, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 24, | |
| "words": 81, | |
| "wer": 0.2962962962962963 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 64, | |
| "wer": 0.03125 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 65, | |
| "wer": 0.046153846153846156 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 65, | |
| "wer": 0.046153846153846156 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 163, | |
| "wer": 0.006134969325153374 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 1, | |
| "words": 165, | |
| "wer": 0.006060606060606061 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 7, | |
| "words": 57, | |
| "wer": 0.12280701754385964 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 7, | |
| "words": 57, | |
| "wer": 0.12280701754385964 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 13, | |
| "words": 98, | |
| "wer": 0.1326530612244898 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 13, | |
| "words": 105, | |
| "wer": 0.12380952380952381 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 9, | |
| "words": 251, | |
| "wer": 0.035856573705179286 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 5, | |
| "words": 256, | |
| "wer": 0.01953125 | |
| } | |
| } | |
| } | |
| }, | |
| "nemotron35_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 446, | |
| "words": 6794, | |
| "wer": 0.0656461583750368 | |
| }, | |
| "raw_95ci": [ | |
| 0.05547445255474453, | |
| 0.07639304973037747 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 412, | |
| "words": 6823, | |
| "wer": 0.060383995309980946 | |
| }, | |
| "semantic_95ci": [ | |
| 0.05129658495111237, | |
| 0.07025560474082536 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 7, | |
| "words": 296, | |
| "wer": 0.02364864864864865 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 7, | |
| "words": 309, | |
| "wer": 0.022653721682847898 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 217, | |
| "wer": 0.004608294930875576 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 226, | |
| "wer": 0.004424778761061947 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 7, | |
| "words": 314, | |
| "wer": 0.022292993630573247 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 7, | |
| "words": 329, | |
| "wer": 0.02127659574468085 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 324, | |
| "words": 4250, | |
| "wer": 0.07623529411764705 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 292, | |
| "words": 4215, | |
| "wer": 0.06927639383155397 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 7, | |
| "words": 70, | |
| "wer": 0.1 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 7, | |
| "words": 70, | |
| "wer": 0.1 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 9, | |
| "words": 412, | |
| "wer": 0.021844660194174758 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 7, | |
| "words": 432, | |
| "wer": 0.016203703703703703 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 121, | |
| "wer": 0.03305785123966942 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 121, | |
| "wer": 0.03305785123966942 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 12, | |
| "words": 57, | |
| "wer": 0.21052631578947367 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 12, | |
| "words": 57, | |
| "wer": 0.21052631578947367 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 342, | |
| "wer": 0.0029239766081871343 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 352, | |
| "wer": 0.002840909090909091 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 81, | |
| "wer": 0.04938271604938271 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 64, | |
| "wer": 0.046875 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 8, | |
| "words": 65, | |
| "wer": 0.12307692307692308 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 9, | |
| "words": 65, | |
| "wer": 0.13846153846153847 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 9, | |
| "words": 163, | |
| "wer": 0.05521472392638037 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 9, | |
| "words": 165, | |
| "wer": 0.05454545454545454 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 10, | |
| "words": 57, | |
| "wer": 0.17543859649122806 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 10, | |
| "words": 57, | |
| "wer": 0.17543859649122806 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 31, | |
| "words": 98, | |
| "wer": 0.3163265306122449 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 31, | |
| "words": 105, | |
| "wer": 0.29523809523809524 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 12, | |
| "words": 251, | |
| "wer": 0.04780876494023904 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 12, | |
| "words": 256, | |
| "wer": 0.046875 | |
| } | |
| } | |
| } | |
| }, | |
| "whisper_large_v3": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 221, | |
| "words": 6794, | |
| "wer": 0.03252870179570209 | |
| }, | |
| "raw_95ci": [ | |
| 0.025667502581501696, | |
| 0.04004781829049611 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 220, | |
| "words": 6823, | |
| "wer": 0.03224388099076653 | |
| }, | |
| "semantic_95ci": [ | |
| 0.02492877492877493, | |
| 0.04039812646370024 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 1, | |
| "words": 296, | |
| "wer": 0.0033783783783783786 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 1, | |
| "words": 309, | |
| "wer": 0.003236245954692557 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 2, | |
| "words": 217, | |
| "wer": 0.009216589861751152 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 226, | |
| "wer": 0.004424778761061947 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 2, | |
| "words": 314, | |
| "wer": 0.006369426751592357 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 2, | |
| "words": 329, | |
| "wer": 0.0060790273556231 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 132, | |
| "words": 4250, | |
| "wer": 0.031058823529411764 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 162, | |
| "words": 4215, | |
| "wer": 0.038434163701067614 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 70, | |
| "wer": 0.02857142857142857 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 70, | |
| "wer": 0.02857142857142857 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 4, | |
| "words": 412, | |
| "wer": 0.009708737864077669 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 4, | |
| "words": 432, | |
| "wer": 0.009259259259259259 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 121, | |
| "wer": 0.008264462809917356 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 121, | |
| "wer": 0.008264462809917356 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 57, | |
| "wer": 0.08771929824561403 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 57, | |
| "wer": 0.08771929824561403 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 342, | |
| "wer": 0.0029239766081871343 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 352, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 32, | |
| "words": 81, | |
| "wer": 0.3950617283950617 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 64, | |
| "wer": 0.078125 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 7, | |
| "words": 163, | |
| "wer": 0.04294478527607362 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 8, | |
| "words": 165, | |
| "wer": 0.048484848484848485 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 57, | |
| "wer": 0.08771929824561403 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 57, | |
| "wer": 0.08771929824561403 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 17, | |
| "words": 98, | |
| "wer": 0.17346938775510204 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 18, | |
| "words": 105, | |
| "wer": 0.17142857142857143 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 10, | |
| "words": 251, | |
| "wer": 0.0398406374501992 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 6, | |
| "words": 256, | |
| "wer": 0.0234375 | |
| } | |
| } | |
| } | |
| }, | |
| "asr_macro_average": { | |
| "wer": 0.040988812350383506, | |
| "wer_95ci": [ | |
| 0.034040477932211655, | |
| 0.04855302374771656 | |
| ] | |
| } | |
| }, | |
| "supertonic3-m2-8step": { | |
| "label": "Supertonic 3 - M2 - 8-step", | |
| "qwen3_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 137, | |
| "words": 6794, | |
| "wer": 0.020164851339417134 | |
| }, | |
| "raw_95ci": [ | |
| 0.014720886168197056, | |
| 0.026397286535909156 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 140, | |
| "words": 6823, | |
| "wer": 0.020518833357760516 | |
| }, | |
| "semantic_95ci": [ | |
| 0.014671013633669235, | |
| 0.02735562310030395 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 0, | |
| "words": 296, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 0, | |
| "words": 309, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 217, | |
| "wer": 0.004608294930875576 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 0, | |
| "words": 226, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 314, | |
| "wer": 0.0031847133757961785 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 329, | |
| "wer": 0.00303951367781155 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 89, | |
| "words": 4250, | |
| "wer": 0.020941176470588234 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 119, | |
| "words": 4215, | |
| "wer": 0.02823250296559905 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 70, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 70, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 3, | |
| "words": 412, | |
| "wer": 0.007281553398058253 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 3, | |
| "words": 432, | |
| "wer": 0.006944444444444444 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 57, | |
| "wer": 0.08771929824561403 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 57, | |
| "wer": 0.08771929824561403 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 342, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 352, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 24, | |
| "words": 81, | |
| "wer": 0.2962962962962963 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 64, | |
| "wer": 0.03125 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 98, | |
| "wer": 0.01020408163265306 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 105, | |
| "wer": 0.009523809523809525 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 9, | |
| "words": 251, | |
| "wer": 0.035856573705179286 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 5, | |
| "words": 256, | |
| "wer": 0.01953125 | |
| } | |
| } | |
| } | |
| }, | |
| "nemotron35_asr": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 284, | |
| "words": 6794, | |
| "wer": 0.04180158963791581 | |
| }, | |
| "raw_95ci": [ | |
| 0.033854950346072825, | |
| 0.050436615477265884 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 243, | |
| "words": 6823, | |
| "wer": 0.035614832185255754 | |
| }, | |
| "semantic_95ci": [ | |
| 0.028567347521782603, | |
| 0.043412264723740136 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 1, | |
| "words": 296, | |
| "wer": 0.0033783783783783786 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 1, | |
| "words": 309, | |
| "wer": 0.003236245954692557 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 3, | |
| "words": 217, | |
| "wer": 0.013824884792626729 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 3, | |
| "words": 226, | |
| "wer": 0.01327433628318584 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 314, | |
| "wer": 0.0031847133757961785 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 329, | |
| "wer": 0.00303951367781155 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 245, | |
| "words": 4250, | |
| "wer": 0.05764705882352941 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 205, | |
| "words": 4215, | |
| "wer": 0.04863582443653618 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 70, | |
| "wer": 0.02857142857142857 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 70, | |
| "wer": 0.02857142857142857 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 4, | |
| "words": 412, | |
| "wer": 0.009708737864077669 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 4, | |
| "words": 432, | |
| "wer": 0.009259259259259259 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 11, | |
| "words": 57, | |
| "wer": 0.19298245614035087 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 11, | |
| "words": 57, | |
| "wer": 0.19298245614035087 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 342, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 352, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 81, | |
| "wer": 0.037037037037037035 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 2, | |
| "words": 64, | |
| "wer": 0.03125 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 57, | |
| "wer": 0.08771929824561403 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 5, | |
| "words": 57, | |
| "wer": 0.08771929824561403 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 98, | |
| "wer": 0.030612244897959183 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 3, | |
| "words": 105, | |
| "wer": 0.02857142857142857 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 6, | |
| "words": 251, | |
| "wer": 0.02390438247011952 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 6, | |
| "words": 256, | |
| "wer": 0.0234375 | |
| } | |
| } | |
| } | |
| }, | |
| "whisper_large_v3": { | |
| "raw": { | |
| "utterances": 400, | |
| "errors": 552, | |
| "words": 6794, | |
| "wer": 0.08124816014130115 | |
| }, | |
| "raw_95ci": [ | |
| 0.017957850994360345, | |
| 0.20512431776834444 | |
| ], | |
| "semantic": { | |
| "utterances": 400, | |
| "errors": 551, | |
| "words": 6823, | |
| "wer": 0.08075626557232889 | |
| }, | |
| "semantic_95ci": [ | |
| 0.0176792479304055, | |
| 0.20577239419769702 | |
| ], | |
| "categories": { | |
| "conversational": { | |
| "raw": { | |
| "utterances": 25, | |
| "errors": 400, | |
| "words": 296, | |
| "wer": 1.3513513513513513 | |
| }, | |
| "semantic": { | |
| "utterances": 25, | |
| "errors": 400, | |
| "words": 309, | |
| "wer": 1.2944983818770226 | |
| } | |
| }, | |
| "descriptive": { | |
| "raw": { | |
| "utterances": 16, | |
| "errors": 1, | |
| "words": 217, | |
| "wer": 0.004608294930875576 | |
| }, | |
| "semantic": { | |
| "utterances": 16, | |
| "errors": 0, | |
| "words": 226, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "emotional": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 314, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 329, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "external_heldout": { | |
| "raw": { | |
| "utterances": 200, | |
| "errors": 93, | |
| "words": 4250, | |
| "wer": 0.02188235294117647 | |
| }, | |
| "semantic": { | |
| "utterances": 200, | |
| "errors": 125, | |
| "words": 4215, | |
| "wer": 0.02965599051008304 | |
| } | |
| }, | |
| "homographs": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 70, | |
| "wer": 0.014285714285714285 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 1, | |
| "words": 70, | |
| "wer": 0.014285714285714285 | |
| } | |
| }, | |
| "long": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 3, | |
| "words": 412, | |
| "wer": 0.007281553398058253 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 3, | |
| "words": 432, | |
| "wer": 0.006944444444444444 | |
| } | |
| }, | |
| "long_range": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 121, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "names_places": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 4, | |
| "words": 57, | |
| "wer": 0.07017543859649122 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 3, | |
| "words": 57, | |
| "wer": 0.05263157894736842 | |
| } | |
| }, | |
| "narrative": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 342, | |
| "wer": 0.0029239766081871343 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 0, | |
| "words": 352, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "numbers": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 32, | |
| "words": 81, | |
| "wer": 0.3950617283950617 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 64, | |
| "wer": 0.09375 | |
| } | |
| }, | |
| "punctuation": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 0, | |
| "words": 65, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "question": { | |
| "raw": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 163, | |
| "wer": 0.0 | |
| }, | |
| "semantic": { | |
| "utterances": 15, | |
| "errors": 0, | |
| "words": 165, | |
| "wer": 0.0 | |
| } | |
| }, | |
| "rare_words": { | |
| "raw": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 57, | |
| "wer": 0.10526315789473684 | |
| }, | |
| "semantic": { | |
| "utterances": 6, | |
| "errors": 6, | |
| "words": 57, | |
| "wer": 0.10526315789473684 | |
| } | |
| }, | |
| "short": { | |
| "raw": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 98, | |
| "wer": 0.01020408163265306 | |
| }, | |
| "semantic": { | |
| "utterances": 24, | |
| "errors": 1, | |
| "words": 105, | |
| "wer": 0.009523809523809525 | |
| } | |
| }, | |
| "technical": { | |
| "raw": { | |
| "utterances": 21, | |
| "errors": 10, | |
| "words": 251, | |
| "wer": 0.0398406374501992 | |
| }, | |
| "semantic": { | |
| "utterances": 21, | |
| "errors": 6, | |
| "words": 256, | |
| "wer": 0.0234375 | |
| } | |
| } | |
| } | |
| }, | |
| "asr_macro_average": { | |
| "wer": 0.04562997703844838, | |
| "wer_95ci": [ | |
| 0.021539744302390213, | |
| 0.0887293754050147 | |
| ] | |
| } | |
| } | |
| } | |
| } | |