Download evaluation/comparison.json from sourojitdotcom/AksharaVP-17M: direct link, hf CLI and curl.
- Browser
- Download file 9.26 kB
-
https://huggingface.co/sourojitdotcom/AksharaVP-17M/resolve/main/evaluation/comparison.json
- Command line
-
hf download hf://sourojitdotcom/AksharaVP-17M/evaluation/comparison.json
-
curl -L -o comparison.json https://huggingface.co/sourojitdotcom/AksharaVP-17M/resolve/main/evaluation/comparison.json
9.26 kB
| { | |
| "models": [ | |
| { | |
| "name": "AksharaVP-17M v2 (this model)", | |
| "ours": true, | |
| "params": 17988627, | |
| "langs": "te \u00b7 hi \u00b7 en, one checkpoint", | |
| "results": { | |
| "te": { | |
| "CER": 6.1974, | |
| "WER": 30.3806, | |
| "UTMOS": 2.6535, | |
| "n": 100 | |
| }, | |
| "hi": { | |
| "CER": 4.1739, | |
| "WER": 12.6582, | |
| "UTMOS": 2.8354, | |
| "n": 100 | |
| }, | |
| "en": { | |
| "CER": 4.7863, | |
| "WER": 9.6513, | |
| "UTMOS": 3.194, | |
| "n": 100 | |
| } | |
| }, | |
| "source": "baseline/r5_E1_say_dev.json", | |
| "fp32_MB": 71.954508 | |
| }, | |
| { | |
| "name": "AksharaVP-17M v1 (previous release)", | |
| "params": 17643026, | |
| "langs": "te \u00b7 hi \u00b7 en, one checkpoint", | |
| "results": { | |
| "te": { | |
| "CER": 6.0372, | |
| "WER": 28.937, | |
| "UTMOS": 2.6385, | |
| "n": 100 | |
| }, | |
| "hi": { | |
| "CER": 4.5652, | |
| "WER": 13.9677, | |
| "UTMOS": 2.733, | |
| "n": 100 | |
| }, | |
| "en": { | |
| "CER": 4.4205, | |
| "WER": 9.4997, | |
| "UTMOS": 3.192, | |
| "n": 100 | |
| } | |
| }, | |
| "source": "baseline/r5_vp17m_say_dev.json", | |
| "fp32_MB": 70.572104 | |
| }, | |
| { | |
| "name": "Hima TTS nano (hi + te checkpoints)", | |
| "params": 72160162, | |
| "langs": "te+en / hi+en, two checkpoints", | |
| "results": { | |
| "te": { | |
| "CER": 6.708, | |
| "WER": 31.2336, | |
| "UTMOS": 3.4974, | |
| "n": 100 | |
| }, | |
| "hi": { | |
| "CER": 4.6957, | |
| "WER": 14.797, | |
| "UTMOS": 3.6089, | |
| "n": 100 | |
| }, | |
| "en": { | |
| "CER": 5.2879, | |
| "WER": 11.4704, | |
| "UTMOS": 4.0328, | |
| "n": 100 | |
| } | |
| }, | |
| "source": "baseline/r5_hima_say_dev.json", | |
| "fp32_MB": 288.640648 | |
| }, | |
| { | |
| "name": "Kyutai Pocket TTS (english)", | |
| "params": 99752802, | |
| "langs": "en only", | |
| "results": { | |
| "en": { | |
| "CER": 2.3513, | |
| "WER": 6.0131, | |
| "UTMOS": 3.9945, | |
| "n": 100 | |
| } | |
| }, | |
| "source": "baseline/enprobe_pocket_english_dev_dev.json", | |
| "fp32_MB": 399.011208 | |
| }, | |
| { | |
| "name": "Kyutai Pocket TTS english_7m_2026-10", | |
| "params": 6862578, | |
| "langs": "en only", | |
| "results": { | |
| "en": { | |
| "CER": 1.9856, | |
| "WER": 6.3669, | |
| "UTMOS": 4.1018, | |
| "n": 100 | |
| } | |
| }, | |
| "source": "baseline/enprobe_pocket_english_7m_2026-10_dev_dev.json", | |
| "fp32_MB": 27.450312 | |
| }, | |
| { | |
| "name": "Kyutai Pocket TTS english_2m_2026-10", | |
| "params": 1922846, | |
| "langs": "en only", | |
| "results": { | |
| "en": { | |
| "CER": 3.8876, | |
| "WER": 9.7019, | |
| "UTMOS": 3.4948, | |
| "n": 100 | |
| } | |
| }, | |
| "source": "baseline/enprobe_pocket_english_2m_2026-10_dev_dev.json", | |
| "fp32_MB": 7.691384 | |
| }, | |
| { | |
| "name": "Meta MMS-TTS tel (VITS)", | |
| "params": 36289776, | |
| "langs": "te only", | |
| "results": { | |
| "te": { | |
| "CER": 10.4826, | |
| "WER": 41.6667, | |
| "UTMOS": 3.2809, | |
| "n": 100 | |
| } | |
| }, | |
| "source": "baseline/mms_tts_dev100_dev.json (facebook/mms-tts-tel, same normalised input text, 16 kHz)", | |
| "fp32_MB": 145.159104 | |
| }, | |
| { | |
| "name": "Meta MMS-TTS hin (VITS)", | |
| "params": 36291120, | |
| "langs": "hi only", | |
| "results": { | |
| "hi": { | |
| "CER": 11.2609, | |
| "WER": 28.6338, | |
| "UTMOS": 3.4889, | |
| "n": 100 | |
| } | |
| }, | |
| "source": "baseline/mms_tts_dev100_dev.json (facebook/mms-tts-hin, same normalised input text, 16 kHz)", | |
| "fp32_MB": 145.16448 | |
| }, | |
| { | |
| "name": "Meta MMS-TTS eng (VITS)", | |
| "params": 36284592, | |
| "langs": "en only", | |
| "results": { | |
| "en": { | |
| "CER": 6.4584, | |
| "WER": 17.332, | |
| "UTMOS": 4.0958, | |
| "n": 100 | |
| } | |
| }, | |
| "source": "baseline/mms_tts_dev100_dev.json (facebook/mms-tts-eng, same normalised input text, 16 kHz)", | |
| "fp32_MB": 145.138368 | |
| } | |
| ], | |
| "bootstrap_vs_hima": { | |
| "te": { | |
| "dCER": [ | |
| -0.5106, | |
| -1.1626, | |
| 0.1103, | |
| 0.1166 | |
| ], | |
| "dWER": [ | |
| -0.853, | |
| -3.1271, | |
| 1.4522, | |
| 0.4922 | |
| ], | |
| "dUTMOS": [ | |
| -0.844, | |
| -0.9148, | |
| -0.7694, | |
| 0.0 | |
| ] | |
| }, | |
| "hi": { | |
| "dCER": [ | |
| -0.5217, | |
| -1.2487, | |
| 0.1766, | |
| 0.1554 | |
| ], | |
| "dWER": [ | |
| -2.1388, | |
| -3.9948, | |
| -0.3097, | |
| 0.024 | |
| ], | |
| "dUTMOS": [ | |
| -0.7736, | |
| -0.8561, | |
| -0.6901, | |
| 0.0 | |
| ] | |
| }, | |
| "en": { | |
| "dCER": [ | |
| -0.5016, | |
| -1.785, | |
| 0.783, | |
| 0.429 | |
| ], | |
| "dWER": [ | |
| -1.8191, | |
| -4.5948, | |
| 0.5647, | |
| 0.1488 | |
| ], | |
| "dUTMOS": [ | |
| -0.8388, | |
| -0.9171, | |
| -0.7585, | |
| 0.0 | |
| ] | |
| } | |
| }, | |
| "v2_vs_v1": { | |
| "te": { | |
| "builtin_CER": [ | |
| 6.1974, | |
| 6.0372, | |
| 0.1602, | |
| -0.4024, | |
| 0.7038, | |
| 0.5772 | |
| ], | |
| "builtin_WER": [ | |
| 30.3806, | |
| 28.937, | |
| 1.4436, | |
| -0.8458, | |
| 3.751, | |
| 0.2176 | |
| ], | |
| "builtin_UTMOS": [ | |
| 2.6535, | |
| 2.6385, | |
| 0.0149, | |
| -0.0273, | |
| 0.0576, | |
| 0.4892 | |
| ], | |
| "clone_CER": [ | |
| 10.424, | |
| 7.8738, | |
| 2.5502, | |
| 0.2448, | |
| 5.6443, | |
| 0.0264 | |
| ], | |
| "clone_WER": [ | |
| 40.0826, | |
| 35.3306, | |
| 4.7521, | |
| 0.4338, | |
| 9.3498, | |
| 0.0356 | |
| ], | |
| "clone_sim_ecapa": [ | |
| 0.4017, | |
| 0.2951, | |
| 0.1066, | |
| 0.0748, | |
| 0.1381, | |
| 0.0 | |
| ], | |
| "clone_sim_wavlm": [ | |
| 0.9134, | |
| 0.8789, | |
| 0.0345, | |
| 0.0146, | |
| 0.0523, | |
| 0.0016 | |
| ], | |
| "clone_utmos": [ | |
| 1.8903, | |
| 2.3775, | |
| -0.4873, | |
| -0.6098, | |
| -0.3669, | |
| 0.0 | |
| ], | |
| "clone_pause150": [ | |
| 0.4667, | |
| 2.0, | |
| -1.5333, | |
| -2.2667, | |
| -0.8333, | |
| 0.0 | |
| ], | |
| "clone_pause300": [ | |
| 0.2333, | |
| 1.2333, | |
| -1.0, | |
| -1.5667, | |
| -0.4667, | |
| 0.0002 | |
| ], | |
| "clone_pause700": [ | |
| 0.0, | |
| 0.1, | |
| -0.1, | |
| -0.2, | |
| 0.0, | |
| 0.0846 | |
| ], | |
| "clone_pause_time": [ | |
| 0.1537, | |
| 0.761, | |
| -0.6073, | |
| -0.881, | |
| -0.3457, | |
| 0.0 | |
| ], | |
| "clone_max_pause": [ | |
| 0.1297, | |
| 0.373, | |
| -0.2433, | |
| -0.314, | |
| -0.1767, | |
| 0.0 | |
| ], | |
| "clone_dropouts": [ | |
| 1.0667, | |
| 11.2333, | |
| -10.1667, | |
| -12.1333, | |
| -8.2333, | |
| 0.0 | |
| ] | |
| }, | |
| "hi": { | |
| "builtin_CER": [ | |
| 4.1739, | |
| 4.5652, | |
| -0.3913, | |
| -0.8562, | |
| 0.1244, | |
| 0.1352 | |
| ], | |
| "builtin_WER": [ | |
| 12.6582, | |
| 13.9677, | |
| -1.3095, | |
| -2.635, | |
| 0.1291, | |
| 0.0756 | |
| ], | |
| "builtin_UTMOS": [ | |
| 2.8354, | |
| 2.733, | |
| 0.1024, | |
| 0.0368, | |
| 0.1694, | |
| 0.0022 | |
| ], | |
| "clone_CER": [ | |
| 6.7245, | |
| 6.3991, | |
| 0.3254, | |
| -1.5108, | |
| 2.0331, | |
| 0.7232 | |
| ], | |
| "clone_WER": [ | |
| 17.9083, | |
| 15.9026, | |
| 2.0057, | |
| -2.381, | |
| 6.2588, | |
| 0.3892 | |
| ], | |
| "clone_sim_ecapa": [ | |
| 0.3633, | |
| 0.2593, | |
| 0.104, | |
| 0.0734, | |
| 0.1344, | |
| 0.0 | |
| ], | |
| "clone_sim_wavlm": [ | |
| 0.9044, | |
| 0.8606, | |
| 0.0438, | |
| 0.012, | |
| 0.0824, | |
| 0.0028 | |
| ], | |
| "clone_utmos": [ | |
| 2.1538, | |
| 2.5419, | |
| -0.3881, | |
| -0.5087, | |
| -0.2636, | |
| 0.0 | |
| ], | |
| "clone_pause150": [ | |
| 0.7, | |
| 2.1333, | |
| -1.4333, | |
| -2.4, | |
| -0.5333, | |
| 0.001 | |
| ], | |
| "clone_pause300": [ | |
| 0.3333, | |
| 1.3333, | |
| -1.0, | |
| -1.6, | |
| -0.5, | |
| 0.0 | |
| ], | |
| "clone_pause700": [ | |
| 0.0, | |
| 0.2, | |
| -0.2, | |
| -0.3333, | |
| -0.0667, | |
| 0.0022 | |
| ], | |
| "clone_pause_time": [ | |
| 0.2323, | |
| 0.87, | |
| -0.6377, | |
| -1.0463, | |
| -0.286, | |
| 0.0 | |
| ], | |
| "clone_max_pause": [ | |
| 0.193, | |
| 0.4317, | |
| -0.2387, | |
| -0.348, | |
| -0.1403, | |
| 0.0 | |
| ], | |
| "clone_dropouts": [ | |
| 2.2333, | |
| 10.7667, | |
| -8.5333, | |
| -10.3333, | |
| -6.7, | |
| 0.0 | |
| ] | |
| }, | |
| "en": { | |
| "builtin_CER": [ | |
| 4.7863, | |
| 4.4205, | |
| 0.3658, | |
| -0.2777, | |
| 1.134, | |
| 0.3148 | |
| ], | |
| "builtin_WER": [ | |
| 9.6513, | |
| 9.4997, | |
| 0.1516, | |
| -1.4306, | |
| 1.7903, | |
| 0.8956 | |
| ], | |
| "builtin_UTMOS": [ | |
| 3.194, | |
| 3.192, | |
| 0.0019, | |
| -0.0665, | |
| 0.0674, | |
| 0.9334 | |
| ], | |
| "clone_CER": [ | |
| 9.7459, | |
| 4.3857, | |
| 5.3603, | |
| -0.3097, | |
| 13.0973, | |
| 0.0732 | |
| ], | |
| "clone_WER": [ | |
| 17.1717, | |
| 10.6061, | |
| 6.5657, | |
| -0.3484, | |
| 14.7356, | |
| 0.073 | |
| ], | |
| "clone_sim_ecapa": [ | |
| 0.3025, | |
| 0.1793, | |
| 0.1232, | |
| 0.0746, | |
| 0.1688, | |
| 0.0 | |
| ], | |
| "clone_sim_wavlm": [ | |
| 0.8893, | |
| 0.8599, | |
| 0.0294, | |
| -0.0229, | |
| 0.0655, | |
| 0.218 | |
| ], | |
| "clone_utmos": [ | |
| 2.4402, | |
| 3.0247, | |
| -0.5844, | |
| -0.7912, | |
| -0.3882, | |
| 0.0 | |
| ], | |
| "clone_pause150": [ | |
| 1.0333, | |
| 1.5, | |
| -0.4667, | |
| -1.2, | |
| 0.2, | |
| 0.2066 | |
| ], | |
| "clone_pause300": [ | |
| 0.6333, | |
| 0.6667, | |
| -0.0333, | |
| -0.6667, | |
| 0.5333, | |
| 0.9934 | |
| ], | |
| "clone_pause700": [ | |
| 0.2333, | |
| 0.0667, | |
| 0.1667, | |
| -0.1, | |
| 0.4333, | |
| 0.2548 | |
| ], | |
| "clone_pause_time": [ | |
| 0.7593, | |
| 0.509, | |
| 0.2503, | |
| -0.351, | |
| 1.0964, | |
| 0.5646 | |
| ], | |
| "clone_max_pause": [ | |
| 0.606, | |
| 0.3043, | |
| 0.3017, | |
| -0.1, | |
| 0.9853, | |
| 0.4234 | |
| ], | |
| "clone_dropouts": [ | |
| 1.7667, | |
| 7.9, | |
| -6.1333, | |
| -7.5667, | |
| -4.7667, | |
| 0.0 | |
| ] | |
| } | |
| }, | |
| "notes": "Notes: the AksharaVP and Hima rows were generated through each model's published Python API (sentence split, automatic retake when a take runs on or goes silent); the Pocket TTS and MMS-TTS rows use those models' own default generation. Hima TTS nano ships separate Telugu and Hindi checkpoints of 72,160,162 parameters each (English is spoken by the Hindi one; its English row uses that checkpoint). Pocket TTS rows are English-only models speaking with their own (non-Indian) built-in voices, so English accent differs from the Indian voices of the other rows. Meta MMS-TTS rows are three separate single-language VITS checkpoints (16 kHz output, no cloning, CC-BY-NC-4.0), given the same normalised input text as AksharaVP; their ASR judge (mms-1b-all) comes from the same MMS project. UTMOS is an English-centric predictor; compare systems within a language column, not across columns." | |
| } |