AksharaVP-17M / evaluation /comparison.json
sourojitdotcom's picture
AksharaVP round 5: better cloning, fewer gaps
7384ac1 verified
Raw History Blame Contribute Delete
9.26 kB
{
"models": [
{
"name": "AksharaVP-17M v2 (this model)",
"ours": true,
"params": 17988627,
"langs": "te \u00b7 hi \u00b7 en, one checkpoint",
"results": {
"te": {
"CER": 6.1974,
"WER": 30.3806,
"UTMOS": 2.6535,
"n": 100
},
"hi": {
"CER": 4.1739,
"WER": 12.6582,
"UTMOS": 2.8354,
"n": 100
},
"en": {
"CER": 4.7863,
"WER": 9.6513,
"UTMOS": 3.194,
"n": 100
}
},
"source": "baseline/r5_E1_say_dev.json",
"fp32_MB": 71.954508
},
{
"name": "AksharaVP-17M v1 (previous release)",
"params": 17643026,
"langs": "te \u00b7 hi \u00b7 en, one checkpoint",
"results": {
"te": {
"CER": 6.0372,
"WER": 28.937,
"UTMOS": 2.6385,
"n": 100
},
"hi": {
"CER": 4.5652,
"WER": 13.9677,
"UTMOS": 2.733,
"n": 100
},
"en": {
"CER": 4.4205,
"WER": 9.4997,
"UTMOS": 3.192,
"n": 100
}
},
"source": "baseline/r5_vp17m_say_dev.json",
"fp32_MB": 70.572104
},
{
"name": "Hima TTS nano (hi + te checkpoints)",
"params": 72160162,
"langs": "te+en / hi+en, two checkpoints",
"results": {
"te": {
"CER": 6.708,
"WER": 31.2336,
"UTMOS": 3.4974,
"n": 100
},
"hi": {
"CER": 4.6957,
"WER": 14.797,
"UTMOS": 3.6089,
"n": 100
},
"en": {
"CER": 5.2879,
"WER": 11.4704,
"UTMOS": 4.0328,
"n": 100
}
},
"source": "baseline/r5_hima_say_dev.json",
"fp32_MB": 288.640648
},
{
"name": "Kyutai Pocket TTS (english)",
"params": 99752802,
"langs": "en only",
"results": {
"en": {
"CER": 2.3513,
"WER": 6.0131,
"UTMOS": 3.9945,
"n": 100
}
},
"source": "baseline/enprobe_pocket_english_dev_dev.json",
"fp32_MB": 399.011208
},
{
"name": "Kyutai Pocket TTS english_7m_2026-10",
"params": 6862578,
"langs": "en only",
"results": {
"en": {
"CER": 1.9856,
"WER": 6.3669,
"UTMOS": 4.1018,
"n": 100
}
},
"source": "baseline/enprobe_pocket_english_7m_2026-10_dev_dev.json",
"fp32_MB": 27.450312
},
{
"name": "Kyutai Pocket TTS english_2m_2026-10",
"params": 1922846,
"langs": "en only",
"results": {
"en": {
"CER": 3.8876,
"WER": 9.7019,
"UTMOS": 3.4948,
"n": 100
}
},
"source": "baseline/enprobe_pocket_english_2m_2026-10_dev_dev.json",
"fp32_MB": 7.691384
},
{
"name": "Meta MMS-TTS tel (VITS)",
"params": 36289776,
"langs": "te only",
"results": {
"te": {
"CER": 10.4826,
"WER": 41.6667,
"UTMOS": 3.2809,
"n": 100
}
},
"source": "baseline/mms_tts_dev100_dev.json (facebook/mms-tts-tel, same normalised input text, 16 kHz)",
"fp32_MB": 145.159104
},
{
"name": "Meta MMS-TTS hin (VITS)",
"params": 36291120,
"langs": "hi only",
"results": {
"hi": {
"CER": 11.2609,
"WER": 28.6338,
"UTMOS": 3.4889,
"n": 100
}
},
"source": "baseline/mms_tts_dev100_dev.json (facebook/mms-tts-hin, same normalised input text, 16 kHz)",
"fp32_MB": 145.16448
},
{
"name": "Meta MMS-TTS eng (VITS)",
"params": 36284592,
"langs": "en only",
"results": {
"en": {
"CER": 6.4584,
"WER": 17.332,
"UTMOS": 4.0958,
"n": 100
}
},
"source": "baseline/mms_tts_dev100_dev.json (facebook/mms-tts-eng, same normalised input text, 16 kHz)",
"fp32_MB": 145.138368
}
],
"bootstrap_vs_hima": {
"te": {
"dCER": [
-0.5106,
-1.1626,
0.1103,
0.1166
],
"dWER": [
-0.853,
-3.1271,
1.4522,
0.4922
],
"dUTMOS": [
-0.844,
-0.9148,
-0.7694,
0.0
]
},
"hi": {
"dCER": [
-0.5217,
-1.2487,
0.1766,
0.1554
],
"dWER": [
-2.1388,
-3.9948,
-0.3097,
0.024
],
"dUTMOS": [
-0.7736,
-0.8561,
-0.6901,
0.0
]
},
"en": {
"dCER": [
-0.5016,
-1.785,
0.783,
0.429
],
"dWER": [
-1.8191,
-4.5948,
0.5647,
0.1488
],
"dUTMOS": [
-0.8388,
-0.9171,
-0.7585,
0.0
]
}
},
"v2_vs_v1": {
"te": {
"builtin_CER": [
6.1974,
6.0372,
0.1602,
-0.4024,
0.7038,
0.5772
],
"builtin_WER": [
30.3806,
28.937,
1.4436,
-0.8458,
3.751,
0.2176
],
"builtin_UTMOS": [
2.6535,
2.6385,
0.0149,
-0.0273,
0.0576,
0.4892
],
"clone_CER": [
10.424,
7.8738,
2.5502,
0.2448,
5.6443,
0.0264
],
"clone_WER": [
40.0826,
35.3306,
4.7521,
0.4338,
9.3498,
0.0356
],
"clone_sim_ecapa": [
0.4017,
0.2951,
0.1066,
0.0748,
0.1381,
0.0
],
"clone_sim_wavlm": [
0.9134,
0.8789,
0.0345,
0.0146,
0.0523,
0.0016
],
"clone_utmos": [
1.8903,
2.3775,
-0.4873,
-0.6098,
-0.3669,
0.0
],
"clone_pause150": [
0.4667,
2.0,
-1.5333,
-2.2667,
-0.8333,
0.0
],
"clone_pause300": [
0.2333,
1.2333,
-1.0,
-1.5667,
-0.4667,
0.0002
],
"clone_pause700": [
0.0,
0.1,
-0.1,
-0.2,
0.0,
0.0846
],
"clone_pause_time": [
0.1537,
0.761,
-0.6073,
-0.881,
-0.3457,
0.0
],
"clone_max_pause": [
0.1297,
0.373,
-0.2433,
-0.314,
-0.1767,
0.0
],
"clone_dropouts": [
1.0667,
11.2333,
-10.1667,
-12.1333,
-8.2333,
0.0
]
},
"hi": {
"builtin_CER": [
4.1739,
4.5652,
-0.3913,
-0.8562,
0.1244,
0.1352
],
"builtin_WER": [
12.6582,
13.9677,
-1.3095,
-2.635,
0.1291,
0.0756
],
"builtin_UTMOS": [
2.8354,
2.733,
0.1024,
0.0368,
0.1694,
0.0022
],
"clone_CER": [
6.7245,
6.3991,
0.3254,
-1.5108,
2.0331,
0.7232
],
"clone_WER": [
17.9083,
15.9026,
2.0057,
-2.381,
6.2588,
0.3892
],
"clone_sim_ecapa": [
0.3633,
0.2593,
0.104,
0.0734,
0.1344,
0.0
],
"clone_sim_wavlm": [
0.9044,
0.8606,
0.0438,
0.012,
0.0824,
0.0028
],
"clone_utmos": [
2.1538,
2.5419,
-0.3881,
-0.5087,
-0.2636,
0.0
],
"clone_pause150": [
0.7,
2.1333,
-1.4333,
-2.4,
-0.5333,
0.001
],
"clone_pause300": [
0.3333,
1.3333,
-1.0,
-1.6,
-0.5,
0.0
],
"clone_pause700": [
0.0,
0.2,
-0.2,
-0.3333,
-0.0667,
0.0022
],
"clone_pause_time": [
0.2323,
0.87,
-0.6377,
-1.0463,
-0.286,
0.0
],
"clone_max_pause": [
0.193,
0.4317,
-0.2387,
-0.348,
-0.1403,
0.0
],
"clone_dropouts": [
2.2333,
10.7667,
-8.5333,
-10.3333,
-6.7,
0.0
]
},
"en": {
"builtin_CER": [
4.7863,
4.4205,
0.3658,
-0.2777,
1.134,
0.3148
],
"builtin_WER": [
9.6513,
9.4997,
0.1516,
-1.4306,
1.7903,
0.8956
],
"builtin_UTMOS": [
3.194,
3.192,
0.0019,
-0.0665,
0.0674,
0.9334
],
"clone_CER": [
9.7459,
4.3857,
5.3603,
-0.3097,
13.0973,
0.0732
],
"clone_WER": [
17.1717,
10.6061,
6.5657,
-0.3484,
14.7356,
0.073
],
"clone_sim_ecapa": [
0.3025,
0.1793,
0.1232,
0.0746,
0.1688,
0.0
],
"clone_sim_wavlm": [
0.8893,
0.8599,
0.0294,
-0.0229,
0.0655,
0.218
],
"clone_utmos": [
2.4402,
3.0247,
-0.5844,
-0.7912,
-0.3882,
0.0
],
"clone_pause150": [
1.0333,
1.5,
-0.4667,
-1.2,
0.2,
0.2066
],
"clone_pause300": [
0.6333,
0.6667,
-0.0333,
-0.6667,
0.5333,
0.9934
],
"clone_pause700": [
0.2333,
0.0667,
0.1667,
-0.1,
0.4333,
0.2548
],
"clone_pause_time": [
0.7593,
0.509,
0.2503,
-0.351,
1.0964,
0.5646
],
"clone_max_pause": [
0.606,
0.3043,
0.3017,
-0.1,
0.9853,
0.4234
],
"clone_dropouts": [
1.7667,
7.9,
-6.1333,
-7.5667,
-4.7667,
0.0
]
}
},
"notes": "Notes: the AksharaVP and Hima rows were generated through each model's published Python API (sentence split, automatic retake when a take runs on or goes silent); the Pocket TTS and MMS-TTS rows use those models' own default generation. Hima TTS nano ships separate Telugu and Hindi checkpoints of 72,160,162 parameters each (English is spoken by the Hindi one; its English row uses that checkpoint). Pocket TTS rows are English-only models speaking with their own (non-Indian) built-in voices, so English accent differs from the Indian voices of the other rows. Meta MMS-TTS rows are three separate single-language VITS checkpoints (16 kHz output, no cloning, CC-BY-NC-4.0), given the same normalised input text as AksharaVP; their ASR judge (mms-1b-all) comes from the same MMS project. UTMOS is an English-centric predictor; compare systems within a language column, not across columns."
}