Download evaluation/round5_bootstrap_dev.json from sourojitdotcom/AksharaVP-17M: direct link, hf CLI and curl.
- Browser
- Download file 43.8 kB
-
https://huggingface.co/sourojitdotcom/AksharaVP-17M/resolve/main/evaluation/round5_bootstrap_dev.json
- Command line
-
hf download hf://sourojitdotcom/AksharaVP-17M/evaluation/round5_bootstrap_dev.json
-
curl -L -o round5_bootstrap_dev.json https://huggingface.co/sourojitdotcom/AksharaVP-17M/resolve/main/evaluation/round5_bootstrap_dev.json
43.8 kB
{
"note": "DEV only (FLEURS dev). Round-5 paired bootstrap: r5 E1 (DCpS) and E2 (DCpSL) vs AksharaVP-17M (published r4 DCp) and vs Hima, per language.\nBuiltin: CER/WER (corpus, MMS-1b-all CTC) + UTMOS from baseline/r5_*_say_dev.json (eval_tts.py), ids 0..99/lang.\nClone: ECAPA/WavLM sim, UTMOS, pause150/300/700, pause_time, max_pause, dropouts (energy dips) from diag/results/{r5_E1_say,r5_E2_say,vp_say,hima_say}.json\n(analyze.py), ids 0..29/lang; clone CER/WER from the eval_tts json too.\nPaired over identical utterance ids, 10,000 resamples (rng seed 1), diff = A - B, 95% percentile CI, two-sided p = min(1, 2*min(P(d<=0), P(d>=0))).\nWrites baseline/r5_bootstrap_dev.json and diag/results/r5_table.md",
"absolute": {
"E1/hi": {
"builtin_n": 100,
"CER": 4.173913043478261,
"WER": 12.658227848101266,
"UTMOS": 2.835365197658539,
"clone_n": 30,
"clone_CER": 6.724511930585683,
"clone_WER": 17.908309455587393,
"clone_sim_ecapa": 0.36334028194348017,
"clone_sim_wavlm": 0.9043956021467845,
"clone_utmos": 2.1537877678871156,
"clone_pause150": 0.7,
"clone_pause300": 0.3333333333333333,
"clone_pause700": 0.0,
"clone_pause_time": 0.2323333333333333,
"clone_max_pause": 0.19299999999999995,
"clone_dropouts": 2.2333333333333334
},
"E1/te": {
"builtin_n": 100,
"CER": 6.197436924309171,
"WER": 30.38057742782152,
"UTMOS": 2.653467684984207,
"clone_n": 30,
"clone_CER": 10.423971947720752,
"clone_WER": 40.082644628099175,
"clone_sim_ecapa": 0.4017001857360204,
"clone_sim_wavlm": 0.9134249667326609,
"clone_utmos": 1.890282436211904,
"clone_pause150": 0.4666666666666667,
"clone_pause300": 0.23333333333333334,
"clone_pause700": 0.0,
"clone_pause_time": 0.15366666666666667,
"clone_max_pause": 0.12966666666666665,
"clone_dropouts": 1.0666666666666667
},
"E1/en": {
"builtin_n": 100,
"CER": 4.7862890584178075,
"WER": 9.651339060131379,
"UTMOS": 3.19397074341774,
"clone_n": 30,
"clone_CER": 9.745910198398887,
"clone_WER": 17.171717171717173,
"clone_sim_ecapa": 0.30246616067985693,
"clone_sim_wavlm": 0.8892902890841167,
"clone_utmos": 2.4402199387550354,
"clone_pause150": 1.0333333333333334,
"clone_pause300": 0.6333333333333333,
"clone_pause700": 0.23333333333333334,
"clone_pause_time": 0.7593333333333334,
"clone_max_pause": 0.6059999999999999,
"clone_dropouts": 1.7666666666666666
},
"E2/hi": {
"builtin_n": 100,
"CER": 4.315217391304348,
"WER": 12.789175032736797,
"UTMOS": 2.7713782811164855,
"clone_n": 30,
"clone_CER": 8.459869848156183,
"clone_WER": 21.63323782234957,
"clone_sim_ecapa": 0.35160381346940994,
"clone_sim_wavlm": 0.9089426139990489,
"clone_utmos": 2.1057937304178873,
"clone_pause150": 1.4,
"clone_pause300": 0.6333333333333333,
"clone_pause700": 0.1,
"clone_pause_time": 0.8969999999999999,
"clone_max_pause": 0.4873333333333333,
"clone_dropouts": 2.0
},
"E2/te": {
"builtin_n": 100,
"CER": 5.826992390869043,
"WER": 28.1496062992126,
"UTMOS": 2.6618559622764586,
"clone_n": 30,
"clone_CER": 12.719158431622569,
"clone_WER": 43.388429752066116,
"clone_sim_ecapa": 0.40696775913238525,
"clone_sim_wavlm": 0.9167420009771983,
"clone_utmos": 1.860637084643046,
"clone_pause150": 0.16666666666666666,
"clone_pause300": 0.0,
"clone_pause700": 0.0,
"clone_pause_time": 0.035333333333333335,
"clone_max_pause": 0.08499999999999999,
"clone_dropouts": 1.0333333333333334
},
"E2/en": {
"builtin_n": 100,
"CER": 4.639983279339534,
"WER": 9.146033350176857,
"UTMOS": 3.1779002070426943,
"clone_n": 30,
"clone_CER": 16.394013226592413,
"clone_WER": 25.252525252525253,
"clone_sim_ecapa": 0.2995812490582466,
"clone_sim_wavlm": 0.8774400850137075,
"clone_utmos": 2.310932183265686,
"clone_pause150": 1.7,
"clone_pause300": 0.7333333333333333,
"clone_pause700": 0.36666666666666664,
"clone_pause_time": 1.1623333333333334,
"clone_max_pause": 0.7013333333333333,
"clone_dropouts": 1.7
},
"VP17M/hi": {
"builtin_n": 100,
"CER": 4.565217391304348,
"WER": 13.967699694456568,
"UTMOS": 2.7329677879810332,
"clone_n": 30,
"clone_CER": 6.399132321041215,
"clone_WER": 15.902578796561604,
"clone_sim_ecapa": 0.2593378665546576,
"clone_sim_wavlm": 0.8606199562549591,
"clone_utmos": 2.5419195175170897,
"clone_pause150": 2.1333333333333333,
"clone_pause300": 1.3333333333333333,
"clone_pause700": 0.2,
"clone_pause_time": 0.8699999999999998,
"clone_max_pause": 0.43166666666666664,
"clone_dropouts": 10.766666666666667
},
"VP17M/te": {
"builtin_n": 100,
"CER": 6.037244693632359,
"WER": 28.937007874015748,
"UTMOS": 2.6385431146621703,
"clone_n": 30,
"clone_CER": 7.8737647433854,
"clone_WER": 35.33057851239669,
"clone_sim_ecapa": 0.2951168326040109,
"clone_sim_wavlm": 0.8788889686266581,
"clone_utmos": 2.377547498544057,
"clone_pause150": 2.0,
"clone_pause300": 1.2333333333333334,
"clone_pause700": 0.1,
"clone_pause_time": 0.7609999999999998,
"clone_max_pause": 0.373,
"clone_dropouts": 11.233333333333333
},
"VP17M/en": {
"builtin_n": 100,
"CER": 4.420524610722124,
"WER": 9.499747347145023,
"UTMOS": 3.192048554420471,
"clone_n": 30,
"clone_CER": 4.3856595892794985,
"clone_WER": 10.606060606060606,
"clone_sim_ecapa": 0.17925435736154516,
"clone_sim_wavlm": 0.859893012046814,
"clone_utmos": 3.024665109316508,
"clone_pause150": 1.5,
"clone_pause300": 0.6666666666666666,
"clone_pause700": 0.06666666666666667,
"clone_pause_time": 0.509,
"clone_max_pause": 0.30433333333333334,
"clone_dropouts": 7.9
},
"Hima/hi": {
"builtin_n": 100,
"CER": 4.695652173913044,
"WER": 14.797031863814928,
"UTMOS": 3.608945007324219,
"clone_n": 30,
"clone_CER": 4.844540853217643,
"clone_WER": 14.469914040114613,
"clone_sim_ecapa": 0.4921727786461512,
"clone_sim_wavlm": 0.9399481952190399,
"clone_utmos": 3.066835403442383,
"clone_pause150": 1.8,
"clone_pause300": 1.2666666666666666,
"clone_pause700": 0.06666666666666667,
"clone_pause_time": 0.75,
"clone_max_pause": 0.35699999999999993,
"clone_dropouts": 5.033333333333333
},
"Hima/te": {
"builtin_n": 100,
"CER": 6.70804965959151,
"WER": 31.233595800524935,
"UTMOS": 3.4974388909339904,
"clone_n": 30,
"clone_CER": 9.180745935607268,
"clone_WER": 36.77685950413223,
"clone_sim_ecapa": 0.559562333424886,
"clone_sim_wavlm": 0.9504657725493113,
"clone_utmos": 2.9912472208340963,
"clone_pause150": 1.7666666666666666,
"clone_pause300": 1.5,
"clone_pause700": 0.13333333333333333,
"clone_pause_time": 0.7733333333333333,
"clone_max_pause": 0.391,
"clone_dropouts": 3.1
},
"Hima/en": {
"builtin_n": 100,
"CER": 5.28790887240046,
"WER": 11.47043961596766,
"UTMOS": 4.032773208618164,
"clone_n": 30,
"clone_CER": 3.4110685694396103,
"clone_WER": 9.25925925925926,
"clone_sim_ecapa": 0.4206847667694092,
"clone_sim_wavlm": 0.9357598861058553,
"clone_utmos": 3.38255455493927,
"clone_pause150": 1.4666666666666666,
"clone_pause300": 0.9,
"clone_pause700": 0.36666666666666664,
"clone_pause_time": 0.7656666666666666,
"clone_max_pause": 0.40366666666666656,
"clone_dropouts": 3.5
}
},
"paired": {
"E1_vs_VP17M_hi": {
"builtin_CER": {
"n": 100,
"a": 4.1739,
"b": 4.5652,
"diff": -0.3913,
"boot_mean": -0.3897,
"lo": -0.8562,
"hi": 0.1244,
"p": 0.1352
},
"builtin_WER": {
"n": 100,
"a": 12.6582,
"b": 13.9677,
"diff": -1.3095,
"boot_mean": -1.2998,
"lo": -2.635,
"hi": 0.1291,
"p": 0.0756
},
"builtin_UTMOS": {
"n": 100,
"a": 2.8354,
"b": 2.733,
"diff": 0.1024,
"boot_mean": 0.1027,
"lo": 0.0368,
"hi": 0.1694,
"p": 0.0022
},
"clone_CER": {
"n": 30,
"a": 6.7245,
"b": 6.3991,
"diff": 0.3254,
"boot_mean": 0.307,
"lo": -1.5108,
"hi": 2.0331,
"p": 0.7232
},
"clone_WER": {
"n": 30,
"a": 17.9083,
"b": 15.9026,
"diff": 2.0057,
"boot_mean": 1.9876,
"lo": -2.381,
"hi": 6.2588,
"p": 0.3892
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.3633,
"b": 0.2593,
"diff": 0.104,
"boot_mean": 0.1042,
"lo": 0.0734,
"hi": 0.1344,
"p": 0.0
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.9044,
"b": 0.8606,
"diff": 0.0438,
"boot_mean": 0.044,
"lo": 0.012,
"hi": 0.0824,
"p": 0.0028
},
"clone_utmos": {
"n": 30,
"a": 2.1538,
"b": 2.5419,
"diff": -0.3881,
"boot_mean": -0.3877,
"lo": -0.5087,
"hi": -0.2636,
"p": 0.0
},
"clone_pause150": {
"n": 30,
"a": 0.7,
"b": 2.1333,
"diff": -1.4333,
"boot_mean": -1.4351,
"lo": -2.4,
"hi": -0.5333,
"p": 0.001
},
"clone_pause300": {
"n": 30,
"a": 0.3333,
"b": 1.3333,
"diff": -1.0,
"boot_mean": -1.0008,
"lo": -1.6,
"hi": -0.5,
"p": 0.0
},
"clone_pause700": {
"n": 30,
"a": 0.0,
"b": 0.2,
"diff": -0.2,
"boot_mean": -0.1998,
"lo": -0.3333,
"hi": -0.0667,
"p": 0.0022
},
"clone_pause_time": {
"n": 30,
"a": 0.2323,
"b": 0.87,
"diff": -0.6377,
"boot_mean": -0.6386,
"lo": -1.0463,
"hi": -0.286,
"p": 0.0
},
"clone_max_pause": {
"n": 30,
"a": 0.193,
"b": 0.4317,
"diff": -0.2387,
"boot_mean": -0.2389,
"lo": -0.348,
"hi": -0.1403,
"p": 0.0
},
"clone_dropouts": {
"n": 30,
"a": 2.2333,
"b": 10.7667,
"diff": -8.5333,
"boot_mean": -8.5331,
"lo": -10.3333,
"hi": -6.7,
"p": 0.0
}
},
"E1_vs_VP17M_te": {
"builtin_CER": {
"n": 100,
"a": 6.1974,
"b": 6.0372,
"diff": 0.1602,
"boot_mean": 0.1602,
"lo": -0.4024,
"hi": 0.7038,
"p": 0.5772
},
"builtin_WER": {
"n": 100,
"a": 30.3806,
"b": 28.937,
"diff": 1.4436,
"boot_mean": 1.4478,
"lo": -0.8458,
"hi": 3.751,
"p": 0.2176
},
"builtin_UTMOS": {
"n": 100,
"a": 2.6535,
"b": 2.6385,
"diff": 0.0149,
"boot_mean": 0.0152,
"lo": -0.0273,
"hi": 0.0576,
"p": 0.4892
},
"clone_CER": {
"n": 30,
"a": 10.424,
"b": 7.8738,
"diff": 2.5502,
"boot_mean": 2.5489,
"lo": 0.2448,
"hi": 5.6443,
"p": 0.0264
},
"clone_WER": {
"n": 30,
"a": 40.0826,
"b": 35.3306,
"diff": 4.7521,
"boot_mean": 4.6875,
"lo": 0.4338,
"hi": 9.3498,
"p": 0.0356
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.4017,
"b": 0.2951,
"diff": 0.1066,
"boot_mean": 0.1065,
"lo": 0.0748,
"hi": 0.1381,
"p": 0.0
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.9134,
"b": 0.8789,
"diff": 0.0345,
"boot_mean": 0.0344,
"lo": 0.0146,
"hi": 0.0523,
"p": 0.0016
},
"clone_utmos": {
"n": 30,
"a": 1.8903,
"b": 2.3775,
"diff": -0.4873,
"boot_mean": -0.4872,
"lo": -0.6098,
"hi": -0.3669,
"p": 0.0
},
"clone_pause150": {
"n": 30,
"a": 0.4667,
"b": 2.0,
"diff": -1.5333,
"boot_mean": -1.5354,
"lo": -2.2667,
"hi": -0.8333,
"p": 0.0
},
"clone_pause300": {
"n": 30,
"a": 0.2333,
"b": 1.2333,
"diff": -1.0,
"boot_mean": -0.9995,
"lo": -1.5667,
"hi": -0.4667,
"p": 0.0002
},
"clone_pause700": {
"n": 30,
"a": 0.0,
"b": 0.1,
"diff": -0.1,
"boot_mean": -0.0996,
"lo": -0.2,
"hi": 0.0,
"p": 0.0846
},
"clone_pause_time": {
"n": 30,
"a": 0.1537,
"b": 0.761,
"diff": -0.6073,
"boot_mean": -0.6074,
"lo": -0.881,
"hi": -0.3457,
"p": 0.0
},
"clone_max_pause": {
"n": 30,
"a": 0.1297,
"b": 0.373,
"diff": -0.2433,
"boot_mean": -0.2433,
"lo": -0.314,
"hi": -0.1767,
"p": 0.0
},
"clone_dropouts": {
"n": 30,
"a": 1.0667,
"b": 11.2333,
"diff": -10.1667,
"boot_mean": -10.1591,
"lo": -12.1333,
"hi": -8.2333,
"p": 0.0
}
},
"E1_vs_VP17M_en": {
"builtin_CER": {
"n": 100,
"a": 4.7863,
"b": 4.4205,
"diff": 0.3658,
"boot_mean": 0.3637,
"lo": -0.2777,
"hi": 1.134,
"p": 0.3148
},
"builtin_WER": {
"n": 100,
"a": 9.6513,
"b": 9.4997,
"diff": 0.1516,
"boot_mean": 0.1454,
"lo": -1.4306,
"hi": 1.7903,
"p": 0.8956
},
"builtin_UTMOS": {
"n": 100,
"a": 3.194,
"b": 3.192,
"diff": 0.0019,
"boot_mean": 0.0025,
"lo": -0.0665,
"hi": 0.0674,
"p": 0.9334
},
"clone_CER": {
"n": 30,
"a": 9.7459,
"b": 4.3857,
"diff": 5.3603,
"boot_mean": 5.3145,
"lo": -0.3097,
"hi": 13.0973,
"p": 0.0732
},
"clone_WER": {
"n": 30,
"a": 17.1717,
"b": 10.6061,
"diff": 6.5657,
"boot_mean": 6.5305,
"lo": -0.3484,
"hi": 14.7356,
"p": 0.073
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.3025,
"b": 0.1793,
"diff": 0.1232,
"boot_mean": 0.123,
"lo": 0.0746,
"hi": 0.1688,
"p": 0.0
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.8893,
"b": 0.8599,
"diff": 0.0294,
"boot_mean": 0.0295,
"lo": -0.0229,
"hi": 0.0655,
"p": 0.218
},
"clone_utmos": {
"n": 30,
"a": 2.4402,
"b": 3.0247,
"diff": -0.5844,
"boot_mean": -0.5841,
"lo": -0.7912,
"hi": -0.3882,
"p": 0.0
},
"clone_pause150": {
"n": 30,
"a": 1.0333,
"b": 1.5,
"diff": -0.4667,
"boot_mean": -0.465,
"lo": -1.2,
"hi": 0.2,
"p": 0.2066
},
"clone_pause300": {
"n": 30,
"a": 0.6333,
"b": 0.6667,
"diff": -0.0333,
"boot_mean": -0.0324,
"lo": -0.6667,
"hi": 0.5333,
"p": 0.9934
},
"clone_pause700": {
"n": 30,
"a": 0.2333,
"b": 0.0667,
"diff": 0.1667,
"boot_mean": 0.1651,
"lo": -0.1,
"hi": 0.4333,
"p": 0.2548
},
"clone_pause_time": {
"n": 30,
"a": 0.7593,
"b": 0.509,
"diff": 0.2503,
"boot_mean": 0.2479,
"lo": -0.351,
"hi": 1.0964,
"p": 0.5646
},
"clone_max_pause": {
"n": 30,
"a": 0.606,
"b": 0.3043,
"diff": 0.3017,
"boot_mean": 0.299,
"lo": -0.1,
"hi": 0.9853,
"p": 0.4234
},
"clone_dropouts": {
"n": 30,
"a": 1.7667,
"b": 7.9,
"diff": -6.1333,
"boot_mean": -6.1281,
"lo": -7.5667,
"hi": -4.7667,
"p": 0.0
}
},
"E1_vs_Hima_hi": {
"builtin_CER": {
"n": 100,
"a": 4.1739,
"b": 4.6957,
"diff": -0.5217,
"boot_mean": -0.5208,
"lo": -1.2487,
"hi": 0.1766,
"p": 0.1554
},
"builtin_WER": {
"n": 100,
"a": 12.6582,
"b": 14.797,
"diff": -2.1388,
"boot_mean": -2.1129,
"lo": -3.9948,
"hi": -0.3097,
"p": 0.024
},
"builtin_UTMOS": {
"n": 100,
"a": 2.8354,
"b": 3.6089,
"diff": -0.7736,
"boot_mean": -0.7736,
"lo": -0.8561,
"hi": -0.6901,
"p": 0.0
},
"clone_CER": {
"n": 30,
"a": 6.7245,
"b": 4.8445,
"diff": 1.88,
"boot_mean": 1.8778,
"lo": 0.2779,
"hi": 3.4575,
"p": 0.0224
},
"clone_WER": {
"n": 30,
"a": 17.9083,
"b": 14.4699,
"diff": 3.4384,
"boot_mean": 3.4438,
"lo": 0.2869,
"hi": 6.9527,
"p": 0.037
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.3633,
"b": 0.4922,
"diff": -0.1288,
"boot_mean": -0.1285,
"lo": -0.1691,
"hi": -0.0874,
"p": 0.0
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.9044,
"b": 0.9399,
"diff": -0.0356,
"boot_mean": -0.0355,
"lo": -0.05,
"hi": -0.0224,
"p": 0.0
},
"clone_utmos": {
"n": 30,
"a": 2.1538,
"b": 3.0668,
"diff": -0.913,
"boot_mean": -0.913,
"lo": -1.0775,
"hi": -0.7425,
"p": 0.0
},
"clone_pause150": {
"n": 30,
"a": 0.7,
"b": 1.8,
"diff": -1.1,
"boot_mean": -1.1004,
"lo": -1.7667,
"hi": -0.5,
"p": 0.0002
},
"clone_pause300": {
"n": 30,
"a": 0.3333,
"b": 1.2667,
"diff": -0.9333,
"boot_mean": -0.9313,
"lo": -1.5667,
"hi": -0.4,
"p": 0.0006
},
"clone_pause700": {
"n": 30,
"a": 0.0,
"b": 0.0667,
"diff": -0.0667,
"boot_mean": -0.0666,
"lo": -0.1667,
"hi": 0.0,
"p": 0.2524
},
"clone_pause_time": {
"n": 30,
"a": 0.2323,
"b": 0.75,
"diff": -0.5177,
"boot_mean": -0.5172,
"lo": -0.829,
"hi": -0.2473,
"p": 0.0004
},
"clone_max_pause": {
"n": 30,
"a": 0.193,
"b": 0.357,
"diff": -0.164,
"boot_mean": -0.1641,
"lo": -0.2737,
"hi": -0.0593,
"p": 0.002
},
"clone_dropouts": {
"n": 30,
"a": 2.2333,
"b": 5.0333,
"diff": -2.8,
"boot_mean": -2.7996,
"lo": -4.4667,
"hi": -1.3333,
"p": 0.0
}
},
"E1_vs_Hima_te": {
"builtin_CER": {
"n": 100,
"a": 6.1974,
"b": 6.708,
"diff": -0.5106,
"boot_mean": -0.5044,
"lo": -1.1626,
"hi": 0.1103,
"p": 0.1166
},
"builtin_WER": {
"n": 100,
"a": 30.3806,
"b": 31.2336,
"diff": -0.853,
"boot_mean": -0.8416,
"lo": -3.1271,
"hi": 1.4522,
"p": 0.4922
},
"builtin_UTMOS": {
"n": 100,
"a": 2.6535,
"b": 3.4974,
"diff": -0.844,
"boot_mean": -0.8435,
"lo": -0.9148,
"hi": -0.7694,
"p": 0.0
},
"clone_CER": {
"n": 30,
"a": 10.424,
"b": 9.1807,
"diff": 1.2432,
"boot_mean": 1.2374,
"lo": -1.4969,
"hi": 4.5675,
"p": 0.46
},
"clone_WER": {
"n": 30,
"a": 40.0826,
"b": 36.7769,
"diff": 3.3058,
"boot_mean": 3.2883,
"lo": -3.711,
"hi": 10.5161,
"p": 0.3872
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.4017,
"b": 0.5596,
"diff": -0.1579,
"boot_mean": -0.1579,
"lo": -0.1821,
"hi": -0.1343,
"p": 0.0
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.9134,
"b": 0.9505,
"diff": -0.037,
"boot_mean": -0.0371,
"lo": -0.0548,
"hi": -0.0226,
"p": 0.0
},
"clone_utmos": {
"n": 30,
"a": 1.8903,
"b": 2.9912,
"diff": -1.101,
"boot_mean": -1.1015,
"lo": -1.2668,
"hi": -0.9342,
"p": 0.0
},
"clone_pause150": {
"n": 30,
"a": 0.4667,
"b": 1.7667,
"diff": -1.3,
"boot_mean": -1.3061,
"lo": -2.0667,
"hi": -0.6667,
"p": 0.0
},
"clone_pause300": {
"n": 30,
"a": 0.2333,
"b": 1.5,
"diff": -1.2667,
"boot_mean": -1.2721,
"lo": -1.9667,
"hi": -0.7,
"p": 0.0
},
"clone_pause700": {
"n": 30,
"a": 0.0,
"b": 0.1333,
"diff": -0.1333,
"boot_mean": -0.1329,
"lo": -0.3,
"hi": 0.0,
"p": 0.0882
},
"clone_pause_time": {
"n": 30,
"a": 0.1537,
"b": 0.7733,
"diff": -0.6197,
"boot_mean": -0.6219,
"lo": -0.966,
"hi": -0.3243,
"p": 0.0
},
"clone_max_pause": {
"n": 30,
"a": 0.1297,
"b": 0.391,
"diff": -0.2613,
"boot_mean": -0.2615,
"lo": -0.3953,
"hi": -0.148,
"p": 0.0
},
"clone_dropouts": {
"n": 30,
"a": 1.0667,
"b": 3.1,
"diff": -2.0333,
"boot_mean": -2.0318,
"lo": -3.3333,
"hi": -0.9667,
"p": 0.0
}
},
"E1_vs_Hima_en": {
"builtin_CER": {
"n": 100,
"a": 4.7863,
"b": 5.2879,
"diff": -0.5016,
"boot_mean": -0.5054,
"lo": -1.785,
"hi": 0.783,
"p": 0.429
},
"builtin_WER": {
"n": 100,
"a": 9.6513,
"b": 11.4704,
"diff": -1.8191,
"boot_mean": -1.8339,
"lo": -4.5948,
"hi": 0.5647,
"p": 0.1488
},
"builtin_UTMOS": {
"n": 100,
"a": 3.194,
"b": 4.0328,
"diff": -0.8388,
"boot_mean": -0.8383,
"lo": -0.9171,
"hi": -0.7585,
"p": 0.0
},
"clone_CER": {
"n": 30,
"a": 9.7459,
"b": 3.4111,
"diff": 6.3348,
"boot_mean": 6.2845,
"lo": 0.8558,
"hi": 14.1825,
"p": 0.0062
},
"clone_WER": {
"n": 30,
"a": 17.1717,
"b": 9.2593,
"diff": 7.9125,
"boot_mean": 7.8561,
"lo": 1.0507,
"hi": 16.6059,
"p": 0.015
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.3025,
"b": 0.4207,
"diff": -0.1182,
"boot_mean": -0.1182,
"lo": -0.1505,
"hi": -0.085,
"p": 0.0
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.8893,
"b": 0.9358,
"diff": -0.0465,
"boot_mean": -0.0464,
"lo": -0.0902,
"hi": -0.0169,
"p": 0.0
},
"clone_utmos": {
"n": 30,
"a": 2.4402,
"b": 3.3826,
"diff": -0.9423,
"boot_mean": -0.9439,
"lo": -1.1784,
"hi": -0.6947,
"p": 0.0
},
"clone_pause150": {
"n": 30,
"a": 1.0333,
"b": 1.4667,
"diff": -0.4333,
"boot_mean": -0.4309,
"lo": -1.2,
"hi": 0.3,
"p": 0.2642
},
"clone_pause300": {
"n": 30,
"a": 0.6333,
"b": 0.9,
"diff": -0.2667,
"boot_mean": -0.2671,
"lo": -0.7333,
"hi": 0.2,
"p": 0.2856
},
"clone_pause700": {
"n": 30,
"a": 0.2333,
"b": 0.3667,
"diff": -0.1333,
"boot_mean": -0.1336,
"lo": -0.3333,
"hi": 0.0667,
"p": 0.245
},
"clone_pause_time": {
"n": 30,
"a": 0.7593,
"b": 0.7657,
"diff": -0.0063,
"boot_mean": -0.0083,
"lo": -0.561,
"hi": 0.6984,
"p": 0.916
},
"clone_max_pause": {
"n": 30,
"a": 0.606,
"b": 0.4037,
"diff": 0.2023,
"boot_mean": 0.1991,
"lo": -0.2143,
"hi": 0.867,
"p": 0.6368
},
"clone_dropouts": {
"n": 30,
"a": 1.7667,
"b": 3.5,
"diff": -1.7333,
"boot_mean": -1.7407,
"lo": -2.8,
"hi": -0.8,
"p": 0.0
}
},
"E2_vs_VP17M_hi": {
"builtin_CER": {
"n": 100,
"a": 4.3152,
"b": 4.5652,
"diff": -0.25,
"boot_mean": -0.256,
"lo": -0.8423,
"hi": 0.4322,
"p": 0.4346
},
"builtin_WER": {
"n": 100,
"a": 12.7892,
"b": 13.9677,
"diff": -1.1785,
"boot_mean": -1.181,
"lo": -2.6382,
"hi": 0.3015,
"p": 0.1244
},
"builtin_UTMOS": {
"n": 100,
"a": 2.7714,
"b": 2.733,
"diff": 0.0384,
"boot_mean": 0.0387,
"lo": -0.016,
"hi": 0.0919,
"p": 0.1592
},
"clone_CER": {
"n": 30,
"a": 8.4599,
"b": 6.3991,
"diff": 2.0607,
"boot_mean": 2.0259,
"lo": -1.7139,
"hi": 6.5143,
"p": 0.3488
},
"clone_WER": {
"n": 30,
"a": 21.6332,
"b": 15.9026,
"diff": 5.7307,
"boot_mean": 5.6696,
"lo": -0.6052,
"hi": 12.8574,
"p": 0.088
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.3516,
"b": 0.2593,
"diff": 0.0923,
"boot_mean": 0.0923,
"lo": 0.059,
"hi": 0.127,
"p": 0.0
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.9089,
"b": 0.8606,
"diff": 0.0483,
"boot_mean": 0.0485,
"lo": 0.0158,
"hi": 0.0878,
"p": 0.0004
},
"clone_utmos": {
"n": 30,
"a": 2.1058,
"b": 2.5419,
"diff": -0.4361,
"boot_mean": -0.4359,
"lo": -0.5634,
"hi": -0.311,
"p": 0.0
},
"clone_pause150": {
"n": 30,
"a": 1.4,
"b": 2.1333,
"diff": -0.7333,
"boot_mean": -0.7382,
"lo": -1.7667,
"hi": 0.5667,
"p": 0.2498
},
"clone_pause300": {
"n": 30,
"a": 0.6333,
"b": 1.3333,
"diff": -0.7,
"boot_mean": -0.7032,
"lo": -1.3667,
"hi": 0.0333,
"p": 0.0702
},
"clone_pause700": {
"n": 30,
"a": 0.1,
"b": 0.2,
"diff": -0.1,
"boot_mean": -0.1,
"lo": -0.3333,
"hi": 0.1667,
"p": 0.4804
},
"clone_pause_time": {
"n": 30,
"a": 0.897,
"b": 0.87,
"diff": 0.027,
"boot_mean": 0.0239,
"lo": -0.868,
"hi": 1.465,
"p": 0.964
},
"clone_max_pause": {
"n": 30,
"a": 0.4873,
"b": 0.4317,
"diff": 0.0557,
"boot_mean": 0.0545,
"lo": -0.329,
"hi": 0.7283,
"p": 0.8044
},
"clone_dropouts": {
"n": 30,
"a": 2.0,
"b": 10.7667,
"diff": -8.7667,
"boot_mean": -8.7663,
"lo": -10.4667,
"hi": -7.0667,
"p": 0.0
}
},
"E2_vs_VP17M_te": {
"builtin_CER": {
"n": 100,
"a": 5.827,
"b": 6.0372,
"diff": -0.2103,
"boot_mean": -0.208,
"lo": -0.7252,
"hi": 0.28,
"p": 0.4326
},
"builtin_WER": {
"n": 100,
"a": 28.1496,
"b": 28.937,
"diff": -0.7874,
"boot_mean": -0.7676,
"lo": -2.8534,
"hi": 1.3708,
"p": 0.4974
},
"builtin_UTMOS": {
"n": 100,
"a": 2.6619,
"b": 2.6385,
"diff": 0.0233,
"boot_mean": 0.0235,
"lo": -0.0254,
"hi": 0.0718,
"p": 0.3512
},
"clone_CER": {
"n": 30,
"a": 12.7192,
"b": 7.8738,
"diff": 4.8454,
"boot_mean": 4.8711,
"lo": 1.9867,
"hi": 8.2234,
"p": 0.0002
},
"clone_WER": {
"n": 30,
"a": 43.3884,
"b": 35.3306,
"diff": 8.0579,
"boot_mean": 8.0811,
"lo": 2.7776,
"hi": 13.4921,
"p": 0.0016
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.407,
"b": 0.2951,
"diff": 0.1119,
"boot_mean": 0.1118,
"lo": 0.0844,
"hi": 0.1393,
"p": 0.0
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.9167,
"b": 0.8789,
"diff": 0.0379,
"boot_mean": 0.0377,
"lo": 0.0208,
"hi": 0.0542,
"p": 0.0002
},
"clone_utmos": {
"n": 30,
"a": 1.8606,
"b": 2.3775,
"diff": -0.5169,
"boot_mean": -0.5169,
"lo": -0.6437,
"hi": -0.392,
"p": 0.0
},
"clone_pause150": {
"n": 30,
"a": 0.1667,
"b": 2.0,
"diff": -1.8333,
"boot_mean": -1.8328,
"lo": -2.5667,
"hi": -1.1333,
"p": 0.0
},
"clone_pause300": {
"n": 30,
"a": 0.0,
"b": 1.2333,
"diff": -1.2333,
"boot_mean": -1.2306,
"lo": -1.7667,
"hi": -0.7333,
"p": 0.0
},
"clone_pause700": {
"n": 30,
"a": 0.0,
"b": 0.1,
"diff": -0.1,
"boot_mean": -0.0996,
"lo": -0.2,
"hi": 0.0,
"p": 0.0846
},
"clone_pause_time": {
"n": 30,
"a": 0.0353,
"b": 0.761,
"diff": -0.7257,
"boot_mean": -0.7248,
"lo": -1.0113,
"hi": -0.4553,
"p": 0.0
},
"clone_max_pause": {
"n": 30,
"a": 0.085,
"b": 0.373,
"diff": -0.288,
"boot_mean": -0.2877,
"lo": -0.3667,
"hi": -0.2097,
"p": 0.0
},
"clone_dropouts": {
"n": 30,
"a": 1.0333,
"b": 11.2333,
"diff": -10.2,
"boot_mean": -10.193,
"lo": -12.1333,
"hi": -8.2667,
"p": 0.0
}
},
"E2_vs_VP17M_en": {
"builtin_CER": {
"n": 100,
"a": 4.64,
"b": 4.4205,
"diff": 0.2195,
"boot_mean": 0.2203,
"lo": -0.3538,
"hi": 0.8509,
"p": 0.489
},
"builtin_WER": {
"n": 100,
"a": 9.146,
"b": 9.4997,
"diff": -0.3537,
"boot_mean": -0.3519,
"lo": -1.6992,
"hi": 0.9481,
"p": 0.6384
},
"builtin_UTMOS": {
"n": 100,
"a": 3.1779,
"b": 3.192,
"diff": -0.0141,
"boot_mean": -0.0144,
"lo": -0.0789,
"hi": 0.0537,
"p": 0.6534
},
"clone_CER": {
"n": 30,
"a": 16.394,
"b": 4.3857,
"diff": 12.0084,
"boot_mean": 11.8757,
"lo": 3.4071,
"hi": 21.8074,
"p": 0.002
},
"clone_WER": {
"n": 30,
"a": 25.2525,
"b": 10.6061,
"diff": 14.6465,
"boot_mean": 14.5013,
"lo": 5.4143,
"hi": 24.5378,
"p": 0.001
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.2996,
"b": 0.1793,
"diff": 0.1203,
"boot_mean": 0.1202,
"lo": 0.0705,
"hi": 0.168,
"p": 0.0
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.8774,
"b": 0.8599,
"diff": 0.0175,
"boot_mean": 0.0179,
"lo": -0.0366,
"hi": 0.0592,
"p": 0.4446
},
"clone_utmos": {
"n": 30,
"a": 2.3109,
"b": 3.0247,
"diff": -0.7137,
"boot_mean": -0.7129,
"lo": -0.9257,
"hi": -0.5115,
"p": 0.0
},
"clone_pause150": {
"n": 30,
"a": 1.7,
"b": 1.5,
"diff": 0.2,
"boot_mean": 0.1971,
"lo": -0.8,
"hi": 1.3333,
"p": 0.7648
},
"clone_pause300": {
"n": 30,
"a": 0.7333,
"b": 0.6667,
"diff": 0.0667,
"boot_mean": 0.0659,
"lo": -0.6333,
"hi": 0.7,
"p": 0.8712
},
"clone_pause700": {
"n": 30,
"a": 0.3667,
"b": 0.0667,
"diff": 0.3,
"boot_mean": 0.2977,
"lo": 0.0,
"hi": 0.6333,
"p": 0.065
},
"clone_pause_time": {
"n": 30,
"a": 1.1623,
"b": 0.509,
"diff": 0.6533,
"boot_mean": 0.6475,
"lo": -0.204,
"hi": 1.8863,
"p": 0.2086
},
"clone_max_pause": {
"n": 30,
"a": 0.7013,
"b": 0.3043,
"diff": 0.397,
"boot_mean": 0.3938,
"lo": -0.0743,
"hi": 1.1867,
"p": 0.216
},
"clone_dropouts": {
"n": 30,
"a": 1.7,
"b": 7.9,
"diff": -6.2,
"boot_mean": -6.195,
"lo": -7.6667,
"hi": -4.7667,
"p": 0.0
}
},
"E2_vs_Hima_hi": {
"builtin_CER": {
"n": 100,
"a": 4.3152,
"b": 4.6957,
"diff": -0.3804,
"boot_mean": -0.3872,
"lo": -1.1476,
"hi": 0.5042,
"p": 0.3578
},
"builtin_WER": {
"n": 100,
"a": 12.7892,
"b": 14.797,
"diff": -2.0079,
"boot_mean": -1.9942,
"lo": -3.6681,
"hi": -0.381,
"p": 0.0146
},
"builtin_UTMOS": {
"n": 100,
"a": 2.7714,
"b": 3.6089,
"diff": -0.8376,
"boot_mean": -0.8376,
"lo": -0.9098,
"hi": -0.7648,
"p": 0.0
},
"clone_CER": {
"n": 30,
"a": 8.4599,
"b": 4.8445,
"diff": 3.6153,
"boot_mean": 3.5967,
"lo": 0.3269,
"hi": 7.7786,
"p": 0.0268
},
"clone_WER": {
"n": 30,
"a": 21.6332,
"b": 14.4699,
"diff": 7.1633,
"boot_mean": 7.1257,
"lo": 1.7518,
"hi": 13.5644,
"p": 0.0028
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.3516,
"b": 0.4922,
"diff": -0.1406,
"boot_mean": -0.1404,
"lo": -0.1775,
"hi": -0.103,
"p": 0.0
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.9089,
"b": 0.9399,
"diff": -0.031,
"boot_mean": -0.031,
"lo": -0.0454,
"hi": -0.0189,
"p": 0.0
},
"clone_utmos": {
"n": 30,
"a": 2.1058,
"b": 3.0668,
"diff": -0.961,
"boot_mean": -0.9612,
"lo": -1.1294,
"hi": -0.787,
"p": 0.0
},
"clone_pause150": {
"n": 30,
"a": 1.4,
"b": 1.8,
"diff": -0.4,
"boot_mean": -0.4035,
"lo": -1.3667,
"hi": 0.6667,
"p": 0.4512
},
"clone_pause300": {
"n": 30,
"a": 0.6333,
"b": 1.2667,
"diff": -0.6333,
"boot_mean": -0.6337,
"lo": -1.3333,
"hi": 0.1,
"p": 0.095
},
"clone_pause700": {
"n": 30,
"a": 0.1,
"b": 0.0667,
"diff": 0.0333,
"boot_mean": 0.0332,
"lo": -0.1333,
"hi": 0.2667,
"p": 0.9686
},
"clone_pause_time": {
"n": 30,
"a": 0.897,
"b": 0.75,
"diff": 0.147,
"boot_mean": 0.1453,
"lo": -0.687,
"hi": 1.4897,
"p": 0.8442
},
"clone_max_pause": {
"n": 30,
"a": 0.4873,
"b": 0.357,
"diff": 0.1303,
"boot_mean": 0.1293,
"lo": -0.259,
"hi": 0.7943,
"p": 0.7282
},
"clone_dropouts": {
"n": 30,
"a": 2.0,
"b": 5.0333,
"diff": -3.0333,
"boot_mean": -3.0328,
"lo": -4.6667,
"hi": -1.6,
"p": 0.0
}
},
"E2_vs_Hima_te": {
"builtin_CER": {
"n": 100,
"a": 5.827,
"b": 6.708,
"diff": -0.8811,
"boot_mean": -0.8726,
"lo": -1.5885,
"hi": -0.1702,
"p": 0.0144
},
"builtin_WER": {
"n": 100,
"a": 28.1496,
"b": 31.2336,
"diff": -3.084,
"boot_mean": -3.057,
"lo": -5.3415,
"hi": -0.8311,
"p": 0.0084
},
"builtin_UTMOS": {
"n": 100,
"a": 2.6619,
"b": 3.4974,
"diff": -0.8356,
"boot_mean": -0.8351,
"lo": -0.9119,
"hi": -0.7542,
"p": 0.0
},
"clone_CER": {
"n": 30,
"a": 12.7192,
"b": 9.1807,
"diff": 3.5384,
"boot_mean": 3.5596,
"lo": 0.8246,
"hi": 6.6667,
"p": 0.0074
},
"clone_WER": {
"n": 30,
"a": 43.3884,
"b": 36.7769,
"diff": 6.6116,
"boot_mean": 6.682,
"lo": 0.1873,
"hi": 13.5516,
"p": 0.0486
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.407,
"b": 0.5596,
"diff": -0.1526,
"boot_mean": -0.1527,
"lo": -0.1801,
"hi": -0.1238,
"p": 0.0
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.9167,
"b": 0.9505,
"diff": -0.0337,
"boot_mean": -0.0338,
"lo": -0.0484,
"hi": -0.0209,
"p": 0.0
},
"clone_utmos": {
"n": 30,
"a": 1.8606,
"b": 2.9912,
"diff": -1.1306,
"boot_mean": -1.1312,
"lo": -1.2851,
"hi": -0.9855,
"p": 0.0
},
"clone_pause150": {
"n": 30,
"a": 0.1667,
"b": 1.7667,
"diff": -1.6,
"boot_mean": -1.6035,
"lo": -2.4333,
"hi": -0.8667,
"p": 0.0
},
"clone_pause300": {
"n": 30,
"a": 0.0,
"b": 1.5,
"diff": -1.5,
"boot_mean": -1.5031,
"lo": -2.2333,
"hi": -0.8667,
"p": 0.0
},
"clone_pause700": {
"n": 30,
"a": 0.0,
"b": 0.1333,
"diff": -0.1333,
"boot_mean": -0.1329,
"lo": -0.3,
"hi": 0.0,
"p": 0.0882
},
"clone_pause_time": {
"n": 30,
"a": 0.0353,
"b": 0.7733,
"diff": -0.738,
"boot_mean": -0.7392,
"lo": -1.1057,
"hi": -0.417,
"p": 0.0
},
"clone_max_pause": {
"n": 30,
"a": 0.085,
"b": 0.391,
"diff": -0.306,
"boot_mean": -0.3059,
"lo": -0.4327,
"hi": -0.1977,
"p": 0.0
},
"clone_dropouts": {
"n": 30,
"a": 1.0333,
"b": 3.1,
"diff": -2.0667,
"boot_mean": -2.0656,
"lo": -3.3667,
"hi": -0.9667,
"p": 0.0
}
},
"E2_vs_Hima_en": {
"builtin_CER": {
"n": 100,
"a": 4.64,
"b": 5.2879,
"diff": -0.6479,
"boot_mean": -0.6488,
"lo": -1.8918,
"hi": 0.4538,
"p": 0.2674
},
"builtin_WER": {
"n": 100,
"a": 9.146,
"b": 11.4704,
"diff": -2.3244,
"boot_mean": -2.3313,
"lo": -5.1547,
"hi": -0.0519,
"p": 0.045
},
"builtin_UTMOS": {
"n": 100,
"a": 3.1779,
"b": 4.0328,
"diff": -0.8549,
"boot_mean": -0.8552,
"lo": -0.9334,
"hi": -0.7777,
"p": 0.0
},
"clone_CER": {
"n": 30,
"a": 16.394,
"b": 3.4111,
"diff": 12.9829,
"boot_mean": 12.8457,
"lo": 4.6165,
"hi": 22.6889,
"p": 0.0
},
"clone_WER": {
"n": 30,
"a": 25.2525,
"b": 9.2593,
"diff": 15.9933,
"boot_mean": 15.8269,
"lo": 7.1801,
"hi": 25.8622,
"p": 0.0
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.2996,
"b": 0.4207,
"diff": -0.1211,
"boot_mean": -0.121,
"lo": -0.1527,
"hi": -0.0878,
"p": 0.0
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.8774,
"b": 0.9358,
"diff": -0.0583,
"boot_mean": -0.0581,
"lo": -0.1041,
"hi": -0.0222,
"p": 0.0
},
"clone_utmos": {
"n": 30,
"a": 2.3109,
"b": 3.3826,
"diff": -1.0716,
"boot_mean": -1.0728,
"lo": -1.2986,
"hi": -0.8416,
"p": 0.0
},
"clone_pause150": {
"n": 30,
"a": 1.7,
"b": 1.4667,
"diff": 0.2333,
"boot_mean": 0.2312,
"lo": -0.5,
"hi": 1.0,
"p": 0.578
},
"clone_pause300": {
"n": 30,
"a": 0.7333,
"b": 0.9,
"diff": -0.1667,
"boot_mean": -0.1688,
"lo": -0.5333,
"hi": 0.1667,
"p": 0.4046
},
"clone_pause700": {
"n": 30,
"a": 0.3667,
"b": 0.3667,
"diff": 0.0,
"boot_mean": -0.001,
"lo": -0.2667,
"hi": 0.2667,
"p": 1.0
},
"clone_pause_time": {
"n": 30,
"a": 1.1623,
"b": 0.7657,
"diff": 0.3967,
"boot_mean": 0.3912,
"lo": -0.2397,
"hi": 1.3854,
"p": 0.4262
},
"clone_max_pause": {
"n": 30,
"a": 0.7013,
"b": 0.4037,
"diff": 0.2977,
"boot_mean": 0.2938,
"lo": -0.1607,
"hi": 1.045,
"p": 0.4812
},
"clone_dropouts": {
"n": 30,
"a": 1.7,
"b": 3.5,
"diff": -1.8,
"boot_mean": -1.8077,
"lo": -2.9667,
"hi": -0.8,
"p": 0.0
}
},
"E2_vs_E1_hi": {
"builtin_CER": {
"n": 100,
"a": 4.3152,
"b": 4.1739,
"diff": 0.1413,
"boot_mean": 0.1336,
"lo": -0.4988,
"hi": 0.9366,
"p": 0.7942
},
"builtin_WER": {
"n": 100,
"a": 12.7892,
"b": 12.6582,
"diff": 0.1309,
"boot_mean": 0.1188,
"lo": -1.2082,
"hi": 1.4546,
"p": 0.8996
},
"builtin_UTMOS": {
"n": 100,
"a": 2.7714,
"b": 2.8354,
"diff": -0.064,
"boot_mean": -0.064,
"lo": -0.1205,
"hi": -0.0089,
"p": 0.023
},
"clone_CER": {
"n": 30,
"a": 8.4599,
"b": 6.7245,
"diff": 1.7354,
"boot_mean": 1.7189,
"lo": -0.7904,
"hi": 4.9885,
"p": 0.251
},
"clone_WER": {
"n": 30,
"a": 21.6332,
"b": 17.9083,
"diff": 3.7249,
"boot_mean": 3.6819,
"lo": 0.2778,
"hi": 7.8507,
"p": 0.035
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.3516,
"b": 0.3633,
"diff": -0.0117,
"boot_mean": -0.0118,
"lo": -0.0324,
"hi": 0.0082,
"p": 0.2498
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.9089,
"b": 0.9044,
"diff": 0.0045,
"boot_mean": 0.0045,
"lo": -0.0022,
"hi": 0.012,
"p": 0.2038
},
"clone_utmos": {
"n": 30,
"a": 2.1058,
"b": 2.1538,
"diff": -0.048,
"boot_mean": -0.0481,
"lo": -0.122,
"hi": 0.0263,
"p": 0.2084
},
"clone_pause150": {
"n": 30,
"a": 1.4,
"b": 0.7,
"diff": 0.7,
"boot_mean": 0.6969,
"lo": 0.0333,
"hi": 1.5667,
"p": 0.0424
},
"clone_pause300": {
"n": 30,
"a": 0.6333,
"b": 0.3333,
"diff": 0.3,
"boot_mean": 0.2977,
"lo": -0.0667,
"hi": 0.8333,
"p": 0.2192
},
"clone_pause700": {
"n": 30,
"a": 0.1,
"b": 0.0,
"diff": 0.1,
"boot_mean": 0.0998,
"lo": 0.0,
"hi": 0.3,
"p": 0.724
},
"clone_pause_time": {
"n": 30,
"a": 0.897,
"b": 0.2323,
"diff": 0.6647,
"boot_mean": 0.6625,
"lo": -0.022,
"hi": 1.9117,
"p": 0.1008
},
"clone_max_pause": {
"n": 30,
"a": 0.4873,
"b": 0.193,
"diff": 0.2943,
"boot_mean": 0.2934,
"lo": -0.055,
"hi": 0.9297,
"p": 0.4654
},
"clone_dropouts": {
"n": 30,
"a": 2.0,
"b": 2.2333,
"diff": -0.2333,
"boot_mean": -0.2332,
"lo": -0.6667,
"hi": 0.2667,
"p": 0.364
}
},
"E2_vs_E1_te": {
"builtin_CER": {
"n": 100,
"a": 5.827,
"b": 6.1974,
"diff": -0.3704,
"boot_mean": -0.3683,
"lo": -0.841,
"hi": 0.0701,
"p": 0.1096
},
"builtin_WER": {
"n": 100,
"a": 28.1496,
"b": 30.3806,
"diff": -2.231,
"boot_mean": -2.2154,
"lo": -4.273,
"hi": -0.1965,
"p": 0.0336
},
"builtin_UTMOS": {
"n": 100,
"a": 2.6619,
"b": 2.6535,
"diff": 0.0084,
"boot_mean": 0.0084,
"lo": -0.0422,
"hi": 0.0585,
"p": 0.7444
},
"clone_CER": {
"n": 30,
"a": 12.7192,
"b": 10.424,
"diff": 2.2952,
"boot_mean": 2.3222,
"lo": 0.0,
"hi": 4.9573,
"p": 0.0518
},
"clone_WER": {
"n": 30,
"a": 43.3884,
"b": 40.0826,
"diff": 3.3058,
"boot_mean": 3.3936,
"lo": -2.2918,
"hi": 9.3025,
"p": 0.258
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.407,
"b": 0.4017,
"diff": 0.0053,
"boot_mean": 0.0052,
"lo": -0.0092,
"hi": 0.0199,
"p": 0.4874
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.9167,
"b": 0.9134,
"diff": 0.0033,
"boot_mean": 0.0033,
"lo": -0.0029,
"hi": 0.0096,
"p": 0.2954
},
"clone_utmos": {
"n": 30,
"a": 1.8606,
"b": 1.8903,
"diff": -0.0296,
"boot_mean": -0.0298,
"lo": -0.0815,
"hi": 0.0226,
"p": 0.269
},
"clone_pause150": {
"n": 30,
"a": 0.1667,
"b": 0.4667,
"diff": -0.3,
"boot_mean": -0.2974,
"lo": -0.7667,
"hi": 0.0333,
"p": 0.1462
},
"clone_pause300": {
"n": 30,
"a": 0.0,
"b": 0.2333,
"diff": -0.2333,
"boot_mean": -0.2311,
"lo": -0.6,
"hi": 0.0,
"p": 0.0892
},
"clone_pause700": {
"n": 30,
"a": 0.0,
"b": 0.0,
"diff": 0.0,
"boot_mean": 0.0,
"lo": 0.0,
"hi": 0.0,
"p": 1.0
},
"clone_pause_time": {
"n": 30,
"a": 0.0353,
"b": 0.1537,
"diff": -0.1183,
"boot_mean": -0.1173,
"lo": -0.2927,
"hi": -0.0017,
"p": 0.0426
},
"clone_max_pause": {
"n": 30,
"a": 0.085,
"b": 0.1297,
"diff": -0.0447,
"boot_mean": -0.0444,
"lo": -0.0993,
"hi": -0.0007,
"p": 0.0462
},
"clone_dropouts": {
"n": 30,
"a": 1.0333,
"b": 1.0667,
"diff": -0.0333,
"boot_mean": -0.0338,
"lo": -0.2333,
"hi": 0.2,
"p": 0.851
}
},
"E2_vs_E1_en": {
"builtin_CER": {
"n": 100,
"a": 4.64,
"b": 4.7863,
"diff": -0.1463,
"boot_mean": -0.1434,
"lo": -1.07,
"hi": 0.6923,
"p": 0.7768
},
"builtin_WER": {
"n": 100,
"a": 9.146,
"b": 9.6513,
"diff": -0.5053,
"boot_mean": -0.4973,
"lo": -2.0186,
"hi": 0.9055,
"p": 0.538
},
"builtin_UTMOS": {
"n": 100,
"a": 3.1779,
"b": 3.194,
"diff": -0.0161,
"boot_mean": -0.0169,
"lo": -0.0836,
"hi": 0.0514,
"p": 0.6206
},
"clone_CER": {
"n": 30,
"a": 16.394,
"b": 9.7459,
"diff": 6.6481,
"boot_mean": 6.5612,
"lo": 1.3289,
"hi": 13.2512,
"p": 0.0016
},
"clone_WER": {
"n": 30,
"a": 25.2525,
"b": 17.1717,
"diff": 8.0808,
"boot_mean": 7.9708,
"lo": 1.8425,
"hi": 15.3974,
"p": 0.0046
},
"clone_sim_ecapa": {
"n": 30,
"a": 0.2996,
"b": 0.3025,
"diff": -0.0029,
"boot_mean": -0.0028,
"lo": -0.0213,
"hi": 0.015,
"p": 0.7828
},
"clone_sim_wavlm": {
"n": 30,
"a": 0.8774,
"b": 0.8893,
"diff": -0.0119,
"boot_mean": -0.0117,
"lo": -0.0366,
"hi": 0.0058,
"p": 0.2796
},
"clone_utmos": {
"n": 30,
"a": 2.3109,
"b": 2.4402,
"diff": -0.1293,
"boot_mean": -0.1288,
"lo": -0.2205,
"hi": -0.0436,
"p": 0.0018
},
"clone_pause150": {
"n": 30,
"a": 1.7,
"b": 1.0333,
"diff": 0.6667,
"boot_mean": 0.6621,
"lo": 0.0667,
"hi": 1.4333,
"p": 0.0316
},
"clone_pause300": {
"n": 30,
"a": 0.7333,
"b": 0.6333,
"diff": 0.1,
"boot_mean": 0.0983,
"lo": -0.2667,
"hi": 0.4667,
"p": 0.6758
},
"clone_pause700": {
"n": 30,
"a": 0.3667,
"b": 0.2333,
"diff": 0.1333,
"boot_mean": 0.1326,
"lo": -0.0667,
"hi": 0.3333,
"p": 0.2448
},
"clone_pause_time": {
"n": 30,
"a": 1.1623,
"b": 0.7593,
"diff": 0.403,
"boot_mean": 0.3996,
"lo": 0.0133,
"hi": 0.8663,
"p": 0.0402
},
"clone_max_pause": {
"n": 30,
"a": 0.7013,
"b": 0.606,
"diff": 0.0953,
"boot_mean": 0.0948,
"lo": -0.055,
"hi": 0.257,
"p": 0.2258
},
"clone_dropouts": {
"n": 30,
"a": 1.7,
"b": 1.7667,
"diff": -0.0667,
"boot_mean": -0.0669,
"lo": -0.4,
"hi": 0.2333,
"p": 0.7612
}
}
}
}
{
"note": "DEV only (FLEURS dev). Round-5 paired bootstrap: r5 E1 (DCpS) and E2 (DCpSL) vs AksharaVP-17M (published r4 DCp) and vs Hima, per language.\nBuiltin: CER/WER (corpus, MMS-1b-all CTC) + UTMOS from baseline/r5_*_say_dev.json (eval_tts.py), ids 0..99/lang.\nClone: ECAPA/WavLM sim, UTMOS, pause150/300/700, pause_time, max_pause, dropouts (energy dips) from diag/results/{r5_E1_say,r5_E2_say,vp_say,hima_say}.json\n(analyze.py), ids 0..29/lang; clone CER/WER from the eval_tts json too.\nPaired over identical utterance ids, 10,000 resamples (rng seed 1), diff = A - B, 95% percentile CI, two-sided p = min(1, 2*min(P(d<=0), P(d>=0))).\nWrites baseline/r5_bootstrap_dev.json and diag/results/r5_table.md",
"absolute": {
"E1/hi": {
"builtin_n": 100,
"CER": 4.173913043478261,
"WER": 12.658227848101266,
"UTMOS": 2.835365197658539,
"clone_n": 30,
"clone_CER": 6.724511930585683,
"clone_WER": 17.908309455587393,
"clone_sim_ecapa": 0.36334028194348017,
"clone_sim_wavlm": 0.9043956021467845,
"clone_utmos": 2.1537877678871156,
"clone_pause150": 0.7,
"clone_pause300": 0.3333333333333333,
"clone_pause700": 0.0,
"clone_pause_time": 0.2323333333333333,
"clone_max_pause": 0.19299999999999995,
"clone_dropouts": 2.2333333333333334
},
"E1/te": {
"builtin_n": 100,
"CER": 6.197436924309171,
"WER": 30.38057742782152,
"UTMOS": 2.653467684984207,
"clone_n": 30,
"clone_CER": 10.423971947720752,
"clone_WER": 40.082644628099175,
"clone_sim_ecapa": 0.4017001857360204,
"clone_sim_wavlm": 0.9134249667326609,
"clone_utmos": 1.890282436211904,
"clone_pause150": 0.4666666666666667,
"clone_pause300": 0.23333333333333334,
"clone_pause700": 0.0,
"clone_pause_time": 0.15366666666666667,
"clone_max_pause": 0.12966666666666665,
"clone_dropouts": 1.0666666666666667
},
"E1/en": {
"builtin_n": 100,
"CER": 4.7862890584178075,
"WER": 9.651339060131379,
"UTMOS": 3.19397074341774,
"clone_n": 30,
"clone_CER": 9.745910198398887,
"clone_WER": 17.171717171717173,
"clone_sim_ecapa": 0.30246616067985693,
"clone_sim_wavlm": 0.8892902890841167,
"clone_utmos": 2.4402199387550354,
"clone_pause150": 1.0333333333333334,
"clone_pause300": 0.6333333333333333,
"clone_pause700": 0.23333333333333334,
"clone_pause_time": 0.7593333333333334,
"clone_max_pause": 0.6059999999999999,
"clone_dropouts": 1.7666666666666666
},
"E2/hi": {
"builtin_n": 100,
"CER": 4.315217391304348,
"WER": 12.789175032736797,
"UTMOS": 2.7713782811164855,
"clone_n": 30,
"clone_CER": 8.459869848156183,
"clone_WER": 21.63323782234957,
"clone_sim_ecapa": 0.35160381346940994,
"clone_sim_wavlm": 0.9089426139990489,
"clone_utmos": 2.1057937304178873,
"clone_pause150": 1.4,
"clone_pause300": 0.6333333333333333,
"clone_pause700": 0.1,
"clone_pause_time": 0.8969999999999999,
"clone_max_pause": 0.4873333333333333,
"clone_dropouts": 2.0
},
"E2/te": {
"builtin_n": 100,
"CER": 5.826992390869043,
"WER": 28.1496062992126,
"UTMOS": 2.6618559622764586,
"clone_n": 30,
"clone_CER": 12.719158431622569,
"clone_WER": 43.388429752066116,
"clone_sim_ecapa": 0.40696775913238525,
"clone_sim_wavlm": 0.9167420009771983,
"clone_utmos": 1.860637084643046,
"clone_pause150": 0.16666666666666666,
"clone_pause300": 0.0,
"clone_pause700": 0.0,
"clone_pause_time": 0.035333333333333335,
"clone_max_pause": 0.08499999999999999,
"clone_dropouts": 1.0333333333333334
},
"E2/en": {
"builtin_n": 100,
"CER": 4.639983279339534,
"WER": 9.146033350176857,
"UTMOS": 3.1779002070426943,
"clone_n": 30,
"clone_CER": 16.394013226592413,