Fleet-2 / evaluation.json
Lucasgitguy-co's picture
Fleet-2: Whistle student of Whisper-large-v3-turbo
15e183b verified
Raw History Blame Contribute Delete
16.9 kB
{
"selection": {
"step": 2163,
"macro_validation": 0.22121288067005115,
"start_macro_validation": 0.22833260484316537,
"reason": "Lowest macro validation error, better than the start model"
},
"init": {
"chosen": "fleet1",
"macro_validation": {
"sky": 0.2466074807442024,
"fleet1": 0.22833260484316537
},
"rule": "lower 8-language mean validation error; tie keeps Sky-Kim",
"adapter": {
"repo": "Lucasgitguy-co/Fleet-1",
"revision": "0cc581616f7e5493e2e3999e71c134cedc879a6d",
"adapter": "fleet_lora.safetensors"
},
"rebuilt_macro": 0.22833260484316537
},
"filter": {
"fleurs/en": {
"candidates": 200,
"kept": 178,
"rejected": {
"disagrees_with_gold": 22
}
},
"fleurs/de": {
"candidates": 200,
"kept": 164,
"rejected": {
"disagrees_with_gold": 36
}
},
"fleurs/fr": {
"candidates": 200,
"kept": 165,
"rejected": {
"disagrees_with_gold": 35
}
},
"fleurs/es": {
"candidates": 200,
"kept": 158,
"rejected": {
"disagrees_with_gold": 42
}
},
"fleurs/it": {
"candidates": 200,
"kept": 169,
"rejected": {
"disagrees_with_gold": 31
}
},
"fleurs/nl": {
"candidates": 200,
"kept": 137,
"rejected": {
"disagrees_with_gold": 63
}
},
"fleurs/pl": {
"candidates": 200,
"kept": 163,
"rejected": {
"disagrees_with_gold": 37
}
},
"fleurs/ko": {
"candidates": 200,
"kept": 178,
"rejected": {
"disagrees_with_gold": 22
}
},
"voxpopuli/en": {
"candidates": 600,
"kept": 400,
"rejected": {
"disagrees_with_gold": 141,
"over_quota": 59
}
},
"voxpopuli/de": {
"candidates": 600,
"kept": 346,
"rejected": {
"disagrees_with_gold": 254
}
},
"voxpopuli/fr": {
"candidates": 600,
"kept": 400,
"rejected": {
"disagrees_with_gold": 163,
"over_quota": 37
}
},
"voxpopuli/es": {
"candidates": 600,
"kept": 400,
"rejected": {
"disagrees_with_gold": 121,
"over_quota": 79
}
},
"voxpopuli/it": {
"candidates": 600,
"kept": 350,
"rejected": {
"disagrees_with_gold": 250
}
},
"voxpopuli/nl": {
"candidates": 600,
"kept": 318,
"rejected": {
"disagrees_with_gold": 280,
"avg_logprob": 2
}
},
"voxpopuli/pl": {
"candidates": 600,
"kept": 400,
"rejected": {
"disagrees_with_gold": 165,
"over_quota": 35
}
},
"zeroth/ko": {
"candidates": 600,
"kept": 400,
"rejected": {
"disagrees_with_gold": 74,
"over_quota": 126
}
}
},
"results": {
"test": {
"Fleet-2": {
"en": {
"metric": "WER",
"error_rate": 0.14496573537163943,
"edits": 275,
"reference_units": 1897,
"clips": 100,
"language_accuracy": 1.0
},
"de": {
"metric": "WER",
"error_rate": 0.22021276595744682,
"edits": 414,
"reference_units": 1880,
"clips": 100,
"language_accuracy": 1.0
},
"fr": {
"metric": "WER",
"error_rate": 0.24926315789473685,
"edits": 592,
"reference_units": 2375,
"clips": 100,
"language_accuracy": 1.0
},
"es": {
"metric": "WER",
"error_rate": 0.11927528938097634,
"edits": 237,
"reference_units": 1987,
"clips": 100,
"language_accuracy": 1.0
},
"it": {
"metric": "WER",
"error_rate": 0.16657381615598885,
"edits": 299,
"reference_units": 1795,
"clips": 100,
"language_accuracy": 1.0
},
"nl": {
"metric": "WER",
"error_rate": 0.30062530062530063,
"edits": 625,
"reference_units": 2079,
"clips": 100,
"language_accuracy": 1.0
},
"pl": {
"metric": "WER",
"error_rate": 0.4161603888213852,
"edits": 685,
"reference_units": 1646,
"clips": 100,
"language_accuracy": 0.99
},
"ko": {
"metric": "CER",
"error_rate": 0.42800114057599087,
"edits": 1501,
"reference_units": 3507,
"clips": 100,
"language_accuracy": 0.97
}
},
"Fleet-1": {
"en": {
"metric": "WER",
"error_rate": 0.14391143911439114,
"edits": 273,
"reference_units": 1897,
"clips": 100,
"language_accuracy": 1.0
},
"de": {
"metric": "WER",
"error_rate": 0.22180851063829787,
"edits": 417,
"reference_units": 1880,
"clips": 100,
"language_accuracy": 1.0
},
"fr": {
"metric": "WER",
"error_rate": 0.2517894736842105,
"edits": 598,
"reference_units": 2375,
"clips": 100,
"language_accuracy": 1.0
},
"es": {
"metric": "WER",
"error_rate": 0.11927528938097634,
"edits": 237,
"reference_units": 1987,
"clips": 100,
"language_accuracy": 1.0
},
"it": {
"metric": "WER",
"error_rate": 0.16323119777158773,
"edits": 293,
"reference_units": 1795,
"clips": 100,
"language_accuracy": 1.0
},
"nl": {
"metric": "WER",
"error_rate": 0.29533429533429534,
"edits": 614,
"reference_units": 2079,
"clips": 100,
"language_accuracy": 0.99
},
"pl": {
"metric": "WER",
"error_rate": 0.4313487241798299,
"edits": 710,
"reference_units": 1646,
"clips": 100,
"language_accuracy": 0.99
},
"ko": {
"metric": "CER",
"error_rate": 0.523524379811805,
"edits": 1836,
"reference_units": 3507,
"clips": 100,
"language_accuracy": 0.98
}
},
"Sky-Kim": {
"en": {
"metric": "WER",
"error_rate": 0.14865577227200844,
"edits": 282,
"reference_units": 1897,
"clips": 100,
"language_accuracy": 0.99
},
"de": {
"metric": "WER",
"error_rate": 0.2271276595744681,
"edits": 427,
"reference_units": 1880,
"clips": 100,
"language_accuracy": 0.99
},
"fr": {
"metric": "WER",
"error_rate": 0.25726315789473686,
"edits": 611,
"reference_units": 2375,
"clips": 100,
"language_accuracy": 1.0
},
"es": {
"metric": "WER",
"error_rate": 0.1182687468545546,
"edits": 235,
"reference_units": 1987,
"clips": 100,
"language_accuracy": 1.0
},
"it": {
"metric": "WER",
"error_rate": 0.166016713091922,
"edits": 298,
"reference_units": 1795,
"clips": 100,
"language_accuracy": 1.0
},
"nl": {
"metric": "WER",
"error_rate": 0.29389129389129387,
"edits": 611,
"reference_units": 2079,
"clips": 100,
"language_accuracy": 0.99
},
"pl": {
"metric": "WER",
"error_rate": 0.43681652490886996,
"edits": 719,
"reference_units": 1646,
"clips": 100,
"language_accuracy": 0.99
},
"ko": {
"metric": "CER",
"error_rate": 0.3367550613059595,
"edits": 1181,
"reference_units": 3507,
"clips": 100,
"language_accuracy": 0.98
}
},
"Whistle": {
"en": {
"metric": "WER",
"error_rate": 0.13863995782814972,
"edits": 263,
"reference_units": 1897,
"clips": 100,
"language_accuracy": 1.0
},
"de": {
"metric": "WER",
"error_rate": 0.22446808510638297,
"edits": 422,
"reference_units": 1880,
"clips": 100,
"language_accuracy": 1.0
},
"fr": {
"metric": "WER",
"error_rate": 0.22147368421052632,
"edits": 526,
"reference_units": 2375,
"clips": 100,
"language_accuracy": 1.0
},
"es": {
"metric": "WER",
"error_rate": 0.1182687468545546,
"edits": 235,
"reference_units": 1987,
"clips": 100,
"language_accuracy": 1.0
},
"it": {
"metric": "WER",
"error_rate": 0.17437325905292478,
"edits": 313,
"reference_units": 1795,
"clips": 100,
"language_accuracy": 1.0
},
"nl": {
"metric": "WER",
"error_rate": 0.3294853294853295,
"edits": 685,
"reference_units": 2079,
"clips": 100,
"language_accuracy": 0.97
},
"pl": {
"metric": "WER",
"error_rate": 0.4538274605103281,
"edits": 747,
"reference_units": 1646,
"clips": 100,
"language_accuracy": 0.95
}
}
},
"ood": {
"Fleet-2": {
"en": {
"metric": "WER",
"error_rate": 0.11981020166073547,
"edits": 101,
"reference_units": 843,
"clips": 50,
"language_accuracy": 0.98
},
"de": {
"metric": "WER",
"error_rate": 0.18359375,
"edits": 141,
"reference_units": 768,
"clips": 50,
"language_accuracy": 1.0
},
"fr": {
"metric": "WER",
"error_rate": 0.1357142857142857,
"edits": 133,
"reference_units": 980,
"clips": 50,
"language_accuracy": 1.0
},
"es": {
"metric": "WER",
"error_rate": 0.11037107516650808,
"edits": 116,
"reference_units": 1051,
"clips": 50,
"language_accuracy": 0.98
},
"it": {
"metric": "WER",
"error_rate": 0.3319371727748691,
"edits": 317,
"reference_units": 955,
"clips": 50,
"language_accuracy": 0.98
},
"nl": {
"metric": "WER",
"error_rate": 0.22945205479452055,
"edits": 201,
"reference_units": 876,
"clips": 50,
"language_accuracy": 0.96
},
"pl": {
"metric": "WER",
"error_rate": 0.23529411764705882,
"edits": 172,
"reference_units": 731,
"clips": 50,
"language_accuracy": 0.98
},
"ko": {
"metric": "CER",
"error_rate": 0.1323529411764706,
"edits": 279,
"reference_units": 2108,
"clips": 50,
"language_accuracy": 1.0
}
},
"Fleet-1": {
"en": {
"metric": "WER",
"error_rate": 0.10913404507710557,
"edits": 92,
"reference_units": 843,
"clips": 50,
"language_accuracy": 0.96
},
"de": {
"metric": "WER",
"error_rate": 0.1953125,
"edits": 150,
"reference_units": 768,
"clips": 50,
"language_accuracy": 1.0
},
"fr": {
"metric": "WER",
"error_rate": 0.12040816326530612,
"edits": 118,
"reference_units": 980,
"clips": 50,
"language_accuracy": 1.0
},
"es": {
"metric": "WER",
"error_rate": 0.1170313986679353,
"edits": 123,
"reference_units": 1051,
"clips": 50,
"language_accuracy": 0.98
},
"it": {
"metric": "WER",
"error_rate": 0.38219895287958117,
"edits": 365,
"reference_units": 955,
"clips": 50,
"language_accuracy": 0.98
},
"nl": {
"metric": "WER",
"error_rate": 0.2317351598173516,
"edits": 203,
"reference_units": 876,
"clips": 50,
"language_accuracy": 0.96
},
"pl": {
"metric": "WER",
"error_rate": 0.22982216142270862,
"edits": 168,
"reference_units": 731,
"clips": 50,
"language_accuracy": 0.96
},
"ko": {
"metric": "CER",
"error_rate": 0.15654648956356737,
"edits": 330,
"reference_units": 2108,
"clips": 50,
"language_accuracy": 1.0
}
},
"Sky-Kim": {
"en": {
"metric": "WER",
"error_rate": 0.1269276393831554,
"edits": 107,
"reference_units": 843,
"clips": 50,
"language_accuracy": 0.94
},
"de": {
"metric": "WER",
"error_rate": 0.19661458333333334,
"edits": 151,
"reference_units": 768,
"clips": 50,
"language_accuracy": 1.0
},
"fr": {
"metric": "WER",
"error_rate": 0.12244897959183673,
"edits": 120,
"reference_units": 980,
"clips": 50,
"language_accuracy": 1.0
},
"es": {
"metric": "WER",
"error_rate": 0.11893434823977164,
"edits": 125,
"reference_units": 1051,
"clips": 50,
"language_accuracy": 0.98
},
"it": {
"metric": "WER",
"error_rate": 0.40209424083769635,
"edits": 384,
"reference_units": 955,
"clips": 50,
"language_accuracy": 0.98
},
"nl": {
"metric": "WER",
"error_rate": 0.2328767123287671,
"edits": 204,
"reference_units": 876,
"clips": 50,
"language_accuracy": 0.96
},
"pl": {
"metric": "WER",
"error_rate": 0.23529411764705882,
"edits": 172,
"reference_units": 731,
"clips": 50,
"language_accuracy": 0.98
},
"ko": {
"metric": "CER",
"error_rate": 0.2518975332068311,
"edits": 531,
"reference_units": 2108,
"clips": 50,
"language_accuracy": 1.0
}
},
"Whistle": {
"en": {
"metric": "WER",
"error_rate": 0.1055753262158956,
"edits": 89,
"reference_units": 843,
"clips": 50,
"language_accuracy": 1.0
},
"de": {
"metric": "WER",
"error_rate": 0.17317708333333334,
"edits": 133,
"reference_units": 768,
"clips": 50,
"language_accuracy": 1.0
},
"fr": {
"metric": "WER",
"error_rate": 0.1306122448979592,
"edits": 128,
"reference_units": 980,
"clips": 50,
"language_accuracy": 1.0
},
"es": {
"metric": "WER",
"error_rate": 0.09990485252140818,
"edits": 105,
"reference_units": 1051,
"clips": 50,
"language_accuracy": 1.0
},
"it": {
"metric": "WER",
"error_rate": 0.34450261780104713,
"edits": 329,
"reference_units": 955,
"clips": 50,
"language_accuracy": 0.98
},
"nl": {
"metric": "WER",
"error_rate": 0.23059360730593606,
"edits": 202,
"reference_units": 876,
"clips": 50,
"language_accuracy": 0.98
},
"pl": {
"metric": "WER",
"error_rate": 0.22024623803009577,
"edits": 161,
"reference_units": 731,
"clips": 50,
"language_accuracy": 0.96
}
}
}
},
"gate": {
"passed": false,
"beats_fleet1_in": [
"de",
"fr",
"pl",
"ko"
],
"korean_not_worse_than_sky": false,
"rule": ">=5 of 8 languages better than Fleet-1 and Korean CER <= Sky-Kim, on Fleet-1's FLEURS test clips"
}
}