{ "selection": { "step": 2163, "macro_validation": 0.22121288067005115, "start_macro_validation": 0.22833260484316537, "reason": "Lowest macro validation error, better than the start model" }, "init": { "chosen": "fleet1", "macro_validation": { "sky": 0.2466074807442024, "fleet1": 0.22833260484316537 }, "rule": "lower 8-language mean validation error; tie keeps Sky-Kim", "adapter": { "repo": "Lucasgitguy-co/Fleet-1", "revision": "0cc581616f7e5493e2e3999e71c134cedc879a6d", "adapter": "fleet_lora.safetensors" }, "rebuilt_macro": 0.22833260484316537 }, "filter": { "fleurs/en": { "candidates": 200, "kept": 178, "rejected": { "disagrees_with_gold": 22 } }, "fleurs/de": { "candidates": 200, "kept": 164, "rejected": { "disagrees_with_gold": 36 } }, "fleurs/fr": { "candidates": 200, "kept": 165, "rejected": { "disagrees_with_gold": 35 } }, "fleurs/es": { "candidates": 200, "kept": 158, "rejected": { "disagrees_with_gold": 42 } }, "fleurs/it": { "candidates": 200, "kept": 169, "rejected": { "disagrees_with_gold": 31 } }, "fleurs/nl": { "candidates": 200, "kept": 137, "rejected": { "disagrees_with_gold": 63 } }, "fleurs/pl": { "candidates": 200, "kept": 163, "rejected": { "disagrees_with_gold": 37 } }, "fleurs/ko": { "candidates": 200, "kept": 178, "rejected": { "disagrees_with_gold": 22 } }, "voxpopuli/en": { "candidates": 600, "kept": 400, "rejected": { "disagrees_with_gold": 141, "over_quota": 59 } }, "voxpopuli/de": { "candidates": 600, "kept": 346, "rejected": { "disagrees_with_gold": 254 } }, "voxpopuli/fr": { "candidates": 600, "kept": 400, "rejected": { "disagrees_with_gold": 163, "over_quota": 37 } }, "voxpopuli/es": { "candidates": 600, "kept": 400, "rejected": { "disagrees_with_gold": 121, "over_quota": 79 } }, "voxpopuli/it": { "candidates": 600, "kept": 350, "rejected": { "disagrees_with_gold": 250 } }, "voxpopuli/nl": { "candidates": 600, "kept": 318, "rejected": { "disagrees_with_gold": 280, "avg_logprob": 2 } }, "voxpopuli/pl": { "candidates": 600, "kept": 400, "rejected": { "disagrees_with_gold": 165, "over_quota": 35 } }, "zeroth/ko": { "candidates": 600, "kept": 400, "rejected": { "disagrees_with_gold": 74, "over_quota": 126 } } }, "results": { "test": { "Fleet-2": { "en": { "metric": "WER", "error_rate": 0.14496573537163943, "edits": 275, "reference_units": 1897, "clips": 100, "language_accuracy": 1.0 }, "de": { "metric": "WER", "error_rate": 0.22021276595744682, "edits": 414, "reference_units": 1880, "clips": 100, "language_accuracy": 1.0 }, "fr": { "metric": "WER", "error_rate": 0.24926315789473685, "edits": 592, "reference_units": 2375, "clips": 100, "language_accuracy": 1.0 }, "es": { "metric": "WER", "error_rate": 0.11927528938097634, "edits": 237, "reference_units": 1987, "clips": 100, "language_accuracy": 1.0 }, "it": { "metric": "WER", "error_rate": 0.16657381615598885, "edits": 299, "reference_units": 1795, "clips": 100, "language_accuracy": 1.0 }, "nl": { "metric": "WER", "error_rate": 0.30062530062530063, "edits": 625, "reference_units": 2079, "clips": 100, "language_accuracy": 1.0 }, "pl": { "metric": "WER", "error_rate": 0.4161603888213852, "edits": 685, "reference_units": 1646, "clips": 100, "language_accuracy": 0.99 }, "ko": { "metric": "CER", "error_rate": 0.42800114057599087, "edits": 1501, "reference_units": 3507, "clips": 100, "language_accuracy": 0.97 } }, "Fleet-1": { "en": { "metric": "WER", "error_rate": 0.14391143911439114, "edits": 273, "reference_units": 1897, "clips": 100, "language_accuracy": 1.0 }, "de": { "metric": "WER", "error_rate": 0.22180851063829787, "edits": 417, "reference_units": 1880, "clips": 100, "language_accuracy": 1.0 }, "fr": { "metric": "WER", "error_rate": 0.2517894736842105, "edits": 598, "reference_units": 2375, "clips": 100, "language_accuracy": 1.0 }, "es": { "metric": "WER", "error_rate": 0.11927528938097634, "edits": 237, "reference_units": 1987, "clips": 100, "language_accuracy": 1.0 }, "it": { "metric": "WER", "error_rate": 0.16323119777158773, "edits": 293, "reference_units": 1795, "clips": 100, "language_accuracy": 1.0 }, "nl": { "metric": "WER", "error_rate": 0.29533429533429534, "edits": 614, "reference_units": 2079, "clips": 100, "language_accuracy": 0.99 }, "pl": { "metric": "WER", "error_rate": 0.4313487241798299, "edits": 710, "reference_units": 1646, "clips": 100, "language_accuracy": 0.99 }, "ko": { "metric": "CER", "error_rate": 0.523524379811805, "edits": 1836, "reference_units": 3507, "clips": 100, "language_accuracy": 0.98 } }, "Sky-Kim": { "en": { "metric": "WER", "error_rate": 0.14865577227200844, "edits": 282, "reference_units": 1897, "clips": 100, "language_accuracy": 0.99 }, "de": { "metric": "WER", "error_rate": 0.2271276595744681, "edits": 427, "reference_units": 1880, "clips": 100, "language_accuracy": 0.99 }, "fr": { "metric": "WER", "error_rate": 0.25726315789473686, "edits": 611, "reference_units": 2375, "clips": 100, "language_accuracy": 1.0 }, "es": { "metric": "WER", "error_rate": 0.1182687468545546, "edits": 235, "reference_units": 1987, "clips": 100, "language_accuracy": 1.0 }, "it": { "metric": "WER", "error_rate": 0.166016713091922, "edits": 298, "reference_units": 1795, "clips": 100, "language_accuracy": 1.0 }, "nl": { "metric": "WER", "error_rate": 0.29389129389129387, "edits": 611, "reference_units": 2079, "clips": 100, "language_accuracy": 0.99 }, "pl": { "metric": "WER", "error_rate": 0.43681652490886996, "edits": 719, "reference_units": 1646, "clips": 100, "language_accuracy": 0.99 }, "ko": { "metric": "CER", "error_rate": 0.3367550613059595, "edits": 1181, "reference_units": 3507, "clips": 100, "language_accuracy": 0.98 } }, "Whistle": { "en": { "metric": "WER", "error_rate": 0.13863995782814972, "edits": 263, "reference_units": 1897, "clips": 100, "language_accuracy": 1.0 }, "de": { "metric": "WER", "error_rate": 0.22446808510638297, "edits": 422, "reference_units": 1880, "clips": 100, "language_accuracy": 1.0 }, "fr": { "metric": "WER", "error_rate": 0.22147368421052632, "edits": 526, "reference_units": 2375, "clips": 100, "language_accuracy": 1.0 }, "es": { "metric": "WER", "error_rate": 0.1182687468545546, "edits": 235, "reference_units": 1987, "clips": 100, "language_accuracy": 1.0 }, "it": { "metric": "WER", "error_rate": 0.17437325905292478, "edits": 313, "reference_units": 1795, "clips": 100, "language_accuracy": 1.0 }, "nl": { "metric": "WER", "error_rate": 0.3294853294853295, "edits": 685, "reference_units": 2079, "clips": 100, "language_accuracy": 0.97 }, "pl": { "metric": "WER", "error_rate": 0.4538274605103281, "edits": 747, "reference_units": 1646, "clips": 100, "language_accuracy": 0.95 } } }, "ood": { "Fleet-2": { "en": { "metric": "WER", "error_rate": 0.11981020166073547, "edits": 101, "reference_units": 843, "clips": 50, "language_accuracy": 0.98 }, "de": { "metric": "WER", "error_rate": 0.18359375, "edits": 141, "reference_units": 768, "clips": 50, "language_accuracy": 1.0 }, "fr": { "metric": "WER", "error_rate": 0.1357142857142857, "edits": 133, "reference_units": 980, "clips": 50, "language_accuracy": 1.0 }, "es": { "metric": "WER", "error_rate": 0.11037107516650808, "edits": 116, "reference_units": 1051, "clips": 50, "language_accuracy": 0.98 }, "it": { "metric": "WER", "error_rate": 0.3319371727748691, "edits": 317, "reference_units": 955, "clips": 50, "language_accuracy": 0.98 }, "nl": { "metric": "WER", "error_rate": 0.22945205479452055, "edits": 201, "reference_units": 876, "clips": 50, "language_accuracy": 0.96 }, "pl": { "metric": "WER", "error_rate": 0.23529411764705882, "edits": 172, "reference_units": 731, "clips": 50, "language_accuracy": 0.98 }, "ko": { "metric": "CER", "error_rate": 0.1323529411764706, "edits": 279, "reference_units": 2108, "clips": 50, "language_accuracy": 1.0 } }, "Fleet-1": { "en": { "metric": "WER", "error_rate": 0.10913404507710557, "edits": 92, "reference_units": 843, "clips": 50, "language_accuracy": 0.96 }, "de": { "metric": "WER", "error_rate": 0.1953125, "edits": 150, "reference_units": 768, "clips": 50, "language_accuracy": 1.0 }, "fr": { "metric": "WER", "error_rate": 0.12040816326530612, "edits": 118, "reference_units": 980, "clips": 50, "language_accuracy": 1.0 }, "es": { "metric": "WER", "error_rate": 0.1170313986679353, "edits": 123, "reference_units": 1051, "clips": 50, "language_accuracy": 0.98 }, "it": { "metric": "WER", "error_rate": 0.38219895287958117, "edits": 365, "reference_units": 955, "clips": 50, "language_accuracy": 0.98 }, "nl": { "metric": "WER", "error_rate": 0.2317351598173516, "edits": 203, "reference_units": 876, "clips": 50, "language_accuracy": 0.96 }, "pl": { "metric": "WER", "error_rate": 0.22982216142270862, "edits": 168, "reference_units": 731, "clips": 50, "language_accuracy": 0.96 }, "ko": { "metric": "CER", "error_rate": 0.15654648956356737, "edits": 330, "reference_units": 2108, "clips": 50, "language_accuracy": 1.0 } }, "Sky-Kim": { "en": { "metric": "WER", "error_rate": 0.1269276393831554, "edits": 107, "reference_units": 843, "clips": 50, "language_accuracy": 0.94 }, "de": { "metric": "WER", "error_rate": 0.19661458333333334, "edits": 151, "reference_units": 768, "clips": 50, "language_accuracy": 1.0 }, "fr": { "metric": "WER", "error_rate": 0.12244897959183673, "edits": 120, "reference_units": 980, "clips": 50, "language_accuracy": 1.0 }, "es": { "metric": "WER", "error_rate": 0.11893434823977164, "edits": 125, "reference_units": 1051, "clips": 50, "language_accuracy": 0.98 }, "it": { "metric": "WER", "error_rate": 0.40209424083769635, "edits": 384, "reference_units": 955, "clips": 50, "language_accuracy": 0.98 }, "nl": { "metric": "WER", "error_rate": 0.2328767123287671, "edits": 204, "reference_units": 876, "clips": 50, "language_accuracy": 0.96 }, "pl": { "metric": "WER", "error_rate": 0.23529411764705882, "edits": 172, "reference_units": 731, "clips": 50, "language_accuracy": 0.98 }, "ko": { "metric": "CER", "error_rate": 0.2518975332068311, "edits": 531, "reference_units": 2108, "clips": 50, "language_accuracy": 1.0 } }, "Whistle": { "en": { "metric": "WER", "error_rate": 0.1055753262158956, "edits": 89, "reference_units": 843, "clips": 50, "language_accuracy": 1.0 }, "de": { "metric": "WER", "error_rate": 0.17317708333333334, "edits": 133, "reference_units": 768, "clips": 50, "language_accuracy": 1.0 }, "fr": { "metric": "WER", "error_rate": 0.1306122448979592, "edits": 128, "reference_units": 980, "clips": 50, "language_accuracy": 1.0 }, "es": { "metric": "WER", "error_rate": 0.09990485252140818, "edits": 105, "reference_units": 1051, "clips": 50, "language_accuracy": 1.0 }, "it": { "metric": "WER", "error_rate": 0.34450261780104713, "edits": 329, "reference_units": 955, "clips": 50, "language_accuracy": 0.98 }, "nl": { "metric": "WER", "error_rate": 0.23059360730593606, "edits": 202, "reference_units": 876, "clips": 50, "language_accuracy": 0.98 }, "pl": { "metric": "WER", "error_rate": 0.22024623803009577, "edits": 161, "reference_units": 731, "clips": 50, "language_accuracy": 0.96 } } } }, "gate": { "passed": false, "beats_fleet1_in": [ "de", "fr", "pl", "ko" ], "korean_not_worse_than_sky": false, "rule": ">=5 of 8 languages better than Fleet-1 and Korean CER <= Sky-Kim, on Fleet-1's FLEURS test clips" } }