File size: 3,441 Bytes
19eb87a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 | {
"model": "Dynamic ASR Pocket",
"architecture": "omniASR_CTC_300M_v2 student distilled from a 1B Dynamic ASR teacher (omniASR_CTC_1B_v2 fine-tune)",
"checkpoint": "P2, step 2500",
"decoding": "greedy CTC, no language model",
"input": "16 kHz mono float32",
"scoring": "CER after: NFC, lowercase, punctuation and apostrophes to spaces, English number words to digits, dot-below variants unified, Igbo tone marks stripped from both sides (Yoruba tone marks kept)",
"eval_sets": {
"gdev": "general dev set (WAXAL validation + OpenSLR-70; 187 clips, 1269 s; eng/hau/ibo/pcm/yor). Used for checkpoint selection, so it is a dev set, not a test set.",
"real_codeswitch": "held-out real code-switched speech test set (94 clips, 758 s); never used for training or selection"
},
"results": {
"pocket_int8_onnx": {
"gdev_cer": {
"eng": 0.0571,
"hau": 0.0522,
"ibo": 0.0595,
"pcm": 0.0963,
"yor": 0.1359,
"mean": 0.0802
},
"real_codeswitch_cer": 0.1558,
"real_codeswitch_wer": 0.4311,
"real_codeswitch_english_word_recall": 0.3009
},
"pocket_fp32_onnx": {
"gdev_cer": {
"eng": 0.0571,
"hau": 0.0514,
"ibo": 0.0595,
"pcm": 0.0925,
"yor": 0.135,
"mean": 0.0791
},
"real_codeswitch_cer": 0.1558,
"real_codeswitch_wer": 0.4333,
"real_codeswitch_english_word_recall": 0.3009
},
"pocket_pytorch_fp32": {
"gdev_cer": {
"eng": 0.0571,
"hau": 0.0514,
"ibo": 0.0595,
"pcm": 0.0925,
"yor": 0.135,
"mean": 0.0791
},
"real_codeswitch_cer": 0.1558,
"real_codeswitch_wer": 0.4333,
"real_codeswitch_english_word_recall": 0.3009
},
"teacher_1b_greedy": {
"gdev_cer": {
"eng": 0.0294,
"hau": 0.0489,
"ibo": 0.0598,
"pcm": 0.0823,
"yor": 0.1288,
"mean": 0.0698
},
"real_codeswitch_cer": 0.1375,
"real_codeswitch_wer": 0.3968,
"real_codeswitch_english_word_recall": 0.4248
}
},
"sizes_bytes": {
"model_int8.onnx": 366630967,
"model_fp32.onnx": 1304478206,
"model.pt": 1304038762
},
"cpu_benchmark": {
"cpu": "Intel Xeon Platinum 8480+",
"batch": 1,
"clips": 20,
"audio_s": 110.1,
"runs": [
{
"model": "pocket_int8.onnx",
"threads": 4,
"rtf": 0.0706,
"ms_median": 357.7,
"ms_p90": 509.3,
"peak_rss_mb": 939.1,
"load_s": 1.02
},
{
"model": "pocket_int8.onnx",
"threads": 2,
"rtf": 0.133,
"ms_median": 693.8,
"ms_p90": 947.1,
"peak_rss_mb": 935.2,
"load_s": 1.0
},
{
"model": "pocket_int8.onnx",
"threads": 1,
"rtf": 0.139,
"ms_median": 708.1,
"ms_p90": 995.2,
"peak_rss_mb": 935.2,
"load_s": 0.92
},
{
"model": "pocket_fp32.onnx",
"threads": 4,
"rtf": 0.1857,
"ms_median": 952.1,
"ms_p90": 1294.9,
"peak_rss_mb": 2287.3,
"load_s": 3.21
}
]
},
"notes": [
"The 1B teacher is an internal checkpoint used for distillation; it is not one of the model files in this repository.",
"English is the largest gap between Pocket and the teacher."
],
"training_data": "licensed real speech (same sources as the 1B models), real-voice splice code-switch clips built from that licensed speech, and an oversample of 0.9 h of real code-switched training clips; no synthetic TTS audio; no African Voices data",
"distillation": "CTC loss on references + KL divergence to the 1B teacher's per-frame distribution; CNN feature encoder frozen"
} |