| { |
| "schema": "jyoti-public-evaluation-v1", |
| "arm": "D3-manifest50-extra-p050", |
| "inference_checkpoint_sha256": "417ed68677b85e644dfeea2415f4d0349322861f7201dcbd9d16435f18a04e00", |
| "training_checkpoint_sha256": "c4fe5a6cfe741da64ff74c928a05189c8a62ab6f3556d72042b404eb9e136b9d", |
| "step": 100000, |
| "quality_qualified": false, |
| "sealed_final_cohort_opened": false, |
| "english": { |
| "asr": "facebook/wav2vec2-base-960h @ 22aad52d435eb6dbaf354bdad9b0da84ce7d6156 (greedy CTC)", |
| "pools": { |
| "clean": "128 LibriSpeech dev-clean utterances", |
| "difficult": "128 LibriSpeech dev-other utterances", |
| "pink_10db": "the clean pool + synthetic pink noise at 10 dB SNR (codec input)", |
| "pink_5db": "the clean pool + synthetic pink noise at 5 dB SNR" |
| }, |
| "mode": "packed bitstream (encode -> bytes -> decode)", |
| "results": { |
| "clean": { |
| "codec_wer": 0.06684611933602512, |
| "original_wer": 0.02826379542395693, |
| "codec_cer": 0.02497897392767031, |
| "original_cer": 0.007485281749369218, |
| "wer_increase_pp": 3.8582323912068186, |
| "mean_stoi": 0.8995658662229111, |
| "mean_pesq_wb": 2.111289313994348, |
| "utterances": 128 |
| }, |
| "difficult": { |
| "codec_wer": 0.20997732426303856, |
| "original_wer": 0.07936507936507936, |
| "codec_cer": 0.09342347879532882, |
| "original_cer": 0.029765563262797435, |
| "wer_increase_pp": 13.061224489795919, |
| "mean_stoi": 0.8848347751522835, |
| "mean_pesq_wb": 1.973732240498066, |
| "utterances": 128 |
| }, |
| "pink_10db": { |
| "codec_wer": 0.24136384028712426, |
| "original_wer": 0.16913414087034545, |
| "codec_cer": 0.11648444070647603, |
| "original_cer": 0.07846930193439866, |
| "wer_increase_pp": 7.222969941677881, |
| "mean_stoi": 0.7282501492237068, |
| "mean_pesq_wb": 1.2500946335494518, |
| "utterances": 128 |
| }, |
| "pink_5db": { |
| "codec_wer": 0.4064602960969044, |
| "original_wer": 0.5118887393449978, |
| "codec_cer": 0.21412952060555088, |
| "original_cer": 0.30538267451640033, |
| "wer_increase_pp": -10.542844324809336, |
| "mean_stoi": 0.6389370863985571, |
| "mean_pesq_wb": 1.1396433534100652, |
| "utterances": 128 |
| } |
| } |
| }, |
| "development_gates": { |
| "note": "Engineering targets registered before training; not values from any paper. Unchanged.", |
| "gates": [ |
| { |
| "gate": "clean_wer_increase_pp", |
| "value": 3.8582323912068186, |
| "operator": "<=", |
| "threshold": 5.0, |
| "passed": true |
| }, |
| { |
| "gate": "difficult_wer_increase_pp", |
| "value": 13.061224489795919, |
| "operator": "<=", |
| "threshold": 10.0, |
| "passed": false |
| }, |
| { |
| "gate": "pink_10db_wer_increase_pp", |
| "value": 7.222969941677881, |
| "operator": "<=", |
| "threshold": 10.0, |
| "passed": true |
| }, |
| { |
| "gate": "pink_5db_wer_increase_pp", |
| "value": -10.542844324809336, |
| "operator": "<=", |
| "threshold": 10.0, |
| "passed": true |
| }, |
| { |
| "gate": "clean_mean_stoi", |
| "value": 0.8995658662229111, |
| "operator": ">=", |
| "threshold": 0.85, |
| "passed": true |
| } |
| ], |
| "passed": 4, |
| "total": 5 |
| }, |
| "nepali_supplemental": { |
| "asr": "ampixa/nepali-conformer-offline @ a7830b7ab16fbc2659b590b4e5e4030737c4caf9", |
| "pool": "128 SLR54 utterances from 32 held-out development speakers (not in training)", |
| "codec": { |
| "word_errors": 242, |
| "reference_words": 378, |
| "character_errors": 596, |
| "reference_characters": 2106, |
| "wer": 0.6402116402116402, |
| "cer": 0.28300094966761635 |
| }, |
| "original_audio": { |
| "word_errors": 154, |
| "reference_words": 378, |
| "character_errors": 310, |
| "reference_characters": 2106, |
| "wer": 0.4074074074074074, |
| "cer": 0.14719848053181386 |
| } |
| }, |
| "sources": { |
| "english_summary_sha256": "15f08d1ee80caa41b4a95ae5a473d5196205263f99e1ec089b6256eff8fe7abe", |
| "nepali_summary_sha256": "996f741f1c5c816b618503a00bc3140bbb246cbe22ef1f5e73352cab00496b1c", |
| "nepali_original_summary_sha256": "80f7fef5e23d2e5932d0de600c8268407d81727fda3f6592923b6a664de49d38", |
| "qualification_targets_sha256": "23e14e40b027de87c217488ee2c7989176e2930277ecc991395de5ef845b5145" |
| } |
| } |
|
|
