SpeedAudio commited on
Commit
c830bfd
·
verified ·
1 Parent(s): c2d6f53

Publish SpeedASR-v1-65M weights, portable inference and Open ASR evaluation package

Browse files

Original epoch-30-avg-9 checkpoint and tokenizer; FP32 offline beam-4 inference; public results and validation; pinned Docker/Jobs submission adapter. Existing Apache 2.0 LICENSE preserved.

This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .dockerignore +13 -0
  2. .gitattributes +3 -0
  3. README.md +190 -0
  4. artifact_manifest.json +269 -0
  5. benchmark_results/adapter_validation.json +54 -0
  6. benchmark_results/dependency_validation.json +100 -0
  7. benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_ArtificialAnalysis-Earnings22-Cleaned-AA-chunked_earnings22_cleaned_aa_chunked_test.jsonl +3 -0
  8. benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_VoiceArena-Monsoon_en_IN_test_monsoon_en_in_test.jsonl +0 -0
  9. benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_ami_cleaned_test.jsonl +0 -0
  10. benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_gigaspeech_cleaned_test.jsonl +3 -0
  11. benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_spgispeech_test.jsonl +3 -0
  12. benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_clean_test.jsonl +0 -0
  13. benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_test.jsonl +0 -0
  14. benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_voxpopuli_cleaned_aa_test.jsonl +0 -0
  15. benchmark_results/local_3090ti_20261119/REPRODUCIBILITY.txt +12 -0
  16. benchmark_results/local_3090ti_20261119/current_scores.json +170 -0
  17. benchmark_results/local_3090ti_20261119/export_manifest.json +62 -0
  18. benchmark_results/local_3090ti_20261119/independent_validation.json +162 -0
  19. benchmark_results/local_3090ti_20261119/input_provenance.json +16 -0
  20. benchmark_results/local_3090ti_20261119/run_config.json +72 -0
  21. benchmark_results/portable_validation.json +228 -0
  22. inference.py +407 -0
  23. leaderboard/Dockerfile +32 -0
  24. leaderboard/cloud_submission.txt +191 -0
  25. leaderboard/dockerignore.txt +13 -0
  26. leaderboard/model_metadata.json +53 -0
  27. leaderboard/requirements.txt +12 -0
  28. leaderboard/run_eval.py +264 -0
  29. leaderboard/space_app.py +26 -0
  30. leaderboard/space_card.txt +15 -0
  31. leaderboard/submit_jobs.sh +95 -0
  32. model_config.json +59 -0
  33. onnx/decoder-epoch-99-avg-1.onnx +3 -0
  34. onnx/encoder-epoch-99-avg-1.onnx +3 -0
  35. onnx/joiner-epoch-99-avg-1.onnx +3 -0
  36. pretrained.pt +3 -0
  37. requirements-core.txt +5 -0
  38. requirements-linux-cu124.txt +7 -0
  39. requirements.txt +3 -0
  40. runtime_source_manifest.json +70 -0
  41. speedasr_core/LICENSE +202 -0
  42. speedasr_core/__init__.py +4 -0
  43. speedasr_core/beam_search.py +423 -0
  44. speedasr_core/decoder.py +134 -0
  45. speedasr_core/encoder_interface.py +43 -0
  46. speedasr_core/joiner.py +67 -0
  47. speedasr_core/model.py +152 -0
  48. speedasr_core/scaling.py +1913 -0
  49. speedasr_core/subsampling.py +406 -0
  50. speedasr_core/utils.py +92 -0
.dockerignore ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ .git
2
+ *.pt
3
+ *.onnx
4
+ *.safetensors
5
+ benchmarks
6
+ results
7
+ __pycache__
8
+ *.pyc
9
+ .env
10
+ *token*
11
+ !tokens.txt
12
+
13
+ benchmark_results
.gitattributes CHANGED
@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_ArtificialAnalysis-Earnings22-Cleaned-AA-chunked_earnings22_cleaned_aa_chunked_test.jsonl filter=lfs diff=lfs merge=lfs -text
37
+ benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_gigaspeech_cleaned_test.jsonl filter=lfs diff=lfs merge=lfs -text
38
+ benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_spgispeech_test.jsonl filter=lfs diff=lfs merge=lfs -text
README.md CHANGED
@@ -1,3 +1,193 @@
1
  ---
2
  license: apache-2.0
 
 
 
 
 
 
 
 
 
 
 
 
3
  ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
  license: apache-2.0
3
+ language:
4
+ - en
5
+ pipeline_tag: automatic-speech-recognition
6
+ library_name: pytorch
7
+ tags:
8
+ - icefall
9
+ - zipformer
10
+ - transducer
11
+ - offline-asr
12
+ - english
13
+ metrics:
14
+ - wer
15
  ---
16
+
17
+ # SpeedASR-v1-65M
18
+
19
+ SpeedASR-v1-65M is an **offline English Zipformer transducer** with
20
+ **65,551,059 parameters (65.55M)**. This release contains the original FP32
21
+ checkpoint, its 500-entry token vocabulary, portable batched inference code,
22
+ and public Open ASR Leaderboard evaluation artifacts.
23
+
24
+ Original encoder/decoder/joiner ONNX exports are also retained in [onnx](onnx).
25
+ Their `epoch-99` filenames are export labels around the epoch-30-avg-9 checkpoint
26
+ alias, not a claim that this is an epoch-99 model. The reported scores and cloud
27
+ adapter use PyTorch; these ONNX files have not been evaluated for numerical
28
+ parity by this release's checks.
29
+
30
+ The non-causal encoder processes complete recordings. This release does not
31
+ provide a native streaming recognizer or claim a measured phone/NPU benchmark.
32
+
33
+ | Property | Value |
34
+ |---|---|
35
+ | Language | English |
36
+ | Encoder | Zipformer2, six stacks / 16 encoder layers; non-causal |
37
+ | Prediction network | Transducer, context size 3, dimension 512 |
38
+ | Joiner | Dimension 512 |
39
+ | Parameters | 65,551,059 total |
40
+ | Audio | 16 kHz, mono, floating-point samples |
41
+ | Features | 80-bin kaldifeat Fbank, dither 0, `snip_edges=False`, `high_freq=-400` |
42
+ | Decoding | Modified beam search, beam size 4, FP32 |
43
+ | Text formatting | VoiceA casing/punctuation tags restored; no benchmark WER normalization inside inference |
44
+ | External language model | None |
45
+ | Checkpoint | `pretrained.pt`, 262,613,934 bytes; epoch-30-avg-9 |
46
+
47
+ The complete frozen construction and inference settings are in
48
+ [model_config.json](model_config.json). The repository's existing Apache 2.0
49
+ license is preserved. Vendored Icefall components retain their upstream notices
50
+ and [Apache 2.0 license](speedasr_core/LICENSE).
51
+
52
+ ## Inference
53
+
54
+ The pinned dependency set targets **Linux x86-64, Python 3.10, CUDA 12.4 and
55
+ PyTorch 2.6.0**. The k2 and kaldifeat wheels must match the PyTorch/CUDA ABI.
56
+
57
+ ```bash
58
+ python3.10 -m venv .venv
59
+ source .venv/bin/activate
60
+ python -m pip install huggingface_hub==1.18.0
61
+ hf download SpeedAudio261119/SpeedASR-v1-65M --local-dir SpeedASR-v1-65M
62
+ cd SpeedASR-v1-65M
63
+ python -m pip install -r requirements-linux-cu124.txt
64
+ python inference.py /path/to/16khz.wav --device cuda:0
65
+ ```
66
+
67
+ From a downloaded copy of this repository:
68
+
69
+ ```python
70
+ import soundfile as sf
71
+ from inference import SpeedASR
72
+
73
+ audio, sample_rate = sf.read("sample.wav", dtype="float32")
74
+ model = SpeedASR(device="cuda:0")
75
+ text = model.transcribe(audio, sample_rate=sample_rate)
76
+ texts = model.transcribe_batch([audio, audio], sample_rate=sample_rate)
77
+ print(text)
78
+ ```
79
+
80
+ Inputs must be mono and 16 kHz; resample other audio before calling the API.
81
+ The WAV CLI uses the first channel, matching the benchmark frontend.
82
+ Inputs shorter than one second are padded; inputs longer than 30 seconds are
83
+ decoded in consecutive 30-second pieces and joined without overlap. The same
84
+ policy and decoder settings apply to all datasets.
85
+
86
+ The loader checks the checkpoint/token hashes, loads the state dictionary
87
+ strictly, and verifies the parameter count. `SpeedASR.from_pretrained` can
88
+ download weights, tokens and configuration for an explicitly pinned Hub commit;
89
+ the local inference module and `speedasr_core` must already be available.
90
+
91
+ ## Public benchmark results
92
+
93
+ The following **local measurements** were completed on **19 November 2026**
94
+ using an **NVIDIA GeForce RTX 3090 Ti, FP32**, with the official scorer at
95
+ [`4a113dd90c9ffd011d7d3d105da76980d511d029`](https://github.com/huggingface/open_asr_leaderboard/tree/4a113dd90c9ffd011d7d3d105da76980d511d029).
96
+ They are not official H200/cloud results or an official leaderboard ranking.
97
+
98
+ | Public testset | Clips | WER (%) |
99
+ |---|---:|---:|
100
+ | AMI Cleaned | 7,805 | 9.59 |
101
+ | GigaSpeech Cleaned | 18,768 | 8.07 |
102
+ | VoxPopuli Cleaned AA | 628 | 3.27 |
103
+ | Earnings22 Cleaned AA Chunked | 341 | 6.78 |
104
+ | SPGISpeech | 39,341 | 3.89 |
105
+ | Voice Arena Monsoon English | 2,102 | 4.44 |
106
+ | URGENT2024 | 1,000 | 12.29 |
107
+ | **Seven-set arithmetic mean** | **69,985** | **6.9043** |
108
+ | URGENT2024 Clean — optional, excluded from mean | 1,000 | 5.54 |
109
+
110
+ Earnings22 predictions are reconstructed and scored as **six parent calls**,
111
+ not 341 independent references. The two private score groups are unavailable
112
+ locally and are excluded from this public average.
113
+
114
+ Seven-set pooled local throughput was **192.6293 RTFx**, using up to 2,048
115
+ streams, a 5,000 padded-audio-second budget, and 512 long-audio chunks per
116
+ internal batch. This is batched GPU throughput, not single-request latency or
117
+ on-device efficiency. Cloud throughput must be measured separately.
118
+
119
+ The [benchmark_results/local_3090ti_20261119](benchmark_results/local_3090ti_20261119)
120
+ directory includes all **70,985** raw public prediction rows, final scores,
121
+ dataset revisions, sanitized historical configuration, and verification data.
122
+ Only machine-local audio paths and model-id filenames were changed for release;
123
+ raw references/predictions, timing and chunk metadata were preserved.
124
+ [export_manifest.json](benchmark_results/local_3090ti_20261119/export_manifest.json)
125
+ records original and released hashes. It contains no audio corpus or private
126
+ testset data.
127
+
128
+ The portable runtime passed strict loading and exact prediction comparisons
129
+ against the original runner on 24 representative short/middle/long examples
130
+ covering all eight public sets, plus a 16-example identical-batch comparison.
131
+ See [portable_validation.json](benchmark_results/portable_validation.json).
132
+ This is a targeted portability check, not a rerun of all testsets with the new
133
+ cloud adapter. Different batch grouping can cause numerical differences.
134
+
135
+ The [adapter smoke check](benchmark_results/adapter_validation.json) also passed
136
+ official data preparation, raw manifest writing and scoring using four cached
137
+ AMI utterances and one complete 51-chunk Earnings22 parent call. Dataset transport
138
+ and metadata requests were replaced with local test fixtures for that check.
139
+ Docker was unavailable on the validation host, so a clean image build and an
140
+ actual H200 Job remain subsequent validation steps.
141
+ The combined pinned inference/evaluation dependencies passed a fresh
142
+ `pip --dry-run --ignore-installed` resolution; see
143
+ [dependency_validation.json](benchmark_results/dependency_validation.json).
144
+
145
+ ## Open ASR cloud evaluation
146
+
147
+ [leaderboard/run_eval.py](leaderboard/run_eval.py) uses the official data loader
148
+ and manifest writer, retains raw references and predictions, preserves
149
+ Earnings22 parent/chunk metadata, and uses one scoring worker. It fixes the
150
+ decoder settings above across datasets and records synchronized inference timing
151
+ after synthetic warmup.
152
+
153
+ The [Dockerfile](leaderboard/Dockerfile),
154
+ [HF Jobs launcher](leaderboard/submit_jobs.sh), and
155
+ [submission instructions](leaderboard/cloud_submission.txt) prepare a Docker
156
+ Space and the seven public cloud evaluation jobs. Build with:
157
+
158
+ ```bash
159
+ docker build -f leaderboard/Dockerfile -t speedasr-open-asr .
160
+ ```
161
+
162
+ Use an immutable model commit SHA for submitted runs. H200 batch settings should
163
+ be checked for the maximum practical throughput; the included defaults reproduce
164
+ the local limits and are not claimed to saturate an H200. Uploading this package
165
+ does not itself build a Space, launch paid Jobs, submit a GitHub PR, evaluate
166
+ private sets or register an official leaderboard entry.
167
+
168
+ ## Training data
169
+
170
+ The author confirms that **all seven default public benchmark testsets listed
171
+ above were excluded from training**. This is an author-provided statement, not
172
+ an independent overlap audit. The full training-dataset composition has not
173
+ been disclosed in this release. No exclusion claim is made here for unavailable
174
+ private datasets or the optional eighth testset.
175
+
176
+ ## Artifact identity
177
+
178
+ ```text
179
+ pretrained.pt SHA-256
180
+ b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01
181
+
182
+ tokens.txt SHA-256
183
+ a238b7a0fa67aa6be998ca13dd129d69c50639970583e32232dd149e5448210a
184
+ ```
185
+
186
+ [runtime_source_manifest.json](runtime_source_manifest.json) records vendored
187
+ source origins and transformations. The upstream model-code revision is
188
+ `3f848bb6d0acc970c9b294a30ca0a04a7c9c78d1`. The release artifact manifest
189
+ records the files actually published.
190
+
191
+ [leaderboard/model_metadata.json](leaderboard/model_metadata.json) provides
192
+ structured submission metadata and parameter counts by component, including
193
+ auxiliary training projections retained in the unchanged checkpoint.
artifact_manifest.json ADDED
@@ -0,0 +1,269 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_id": "SpeedAudio261119/SpeedASR-v1-65M",
3
+ "release_date": "2026-11-19",
4
+ "source_repository_parent_commit": "c2d6f53b50895c81f8e624789da1239edf050e99",
5
+ "note": "This manifest covers uploaded package files except itself. Existing Hub LICENSE and .gitattributes are preserved and are not overwritten.",
6
+ "validation": {
7
+ "checkpoint_and_token_hashes_match_original_benchmark": true,
8
+ "portable_original_prediction_cases": 24,
9
+ "identical_batch_cases": 16,
10
+ "portable_exact_prediction_parity": true,
11
+ "adapter_official_preparation_writer_scoring_smoke_passed": true,
12
+ "clean_docker_build_performed": false,
13
+ "official_cloud_or_private_evaluation_performed": false,
14
+ "credential_scan_passed": true,
15
+ "fresh_dependency_resolution_passed": true
16
+ },
17
+ "files": [
18
+ {
19
+ "path": ".dockerignore",
20
+ "bytes": 112,
21
+ "sha256": "f5cbe531d1fdf5f84517ca8b9014f38b99a6dbc2fd7d71093fc4d0e5648780fe"
22
+ },
23
+ {
24
+ "path": "README.md",
25
+ "bytes": 8566,
26
+ "sha256": "628abdfa3846a092ad369eaa3f01e18f8cc5a182c29b69e88197205d06f41bfc"
27
+ },
28
+ {
29
+ "path": "benchmark_results/adapter_validation.json",
30
+ "bytes": 1435,
31
+ "sha256": "3d1622c48c966aec8e7a8239407108232217f7d9b999b5c9a12cc9d6a29506b4"
32
+ },
33
+ {
34
+ "path": "benchmark_results/dependency_validation.json",
35
+ "bytes": 2733,
36
+ "sha256": "7bd98fc8178d36342d48928ada975abfa42aae73f2ac2ed83f8569de0f4a8b97"
37
+ },
38
+ {
39
+ "path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_ArtificialAnalysis-Earnings22-Cleaned-AA-chunked_earnings22_cleaned_aa_chunked_test.jsonl",
40
+ "bytes": 11172360,
41
+ "sha256": "5010bb0dcc3024b9d3dda5956734044a0d35ccaf9bb3214271efb9f03cc6ea92"
42
+ },
43
+ {
44
+ "path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_VoiceArena-Monsoon_en_IN_test_monsoon_en_in_test.jsonl",
45
+ "bytes": 1922498,
46
+ "sha256": "fd947f47d55206b7e720a96e8ce2d704b5213c3e6709c83a35a1d8138df0611c"
47
+ },
48
+ {
49
+ "path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_ami_cleaned_test.jsonl",
50
+ "bytes": 5349141,
51
+ "sha256": "73b4f9d3c03e519f70a5897ae20b5195a9733a1f9baa8b9ff0618c590d86fb14"
52
+ },
53
+ {
54
+ "path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_gigaspeech_cleaned_test.jsonl",
55
+ "bytes": 16540413,
56
+ "sha256": "e25623081b67f8c37db13966077e8f44eff25e47de61f96db2c5f2938ae1aaa0"
57
+ },
58
+ {
59
+ "path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_spgispeech_test.jsonl",
60
+ "bytes": 41084800,
61
+ "sha256": "ce7512cd10eff18d797cdc9efb167ada34b37c4beda1b5ba4bcde206ced0330b"
62
+ },
63
+ {
64
+ "path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_clean_test.jsonl",
65
+ "bytes": 799100,
66
+ "sha256": "bf1f804714c7ae77d742a24cb499a3ddb50934e03ca8ce8b91c0c92928a1321a"
67
+ },
68
+ {
69
+ "path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_test.jsonl",
70
+ "bytes": 786374,
71
+ "sha256": "30edc6b2c7dcf13c664682f5ca06fb7b2dace5bb92cde4e8710688729206a887"
72
+ },
73
+ {
74
+ "path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_voxpopuli_cleaned_aa_test.jsonl",
75
+ "bytes": 739478,
76
+ "sha256": "1bb76945bcb6f8b6fb89420e10615b1ab310160391894062d53f443b3ff79d51"
77
+ },
78
+ {
79
+ "path": "benchmark_results/local_3090ti_20261119/REPRODUCIBILITY.txt",
80
+ "bytes": 1353,
81
+ "sha256": "a441630f8054b9901b65209b447a0128e07bb09a22319fccd7625a4d95487b44"
82
+ },
83
+ {
84
+ "path": "benchmark_results/local_3090ti_20261119/current_scores.json",
85
+ "bytes": 6358,
86
+ "sha256": "a3e165a0a543a881ba210ae448e0d77fd209a963fe9c4a1d17410023ae19ba99"
87
+ },
88
+ {
89
+ "path": "benchmark_results/local_3090ti_20261119/export_manifest.json",
90
+ "bytes": 4555,
91
+ "sha256": "b6a4c38ad79119ed90b898986d4342c5f0f3af170a46639d5d9b0b83ce1873a4"
92
+ },
93
+ {
94
+ "path": "benchmark_results/local_3090ti_20261119/independent_validation.json",
95
+ "bytes": 6095,
96
+ "sha256": "8e1e46acf345955e3290b5087416f9650d4e98dc5cf2a5acc749e7966ff4e70e"
97
+ },
98
+ {
99
+ "path": "benchmark_results/local_3090ti_20261119/input_provenance.json",
100
+ "bytes": 1008,
101
+ "sha256": "6936df47c16498b02e11bbff5c0bd0bc7318323b89a9e5cdbb8dedb7547a1e56"
102
+ },
103
+ {
104
+ "path": "benchmark_results/local_3090ti_20261119/run_config.json",
105
+ "bytes": 2160,
106
+ "sha256": "bed45bbfb0bdad8d9e00dba4c0b7e8e65d6598d728d08eb7ef5bbb7a61045c19"
107
+ },
108
+ {
109
+ "path": "benchmark_results/portable_validation.json",
110
+ "bytes": 16310,
111
+ "sha256": "b8c2677d20c34b75eaa83c3980e464da6ecfeebc594dea47c65db38aef3f1048"
112
+ },
113
+ {
114
+ "path": "inference.py",
115
+ "bytes": 19227,
116
+ "sha256": "17ea7c209dc40b0540b3ccca49cef04e265211b38b3daf00333e1180abdaa39e"
117
+ },
118
+ {
119
+ "path": "leaderboard/Dockerfile",
120
+ "bytes": 1565,
121
+ "sha256": "147627537d4bb763aa4d86e9f55fe66bf31248b90a91e386fef5c6700f4541ff"
122
+ },
123
+ {
124
+ "path": "leaderboard/cloud_submission.txt",
125
+ "bytes": 9805,
126
+ "sha256": "234c74d6688308cf54ab9accab5ee127acb06ed9db06fec786a22fe79dffb362"
127
+ },
128
+ {
129
+ "path": "leaderboard/dockerignore.txt",
130
+ "bytes": 112,
131
+ "sha256": "f5cbe531d1fdf5f84517ca8b9014f38b99a6dbc2fd7d71093fc4d0e5648780fe"
132
+ },
133
+ {
134
+ "path": "leaderboard/model_metadata.json",
135
+ "bytes": 1788,
136
+ "sha256": "8a496e931996e92a5ceb6a4210a8e7015f83a8577371ce43bdfa5d3a52ac10c6"
137
+ },
138
+ {
139
+ "path": "leaderboard/requirements.txt",
140
+ "bytes": 373,
141
+ "sha256": "5a169ebe8356c349bfa076617d5b744a1853adeebbe95a1513c5379871b8c912"
142
+ },
143
+ {
144
+ "path": "leaderboard/run_eval.py",
145
+ "bytes": 12739,
146
+ "sha256": "98aa19d45326ec8ceefd6707f096460e610fa9b886d8585996e4361c7b1bd56e"
147
+ },
148
+ {
149
+ "path": "leaderboard/space_app.py",
150
+ "bytes": 1198,
151
+ "sha256": "11b9fd43d0fde5aaf9974c5564563253f22a51a5ee6735bccc1a718b2f15f913"
152
+ },
153
+ {
154
+ "path": "leaderboard/space_card.txt",
155
+ "bytes": 404,
156
+ "sha256": "02482305533c92b68080a10dd04931e96dbdbdc1224f0b5293eac4568eeb7889"
157
+ },
158
+ {
159
+ "path": "leaderboard/submit_jobs.sh",
160
+ "bytes": 4050,
161
+ "sha256": "8ed64f80d5b1c68eb987faf05a603cd429287c9cae169cf932b55b52f4ae23ab"
162
+ },
163
+ {
164
+ "path": "model_config.json",
165
+ "bytes": 1760,
166
+ "sha256": "e17381d31eb19a4d781c2b56f2060986c6bff02154f9f7e31edb57de11eb0e7f"
167
+ },
168
+ {
169
+ "path": "onnx/decoder-epoch-99-avg-1.onnx",
170
+ "bytes": 2101273,
171
+ "sha256": "1fc497358cc2f63d4dcd3f249da1e4f29a04315c5d7e4aa358cfaf17ea8d5db1"
172
+ },
173
+ {
174
+ "path": "onnx/encoder-epoch-99-avg-1.onnx",
175
+ "bytes": 263111425,
176
+ "sha256": "768fedad168901e872e46477b347adc95cc304ed9ee60ee57c95b60916da42f0"
177
+ },
178
+ {
179
+ "path": "onnx/joiner-epoch-99-avg-1.onnx",
180
+ "bytes": 1026462,
181
+ "sha256": "c26540ac6d9ac00191436193bc22a8dd3ceae7bd3a83058aa2d18e1470bed5f4"
182
+ },
183
+ {
184
+ "path": "pretrained.pt",
185
+ "bytes": 262613934,
186
+ "sha256": "b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01"
187
+ },
188
+ {
189
+ "path": "requirements-core.txt",
190
+ "bytes": 160,
191
+ "sha256": "c0e62985bd29049fce9e9899be94a0edbd7127c85f9405245ac9456542aeaa44"
192
+ },
193
+ {
194
+ "path": "requirements-linux-cu124.txt",
195
+ "bytes": 735,
196
+ "sha256": "2512fc51922748928a8cb4c25a688ab1f3f43f4f2c4a9f2383ab47ee132dbe82"
197
+ },
198
+ {
199
+ "path": "requirements.txt",
200
+ "bytes": 166,
201
+ "sha256": "415a0461b2a9926ec77ef9c595f46c2244506489d79c2cfa56dba4a000e81c8c"
202
+ },
203
+ {
204
+ "path": "runtime_source_manifest.json",
205
+ "bytes": 3840,
206
+ "sha256": "0e7bfe39d7201c2a38629c3395262d7996ea10fb64e239e2e17f9ff0745449ed"
207
+ },
208
+ {
209
+ "path": "speedasr_core/LICENSE",
210
+ "bytes": 11358,
211
+ "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"
212
+ },
213
+ {
214
+ "path": "speedasr_core/__init__.py",
215
+ "bytes": 113,
216
+ "sha256": "596ac4de65fae9944440e22cf00ec4629b80da70c2e9b309e0702e4686ac988e"
217
+ },
218
+ {
219
+ "path": "speedasr_core/beam_search.py",
220
+ "bytes": 13852,
221
+ "sha256": "d2e0297d1626b14b834a9b357caa875a5b11ce8f0aabc475debb7d20dec3b8c1"
222
+ },
223
+ {
224
+ "path": "speedasr_core/decoder.py",
225
+ "bytes": 4756,
226
+ "sha256": "a9aeeeef278f4e5357d4e49253d24f5f1c60df88cfcb623ccbd01481dc1503ce"
227
+ },
228
+ {
229
+ "path": "speedasr_core/encoder_interface.py",
230
+ "bytes": 1652,
231
+ "sha256": "fc70f530c33e2489435ff2cb4bb0dad8b9d6f58a248c425cbd6100c62f628569"
232
+ },
233
+ {
234
+ "path": "speedasr_core/joiner.py",
235
+ "bytes": 2214,
236
+ "sha256": "81faad6e0d07342d1abab28f2271201256f2139f5816b04296d6a8cbe0e2c887"
237
+ },
238
+ {
239
+ "path": "speedasr_core/model.py",
240
+ "bytes": 5938,
241
+ "sha256": "46b8e181fb326e8cd82c92a4c0ffd0917eb1474a26bd5b60b39b48daac28e90f"
242
+ },
243
+ {
244
+ "path": "speedasr_core/scaling.py",
245
+ "bytes": 70936,
246
+ "sha256": "679b1fe2655902ba13ef3f5cf7c924129a16681e3edc4343e766ff7086f5e6d7"
247
+ },
248
+ {
249
+ "path": "speedasr_core/subsampling.py",
250
+ "bytes": 13139,
251
+ "sha256": "dc33ac662d2508e191f4da5562f0c8d23444819b707624736429a85517db9145"
252
+ },
253
+ {
254
+ "path": "speedasr_core/utils.py",
255
+ "bytes": 3046,
256
+ "sha256": "682d0733f0e149be0ae9b62cb8de60d7baef65be4e64816001b6bb089b4e787b"
257
+ },
258
+ {
259
+ "path": "speedasr_core/zipformer.py",
260
+ "bytes": 93872,
261
+ "sha256": "5cea6da601de9f37de2e1db6c2dadcd931220b45341472efaa426a25130a5a36"
262
+ },
263
+ {
264
+ "path": "tokens.txt",
265
+ "bytes": 5180,
266
+ "sha256": "a238b7a0fa67aa6be998ca13dd129d69c50639970583e32232dd149e5448210a"
267
+ }
268
+ ]
269
+ }
benchmark_results/adapter_validation.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "compile": true,
3
+ "docker_available": false,
4
+ "bash_syntax": "passed",
5
+ "tests": [
6
+ {
7
+ "dataset": "ami",
8
+ "rows": 4,
9
+ "raw_text_preserved": true,
10
+ "chunk_metadata_preserved": true,
11
+ "score": {
12
+ "wer": 38.46,
13
+ "ins": 0,
14
+ "del": 3,
15
+ "sub": 2,
16
+ "audio_length": 4.12,
17
+ "inference_time": 0.08191168299890705,
18
+ "rtfx": 50.2981
19
+ }
20
+ },
21
+ {
22
+ "dataset": "earnings",
23
+ "rows": 51,
24
+ "raw_text_preserved": true,
25
+ "chunk_metadata_preserved": true,
26
+ "score": {
27
+ "wer": 6.86,
28
+ "ins": 9,
29
+ "del": 25,
30
+ "sub": 112,
31
+ "audio_length": 985.0013749999999,
32
+ "inference_time": 6.488993022998328,
33
+ "rtfx": 151.7957
34
+ }
35
+ }
36
+ ],
37
+ "help": "passed",
38
+ "component_parameters": {
39
+ "encoder_embed": 611953,
40
+ "encoder": 63382150,
41
+ "decoder": 262144,
42
+ "joiner": 781812,
43
+ "simple_am_proj": 256500,
44
+ "simple_lm_proj": 256500
45
+ },
46
+ "inference_sha256": "17ea7c209dc40b0540b3ccca49cef04e265211b38b3daf00333e1180abdaa39e",
47
+ "run_eval_sha256": "98aa19d45326ec8ceefd6707f096460e610fa9b886d8585996e4361c7b1bd56e",
48
+ "cpu_smoke": {
49
+ "prediction": "Yeah",
50
+ "parameters": 65551059
51
+ },
52
+ "all_passed": true,
53
+ "validation_scope": "Real inference plus official prepare_data/writer/scorer; local cached audio transport and metadata API stubs; not an HF Jobs or clean Docker build."
54
+ }
benchmark_results/dependency_validation.json ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "exit_code": 0,
3
+ "fresh_resolution": true,
4
+ "installed_packages": false,
5
+ "docker_build_performed": false,
6
+ "packages": {
7
+ "k2": "1.24.4.dev20260625+cuda12.4.torch2.6.0",
8
+ "kaldifeat": "1.25.5.dev20260626+cuda12.4.torch2.6.0",
9
+ "torch": "2.6.0+cu124",
10
+ "torchaudio": "2.6.0+cu124",
11
+ "numpy": "1.24.4",
12
+ "soundfile": "0.13.1",
13
+ "huggingface_hub": "1.18.0",
14
+ "datasets": "3.6.0",
15
+ "fsspec": "2025.3.0",
16
+ "librosa": "0.10.2.post1",
17
+ "pandas": "2.2.3",
18
+ "pyarrow": "20.0.0",
19
+ "kaldialign": "0.12.0",
20
+ "num2words": "0.5.14",
21
+ "regex": "2024.11.6",
22
+ "tqdm": "4.66.4",
23
+ "nvidia-cublas-cu12": "12.4.5.8",
24
+ "nvidia-cuda-cupti-cu12": "12.4.127",
25
+ "nvidia-cuda-nvrtc-cu12": "12.4.127",
26
+ "nvidia-cuda-runtime-cu12": "12.4.127",
27
+ "nvidia-cudnn-cu12": "9.1.0.70",
28
+ "nvidia-cufft-cu12": "11.2.1.3",
29
+ "nvidia-curand-cu12": "10.3.5.147",
30
+ "nvidia-cusolver-cu12": "11.6.1.9",
31
+ "nvidia-cusparse-cu12": "12.3.1.170",
32
+ "nvidia-cusparselt-cu12": "0.6.2",
33
+ "nvidia-nccl-cu12": "2.21.5",
34
+ "nvidia-nvjitlink-cu12": "12.4.127",
35
+ "nvidia-nvtx-cu12": "12.4.127",
36
+ "sympy": "1.13.1",
37
+ "triton": "3.2.0",
38
+ "dill": "0.3.8",
39
+ "hf-xet": "1.8.0",
40
+ "httpx": "0.28.1",
41
+ "httpcore": "1.0.9",
42
+ "mpmath": "1.3.0",
43
+ "multiprocess": "0.70.16",
44
+ "typer": "0.25.1",
45
+ "packaging": "26.3",
46
+ "aiohttp": "3.14.5",
47
+ "async-timeout": "5.0.1",
48
+ "multidict": "7.1.0",
49
+ "yarl": "1.25.1",
50
+ "aiohappyeyeballs": "2.7.1",
51
+ "aiosignal": "1.4.0",
52
+ "annotated-doc": "0.0.6",
53
+ "attrs": "26.1.0",
54
+ "audioread": "3.1.0",
55
+ "cffi": "2.1.1",
56
+ "click": "8.5.1",
57
+ "decorator": "5.3.1",
58
+ "docopt": "0.6.2",
59
+ "filelock": "4.1.0",
60
+ "frozenlist": "1.8.0",
61
+ "h11": "0.16.0",
62
+ "idna": "3.20",
63
+ "joblib": "1.6.0",
64
+ "cloudpickle": "3.1.3",
65
+ "lazy-loader": "0.6",
66
+ "msgpack": "1.2.4",
67
+ "numba": "0.68.1",
68
+ "llvmlite": "0.50.0",
69
+ "pooch": "1.9.0",
70
+ "platformdirs": "4.12.4",
71
+ "propcache": "0.5.4",
72
+ "python-dateutil": "3.0.0",
73
+ "pytz": "2026.6",
74
+ "PyYAML": "6.0.3",
75
+ "requests": "2.34.2",
76
+ "charset-normalizer": "3.5.2",
77
+ "urllib3": "2.8.0",
78
+ "certifi": "2026.10.23",
79
+ "rich": "15.0.0",
80
+ "Pygments": "2.21.0",
81
+ "markdown-it-py": "4.3.0",
82
+ "mdurl": "0.1.2",
83
+ "scikit-learn": "1.7.2",
84
+ "scipy": "1.15.3",
85
+ "shellingham": "1.5.4",
86
+ "six": "1.17.0",
87
+ "soxr": "1.1.0",
88
+ "threadpoolctl": "3.8.0",
89
+ "typing_extensions": "4.16.0",
90
+ "tzdata": "2026.6",
91
+ "anyio": "4.16.0",
92
+ "exceptiongroup": "1.3.1",
93
+ "graphviz": "0.21",
94
+ "Jinja2": "3.1.6",
95
+ "MarkupSafe": "3.0.4",
96
+ "networkx": "3.4.2",
97
+ "pycparser": "3.11",
98
+ "xxhash": "4.1.0"
99
+ }
100
+ }
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_ArtificialAnalysis-Earnings22-Cleaned-AA-chunked_earnings22_cleaned_aa_chunked_test.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5010bb0dcc3024b9d3dda5956734044a0d35ccaf9bb3214271efb9f03cc6ea92
3
+ size 11172360
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_VoiceArena-Monsoon_en_IN_test_monsoon_en_in_test.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_ami_cleaned_test.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_gigaspeech_cleaned_test.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e25623081b67f8c37db13966077e8f44eff25e47de61f96db2c5f2938ae1aaa0
3
+ size 16540413
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_spgispeech_test.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce7512cd10eff18d797cdc9efb167ada34b37c4beda1b5ba4bcde206ced0330b
3
+ size 41084800
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_clean_test.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_test.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_voxpopuli_cleaned_aa_test.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
benchmark_results/local_3090ti_20261119/REPRODUCIBILITY.txt ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ This folder contains the completed local RTX 3090 Ti evaluation dated 19 November 2026.
2
+ It is not an official cloud/H200 run and contains no private testset scores.
3
+
4
+ The checkpoint and token SHA256 hashes match the released root files.
5
+ Inference: FP32, causal=False, context_size=3, modified beam search, beam=4, VoiceA formatting, minimum1s input,30s internal chunks.
6
+ Batch configuration: up to2048 streams,5000 padded audio seconds,512 long-audio chunks,16PyTorch threads and8audio workers.
7
+ Final official scoring used one worker at repository commit4a113dd90c9ffd011d7d3d105da76980d511d029 after the original four-worker scorer stalled. Saved inference was not rerun.
8
+
9
+ Filenames were relabelled with the published Hub model id. Machine-specific audio paths were replaced by stable publicsource identifiers. Raw text, predictions, durations, inference timing and Earnings22 chunk metadata were preserved.
10
+ export_manifest.json maps original to released outputs and provides both hash sets. independent_validation.json records validation of the original outputs. current_scores.json is the authoritative final score summary. run_config.json is sanitized historical provenance, not an executable cloud command.
11
+
12
+ Default macro WER uses7publicsets; URGENT2024Clean is optional and excluded. Full-ranking averages may include private sets unavailable here.
benchmark_results/local_3090ti_20261119/current_scores.json ADDED
@@ -0,0 +1,170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "average_note": "Public seven-set macro-average only; the official default ranking also selects private score groups.",
3
+ "checked_at_utc": "2026-11-19T03:54:19.389175+00:00",
4
+ "complete": true,
5
+ "excluded_from_public_average": [
6
+ "urgent2024_clean"
7
+ ],
8
+ "model_file_prefix": "MODEL_SpeedAudio261119-SpeedASR-v1-65M",
9
+ "official_commit": "4a113dd90c9ffd011d7d3d105da76980d511d029",
10
+ "official_default_ranking_average": null,
11
+ "private_results_available": false,
12
+ "public_default_average_wer": 6.904285714285714,
13
+ "public_default_datasets": [
14
+ "ami_cleaned",
15
+ "gigaspeech_cleaned",
16
+ "voxpopuli_cleaned_aa",
17
+ "earnings22_cleaned_aa_chunked",
18
+ "spgispeech",
19
+ "monsoon_en_in",
20
+ "urgent2024"
21
+ ],
22
+ "public_default_pooled_rtfx": 192.6292762644339,
23
+ "test_sets": [
24
+ {
25
+ "audio_length": 28856.700437500032,
26
+ "cuts": 7805,
27
+ "dataset": "ami_cleaned",
28
+ "default_average": true,
29
+ "del": 2334,
30
+ "inference_time": 142.35132213500114,
31
+ "ins": 2053,
32
+ "manifest_sha256": "89462aa1a42190471e8616dd27e7d2403c126d67ed05e6a57e31a86bdee991d6",
33
+ "result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_ami_cleaned_test.jsonl",
34
+ "revision": "de79e57adf9983993bcc72586d7313460825f5eb",
35
+ "rtfx": 202.7147,
36
+ "scoring_rows": 7805,
37
+ "scoring_unit": "utterance",
38
+ "split": "test",
39
+ "sub": 3514,
40
+ "wer": 9.59
41
+ },
42
+ {
43
+ "audio_length": 126528.78700000017,
44
+ "cuts": 18768,
45
+ "dataset": "gigaspeech_cleaned",
46
+ "default_average": true,
47
+ "del": 5098,
48
+ "inference_time": 652.5400200150451,
49
+ "ins": 8548,
50
+ "manifest_sha256": "cbebef3e4f0d28b348ee9aa741090f8a9a48c7925eb08872a519b6e487b910c5",
51
+ "result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_gigaspeech_cleaned_test.jsonl",
52
+ "revision": "de79e57adf9983993bcc72586d7313460825f5eb",
53
+ "rtfx": 193.902,
54
+ "scoring_rows": 18768,
55
+ "scoring_unit": "utterance",
56
+ "split": "test",
57
+ "sub": 18396,
58
+ "wer": 8.07
59
+ },
60
+ {
61
+ "audio_length": 7122.378374999999,
62
+ "cuts": 628,
63
+ "dataset": "voxpopuli_cleaned_aa",
64
+ "default_average": true,
65
+ "del": 226,
66
+ "inference_time": 37.72588397499897,
67
+ "ins": 130,
68
+ "manifest_sha256": "61e855acf35048601fdcaed1c0ca10ba30ac7fb3a887d722c31a456c545605e1",
69
+ "result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_voxpopuli_cleaned_aa_test.jsonl",
70
+ "revision": "de79e57adf9983993bcc72586d7313460825f5eb",
71
+ "rtfx": 188.7929,
72
+ "scoring_rows": 628,
73
+ "scoring_unit": "utterance",
74
+ "split": "test",
75
+ "sub": 225,
76
+ "wer": 3.27
77
+ },
78
+ {
79
+ "audio_length": 6900.9376250000005,
80
+ "cuts": 341,
81
+ "dataset": "earnings22_cleaned_aa_chunked",
82
+ "default_average": true,
83
+ "del": 222,
84
+ "inference_time": 36.58612209400007,
85
+ "ins": 154,
86
+ "manifest_sha256": "d6d297c76ce3fa9208791d989bd15cbaaaff6fed545ad17ab96b7d4492beef70",
87
+ "result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_ArtificialAnalysis-Earnings22-Cleaned-AA-chunked_earnings22_cleaned_aa_chunked_test.jsonl",
88
+ "revision": "27f7d0056d593231974a752ce03c1e965da69b3f",
89
+ "rtfx": 188.6217,
90
+ "scoring_rows": 6,
91
+ "scoring_unit": "parent_session",
92
+ "split": "test",
93
+ "sub": 668,
94
+ "wer": 6.78
95
+ },
96
+ {
97
+ "audio_length": 360007.49950000306,
98
+ "cuts": 39341,
99
+ "dataset": "spgispeech",
100
+ "default_average": true,
101
+ "del": 4664,
102
+ "inference_time": 1881.6296108457523,
103
+ "ins": 19136,
104
+ "manifest_sha256": "f4a401536ffbefa5db8e4c9e7336301357c8d71a3d9eac16e026556456e0f44c",
105
+ "result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_spgispeech_test.jsonl",
106
+ "revision": "de79e57adf9983993bcc72586d7313460825f5eb",
107
+ "rtfx": 191.3275,
108
+ "scoring_rows": 39341,
109
+ "scoring_unit": "utterance",
110
+ "split": "test",
111
+ "sub": 13919,
112
+ "wer": 3.89
113
+ },
114
+ {
115
+ "audio_length": 20247.60743749999,
116
+ "cuts": 2102,
117
+ "dataset": "monsoon_en_in",
118
+ "default_average": true,
119
+ "del": 350,
120
+ "inference_time": 103.29725760700066,
121
+ "ins": 550,
122
+ "manifest_sha256": "571d17a51e0df966785330f3e7e232a39bbe00f3390e47ec20bfd3fb7a74b9b1",
123
+ "result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_VoiceArena-Monsoon_en_IN_test_monsoon_en_in_test.jsonl",
124
+ "revision": "bc1da7b42ef6e2853123c97bf6d22067e4802d11",
125
+ "rtfx": 196.013,
126
+ "scoring_rows": 2102,
127
+ "scoring_unit": "utterance",
128
+ "split": "test",
129
+ "sub": 1270,
130
+ "wer": 4.44
131
+ },
132
+ {
133
+ "audio_length": 6928.642312499995,
134
+ "cuts": 1000,
135
+ "dataset": "urgent2024",
136
+ "default_average": true,
137
+ "del": 814,
138
+ "inference_time": 35.319214283000115,
139
+ "ins": 296,
140
+ "manifest_sha256": "2015c656012b35660a3469702cd54af9a55a853e35ebe017bba6d39e24651983",
141
+ "result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_test.jsonl",
142
+ "revision": "de79e57adf9983993bcc72586d7313460825f5eb",
143
+ "rtfx": 196.172,
144
+ "scoring_rows": 1000,
145
+ "scoring_unit": "utterance",
146
+ "split": "test",
147
+ "sub": 1170,
148
+ "wer": 12.29
149
+ },
150
+ {
151
+ "audio_length": 6928.642312499995,
152
+ "cuts": 1000,
153
+ "dataset": "urgent2024_clean",
154
+ "default_average": false,
155
+ "del": 260,
156
+ "inference_time": 35.618651621000595,
157
+ "ins": 268,
158
+ "manifest_sha256": "c260695c3d61864b7e220fed2993680c6e80f107519d7df0a3c6cb849d9bb535",
159
+ "result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_clean_test.jsonl",
160
+ "revision": "de79e57adf9983993bcc72586d7313460825f5eb",
161
+ "rtfx": 194.5229,
162
+ "scoring_rows": 1000,
163
+ "scoring_unit": "utterance",
164
+ "split": "test",
165
+ "sub": 499,
166
+ "wer": 5.54
167
+ }
168
+ ],
169
+ "export_note": "Local inference and final scoring statistics are unchanged. Result filenames now use the published Hub id, and audio_filepath is a stable public identifier. Original manifest SHA256 values refer to the original scored outputs; released output SHA256 values are in export_manifest.json."
170
+ }
benchmark_results/local_3090ti_20261119/export_manifest.json ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_id": "SpeedAudio261119/SpeedASR-v1-65M",
3
+ "transformation": "Changed filenames to the released model id; audio_filepath replaced with stable public source_id/cut_id. Raw references, predictions, durations, timing and chunk metadata remain unchanged. Original verification hashes refer to the original files; released hashes are recorded here.",
4
+ "files": [
5
+ {
6
+ "original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_ArtificialAnalysis-Earnings22-Cleaned-AA-chunked_earnings22_cleaned_aa_chunked_test.jsonl",
7
+ "original_sha256": "d6d297c76ce3fa9208791d989bd15cbaaaff6fed545ad17ab96b7d4492beef70",
8
+ "released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_ArtificialAnalysis-Earnings22-Cleaned-AA-chunked_earnings22_cleaned_aa_chunked_test.jsonl",
9
+ "released_sha256": "5010bb0dcc3024b9d3dda5956734044a0d35ccaf9bb3214271efb9f03cc6ea92",
10
+ "rows": 341
11
+ },
12
+ {
13
+ "original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_VoiceArena-Monsoon_en_IN_test_monsoon_en_in_test.jsonl",
14
+ "original_sha256": "571d17a51e0df966785330f3e7e232a39bbe00f3390e47ec20bfd3fb7a74b9b1",
15
+ "released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_VoiceArena-Monsoon_en_IN_test_monsoon_en_in_test.jsonl",
16
+ "released_sha256": "fd947f47d55206b7e720a96e8ce2d704b5213c3e6709c83a35a1d8138df0611c",
17
+ "rows": 2102
18
+ },
19
+ {
20
+ "original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_hf-audio-open-asr-leaderboard_ami_cleaned_test.jsonl",
21
+ "original_sha256": "89462aa1a42190471e8616dd27e7d2403c126d67ed05e6a57e31a86bdee991d6",
22
+ "released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_ami_cleaned_test.jsonl",
23
+ "released_sha256": "73b4f9d3c03e519f70a5897ae20b5195a9733a1f9baa8b9ff0618c590d86fb14",
24
+ "rows": 7805
25
+ },
26
+ {
27
+ "original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_hf-audio-open-asr-leaderboard_gigaspeech_cleaned_test.jsonl",
28
+ "original_sha256": "cbebef3e4f0d28b348ee9aa741090f8a9a48c7925eb08872a519b6e487b910c5",
29
+ "released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_gigaspeech_cleaned_test.jsonl",
30
+ "released_sha256": "e25623081b67f8c37db13966077e8f44eff25e47de61f96db2c5f2938ae1aaa0",
31
+ "rows": 18768
32
+ },
33
+ {
34
+ "original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_hf-audio-open-asr-leaderboard_spgispeech_test.jsonl",
35
+ "original_sha256": "f4a401536ffbefa5db8e4c9e7336301357c8d71a3d9eac16e026556456e0f44c",
36
+ "released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_spgispeech_test.jsonl",
37
+ "released_sha256": "ce7512cd10eff18d797cdc9efb167ada34b37c4beda1b5ba4bcde206ced0330b",
38
+ "rows": 39341
39
+ },
40
+ {
41
+ "original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_hf-audio-open-asr-leaderboard_urgent2024_clean_test.jsonl",
42
+ "original_sha256": "c260695c3d61864b7e220fed2993680c6e80f107519d7df0a3c6cb849d9bb535",
43
+ "released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_clean_test.jsonl",
44
+ "released_sha256": "bf1f804714c7ae77d742a24cb499a3ddb50934e03ca8ce8b91c0c92928a1321a",
45
+ "rows": 1000
46
+ },
47
+ {
48
+ "original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_hf-audio-open-asr-leaderboard_urgent2024_test.jsonl",
49
+ "original_sha256": "2015c656012b35660a3469702cd54af9a55a853e35ebe017bba6d39e24651983",
50
+ "released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_test.jsonl",
51
+ "released_sha256": "30edc6b2c7dcf13c664682f5ca06fb7b2dace5bb92cde4e8710688729206a887",
52
+ "rows": 1000
53
+ },
54
+ {
55
+ "original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_hf-audio-open-asr-leaderboard_voxpopuli_cleaned_aa_test.jsonl",
56
+ "original_sha256": "61e855acf35048601fdcaed1c0ca10ba30ac7fb3a887d722c31a456c545605e1",
57
+ "released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_voxpopuli_cleaned_aa_test.jsonl",
58
+ "released_sha256": "1bb76945bcb6f8b6fb89420e10615b1ab310160391894062d53f443b3ff79d51",
59
+ "rows": 628
60
+ }
61
+ ]
62
+ }
benchmark_results/local_3090ti_20261119/independent_validation.json ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "state": "passed",
3
+ "complete": true,
4
+ "checked_at": "2026-11-19T03:54:19.864649+00:00",
5
+ "results_dir": "$RESULTS_DIR",
6
+ "provenance": {
7
+ "checkpoint_sha256": "b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01",
8
+ "tokens_sha256": "a238b7a0fa67aa6be998ca13dd129d69c50639970583e32232dd149e5448210a",
9
+ "runner_sha256": {
10
+ "benchmark_offline_pytorch.py": "e0d9b56d8334f56011ec927ea55411fe26d96317159458e9cff1bcd34601deb6",
11
+ "run_open_asr.py": "a8c336887d97b241cd58ea03f1ef5dc481bbb877ed4425c5dd0898d56f7b49a4",
12
+ "score_open_asr.py": "b3959a5a8e66e86b3f9923996009a7c3f66d62105499518206cf37bae8017d73",
13
+ "supervise_benchmark.py": "426862a8ad2d33d38991f9d56ee0556ad114fdc01bac23e9eaf89dca189004b5"
14
+ },
15
+ "manifest_index_sha256": "b09ba6209d2b96a092f5cc605e8b9045abc1b1d4e3ff566bd57f002b336fbedb",
16
+ "run_config_sha256": "4e3b9b0a0533c8d27de4118fb4a0e104729f386679f6792bb622d89020a58b97",
17
+ "input_provenance_sha256": "170cc1f5caab32df0f7de151fdf46673d3fa6fed7b7b8145a513c1cb7206f1b1",
18
+ "official_commit": "4a113dd90c9ffd011d7d3d105da76980d511d029",
19
+ "model_code_git_commit": "3f848bb6d0acc970c9b294a30ca0a04a7c9c78d1"
20
+ },
21
+ "test_sets": [
22
+ {
23
+ "dataset": "ami_cleaned",
24
+ "rows": 7805,
25
+ "scoring_rows": 7805,
26
+ "default_average": true,
27
+ "result_sha256": "89462aa1a42190471e8616dd27e7d2403c126d67ed05e6a57e31a86bdee991d6",
28
+ "cuts_manifest_sha256": "5153cc565bc9115d0ab8c48a18a14effb5a372ad781740a105ebefc68e98b590",
29
+ "wer": 9.59,
30
+ "ins": 2053,
31
+ "del": 2334,
32
+ "sub": 3514,
33
+ "audio_length": 28856.700437500032,
34
+ "inference_time": 142.35132213500114,
35
+ "rtfx": 202.7147
36
+ },
37
+ {
38
+ "dataset": "gigaspeech_cleaned",
39
+ "rows": 18768,
40
+ "scoring_rows": 18768,
41
+ "default_average": true,
42
+ "result_sha256": "cbebef3e4f0d28b348ee9aa741090f8a9a48c7925eb08872a519b6e487b910c5",
43
+ "cuts_manifest_sha256": "79a86209fcbdbd0e2263a0b28fd58c801a29583fe0073102cec302de8fe025e0",
44
+ "wer": 8.07,
45
+ "ins": 8548,
46
+ "del": 5098,
47
+ "sub": 18396,
48
+ "audio_length": 126528.78700000017,
49
+ "inference_time": 652.5400200150451,
50
+ "rtfx": 193.902
51
+ },
52
+ {
53
+ "dataset": "voxpopuli_cleaned_aa",
54
+ "rows": 628,
55
+ "scoring_rows": 628,
56
+ "default_average": true,
57
+ "result_sha256": "61e855acf35048601fdcaed1c0ca10ba30ac7fb3a887d722c31a456c545605e1",
58
+ "cuts_manifest_sha256": "2e59bbe6fb67984168b3708def118f7b0ca2ece581f5f52f79a291f047e8f92c",
59
+ "wer": 3.27,
60
+ "ins": 130,
61
+ "del": 226,
62
+ "sub": 225,
63
+ "audio_length": 7122.378374999999,
64
+ "inference_time": 37.72588397499897,
65
+ "rtfx": 188.7929
66
+ },
67
+ {
68
+ "dataset": "earnings22_cleaned_aa_chunked",
69
+ "rows": 341,
70
+ "scoring_rows": 6,
71
+ "default_average": true,
72
+ "result_sha256": "d6d297c76ce3fa9208791d989bd15cbaaaff6fed545ad17ab96b7d4492beef70",
73
+ "cuts_manifest_sha256": "05828c77118824dbe506395891e930ac5222b6cadd67f19323eb03f67235e8c5",
74
+ "wer": 6.78,
75
+ "ins": 154,
76
+ "del": 222,
77
+ "sub": 668,
78
+ "audio_length": 6900.9376250000005,
79
+ "inference_time": 36.58612209400007,
80
+ "rtfx": 188.6217,
81
+ "parent_chunk_counts": {
82
+ "4468919_trimmed": 51,
83
+ "4471586": 62,
84
+ "4474955": 44,
85
+ "4475604": 61,
86
+ "4482968": 66,
87
+ "4483046": 57
88
+ },
89
+ "earnings_order_independence": "passed"
90
+ },
91
+ {
92
+ "dataset": "spgispeech",
93
+ "rows": 39341,
94
+ "scoring_rows": 39341,
95
+ "default_average": true,
96
+ "result_sha256": "f4a401536ffbefa5db8e4c9e7336301357c8d71a3d9eac16e026556456e0f44c",
97
+ "cuts_manifest_sha256": "878e6e0c4a5b6bad505d50d5f66ce22dd9e24aec327d6d06a3ec161e66f47112",
98
+ "wer": 3.89,
99
+ "ins": 19136,
100
+ "del": 4664,
101
+ "sub": 13919,
102
+ "audio_length": 360007.49950000306,
103
+ "inference_time": 1881.6296108457523,
104
+ "rtfx": 191.3275
105
+ },
106
+ {
107
+ "dataset": "monsoon_en_in",
108
+ "rows": 2102,
109
+ "scoring_rows": 2102,
110
+ "default_average": true,
111
+ "result_sha256": "571d17a51e0df966785330f3e7e232a39bbe00f3390e47ec20bfd3fb7a74b9b1",
112
+ "cuts_manifest_sha256": "7fac09464f571bf0777fb7d19015704946033fa64bd7ce0fd5a20658dc5fe719",
113
+ "wer": 4.44,
114
+ "ins": 550,
115
+ "del": 350,
116
+ "sub": 1270,
117
+ "audio_length": 20247.60743749999,
118
+ "inference_time": 103.29725760700066,
119
+ "rtfx": 196.013
120
+ },
121
+ {
122
+ "dataset": "urgent2024",
123
+ "rows": 1000,
124
+ "scoring_rows": 1000,
125
+ "default_average": true,
126
+ "result_sha256": "2015c656012b35660a3469702cd54af9a55a853e35ebe017bba6d39e24651983",
127
+ "cuts_manifest_sha256": "bbf812820676c8beaf2b1f7bdfca9a0a1a33b8adadc50a43076e8970d7af6dba",
128
+ "wer": 12.29,
129
+ "ins": 296,
130
+ "del": 814,
131
+ "sub": 1170,
132
+ "audio_length": 6928.642312499995,
133
+ "inference_time": 35.319214283000115,
134
+ "rtfx": 196.172
135
+ },
136
+ {
137
+ "dataset": "urgent2024_clean",
138
+ "rows": 1000,
139
+ "scoring_rows": 1000,
140
+ "default_average": false,
141
+ "result_sha256": "c260695c3d61864b7e220fed2993680c6e80f107519d7df0a3c6cb849d9bb535",
142
+ "cuts_manifest_sha256": "39f91cd92c672659ba4d6f399ff6007e9279811dc4931a7162b82460e51b9f99",
143
+ "wer": 5.54,
144
+ "ins": 268,
145
+ "del": 260,
146
+ "sub": 499,
147
+ "audio_length": 6928.642312499995,
148
+ "inference_time": 35.618651621000595,
149
+ "rtfx": 194.5229
150
+ }
151
+ ],
152
+ "source_alignment_report_sha256": "5c881b63e09401193d48c972668bdde4650c5c8ed000bc4aedfe9acb761e0877",
153
+ "scope": "Independent exported-row/provenance validation and official CPU rescoring; no inference or raw dataset download rehash.",
154
+ "total_rows": 70985,
155
+ "test_set_count": 8,
156
+ "public_default_test_sets": 7,
157
+ "public_default_average_wer": 6.904285714285714,
158
+ "public_default_pooled_rtfx": 192.6292762644339,
159
+ "official_default_ranking_average": null,
160
+ "current_scores_sha256": "b510e95ac08af92186551bbd402e03e6a6dd3e0f1bf3e6812c859c688682364a",
161
+ "official_scorer_sha256": "ddb5bc93e77b3bc2e5397bed09f32bf3aa9953546c822d25f30b957d6a06e9fd"
162
+ }
benchmark_results/local_3090ti_20261119/input_provenance.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "$MODEL_DIR/pretrained.pt",
3
+ "checkpoint_sha256": "b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01",
4
+ "tokens": "$ICEFALL_ROOT/egs/librispeech/ASR/data/lang_bpe_500/tokens.txt",
5
+ "tokens_sha256": "a238b7a0fa67aa6be998ca13dd129d69c50639970583e32232dd149e5448210a",
6
+ "official_commit": "4a113dd90c9ffd011d7d3d105da76980d511d029",
7
+ "manifest_dir": "$BENCHMARK_ROOT/data/english_shortform_20261118",
8
+ "runner_script": "$BENCHMARK_ROOT/scripts/benchmark_epoch30_20261119/run_open_asr.py",
9
+ "runner_sha256": {
10
+ "benchmark_offline_pytorch.py": "e0d9b56d8334f56011ec927ea55411fe26d96317159458e9cff1bcd34601deb6",
11
+ "run_open_asr.py": "a8c336887d97b241cd58ea03f1ef5dc481bbb877ed4425c5dd0898d56f7b49a4",
12
+ "score_open_asr.py": "b3959a5a8e66e86b3f9923996009a7c3f66d62105499518206cf37bae8017d73",
13
+ "supervise_benchmark.py": "426862a8ad2d33d38991f9d56ee0556ad114fdc01bac23e9eaf89dca189004b5"
14
+ },
15
+ "model_code_git_commit": "3f848bb6d0acc970c9b294a30ca0a04a7c9c78d1"
16
+ }
benchmark_results/local_3090ti_20261119/run_config.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "audio_workers": 8,
3
+ "batch_audio_seconds": 5000.0,
4
+ "beam_size": 4,
5
+ "causal": false,
6
+ "checkpoint": "$MODEL_DIR/pretrained.pt",
7
+ "checkpoint_sha256": "b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01",
8
+ "command": [
9
+ "$BENCHMARK_ROOT/.venv/bin/python",
10
+ "-u",
11
+ "-B",
12
+ "$BENCHMARK_ROOT/scripts/benchmark_epoch30_20261119/benchmark_offline_pytorch.py",
13
+ "--work-dir",
14
+ "$BENCHMARK_ROOT",
15
+ "--open-asr-repo",
16
+ "$BENCHMARK_ROOT/open_asr_leaderboard_20261119",
17
+ "--manifest-dir",
18
+ "$BENCHMARK_ROOT/data/english_shortform_20261118",
19
+ "--results-dir",
20
+ "$RESULTS_DIR",
21
+ "--model-id",
22
+ "jianyou/zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9",
23
+ "--model-dir",
24
+ "$MODEL_DIR",
25
+ "--checkpoint",
26
+ "$MODEL_DIR/pretrained.pt",
27
+ "--tokens",
28
+ "$ICEFALL_ROOT/egs/librispeech/ASR/data/lang_bpe_500/tokens.txt",
29
+ "--icefall-root",
30
+ "$ICEFALL_ROOT",
31
+ "--zipformer-dir",
32
+ "$ICEFALL_ROOT/egs/librispeech/ASR/zipformer",
33
+ "--context-size",
34
+ "3",
35
+ "--decoding-method",
36
+ "modified_beam_search",
37
+ "--num-active-paths",
38
+ "4",
39
+ "--num-decode-streams",
40
+ "2048",
41
+ "--batch-audio-seconds",
42
+ "5000.0",
43
+ "--long-chunk-batch-size",
44
+ "512",
45
+ "--torch-threads",
46
+ "16",
47
+ "--audio-workers",
48
+ "8",
49
+ "--device-id",
50
+ "0",
51
+ "--voicea"
52
+ ],
53
+ "context_size": 3,
54
+ "decoding_method": "modified_beam_search",
55
+ "dry_run": false,
56
+ "force": false,
57
+ "gpu": 0,
58
+ "icefall_root": "$ICEFALL_ROOT",
59
+ "long_chunk_batch_size": 512,
60
+ "manifest_dir": "$BENCHMARK_ROOT/data/english_shortform_20261118",
61
+ "manifest_index_sha256": "b09ba6209d2b96a092f5cc605e8b9045abc1b1d4e3ff566bd57f002b336fbedb",
62
+ "model_id": "jianyou/zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9",
63
+ "num_decode_streams": 2048,
64
+ "official_commit": "4a113dd90c9ffd011d7d3d105da76980d511d029",
65
+ "open_asr_repo": "$BENCHMARK_ROOT/open_asr_leaderboard_20261119",
66
+ "precision": "fp32",
67
+ "preflight_only": false,
68
+ "results_dir": "$RESULTS_DIR",
69
+ "score_workers": 4,
70
+ "tokens": "$ICEFALL_ROOT/egs/librispeech/ASR/data/lang_bpe_500/tokens.txt",
71
+ "torch_threads": 16
72
+ }
benchmark_results/portable_validation.json ADDED
@@ -0,0 +1,228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "parameters": 65551059,
3
+ "device": "cuda:0",
4
+ "torch": "2.6.0+cu124",
5
+ "compile": "passed",
6
+ "strict_checkpoint": "passed",
7
+ "cases": [
8
+ {
9
+ "dataset": "ArtificialAnalysis__Earnings22-Cleaned-AA-chunked__earnings22_cleaned_aa_chunked__test.cuts.jsonl.gz",
10
+ "id": "00000124_2b6f28ab-45a2-59fc-b4ba-ff23567dd270_1c6057d858",
11
+ "seconds": 2.322,
12
+ "matches_original": true,
13
+ "original_prediction": "Is slower than expected .",
14
+ "portable_prediction": "Is slower than expected ."
15
+ },
16
+ {
17
+ "dataset": "ArtificialAnalysis__Earnings22-Cleaned-AA-chunked__earnings22_cleaned_aa_chunked__test.cuts.jsonl.gz",
18
+ "id": "00000075_9d3ce16b-47c2-5d75-8ed2-5a4989527b34_652f49e385",
19
+ "seconds": 24.416,
20
+ "matches_original": true,
21
+ "original_prediction": "So in November we received a score of fifty seven and S P Global CSA and sustainables ESG risk rating of twenty five point zero . And also MSCI confirmed our MSCI ESG rating at A . That's that's uh all uh as for the uh management comments , and we are ready to take your questions .",
22
+ "portable_prediction": "So in November we received a score of fifty seven and S P Global CSA and sustainables ESG risk rating of twenty five point zero . And also MSCI confirmed our MSCI ESG rating at A . That's that's uh all uh as for the uh management comments , and we are ready to take your questions ."
23
+ },
24
+ {
25
+ "dataset": "ArtificialAnalysis__Earnings22-Cleaned-AA-chunked__earnings22_cleaned_aa_chunked__test.cuts.jsonl.gz",
26
+ "id": "00000005_10793ea8-44a5-5699-aa20-4e74cd1976d6_ba691e780d",
27
+ "seconds": 29.984,
28
+ "matches_original": true,
29
+ "original_prediction": "We assume no obligation to update or revise any forward looking statements to reflect new or changed events or circumstances . This conference call will also include discussion on non gap financial measures . The full reconciliation to the corresponding financial measures is included in the ernest press release . Now I would like to turn the call over to Sargo . Thank you , Diego . Hello everyone and thank you for showing us today .",
30
+ "portable_prediction": "We assume no obligation to update or revise any forward looking statements to reflect new or changed events or circumstances . This conference call will also include discussion on non gap financial measures . The full reconciliation to the corresponding financial measures is included in the ernest press release . Now I would like to turn the call over to Sargo . Thank you , Diego . Hello everyone and thank you for showing us today ."
31
+ },
32
+ {
33
+ "dataset": "VoiceArena__Monsoon_en_IN_test__monsoon_en_in__test.cuts.jsonl.gz",
34
+ "id": "00000384_80041517_12_72_13_95_86406836b9",
35
+ "seconds": 1.23,
36
+ "matches_original": true,
37
+ "original_prediction": "Can you hear me ?",
38
+ "portable_prediction": "Can you hear me ?"
39
+ },
40
+ {
41
+ "dataset": "VoiceArena__Monsoon_en_IN_test__monsoon_en_in__test.cuts.jsonl.gz",
42
+ "id": "00000881_25383873_120_3_130_71_51f9103de9",
43
+ "seconds": 10.41,
44
+ "matches_original": true,
45
+ "original_prediction": "Yeah . I have also felt that on that particular time everyone is just advising me to left this or to do some other things .",
46
+ "portable_prediction": "Yeah . I have also felt that on that particular time everyone is just advising me to left this or to do some other things ."
47
+ },
48
+ {
49
+ "dataset": "VoiceArena__Monsoon_en_IN_test__monsoon_en_in__test.cuts.jsonl.gz",
50
+ "id": "00000848_92653570_353_07_368_07_602c1b0efd",
51
+ "seconds": 15.0,
52
+ "matches_original": true,
53
+ "original_prediction": "Fruit . My my favorite fruit is apple because I am farmer from Himachal Pradesh , I grow apples in my farms . Which is your favorite fruit ? Oh , mango , very nice choice .",
54
+ "portable_prediction": "Fruit . My my favorite fruit is apple because I am farmer from Himachal Pradesh , I grow apples in my farms . Which is your favorite fruit ? Oh , mango , very nice choice ."
55
+ },
56
+ {
57
+ "dataset": "hf-audio__open-asr-leaderboard__ami_cleaned__test.cuts.jsonl.gz",
58
+ "id": "00007778_AMI_EN2002a_H03_MEE071_0047016_0047116_1cf8ac5dc8",
59
+ "seconds": 1.0,
60
+ "matches_original": true,
61
+ "original_prediction": "I didn't quite understand .",
62
+ "portable_prediction": "I didn't quite understand ."
63
+ },
64
+ {
65
+ "dataset": "hf-audio__open-asr-leaderboard__ami_cleaned__test.cuts.jsonl.gz",
66
+ "id": "00003890_AMI_EN2002a_H01_FEO070_0212701_0212989_7cb11459f8",
67
+ "seconds": 2.88,
68
+ "matches_original": true,
69
+ "original_prediction": "Yeah .",
70
+ "portable_prediction": "Yeah ."
71
+ },
72
+ {
73
+ "dataset": "hf-audio__open-asr-leaderboard__ami_cleaned__test.cuts.jsonl.gz",
74
+ "id": "00000000_AMI_IS1009b_H03_FIO089_0023485_0026102_c32dfaec4b",
75
+ "seconds": 26.1699375,
76
+ "matches_original": true,
77
+ "original_prediction": "Um and then I'll be coming to you as the user interface person to try to tell me from your point of view what are the most friendly features that we could put on it and try to h prior help me with that prioritizing of uh of the features and of the the look and the color and I'll be coming back to you to help weed out those suggestions from that point of view .",
78
+ "portable_prediction": "Um and then I'll be coming to you as the user interface person to try to tell me from your point of view what are the most friendly features that we could put on it and try to h prior help me with that prioritizing of uh of the features and of the the look and the color and I'll be coming back to you to help weed out those suggestions from that point of view ."
79
+ },
80
+ {
81
+ "dataset": "hf-audio__open-asr-leaderboard__gigaspeech_cleaned__test.cuts.jsonl.gz",
82
+ "id": "00018750_POD1000000040_S0000787_13c0467f52",
83
+ "seconds": 1.0,
84
+ "matches_original": true,
85
+ "original_prediction": "Oh I can't see it .",
86
+ "portable_prediction": "Oh I can't see it ."
87
+ },
88
+ {
89
+ "dataset": "hf-audio__open-asr-leaderboard__gigaspeech_cleaned__test.cuts.jsonl.gz",
90
+ "id": "00009382_YOU1000000140_S0000081_47adf75a06",
91
+ "seconds": 6.509,
92
+ "matches_original": true,
93
+ "original_prediction": "it's only y'all two out here , Dad ain't nobody else out here with y'all . So you telling me he that fast ? They not jumping me ? Ain't nobody jumping me ?",
94
+ "portable_prediction": "it's only y'all two out here , Dad ain't nobody else out here with y'all . So you telling me he that fast ? They not jumping me ? Ain't nobody jumping me ?"
95
+ },
96
+ {
97
+ "dataset": "hf-audio__open-asr-leaderboard__gigaspeech_cleaned__test.cuts.jsonl.gz",
98
+ "id": "00000000_POD1000000005_S0000339_7d69478636",
99
+ "seconds": 22.025,
100
+ "matches_original": true,
101
+ "original_prediction": "And , you know , those stores look at trend reports and they look at what's on the runway . And part of their great uh skill in both marketing and manufacturing is being able to take those ideas and turn them around at just a rapid fire pace and get them into stores .",
102
+ "portable_prediction": "And , you know , those stores look at trend reports and they look at what's on the runway . And part of their great uh skill in both marketing and manufacturing is being able to take those ideas and turn them around at just a rapid fire pace and get them into stores ."
103
+ },
104
+ {
105
+ "dataset": "hf-audio__open-asr-leaderboard__spgispeech__test.cuts.jsonl.gz",
106
+ "id": "00039340_403903eb119bfc1cec44d86ec907077a__123.wav_30a8eca6f1",
107
+ "seconds": 5.0099375,
108
+ "matches_original": true,
109
+ "original_prediction": "Has sort of put a little bit of um sand in the gears in the near short term .",
110
+ "portable_prediction": "Has sort of put a little bit of um sand in the gears in the near short term ."
111
+ },
112
+ {
113
+ "dataset": "hf-audio__open-asr-leaderboard__spgispeech__test.cuts.jsonl.gz",
114
+ "id": "00019757_9826162bc1bb67bf8d65e4cf50e656e9__85.wav_351da744aa",
115
+ "seconds": 8.85,
116
+ "matches_original": true,
117
+ "original_prediction": "The first thing that we wanted to drive during the second half was generating more proximity with our customer to drive collaboration .",
118
+ "portable_prediction": "The first thing that we wanted to drive during the second half was generating more proximity with our customer to drive collaboration ."
119
+ },
120
+ {
121
+ "dataset": "hf-audio__open-asr-leaderboard__spgispeech__test.cuts.jsonl.gz",
122
+ "id": "00000051_97b02b7a41ecb22cb04e7331052cefc8__39.wav_ad39d97ffb",
123
+ "seconds": 15.0,
124
+ "matches_original": true,
125
+ "original_prediction": "They'll ask us to look at that . So it's really it's a little bit of a two way street . a lot of our opportunities come from client driven needs that they're talking to our account management and sales team about all the time .",
126
+ "portable_prediction": "They'll ask us to look at that . So it's really it's a little bit of a two way street . a lot of our opportunities come from client driven needs that they're talking to our account management and sales team about all the time ."
127
+ },
128
+ {
129
+ "dataset": "hf-audio__open-asr-leaderboard__urgent2024__test.cuts.jsonl.gz",
130
+ "id": "00000999_fileid_636_94308ea578",
131
+ "seconds": 2.51,
132
+ "matches_original": true,
133
+ "original_prediction": "There were benches for twenty men along each side .",
134
+ "portable_prediction": "There were benches for twenty men along each side ."
135
+ },
136
+ {
137
+ "dataset": "hf-audio__open-asr-leaderboard__urgent2024__test.cuts.jsonl.gz",
138
+ "id": "00000499_fileid_59_7291623332",
139
+ "seconds": 6.9499375,
140
+ "matches_original": true,
141
+ "original_prediction": "Uh I think I should like it of all things replied Philip , some hesitation . But what more ?",
142
+ "portable_prediction": "Uh I think I should like it of all things replied Philip , some hesitation . But what more ?"
143
+ },
144
+ {
145
+ "dataset": "hf-audio__open-asr-leaderboard__urgent2024__test.cuts.jsonl.gz",
146
+ "id": "00000000_fileid_401_ff740dcae0",
147
+ "seconds": 14.73,
148
+ "matches_original": true,
149
+ "original_prediction": "We may even have used words to describe it to ourselves while it was happening . In that case , the habit of these words may fulfill the function of Bergson's true memory , while in reality it is nothing but habit moving .",
150
+ "portable_prediction": "We may even have used words to describe it to ourselves while it was happening . In that case , the habit of these words may fulfill the function of Bergson's true memory , while in reality it is nothing but habit moving ."
151
+ },
152
+ {
153
+ "dataset": "hf-audio__open-asr-leaderboard__urgent2024_clean__test.cuts.jsonl.gz",
154
+ "id": "00000999_fileid_636_f03b4eb095",
155
+ "seconds": 2.51,
156
+ "matches_original": true,
157
+ "original_prediction": "There were benches for twenty men along each side .",
158
+ "portable_prediction": "There were benches for twenty men along each side ."
159
+ },
160
+ {
161
+ "dataset": "hf-audio__open-asr-leaderboard__urgent2024_clean__test.cuts.jsonl.gz",
162
+ "id": "00000499_fileid_59_2c39dc0950",
163
+ "seconds": 6.9499375,
164
+ "matches_original": true,
165
+ "original_prediction": "I think I should like it , of all things , replied Philip , with some hesitation . But what for ?",
166
+ "portable_prediction": "I think I should like it , of all things , replied Philip , with some hesitation . But what for ?"
167
+ },
168
+ {
169
+ "dataset": "hf-audio__open-asr-leaderboard__urgent2024_clean__test.cuts.jsonl.gz",
170
+ "id": "00000000_fileid_401_27b267791c",
171
+ "seconds": 14.73,
172
+ "matches_original": true,
173
+ "original_prediction": "We may even have used words to describe it to ourselves while it was happening . In that case , the habit of these words may fulfill the function of Bergson's true memory , while in reality it is nothing but habit memory .",
174
+ "portable_prediction": "We may even have used words to describe it to ourselves while it was happening . In that case , the habit of these words may fulfill the function of Bergson's true memory , while in reality it is nothing but habit memory ."
175
+ },
176
+ {
177
+ "dataset": "hf-audio__open-asr-leaderboard__voxpopuli_cleaned_aa__test.cuts.jsonl.gz",
178
+ "id": "00000627_20090421-0900-PLENARY-17-en_20090421-22_22_06_2_f1fcb752d2",
179
+ "seconds": 5.0194375,
180
+ "matches_original": true,
181
+ "original_prediction": "Cently though , I have been able to assure people that the policy is undergoing significant change .",
182
+ "portable_prediction": "Cently though , I have been able to assure people that the policy is undergoing significant change ."
183
+ },
184
+ {
185
+ "dataset": "hf-audio__open-asr-leaderboard__voxpopuli_cleaned_aa__test.cuts.jsonl.gz",
186
+ "id": "00000314_20110405-0900-PLENARY-7-en_20110405-15_47_36_0_d327a19898",
187
+ "seconds": 9.94,
188
+ "matches_original": true,
189
+ "original_prediction": "Mr President , the March European Council confirmed the EU role as a promoter of the highest nuclear safety standards worldwide .",
190
+ "portable_prediction": "Mr President , the March European Council confirmed the EU role as a promoter of the highest nuclear safety standards worldwide ."
191
+ },
192
+ {
193
+ "dataset": "hf-audio__open-asr-leaderboard__voxpopuli_cleaned_aa__test.cuts.jsonl.gz",
194
+ "id": "00000000_20181210-0900-PLENARY-en_20181210-22_04_34_4_5456d2323a",
195
+ "seconds": 38.435625,
196
+ "matches_original": true,
197
+ "original_prediction": "Cedefop's role is to contribute to the shaping and implementing of VET , but also shaping skills and qualification policies at Union level . The Agency can do this by providing crucial evidence and services for policy making and knowledge sharing amongst the Union and national actors , in particular to governments and social partners . I believe that the revision of the functioning of this tripartite agency has been a very good one and I fully appreciate the agency the agency's great work in supporting and developing inclusive and quality VAT systems . Now we have to work together that we put in place .",
198
+ "portable_prediction": "Cedefop's role is to contribute to the shaping and implementing of VET , but also shaping skills and qualification policies at Union level . The Agency can do this by providing crucial evidence and services for policy making and knowledge sharing amongst the Union and national actors , in particular to governments and social partners . I believe that the revision of the functioning of this tripartite agency has been a very good one and I fully appreciate the agency the agency's great work in supporting and developing inclusive and quality VAT systems . Now we have to work together that we put in place ."
199
+ }
200
+ ],
201
+ "original_runner_sha256": "e0d9b56d8334f56011ec927ea55411fe26d96317159458e9cff1bcd34601deb6",
202
+ "inference_sha256": "17ea7c209dc40b0540b3ccca49cef04e265211b38b3daf00333e1180abdaa39e",
203
+ "core_sha256": {
204
+ "__init__.py": "596ac4de65fae9944440e22cf00ec4629b80da70c2e9b309e0702e4686ac988e",
205
+ "beam_search.py": "d2e0297d1626b14b834a9b357caa875a5b11ce8f0aabc475debb7d20dec3b8c1",
206
+ "decoder.py": "a9aeeeef278f4e5357d4e49253d24f5f1c60df88cfcb623ccbd01481dc1503ce",
207
+ "encoder_interface.py": "fc70f530c33e2489435ff2cb4bb0dad8b9d6f58a248c425cbd6100c62f628569",
208
+ "joiner.py": "81faad6e0d07342d1abab28f2271201256f2139f5816b04296d6a8cbe0e2c887",
209
+ "model.py": "46b8e181fb326e8cd82c92a4c0ffd0917eb1474a26bd5b60b39b48daac28e90f",
210
+ "scaling.py": "679b1fe2655902ba13ef3f5cf7c924129a16681e3edc4343e766ff7086f5e6d7",
211
+ "subsampling.py": "dc33ac662d2508e191f4da5562f0c8d23444819b707624736429a85517db9145",
212
+ "utils.py": "682d0733f0e149be0ae9b62cb8de60d7baef65be4e64816001b6bb089b4e787b",
213
+ "zipformer.py": "5cea6da601de9f37de2e1db6c2dadcd931220b45341472efaa426a25130a5a36"
214
+ },
215
+ "same_batch_decode": {
216
+ "examples": 16,
217
+ "exact_prediction_match": true
218
+ },
219
+ "public_batch_api": {
220
+ "examples": 16,
221
+ "output_count_ok": true,
222
+ "all_strings": true,
223
+ "empty_batch_ok": true
224
+ },
225
+ "invalid_audio_rejected": true,
226
+ "elapsed_seconds": 14.379630783998437,
227
+ "all_passed": true
228
+ }
inference.py ADDED
@@ -0,0 +1,407 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Portable FP32 offline SpeedASR inference, including batched beam search.
3
+
4
+ model = SpeedASR.from_pretrained(revision="<Hub commit SHA>")
5
+ texts = model.transcribe_batch([mono_float32_audio], sample_rate=16000)
6
+
7
+ This frontend returns decoded text with VoiceA casing/punctuation restored.
8
+ WER normalization belongs to the benchmark scorer, not the recognizer.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import argparse
13
+ import hashlib
14
+ import json
15
+ import logging
16
+ import math
17
+ import re
18
+ from itertools import pairwise
19
+ from pathlib import Path
20
+ from typing import Iterable, Sequence
21
+
22
+ import kaldifeat
23
+ import numpy as np
24
+ import soundfile as sf
25
+ import torch
26
+ from torch.nn.utils.rnn import pad_sequence
27
+
28
+ from speedasr_core.beam_search import modified_beam_search
29
+ from speedasr_core.decoder import Decoder
30
+ from speedasr_core.joiner import Joiner
31
+ from speedasr_core.model import AsrModel
32
+ from speedasr_core.scaling import ScheduledFloat
33
+ from speedasr_core.subsampling import Conv2dSubsampling
34
+ from speedasr_core.zipformer import Zipformer2
35
+
36
+
37
+ def _sha256(path: Path) -> str:
38
+ digest = hashlib.sha256()
39
+ with path.open("rb") as stream:
40
+ for block in iter(lambda: stream.read(1 << 20), b""):
41
+ digest.update(block)
42
+ return digest.hexdigest()
43
+
44
+
45
+ def _read_tokens(path: Path) -> dict[int, str]:
46
+ table = {}
47
+ for line in path.read_text(encoding="utf-8").splitlines():
48
+ parts = line.split()
49
+ if len(parts) >= 2:
50
+ token_id = int(parts[-1])
51
+ if token_id in table:
52
+ raise ValueError(f"Duplicate token ID: {token_id}")
53
+ table[token_id] = " ".join(parts[:-1])
54
+ if not table or "<blk>" not in table.values() or "<unk>" not in table.values():
55
+ raise ValueError("tokens.txt must contain <blk> and <unk>")
56
+ return table
57
+
58
+
59
+ def restore_voicea_text(text: str) -> str:
60
+ """The exact VoiceA postprocessing used by the local benchmark."""
61
+ if not text:
62
+ return text
63
+ processed = (
64
+ text.replace("<,>", ",").replace("<.>", ".")
65
+ .replace("<?>", "?").replace("<!>", "!").replace("<unk>", "")
66
+ )
67
+ words = processed.split()
68
+ casing_set = {"<cap>", "<upp>", "<mix>"}
69
+ ans = []
70
+ for w1, w2 in pairwise(words):
71
+ if w1 in casing_set:
72
+ continue
73
+ if w2 in casing_set:
74
+ if w2 == "<cap>":
75
+ w1 = w1.capitalize()
76
+ elif w2 == "<upp>":
77
+ w1 = w1.upper()
78
+ ans.append(w1)
79
+ if words and words[-1] not in casing_set:
80
+ ans.append(words[-1])
81
+ processed = " ".join(ans)
82
+ processed = re.sub(r"<[^>]+>", "", processed)
83
+ return re.sub(r"\s+", " ", processed).strip()
84
+
85
+
86
+ def _ints(value) -> tuple[int, ...]:
87
+ if isinstance(value, str):
88
+ return tuple(map(int, value.split(",")))
89
+ if isinstance(value, int):
90
+ return (value,)
91
+ return tuple(map(int, value))
92
+
93
+
94
+ def _build_model(config: dict, vocab_size: int, blank_id: int) -> AsrModel:
95
+ """Same construction as the frozen Icefall train.get_model, without training imports."""
96
+ if config.get("causal") is not False or int(config["context_size"]) != 3:
97
+ raise ValueError("This checkpoint requires offline causal=False and context_size=3")
98
+ if not config.get("use_transducer", True) or any(
99
+ config.get(name, False) for name in ("use_ctc", "use_attention_decoder", "use_cr_ctc")
100
+ ):
101
+ raise ValueError("Expected the transducer-only checkpoint")
102
+ encoder_dims = _ints(config["encoder_dim"])
103
+ encoder_embed = Conv2dSubsampling(
104
+ in_channels=int(config.get("feature_dim", 80)),
105
+ out_channels=encoder_dims[0],
106
+ dropout=ScheduledFloat((0.0, 0.3), (20000.0, 0.1)),
107
+ )
108
+ encoder = Zipformer2(
109
+ output_downsampling_factor=2,
110
+ downsampling_factor=_ints(config["downsampling_factor"]),
111
+ num_encoder_layers=_ints(config["num_encoder_layers"]),
112
+ encoder_dim=encoder_dims,
113
+ encoder_unmasked_dim=_ints(config["encoder_unmasked_dim"]),
114
+ query_head_dim=_ints(config["query_head_dim"]),
115
+ pos_head_dim=_ints(config["pos_head_dim"]),
116
+ value_head_dim=_ints(config["value_head_dim"]),
117
+ pos_dim=int(config["pos_dim"]),
118
+ num_heads=_ints(config["num_heads"]),
119
+ feedforward_dim=_ints(config["feedforward_dim"]),
120
+ cnn_module_kernel=_ints(config["cnn_module_kernel"]),
121
+ dropout=ScheduledFloat((0.0, 0.3), (20000.0, 0.1)),
122
+ warmup_batches=4000.0,
123
+ causal=False,
124
+ chunk_size=_ints(config["chunk_size"]),
125
+ left_context_frames=_ints(config["left_context_frames"]),
126
+ )
127
+ decoder_dim = int(config["decoder_dim"])
128
+ decoder = Decoder(vocab_size, decoder_dim, blank_id, int(config["context_size"]))
129
+ joiner = Joiner(max(encoder_dims), decoder_dim, int(config["joiner_dim"]), vocab_size)
130
+ return AsrModel(
131
+ encoder_embed=encoder_embed, encoder=encoder, decoder=decoder, joiner=joiner,
132
+ attention_decoder=None, encoder_dim=max(encoder_dims), decoder_dim=decoder_dim,
133
+ vocab_size=vocab_size, use_transducer=True, use_ctc=False, use_attention_decoder=False,
134
+ )
135
+
136
+
137
+ class SpeedASR:
138
+ """65.55M-parameter offline English recognizer.
139
+
140
+ Inputs are finite, mono, one-dimensional float waveform arrays at 16 kHz.
141
+ ``transcribe_batch`` preserves input order and adapts GPU batches to memory.
142
+ No audio, transcripts or credentials are sent to a service during inference.
143
+ """
144
+
145
+ def __init__(
146
+ self,
147
+ checkpoint: str | Path | None = None,
148
+ tokens: str | Path | None = None,
149
+ config: str | Path | None = None,
150
+ device: str | torch.device = "auto",
151
+ batch_size: int = 2048,
152
+ batch_audio_seconds: float = 5000.0,
153
+ torch_threads: int = 16,
154
+ verify_hashes: bool = True,
155
+ ):
156
+ root = Path(__file__).resolve().parent
157
+ checkpoint_path = Path(checkpoint).expanduser().resolve() if checkpoint else root / "pretrained.pt"
158
+ if checkpoint_path.is_dir():
159
+ checkpoint_path = checkpoint_path / "pretrained.pt"
160
+ tokens_path = Path(tokens).expanduser().resolve() if tokens else checkpoint_path.parent / "tokens.txt"
161
+ config_path = Path(config).expanduser().resolve() if config else root / "model_config.json"
162
+ for path in (checkpoint_path, tokens_path, config_path):
163
+ if not path.is_file():
164
+ raise FileNotFoundError(path)
165
+ self.config = json.loads(config_path.read_text(encoding="utf-8"))
166
+ architecture = self.config.get("model_args", self.config)
167
+ inference = self.config.get("inference", {})
168
+ expected = self.config.get("expected", {})
169
+ if verify_hashes:
170
+ for path, key in ((checkpoint_path, "checkpoint_sha256"), (tokens_path, "tokens_sha256")):
171
+ if expected.get(key) and _sha256(path) != expected[key]:
172
+ raise ValueError(f"Artifact hash mismatch: {path.name}")
173
+ self.token_table = _read_tokens(tokens_path)
174
+ inverse_tokens = {text: token_id for token_id, text in self.token_table.items()}
175
+ # Equivalent to icefall.utils.num_tokens(table) + 1: exclude disambiguation symbols.
176
+ token_ids = [token_id for token_id, token in self.token_table.items()
177
+ if not re.match(r"^#\d+$", token)]
178
+ vocab_size = len(token_ids) - int(0 in token_ids) + 1
179
+ for key, value in (("vocab_size", vocab_size), ("blank_id", inverse_tokens["<blk>"]),
180
+ ("unk_id", inverse_tokens["<unk>"])):
181
+ if key in architecture and int(architecture[key]) != value:
182
+ raise ValueError(f"Token table and model configuration disagree on {key}")
183
+ torch.set_num_threads(int(torch_threads))
184
+ if str(device) == "auto":
185
+ device = "cuda:0" if torch.cuda.is_available() else "cpu"
186
+ self.device = torch.device(device)
187
+ if self.device.type not in {"cpu", "cuda"}:
188
+ raise ValueError("Supported devices are cpu and cuda")
189
+ if self.device.type == "cuda":
190
+ if not torch.cuda.is_available():
191
+ raise RuntimeError("CUDA was requested but is unavailable")
192
+ torch.cuda.set_device(self.device)
193
+ self.model = _build_model(architecture, vocab_size, inverse_tokens["<blk>"])
194
+ checkpoint_data = torch.load(checkpoint_path, map_location="cpu", weights_only=True)
195
+ if not isinstance(checkpoint_data, dict) or not isinstance(checkpoint_data.get("model"), dict):
196
+ raise ValueError("Expected an Icefall checkpoint with a model state dictionary")
197
+ self.model.load_state_dict(checkpoint_data["model"], strict=True)
198
+ del checkpoint_data
199
+ self.num_parameters = sum(parameter.numel() for parameter in self.model.parameters())
200
+ if expected.get("parameters") and self.num_parameters != int(expected["parameters"]):
201
+ raise ValueError(f"Parameter count mismatch: {self.num_parameters}")
202
+ # Deliberately FP32. Quantized/FP16 accuracy was not measured by this benchmark.
203
+ self.model.float().to(self.device).eval()
204
+ self.max_gpu_allocated_bytes = 0
205
+ self.sample_rate = int(inference.get("sample_rate", 16000))
206
+ self.min_input_seconds = float(inference.get("min_input_seconds", 1.0))
207
+ self.long_chunk_seconds = float(inference.get("long_chunk_seconds", 30.0))
208
+ self.long_chunk_batch_size = int(inference.get("long_chunk_batch_size", 512))
209
+ self.beam_size = int(inference.get("beam_size", 4))
210
+ self.batch_size = int(batch_size)
211
+ self.batch_audio_seconds = float(batch_audio_seconds)
212
+ if self.batch_size < 1 or self.batch_audio_seconds <= 0:
213
+ raise ValueError("Batch limits must be positive")
214
+ opts = kaldifeat.FbankOptions()
215
+ opts.device = self.device
216
+ opts.frame_opts.dither = 0
217
+ opts.frame_opts.snip_edges = False
218
+ opts.frame_opts.samp_freq = self.sample_rate
219
+ opts.mel_opts.num_bins = int(architecture.get("feature_dim", 80))
220
+ opts.mel_opts.high_freq = -400
221
+ self.fbank = kaldifeat.Fbank(opts)
222
+
223
+ @classmethod
224
+ def from_pretrained(
225
+ cls,
226
+ repo_id: str = "SpeedAudio261119/SpeedASR-v1-65M",
227
+ revision: str = "main",
228
+ token: str | bool | None = None,
229
+ cache_dir: str | Path | None = None,
230
+ **kwargs,
231
+ ) -> "SpeedASR":
232
+ """Download only weights/tokens/config. Pin a commit SHA for reproducibility."""
233
+ from huggingface_hub import hf_hub_download
234
+ paths = {
235
+ name: hf_hub_download(repo_id=repo_id, filename=name, revision=revision,
236
+ token=token, cache_dir=cache_dir)
237
+ for name in ("pretrained.pt", "tokens.txt", "model_config.json")
238
+ }
239
+ return cls(checkpoint=paths["pretrained.pt"], tokens=paths["tokens.txt"],
240
+ config=paths["model_config.json"], **kwargs)
241
+
242
+ def _wave(self, audio) -> torch.Tensor:
243
+ if isinstance(audio, torch.Tensor):
244
+ if not audio.is_floating_point():
245
+ raise ValueError("Waveforms must contain floating-point samples, not PCM integers")
246
+ wave = audio.detach().to(device="cpu", dtype=torch.float32).contiguous()
247
+ else:
248
+ array = np.asarray(audio)
249
+ if not np.issubdtype(array.dtype, np.floating):
250
+ raise ValueError("Waveforms must contain floating-point samples, not PCM integers")
251
+ wave = torch.from_numpy(np.ascontiguousarray(array, dtype=np.float32))
252
+ if wave.ndim != 1 or wave.numel() == 0:
253
+ raise ValueError("Expected a nonempty one-dimensional mono waveform")
254
+ if not bool(torch.isfinite(wave).all()):
255
+ raise ValueError("Waveform contains non-finite samples")
256
+ return wave
257
+
258
+ def _pad(self, wave: torch.Tensor) -> torch.Tensor:
259
+ minimum = int(round(self.min_input_seconds * self.sample_rate))
260
+ if wave.numel() < minimum:
261
+ wave = torch.nn.functional.pad(wave, (0, minimum - wave.numel()))
262
+ return wave
263
+
264
+ @torch.no_grad()
265
+ def _decode(self, waves: list[torch.Tensor]) -> list[str]:
266
+ waves = [self._pad(wave).to(self.device) for wave in waves]
267
+ features = self.fbank(waves)
268
+ lengths = torch.tensor([f.size(0) for f in features], device=self.device, dtype=torch.int64)
269
+ features = pad_sequence(features, batch_first=True, padding_value=math.log(1e-10))
270
+ with torch.autocast(device_type=self.device.type, enabled=False):
271
+ encoded, encoded_lengths = self.model.forward_encoder(features, lengths)
272
+ hypotheses = modified_beam_search(
273
+ model=self.model, encoder_out=encoded, encoder_out_lens=encoded_lengths,
274
+ beam=self.beam_size,
275
+ )
276
+ if len(hypotheses) != len(waves):
277
+ raise RuntimeError("Decoder returned the wrong number of hypotheses")
278
+ result = []
279
+ for hypothesis in hypotheses:
280
+ text = "".join(self.token_table.get(int(token), "") for token in hypothesis)
281
+ result.append(restore_voicea_text(text.replace("\u2581", " ").strip()))
282
+ if self.device.type == "cuda":
283
+ self.max_gpu_allocated_bytes = max(
284
+ self.max_gpu_allocated_bytes, torch.cuda.max_memory_allocated(self.device)
285
+ )
286
+ return result
287
+
288
+ @staticmethod
289
+ def _is_oom(exc: RuntimeError) -> bool:
290
+ return isinstance(exc, torch.cuda.OutOfMemoryError) or "out of memory" in str(exc).lower()
291
+
292
+ def _decode_long(self, wave: torch.Tensor) -> str:
293
+ samples = max(1, int(round(self.long_chunk_seconds * self.sample_rate)))
294
+ chunks = [wave[start:start + samples] for start in range(0, wave.numel(), samples)]
295
+ batch_size = max(1, min(self.long_chunk_batch_size, len(chunks)))
296
+ output = []
297
+ offset = 0
298
+ while offset < len(chunks):
299
+ count = min(batch_size, len(chunks) - offset)
300
+ try:
301
+ output.extend(self._decode(chunks[offset:offset + count]))
302
+ except RuntimeError as exc:
303
+ if self.device.type == "cuda" and self._is_oom(exc) and count > 1:
304
+ torch.cuda.empty_cache()
305
+ batch_size = max(1, count // 2)
306
+ continue
307
+ raise
308
+ offset += count
309
+ if batch_size < self.long_chunk_batch_size and count == batch_size:
310
+ batch_size = min(self.long_chunk_batch_size, batch_size * 2)
311
+ if self.device.type == "cuda":
312
+ torch.cuda.empty_cache()
313
+ return re.sub(r"\s+", " ", " ".join(part for part in output if part)).strip()
314
+
315
+ @torch.no_grad()
316
+ def transcribe_batch(
317
+ self,
318
+ waveforms: Sequence,
319
+ sample_rate: int = 16000,
320
+ batch_size: int | None = None,
321
+ ) -> list[str]:
322
+ """Decode mono float waveforms and return one string per input, in input order."""
323
+ if int(sample_rate) != self.sample_rate:
324
+ raise ValueError(f"Expected {self.sample_rate} Hz; resample before calling inference")
325
+ waves = [self._wave(wave) for wave in waveforms]
326
+ if not waves:
327
+ return []
328
+ target_batch_size = self.batch_size if batch_size is None else int(batch_size)
329
+ if target_batch_size < 1:
330
+ raise ValueError("batch_size must be positive")
331
+ order = sorted(range(len(waves)), key=lambda i: waves[i].numel(), reverse=True)
332
+ outputs = [""] * len(waves)
333
+ offset = 0
334
+ safe_seconds = self.batch_audio_seconds
335
+ while offset < len(order):
336
+ first = waves[order[offset]]
337
+ seconds = first.numel() / self.sample_rate
338
+ if seconds > self.long_chunk_seconds:
339
+ outputs[order[offset]] = self._decode_long(first)
340
+ offset += 1
341
+ continue
342
+ count = min(target_batch_size, max(1, int(safe_seconds / max(1.0, seconds))))
343
+ count = min(count, len(order) - offset)
344
+ indices = order[offset:offset + count]
345
+ try:
346
+ if self.device.type == "cuda":
347
+ torch.cuda.reset_peak_memory_stats(self.device)
348
+ texts = self._decode([waves[i] for i in indices])
349
+ except RuntimeError as exc:
350
+ if self.device.type != "cuda" or not self._is_oom(exc):
351
+ raise
352
+ torch.cuda.empty_cache()
353
+ if count == 1:
354
+ outputs[indices[0]] = self._decode_long(first)
355
+ offset += 1
356
+ else:
357
+ safe_seconds = max(1, count // 2) * max(1.0, seconds)
358
+ logging.warning("CUDA memory limit; retrying with a smaller batch")
359
+ continue
360
+ for index, text in zip(indices, texts):
361
+ outputs[index] = text
362
+ if self.device.type == "cuda":
363
+ peak = torch.cuda.max_memory_allocated(self.device)
364
+ target_memory = torch.cuda.get_device_properties(self.device).total_memory * 0.80
365
+ if peak > 0 and peak < target_memory * 0.85:
366
+ safe_seconds *= min(1.15, target_memory / peak)
367
+ offset += count
368
+ if self.device.type == "cuda":
369
+ torch.cuda.synchronize(self.device)
370
+ return outputs
371
+
372
+ def transcribe(self, waveform, sample_rate: int = 16000) -> str:
373
+ """Decode one mono float waveform at 16 kHz."""
374
+ return self.transcribe_batch([waveform], sample_rate=sample_rate)[0]
375
+
376
+
377
+ def main() -> None:
378
+ parser = argparse.ArgumentParser(description=__doc__)
379
+ parser.add_argument("audio", nargs="+", type=Path, help="16 kHz WAV/FLAC files")
380
+ parser.add_argument("--checkpoint", type=Path)
381
+ parser.add_argument("--tokens", type=Path)
382
+ parser.add_argument("--config", type=Path)
383
+ parser.add_argument("--repo-id", help="Download model artifacts from this Hub repository")
384
+ parser.add_argument("--revision", default="main", help="Prefer an immutable Hub commit SHA")
385
+ parser.add_argument("--device", default="auto")
386
+ parser.add_argument("--batch-size", type=int, default=2048)
387
+ parser.add_argument("--batch-audio-seconds", type=float, default=5000.0)
388
+ parser.add_argument("--torch-threads", type=int, default=16)
389
+ args = parser.parse_args()
390
+ options = dict(device=args.device, batch_size=args.batch_size,
391
+ batch_audio_seconds=args.batch_audio_seconds, torch_threads=args.torch_threads)
392
+ if args.repo_id:
393
+ model = SpeedASR.from_pretrained(args.repo_id, revision=args.revision, **options)
394
+ else:
395
+ model = SpeedASR(args.checkpoint, args.tokens, args.config, **options)
396
+ waves = []
397
+ for path in args.audio:
398
+ audio, sample_rate = sf.read(path, dtype="float32", always_2d=True)
399
+ if sample_rate != model.sample_rate:
400
+ parser.error(f"{path}: expected {model.sample_rate} Hz, got {sample_rate}")
401
+ waves.append(np.ascontiguousarray(audio[:, 0]))
402
+ for path, text in zip(args.audio, model.transcribe_batch(waves)):
403
+ print(json.dumps({"audio_filepath": str(path), "pred_text": text}, ensure_ascii=False))
404
+
405
+
406
+ if __name__ == "__main__":
407
+ main()
leaderboard/Dockerfile ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Build from the model package root:
2
+ # docker build -f leaderboard/Dockerfile -t speedasr-open-asr .
3
+ FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
4
+ ARG OPEN_ASR_COMMIT=4a113dd90c9ffd011d7d3d105da76980d511d029
5
+ ENV DEBIAN_FRONTEND=noninteractive \
6
+ VIRTUAL_ENV=/opt/venv \
7
+ PATH=/opt/venv/bin:$PATH \
8
+ PYTHONPATH=/app:/opt/open_asr_leaderboard \
9
+ OPEN_ASR_REPO=/opt/open_asr_leaderboard \
10
+ HF_AUDIO_DECODER_BACKEND=soundfile \
11
+ HF_HOME=/tmp/huggingface \
12
+ PYTHONUNBUFFERED=1
13
+ RUN apt-get update && apt-get install -y --no-install-recommends \
14
+ python3.10 python3.10-venv libsndfile1 libgomp1 git ca-certificates \
15
+ && rm -rf /var/lib/apt/lists/* \
16
+ && python3.10 -m venv /opt/venv \
17
+ && python -m pip install --no-cache-dir pip==25.0.1
18
+ WORKDIR /app
19
+ COPY requirements-linux-cu124.txt /app/requirements-linux-cu124.txt
20
+ COPY requirements-core.txt /app/requirements-core.txt
21
+ COPY leaderboard/requirements.txt /app/leaderboard/requirements.txt
22
+ RUN python -m pip install --no-cache-dir \
23
+ -r requirements-linux-cu124.txt -r leaderboard/requirements.txt \
24
+ && python -m pip check
25
+ RUN git init /opt/open_asr_leaderboard \
26
+ && git -C /opt/open_asr_leaderboard remote add origin https://github.com/huggingface/open_asr_leaderboard.git \
27
+ && git -C /opt/open_asr_leaderboard fetch --depth 1 origin ${OPEN_ASR_COMMIT} \
28
+ && git -C /opt/open_asr_leaderboard checkout --detach FETCH_HEAD
29
+ COPY . /app
30
+ EXPOSE 7860
31
+ # The Space stays lightweight; Jobs override this command with run_eval.py.
32
+ CMD ["python", "leaderboard/space_app.py"]
leaderboard/cloud_submission.txt ADDED
@@ -0,0 +1,191 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ SpeedASR-v1-65M: reproducible Open ASR Leaderboard submission package
2
+ Prepared 2026-11-19
3
+
4
+ Model: https://huggingface.co/SpeedAudio261119/SpeedASR-v1-65M
5
+ Official evaluation/scorer revision:
6
+ 4a113dd90c9ffd011d7d3d105da76980d511d029
7
+
8
+ STATUS AND SCOPE
9
+
10
+ Uploading this model package publishes the checkpoint, tokenizer, code and local
11
+ results. It does not automatically create an evaluation Space, run paid HF Jobs,
12
+ open a GitHub pull request, or add a row to the official leaderboard. The files
13
+ in this directory prepare those subsequent steps for an owner/maintainer.
14
+
15
+ The supplied local benchmark was performed on an RTX 3090 Ti in FP32. Its WER
16
+ and throughput must not be labeled as an H200 cloud run. Private-set evaluation
17
+ requires the leaderboard maintainers and is not included in the local results.
18
+
19
+ MODEL AND DECODE SETTINGS
20
+
21
+ Offline Zipformer transducer; 65,551,059 total parameters; 16 kHz mono audio;
22
+ context size 3; modified beam search; beam size 4; FP32; the same decoder and
23
+ text postprocessor for every dataset. No external language model or per-dataset
24
+ prompt/vocabulary. The model package's inference.py exposes:
25
+
26
+ from inference import SpeedASR
27
+ model = SpeedASR.from_pretrained(
28
+ "SpeedAudio261119/SpeedASR-v1-65M", revision="<MODEL_COMMIT_SHA>",
29
+ device="cuda:0", batch_size=2048, batch_audio_seconds=5000,
30
+ )
31
+ predictions = model.transcribe_batch(waveforms, sample_rate=16000)
32
+
33
+ For a local package, pass checkpoint=..., tokens=..., config=... to SpeedASR.
34
+ The adapter uses the same inference code and model settings. Batching can differ
35
+ from the earlier local runner, so throughput is independently measured.
36
+
37
+ 1. BUILD A DOCKER EVALUATION IMAGE / HF SPACE
38
+
39
+ Use a separate directory or checkout for a Docker Space. Copy the model package
40
+ code, vendor sources, model_config.json, requirements files and leaderboard/
41
+ directory into it. Copy leaderboard/Dockerfile to the Space's root Dockerfile,
42
+ leaderboard/space_card.txt to its root README.md, and
43
+ leaderboard/dockerignore.txt to its root .dockerignore. Checkpoint/ONNX weights
44
+ and benchmark artifacts do not need to be baked into the image: the adapter
45
+ downloads the explicitly pinned model snapshot at job startup.
46
+
47
+ The Space README is distinct from the model README; do not replace the model
48
+ card in the model repository. Create a Docker Space in your own account, e.g.
49
+ your-user/open-asr-leaderboard-speedasr, and upload that build directory. A CPU
50
+ Space is sufficient to build the image; the Jobs command selects the H200.
51
+ Wait for the Docker build to succeed. No benchmark starts from the landing page.
52
+
53
+ The Dockerfile uses CUDA 12.4.1 / Ubuntu 22.04 / Python 3.10 and the package's
54
+ requirements-linux-cu124.txt with compatible Torch/k2/kaldifeat wheels. It pins
55
+ the official normalizer checkout to the revision above. Do not independently
56
+ upgrade Torch without installing matching k2 and kaldifeat wheels.
57
+
58
+ Optional local Docker build from this model package root:
59
+
60
+ docker build -f leaderboard/Dockerfile -t speedasr-open-asr .
61
+
62
+ 2. SMOKE TEST THE ADAPTER
63
+
64
+ From a fresh working directory (results are written relative to that directory),
65
+ with the package dependencies installed and an official checkout at the pinned
66
+ revision:
67
+
68
+ python /path/to/SpeedASR-v1-65M/leaderboard/run_eval.py \
69
+ --model_id SpeedAudio261119/SpeedASR-v1-65M \
70
+ --revision <MODEL_COMMIT_SHA> \
71
+ --open-asr-repo /path/to/pinned/open_asr_leaderboard \
72
+ --dataset_path hf-audio/open-asr-leaderboard \
73
+ --dataset ami_cleaned --split test --device 0 \
74
+ --batch_size 16 --batch-audio-seconds 200 --max_eval_samples 16
75
+
76
+ Use --model-dir /path/to/downloaded/model/package instead of --revision to load
77
+ local weights. Do not truncate Earnings22: every chunk is required to reconstruct
78
+ its complete parent sessions. The adapter rejects chunked partial runs.
79
+
80
+ The adapter accepts only the stated official Git revision so a normalizer change
81
+ cannot silently alter scores. Later updates should deliberately update the pin,
82
+ Docker image and score provenance together.
83
+
84
+ 3. LAUNCH THE SEVEN PUBLIC SETS ON HF JOBS (PAID)
85
+
86
+ HF Jobs requires credits and a write token, plus an existing writable results
87
+ bucket. Credentials belong in the process environment / HF Job secrets, not in
88
+ the model repository, model card, scripts or command logs.
89
+
90
+ Install a current huggingface_hub CLI (the working local environment used
91
+ huggingface_hub 1.18.0), then set the following environment variables:
92
+
93
+ export HF_TOKEN=<your_write_token>
94
+ export SPACE=your-user/open-asr-leaderboard-speedasr
95
+ export RESULTS_BUCKET=your-user/speedasr-results
96
+ export MODEL_REVISION=<FULL_MODEL_UPLOAD_COMMIT_SHA>
97
+ bash leaderboard/submit_jobs.sh
98
+
99
+ This script deliberately runs one H200 job per dataset, sequentially. It waits
100
+ for each job, fails on a job error and downloads this run's results. A unique
101
+ UTC RUN_ID keeps repeated result sets separate. ORG_NAME selects organization
102
+ billing if supplied. FLAVOR defaults to h200; TIMEOUT defaults to 8h per job.
103
+
104
+ Before a final official throughput run, determine the maximum practical H200
105
+ batch settings and record them. The shipped defaults reproduce the local
106
+ batch-size/duration limits; they are not claimed to saturate an H200. Override:
107
+
108
+ BATCH_SIZE=... BATCH_AUDIO_SECONDS=... TORCH_THREADS=... \
109
+ bash leaderboard/submit_jobs.sh
110
+
111
+ All decoding settings stay fixed when changing these throughput controls. Do
112
+ not tune decoder settings or prompts separately for individual datasets.
113
+
114
+ To select a subset, e.g. for a complete single-dataset cloud check:
115
+
116
+ ONLY_DATASETS="ami_cleaned" bash leaderboard/submit_jobs.sh
117
+
118
+ The default seven sets are:
119
+ ami_cleaned hf-audio/open-asr-leaderboard
120
+ gigaspeech_cleaned hf-audio/open-asr-leaderboard
121
+ voxpopuli_cleaned_aa hf-audio/open-asr-leaderboard
122
+ earnings22_cleaned_aa_chunked
123
+ ArtificialAnalysis/Earnings22-Cleaned-AA-chunked
124
+ spgispeech hf-audio/open-asr-leaderboard
125
+ urgent2024 hf-audio/open-asr-leaderboard
126
+ monsoon_en_in VoiceArena/Monsoon_en_IN_test
127
+
128
+ All use split=test. Dedicated Earnings22 and Monsoon repositories use an empty
129
+ config name. INCLUDE_URGENT_CLEAN=1 adds urgent2024_clean as an optional eighth
130
+ public set; it is outside the default seven-set average. Legacy LibriSpeech and
131
+ uncleaned variants are not needed for this default seven-set average.
132
+
133
+ 4. OUTPUTS AND SCORING
134
+
135
+ Each run writes official-format JSONL files under ./results. Standard row keys:
136
+ audio_filepath, duration, time, text, pred_text
137
+
138
+ text and pred_text are raw, unnormalized strings. duration is real audio seconds,
139
+ and time is each batch's synchronized wall time divided equally among its rows.
140
+ For Earnings22 the adapter also saves parent_id and chunk_index. The official
141
+ scorer merges chunk predictions in index order and scores against one parent
142
+ reference per call. No chunk or long audio is truncated.
143
+
144
+ The metadata sidecar records model/data revisions, total model parameters,
145
+ environment versions, batching, raw manifest hash, timing scope and GPU details.
146
+ The official chunk loader resolves main internally: observed chunk/parent
147
+ revisions are recorded, but that loader does not offer snapshot locking. The
148
+ adapter explicitly rejects a dataset-revision override for that case.
149
+
150
+ Audio loading/resampling, staging/sorting, model loading, synthetic warmup and
151
+ scoring are excluded from throughput timing. Timing includes the complete
152
+ transcribe_batch call (features, transfers, neural inference, decode and text).
153
+ Synthetic warmup is independent of evaluation examples; any work not covered
154
+ by warmup remains inside the timed evaluation calls.
155
+
156
+ To print a combined public-set Markdown result table, use the pinned official
157
+ repo and the downloaded cloud results (or the included local raw results):
158
+
159
+ PYTHONPATH=/path/to/pinned/open_asr_leaderboard python - <<'PY'
160
+ from normalizer.eval_utils import score_results
161
+ score_results('/path/to/results', 'SpeedAudio261119/SpeedASR-v1-65M',
162
+ families=['public'], num_workers=1)
163
+ PY
164
+
165
+ Serial scoring is intentional; it avoids a multiprocessing alignment stall
166
+ observed on the local machine. Normalization is applied only at scoring time.
167
+ The default public WER is an equal-weight mean across the seven testsets, not
168
+ a duration/word-weighted pooled WER. RTFx uses real audio duration divided by
169
+ measured inference wall time. Private scores are unavailable in this package.
170
+
171
+ 5. PREPARE THE OFFICIAL LEADERBOARD PR
172
+
173
+ Follow the current official PR checklist. Add a model-family folder containing
174
+ the adapter and submit_jobs.sh to the official GitHub repository, point its
175
+ launcher to the built evaluation Space and results bucket, and provide the
176
+ complete seven-set WER/RTFx table and raw result provenance. Provide a public
177
+ model link and metadata: license, total size=0.065551059B, language count=1,
178
+ encoder=Zipformer, decoder=stateless transducer prediction network/joiner, and
179
+ a link to the model card's training-data disclosure. Do not infer an undisclosed
180
+ training-data composition from experiment directory names.
181
+
182
+ Uploading this package has not opened that PR. Maintainers decide inclusion,
183
+ reproduce results and evaluate the held-out private sets separately.
184
+
185
+ PRIMARY REFERENCES
186
+
187
+ https://github.com/huggingface/open_asr_leaderboard/blob/4a113dd90c9ffd011d7d3d105da76980d511d029/.github/PULL_REQUEST_TEMPLATE.md
188
+ https://github.com/huggingface/open_asr_leaderboard/blob/4a113dd90c9ffd011d7d3d105da76980d511d029/README.md
189
+ https://github.com/huggingface/open_asr_leaderboard/blob/4a113dd90c9ffd011d7d3d105da76980d511d029/normalizer/data_utils.py
190
+ https://github.com/huggingface/open_asr_leaderboard/blob/4a113dd90c9ffd011d7d3d105da76980d511d029/normalizer/eval_utils.py
191
+ https://huggingface.co/docs/huggingface_hub/guides/jobs
leaderboard/dockerignore.txt ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ .git
2
+ *.pt
3
+ *.onnx
4
+ *.safetensors
5
+ benchmarks
6
+ results
7
+ __pycache__
8
+ *.pyc
9
+ .env
10
+ *token*
11
+ !tokens.txt
12
+
13
+ benchmark_results
leaderboard/model_metadata.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_id": "SpeedAudio261119/SpeedASR-v1-65M",
3
+ "model_name": "SpeedASR-v1-65M",
4
+ "architecture": "Offline Zipformer2 transducer",
5
+ "total_parameters": 65551059,
6
+ "components": {
7
+ "audio_encoder_including_subsampling": 63994103,
8
+ "prediction_network": 262144,
9
+ "joiner": 781812,
10
+ "auxiliary_training_projections_retained_in_checkpoint": 513000
11
+ },
12
+ "languages": [
13
+ "en"
14
+ ],
15
+ "language_count": 1,
16
+ "license": "apache-2.0",
17
+ "license_basis": "Preserves the pre-existing target repository declaration and LICENSE.",
18
+ "encoder": {
19
+ "type": "Non-causal Zipformer2",
20
+ "stacks": 6,
21
+ "layers": 16
22
+ },
23
+ "decoder": {
24
+ "type": "Transducer stateless prediction network",
25
+ "context_size": 3,
26
+ "dimension": 512
27
+ },
28
+ "inference": {
29
+ "precision": "fp32",
30
+ "sample_rate": 16000,
31
+ "decoding": "modified_beam_search",
32
+ "beam": 4,
33
+ "causal": false,
34
+ "external_language_model": false,
35
+ "voicea_postprocessing": true
36
+ },
37
+ "training_data": {
38
+ "composition": "Not disclosed",
39
+ "default_seven_public_testsets_excluded": true,
40
+ "evidence": "Author confirmation in this release task; not an independent overlap audit."
41
+ },
42
+ "evaluation": {
43
+ "status": "Local public evaluation; official cloud/private evaluation not yet performed",
44
+ "date": "2026-11-19",
45
+ "hardware": "NVIDIA GeForce RTX 3090 Ti",
46
+ "public_seven_set_macro_wer": 6.904285714285714,
47
+ "pooled_public_rtfx": 192.6292762644339,
48
+ "official_scorer_commit": "4a113dd90c9ffd011d7d3d105da76980d511d029",
49
+ "result_directory": "benchmark_results/local_3090ti_20261119"
50
+ },
51
+ "checkpoint_sha256": "b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01",
52
+ "tokens_sha256": "a238b7a0fa67aa6be998ca13dd129d69c50639970583e32232dd149e5448210a"
53
+ }
leaderboard/requirements.txt ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # English-only leaderboard loading and official scorer dependencies.
2
+ # Install together with ../requirements-linux-cu124.txt to retain the tested
3
+ # torch/k2/kaldifeat ABI. No Transformers or training packages are needed.
4
+ datasets==3.6.0
5
+ librosa==0.10.2.post1
6
+ pandas==2.2.3
7
+ pyarrow==20.0.0
8
+ fsspec==2025.3.0
9
+ kaldialign==0.12.0
10
+ num2words==0.5.14
11
+ regex==2024.11.6
12
+ tqdm==4.66.4
leaderboard/run_eval.py ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Batched SpeedASR evaluation using the official Open ASR data/manifest code.
3
+
4
+ Run from the model package root. The Dockerfile installs the official scorer at
5
+ the pinned commit; for a local run set --open-asr-repo to that checkout. Raw
6
+ references/predictions are retained, including Earnings22 session metadata.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import argparse
11
+ import hashlib
12
+ import importlib.metadata
13
+ import json
14
+ import math
15
+ import os
16
+ from pathlib import Path
17
+ import platform
18
+ import subprocess
19
+ import sys
20
+ import time
21
+
22
+
23
+ OFFICIAL_COMMIT = "4a113dd90c9ffd011d7d3d105da76980d511d029"
24
+ PACKAGE_ROOT = Path(__file__).resolve().parents[1]
25
+
26
+
27
+ def parse_args():
28
+ p = argparse.ArgumentParser(description=__doc__)
29
+ p.add_argument("--model_id", "--model-id", default="SpeedAudio261119/SpeedASR-v1-65M")
30
+ p.add_argument("--revision", help="Pinned model commit/tag; required for a Hub download.")
31
+ p.add_argument("--model-dir", type=Path, help="Use an already downloaded package instead of Hub weights.")
32
+ p.add_argument("--dataset_path", "--dataset-path", default="hf-audio/open-asr-leaderboard")
33
+ p.add_argument("--dataset", default="", help="Configuration name, or empty for a dedicated dataset repo.")
34
+ p.add_argument("--dataset-revision", default=None)
35
+ p.add_argument("--split", default="test")
36
+ p.add_argument("--device", default="0", help="CUDA index, cuda:N, or cpu.")
37
+ p.add_argument("--batch_size", "--batch-size", type=int, default=2048)
38
+ p.add_argument("--batch-audio-seconds", type=float, default=5000.0)
39
+ p.add_argument("--torch-threads", type=int, default=16)
40
+ p.add_argument("--warmup-steps", type=int, default=5)
41
+ p.add_argument("--warmup-seconds", type=float, default=10.0,
42
+ help="Fixed synthetic audio duration, independent of the test set.")
43
+ p.add_argument("--max_eval_samples", "--max-eval-samples", type=int, default=-1)
44
+ p.add_argument("--open-asr-repo", type=Path,
45
+ default=Path(os.getenv("OPEN_ASR_REPO", "/opt/open_asr_leaderboard")))
46
+ p.add_argument("--skip-score", action="store_true", help="Write raw manifests without local scoring.")
47
+ a = p.parse_args()
48
+ a.streaming = False
49
+ if (a.batch_size < 1 or not math.isfinite(a.batch_audio_seconds)
50
+ or a.batch_audio_seconds <= 0 or a.torch_threads < 1):
51
+ p.error("Batch size, batch audio seconds and torch threads must be positive.")
52
+ if a.warmup_steps < 0 or not math.isfinite(a.warmup_seconds) or a.warmup_seconds <= 0:
53
+ p.error("Warmup steps must be nonnegative and warmup duration positive.")
54
+ if a.max_eval_samples == 0 or a.max_eval_samples < -1:
55
+ p.error("max_eval_samples must be -1 or positive.")
56
+ if a.model_dir is None and not a.revision:
57
+ p.error("Set --revision to the pinned Hub model commit/tag, or use --model-dir.")
58
+ return a
59
+
60
+
61
+ def sha256(path):
62
+ h = hashlib.sha256()
63
+ with open(path, "rb") as f:
64
+ for block in iter(lambda: f.read(1024 * 1024), b""):
65
+ h.update(block)
66
+ return h.hexdigest()
67
+
68
+
69
+ def prepared_batches(dataset, batch_size, batch_audio_seconds, np):
70
+ """Decode bounded staging windows, then batch by count and padded duration.
71
+
72
+ Audio decoding/resampling and sorting occur before the timed inference call.
73
+ No example is truncated or dropped. Long single examples form their own
74
+ batch, even when they exceed the nominal batch audio budget.
75
+ """
76
+ def stage(samples):
77
+ rows = []
78
+ for sample in samples:
79
+ audio = sample["audio"]
80
+ waveform = np.asarray(audio["array"], dtype=np.float32)
81
+ if waveform.ndim != 1 or waveform.size == 0 or not np.isfinite(waveform).all():
82
+ raise ValueError("Expected nonempty finite mono audio from official prepare_data.")
83
+ if int(audio["sampling_rate"]) != 16000:
84
+ raise ValueError("Official prepare_data must resample audio to 16000 Hz.")
85
+ rows.append((sample, waveform))
86
+ rows.sort(key=lambda r: len(r[1]), reverse=True)
87
+ batch = []
88
+ max_samples = 0
89
+ for row in rows:
90
+ proposed_max = max(max_samples, len(row[1]))
91
+ if batch and (len(batch) >= batch_size or
92
+ proposed_max * (len(batch) + 1) / 16000 > batch_audio_seconds):
93
+ yield batch
94
+ batch, max_samples = [], 0
95
+ batch.append(row)
96
+ max_samples = max(max_samples, len(row[1]))
97
+ if batch:
98
+ yield batch
99
+
100
+ window = []
101
+ for sample in dataset:
102
+ window.append(sample)
103
+ if len(window) >= batch_size:
104
+ yield from stage(window)
105
+ window = []
106
+ if window:
107
+ yield from stage(window)
108
+
109
+
110
+ def main(a):
111
+ sys.path.insert(0, str(PACKAGE_ROOT))
112
+ sys.path.insert(0, str(a.open_asr_repo.resolve()))
113
+ try:
114
+ commit = subprocess.check_output(
115
+ ["git", "-C", str(a.open_asr_repo), "rev-parse", "HEAD"], text=True
116
+ ).strip()
117
+ except (OSError, subprocess.CalledProcessError) as e:
118
+ raise RuntimeError("--open-asr-repo must point to the pinned official Git checkout.") from e
119
+ if commit != OFFICIAL_COMMIT:
120
+ raise RuntimeError(f"Expected official scorer {OFFICIAL_COMMIT}, found {commit}.")
121
+
122
+ import numpy as np
123
+ import torch
124
+ from huggingface_hub import HfApi
125
+ from normalizer import data_utils
126
+ from normalizer.eval_utils import score_manifest
127
+ from inference import SpeedASR
128
+
129
+ is_chunked = data_utils.is_chunked_dataset(a.dataset_path)
130
+ if is_chunked and a.max_eval_samples > 0:
131
+ raise ValueError("Do not truncate Earnings22 chunks: complete parent sessions are required.")
132
+ if is_chunked and a.dataset_revision:
133
+ raise ValueError("This pinned official chunk loader does not support --dataset-revision.")
134
+ if a.device in {"cpu", "-1"}:
135
+ device = "cpu"
136
+ else:
137
+ device = a.device if a.device.startswith("cuda") else f"cuda:{int(a.device)}"
138
+ if not torch.cuda.is_available():
139
+ raise RuntimeError("CUDA was requested but is unavailable.")
140
+ torch.set_num_threads(a.torch_threads)
141
+
142
+ api = HfApi()
143
+ model_revision = a.revision
144
+ if a.model_dir:
145
+ model_dir = a.model_dir.resolve()
146
+ model = SpeedASR(checkpoint=model_dir / "pretrained.pt", tokens=model_dir / "tokens.txt",
147
+ config=model_dir / "model_config.json", device=device,
148
+ batch_size=a.batch_size, batch_audio_seconds=a.batch_audio_seconds,
149
+ torch_threads=a.torch_threads)
150
+ else:
151
+ model_revision = api.model_info(a.model_id, revision=a.revision).sha
152
+ model = SpeedASR.from_pretrained(
153
+ a.model_id, revision=model_revision, device=device, batch_size=a.batch_size,
154
+ batch_audio_seconds=a.batch_audio_seconds, torch_threads=a.torch_threads,
155
+ )
156
+ count = sum(p.numel() for p in model.model.parameters())
157
+ if model.beam_size != 4:
158
+ raise ValueError("This submission uses the same fixed beam size 4 on every dataset.")
159
+ print(f"Model size: {count} parameters ({count / 1e9:.9f}B)", flush=True)
160
+
161
+ dataset_revision = api.dataset_info(a.dataset_path, revision=a.dataset_revision).sha
162
+ parent_revision = None
163
+ if is_chunked:
164
+ parent_revision = api.dataset_info(
165
+ data_utils.CHUNKED_DATASETS[a.dataset_path.lower()]["parent_dataset_path"]
166
+ ).sha
167
+ # Dedicated dataset repos use no config. Keep the manifest config as an empty
168
+ # string while passing None to load_dataset through the official helper.
169
+ load_args = argparse.Namespace(**vars(a))
170
+ load_args.dataset = a.dataset or None
171
+ if not is_chunked:
172
+ load_args.dataset_revision = dataset_revision
173
+ dataset = data_utils.prepare_data(data_utils.load_data(load_args))
174
+ if a.max_eval_samples > 0:
175
+ dataset = dataset.select(range(min(a.max_eval_samples, len(dataset))))
176
+
177
+ def sync():
178
+ if device.startswith("cuda"):
179
+ torch.cuda.synchronize(device)
180
+
181
+ dummy = np.zeros(round(16000 * a.warmup_seconds), dtype=np.float32)
182
+ for _ in range(a.warmup_steps):
183
+ model.transcribe_batch([dummy], sample_rate=16000)
184
+ sync()
185
+ if device.startswith("cuda"):
186
+ torch.cuda.reset_peak_memory_stats(device)
187
+ model.max_gpu_allocated_bytes = 0
188
+
189
+ refs, preds, durations, times, paths = [], [], [], [], []
190
+ extra = {key: [] for key in data_utils.CHUNK_METADATA_KEYS} if is_chunked else {}
191
+ batches = 0
192
+ for batch in prepared_batches(dataset, a.batch_size, a.batch_audio_seconds, np):
193
+ waveforms = [waveform for _, waveform in batch]
194
+ sync()
195
+ started = time.perf_counter()
196
+ output = model.transcribe_batch(waveforms, sample_rate=16000)
197
+ sync()
198
+ runtime = time.perf_counter() - started
199
+ if len(output) != len(batch) or any(not isinstance(text, str) for text in output):
200
+ raise RuntimeError("Inference returned an invalid number/type of predictions.")
201
+ for (sample, waveform), prediction in zip(batch, output):
202
+ refs.append(sample["original_text"])
203
+ preds.append(prediction)
204
+ durations.append(len(waveform) / 16000)
205
+ times.append(runtime / len(batch))
206
+ paths.append(data_utils.extract_audio_filepath_from_sample(sample))
207
+ for key in extra:
208
+ extra[key].append(sample[key])
209
+ batches += 1
210
+ print(f"Finished batch {batches}: {len(refs)} examples", flush=True)
211
+ if not refs:
212
+ raise RuntimeError("No evaluation examples remain after official reference filtering.")
213
+
214
+ manifest = Path(data_utils.write_manifest(
215
+ refs, preds, a.model_id, a.dataset_path, a.dataset or "", a.split,
216
+ audio_length=durations, transcription_time=times,
217
+ audio_filepaths=paths, extra_fields=extra or None,
218
+ ))
219
+ metadata = {
220
+ "model_id": a.model_id, "model_revision": model_revision,
221
+ "model_parameters": count, "dataset_path": a.dataset_path,
222
+ "dataset": a.dataset, "dataset_revision": dataset_revision,
223
+ "parent_dataset_revision_observed": parent_revision,
224
+ "chunk_revision_note": "Official chunk loader resolves main; observed revisions recorded, not a snapshot lock." if is_chunked else None,
225
+ "split": a.split, "official_commit": commit, "examples": len(refs),
226
+ "batches": batches, "precision": "fp32", "causal": False,
227
+ "decoding_method": "modified_beam_search", "beam_size": 4,
228
+ "context_size": 3, "batch_size": a.batch_size,
229
+ "long_chunk_seconds": model.long_chunk_seconds,
230
+ "long_chunk_batch_size": model.long_chunk_batch_size,
231
+ "min_input_seconds": model.min_input_seconds,
232
+ "batch_audio_seconds": a.batch_audio_seconds,
233
+ "warmup_steps": a.warmup_steps, "warmup_seconds": a.warmup_seconds,
234
+ "warmup_input": "synthetic zeros, independent of evaluation data",
235
+ "audio_seconds": sum(durations), "inference_seconds": sum(times),
236
+ "rtfx": sum(durations) / sum(times),
237
+ "timing_scope": "Synchronized wall time of complete transcribe_batch calls; excludes audio loading/resampling, staging/sorting, model loading, warmup and scoring.",
238
+ "device": device,
239
+ "gpu_name": torch.cuda.get_device_name(device) if device.startswith("cuda") else None,
240
+ "peak_gpu_allocated_bytes": model.max_gpu_allocated_bytes if device.startswith("cuda") else None,
241
+ "python": platform.python_version(), "torch": torch.__version__,
242
+ "package_versions": {n: importlib.metadata.version(n) for n in ("k2", "kaldifeat", "numpy", "datasets", "huggingface_hub")},
243
+ "manifest_sha256": sha256(manifest),
244
+ "run_eval_sha256": sha256(Path(__file__)),
245
+ "inference_code_sha256": sha256(PACKAGE_ROOT / "inference.py"),
246
+ "vendor_code_sha256": {p.name: sha256(p) for p in sorted((PACKAGE_ROOT / "speedasr_core").glob("*.py"))},
247
+ "model_configuration": model.config,
248
+ "partial_run": a.max_eval_samples > 0,
249
+ }
250
+ if not a.skip_score:
251
+ rows = data_utils.read_manifest(manifest)
252
+ if is_chunked:
253
+ rows = data_utils.merge_chunked_manifest(rows)
254
+ score = score_manifest(rows, language="en", multilingual=False, num_workers=1)
255
+ metadata["official_score"] = score
256
+ print("Official score:", json.dumps(score, ensure_ascii=False), flush=True)
257
+ manifest.with_suffix(".meta.json").write_text(
258
+ json.dumps(metadata, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
259
+ )
260
+ print(f"Manifest: {manifest.resolve()}\nRTFx: {metadata['rtfx']:.4f}", flush=True)
261
+
262
+
263
+ if __name__ == "__main__":
264
+ main(parse_args())
leaderboard/space_app.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Minimal landing page for the reproducible Docker evaluation Space."""
2
+ from http.server import BaseHTTPRequestHandler, HTTPServer
3
+
4
+ BODY = b"""<!doctype html><html lang=en><meta charset=utf-8>
5
+ <title>SpeedASR Open ASR evaluation image</title>
6
+ <body style="max-width:780px;margin:60px auto;padding:20px;font:17px/1.6 system-ui">
7
+ <h1>SpeedASR Open ASR evaluation image</h1>
8
+ <p>This Docker Space supplies the reproducible inference environment for
9
+ <a href="https://huggingface.co/SpeedAudio261119/SpeedASR-v1-65M">SpeedASR-v1-65M</a>.
10
+ It does not run a benchmark automatically.</p>
11
+ <p>Read <code>leaderboard/cloud_submission.txt</code> in the model repository for
12
+ the pinned evaluation command, result schema and HF Jobs launcher. Jobs use
13
+ separate paid hardware; the Space landing page performs no inference.</p></body></html>"""
14
+
15
+
16
+ class Handler(BaseHTTPRequestHandler):
17
+ def do_GET(self):
18
+ self.send_response(200)
19
+ self.send_header("Content-Type", "text/html; charset=utf-8")
20
+ self.send_header("Content-Length", str(len(BODY)))
21
+ self.end_headers()
22
+ self.wfile.write(BODY)
23
+
24
+
25
+ if __name__ == "__main__":
26
+ HTTPServer(("0.0.0.0", 7860), Handler).serve_forever()
leaderboard/space_card.txt ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: SpeedASR Open ASR evaluation
3
+ emoji: 🎙️
4
+ colorFrom: blue
5
+ colorTo: green
6
+ sdk: docker
7
+ app_port: 7860
8
+ ---
9
+
10
+ Reproducible Docker image for the SpeedASR-v1-65M offline English ASR model.
11
+
12
+ Model: https://huggingface.co/SpeedAudio261119/SpeedASR-v1-65M
13
+
14
+ See leaderboard/cloud_submission.txt for build and HF Jobs commands. The landing
15
+ page does not launch paid jobs or run inference automatically.
leaderboard/submit_jobs.sh ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ # Run this script deliberately: it launches PAID HF Jobs, one H200 per dataset.
3
+ # It does not submit a GitHub PR or register the model on the leaderboard.
4
+ set -euo pipefail
5
+
6
+ : "${HF_TOKEN:?Set a write token in the process environment; do not commit it.}"
7
+ : "${SPACE:?Set the built Docker Space, e.g. your-user/open-asr-leaderboard-speedasr.}"
8
+ : "${RESULTS_BUCKET:?Set an existing writable bucket, e.g. your-user/speedasr-results.}"
9
+ : "${MODEL_REVISION:?Set the full model commit SHA from the Hub upload.}"
10
+
11
+ MODEL_ID=${MODEL_ID:-SpeedAudio261119/SpeedASR-v1-65M}
12
+ FLAVOR=${FLAVOR:-h200}
13
+ TIMEOUT=${TIMEOUT:-8h}
14
+ BATCH_SIZE=${BATCH_SIZE:-2048}
15
+ BATCH_AUDIO_SECONDS=${BATCH_AUDIO_SECONDS:-5000}
16
+ TORCH_THREADS=${TORCH_THREADS:-16}
17
+ RUN_ID=${RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)}
18
+ MODEL_FOLDER=${MODEL_ID//\//-}
19
+ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
20
+
21
+ DATASETS=(
22
+ "ami_cleaned|hf-audio/open-asr-leaderboard|ami_cleaned"
23
+ "gigaspeech_cleaned|hf-audio/open-asr-leaderboard|gigaspeech_cleaned"
24
+ "voxpopuli_cleaned_aa|hf-audio/open-asr-leaderboard|voxpopuli_cleaned_aa"
25
+ "earnings22_cleaned_aa_chunked|ArtificialAnalysis/Earnings22-Cleaned-AA-chunked|"
26
+ "spgispeech|hf-audio/open-asr-leaderboard|spgispeech"
27
+ "urgent2024|hf-audio/open-asr-leaderboard|urgent2024"
28
+ "monsoon_en_in|VoiceArena/Monsoon_en_IN_test|"
29
+ )
30
+ if [[ ${INCLUDE_URGENT_CLEAN:-0} == 1 ]]; then
31
+ DATASETS+=("urgent2024_clean|hf-audio/open-asr-leaderboard|urgent2024_clean")
32
+ fi
33
+ namespace_args=()
34
+ if [[ -n ${ORG_NAME:-} ]]; then
35
+ namespace_args=(--namespace "$ORG_NAME")
36
+ fi
37
+
38
+ # Variables enter the container as environment values; the fixed command below
39
+ # contains no interpolated token or user-supplied shell fragments.
40
+ JOB_COMMAND=$(cat <<'JOB'
41
+ set -euo pipefail
42
+ cd /app
43
+ OUT="/results/${SPEEDASR_MODEL_FOLDER}/${SPEEDASR_RUN_ID}/${SPEEDASR_DATASET_LABEL}"
44
+ mkdir -p "$OUT"
45
+ persist_results() {
46
+ cp results/*.jsonl results/*.meta.json "$OUT/" 2>/dev/null || true
47
+ }
48
+ trap persist_results EXIT
49
+ python leaderboard/run_eval.py \
50
+ --model_id "$SPEEDASR_MODEL_ID" --revision "$SPEEDASR_MODEL_REVISION" \
51
+ --dataset_path "$SPEEDASR_DATASET_PATH" --dataset "$SPEEDASR_DATASET_CONFIG" \
52
+ --split test --device 0 --batch_size "$SPEEDASR_BATCH_SIZE" \
53
+ --batch-audio-seconds "$SPEEDASR_BATCH_AUDIO_SECONDS" \
54
+ --torch-threads "$SPEEDASR_TORCH_THREADS" --warmup-steps 5 \
55
+ --warmup-seconds 10 --max_eval_samples -1 2>&1 | tee "$OUT/job.log"
56
+ JOB
57
+ )
58
+
59
+ selected=0
60
+ for cfg in "${DATASETS[@]}"; do
61
+ IFS='|' read -r label dataset_path dataset_config <<< "$cfg"
62
+ if [[ -n ${ONLY_DATASETS:-} && " ${ONLY_DATASETS} " != *" ${label} "* ]]; then
63
+ continue
64
+ fi
65
+ selected=$((selected + 1))
66
+ printf 'Running %s, model revision %s, flavor %s\n' "$label" "$MODEL_REVISION" "$FLAVOR"
67
+ # Without --detach the current hf CLI waits and returns nonzero on job failure.
68
+ hf jobs run --flavor "$FLAVOR" --timeout "$TIMEOUT" \
69
+ "${namespace_args[@]}" \
70
+ --secrets "HF_TOKEN=$HF_TOKEN" \
71
+ --env "HF_AUDIO_DECODER_BACKEND=soundfile" \
72
+ --env "SPEEDASR_MODEL_ID=$MODEL_ID" \
73
+ --env "SPEEDASR_MODEL_REVISION=$MODEL_REVISION" \
74
+ --env "SPEEDASR_MODEL_FOLDER=$MODEL_FOLDER" \
75
+ --env "SPEEDASR_RUN_ID=$RUN_ID" \
76
+ --env "SPEEDASR_DATASET_LABEL=$label" \
77
+ --env "SPEEDASR_DATASET_PATH=$dataset_path" \
78
+ --env "SPEEDASR_DATASET_CONFIG=$dataset_config" \
79
+ --env "SPEEDASR_BATCH_SIZE=$BATCH_SIZE" \
80
+ --env "SPEEDASR_BATCH_AUDIO_SECONDS=$BATCH_AUDIO_SECONDS" \
81
+ --env "SPEEDASR_TORCH_THREADS=$TORCH_THREADS" \
82
+ --volume "hf://buckets/${RESULTS_BUCKET}:/results" \
83
+ "hf.co/spaces/${SPACE}" bash -c "$JOB_COMMAND"
84
+ done
85
+ if (( selected == 0 )); then
86
+ printf 'No datasets selected; check ONLY_DATASETS.\n' >&2
87
+ exit 2
88
+ fi
89
+
90
+ LOCAL_RESULTS="$SCRIPT_DIR/cloud_results/$RUN_ID"
91
+ mkdir -p "$LOCAL_RESULTS"
92
+ hf buckets sync \
93
+ "hf://buckets/${RESULTS_BUCKET}/${MODEL_FOLDER}/${RUN_ID}" "$LOCAL_RESULTS"
94
+ printf 'Downloaded cloud results to %s\n' "$LOCAL_RESULTS"
95
+ printf 'Score all seven sets using the pinned official scorer; see cloud_submission.txt.\n'
model_config.json ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "model_type": "offline_zipformer_transducer",
4
+ "model_args": {
5
+ "num_encoder_layers": "2,2,3,4,3,2",
6
+ "downsampling_factor": "1,2,4,8,4,2",
7
+ "feedforward_dim": "512,768,1024,1536,1024,768",
8
+ "num_heads": "4,4,4,8,4,4",
9
+ "encoder_dim": "192,256,384,512,384,256",
10
+ "query_head_dim": "32",
11
+ "value_head_dim": "12",
12
+ "pos_head_dim": "4",
13
+ "pos_dim": 48,
14
+ "encoder_unmasked_dim": "192,192,256,256,256,192",
15
+ "cnn_module_kernel": "31,31,15,15,15,31",
16
+ "decoder_dim": 512,
17
+ "joiner_dim": 512,
18
+ "context_size": 3,
19
+ "causal": false,
20
+ "chunk_size": "-1",
21
+ "left_context_frames": "-1",
22
+ "use_transducer": true,
23
+ "use_ctc": false,
24
+ "use_attention_decoder": false,
25
+ "use_cr_ctc": false,
26
+ "feature_dim": 80,
27
+ "attention_decoder_dim": 512,
28
+ "attention_decoder_num_layers": 6,
29
+ "attention_decoder_attention_dim": 512,
30
+ "attention_decoder_num_heads": 8,
31
+ "attention_decoder_feedforward_dim": 2048,
32
+ "blank_id": 0,
33
+ "unk_id": 7,
34
+ "vocab_size": 500
35
+ },
36
+ "inference": {
37
+ "sample_rate": 16000,
38
+ "feature_type": "kaldifeat.Fbank",
39
+ "num_mel_bins": 80,
40
+ "dither": 0,
41
+ "snip_edges": false,
42
+ "high_freq": -400,
43
+ "precision": "fp32",
44
+ "decoding_method": "modified_beam_search",
45
+ "beam_size": 4,
46
+ "voicea_postprocessing": true,
47
+ "min_input_seconds": 1,
48
+ "long_chunk_seconds": 30,
49
+ "long_chunk_batch_size": 512,
50
+ "batch_size": 2048,
51
+ "batch_audio_seconds": 5000,
52
+ "torch_threads": 16
53
+ },
54
+ "expected": {
55
+ "parameters": 65551059,
56
+ "checkpoint_sha256": "b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01",
57
+ "tokens_sha256": "a238b7a0fa67aa6be998ca13dd129d69c50639970583e32232dd149e5448210a"
58
+ }
59
+ }
onnx/decoder-epoch-99-avg-1.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1fc497358cc2f63d4dcd3f249da1e4f29a04315c5d7e4aa358cfaf17ea8d5db1
3
+ size 2101273
onnx/encoder-epoch-99-avg-1.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:768fedad168901e872e46477b347adc95cc304ed9ee60ee57c95b60916da42f0
3
+ size 263111425
onnx/joiner-epoch-99-avg-1.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c26540ac6d9ac00191436193bc22a8dd3ceae7bd3a83058aa2d18e1470bed5f4
3
+ size 1026462
pretrained.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01
3
+ size 262613934
requirements-core.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # Install matching torch / k2 / kaldifeat wheels first. See requirements-linux-cu124.txt.
2
+ numpy==1.24.4
3
+ soundfile==0.13.1
4
+ packaging>=20
5
+ huggingface_hub==1.18.0
requirements-linux-cu124.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ # Reproduces the benchmark ABI: Linux x86_64, CPython 3.10, CUDA 12.4.
2
+ --extra-index-url https://download.pytorch.org/whl/cu124
3
+ torch==2.6.0+cu124
4
+ torchaudio==2.6.0+cu124
5
+ k2 @ https://huggingface.co/csukuangfj2/k2/resolve/main/ubuntu-cuda/1.24.4.dev20260625/k2-1.24.4.dev20260625+cuda12.4.torch2.6.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl#sha256=1057e7c004c332433b422ef7c7744b99515a71d0ec19ae18fc9ff80d46139237
6
+ kaldifeat @ https://huggingface.co/csukuangfj/kaldifeat/resolve/main/cuda/1.25.5.dev20260626/linux/kaldifeat-1.25.5.dev20260626+cuda12.4.torch2.6.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl#sha256=2991f7a70876e69bbf507d22fc8117e887e3098e892e7d28a1febeacee306cdb
7
+ -r requirements-core.txt
requirements.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # Reference CUDA build; use CPython 3.10 on Linux x86_64.
2
+ # CPU/other Python builds require torch-compatible k2 and kaldifeat wheels.
3
+ -r requirements-linux-cu124.txt
runtime_source_manifest.json ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "source_repository": "https://github.com/k2-fsa/icefall",
3
+ "source_git_revision": "3f848bb6d0acc970c9b294a30ca0a04a7c9c78d1",
4
+ "notes": "Inference-only extraction from the actual 3090 model source. Original copyright and Apache2 notices retained. Imports made package-relative; no training/data modules are imported. Text line endings normalized to LF.",
5
+ "files": [
6
+ {
7
+ "path": "speedasr_core/encoder_interface.py",
8
+ "origin": "egs/librispeech/ASR/zipformer/encoder_interface.py",
9
+ "original_sha256": "fc70f530c33e2489435ff2cb4bb0dad8b9d6f58a248c425cbd6100c62f628569",
10
+ "sha256": "fc70f530c33e2489435ff2cb4bb0dad8b9d6f58a248c425cbd6100c62f628569",
11
+ "change": "Relative imports only; implementation retained."
12
+ },
13
+ {
14
+ "path": "speedasr_core/scaling.py",
15
+ "origin": "egs/librispeech/ASR/zipformer/scaling.py",
16
+ "original_sha256": "d5a5db141cd00b4beaa990fe6827599b3248b2fddd87e8dc7809b0dbd9d6633b",
17
+ "sha256": "679b1fe2655902ba13ef3f5cf7c924129a16681e3edc4343e766ff7086f5e6d7",
18
+ "change": "Relative imports only; implementation retained."
19
+ },
20
+ {
21
+ "path": "speedasr_core/subsampling.py",
22
+ "origin": "egs/librispeech/ASR/zipformer/subsampling.py",
23
+ "original_sha256": "e04de281f136560e2d44b5f7df174d0364b5feb2c22608173b60de02630c2a66",
24
+ "sha256": "dc33ac662d2508e191f4da5562f0c8d23444819b707624736429a85517db9145",
25
+ "change": "Relative imports only; implementation retained."
26
+ },
27
+ {
28
+ "path": "speedasr_core/zipformer.py",
29
+ "origin": "egs/librispeech/ASR/zipformer/zipformer.py",
30
+ "original_sha256": "4d5365ed19dee07f5a51fcaaa26e18310f7fafbdf2a391e24a7cc01d1b2e8cfe",
31
+ "sha256": "5cea6da601de9f37de2e1db6c2dadcd931220b45341472efaa426a25130a5a36",
32
+ "change": "Relative imports only; implementation retained."
33
+ },
34
+ {
35
+ "path": "speedasr_core/decoder.py",
36
+ "origin": "egs/librispeech/ASR/zipformer/decoder.py",
37
+ "original_sha256": "92ee967e4b927a3ef2e4cc2d900756609a191f859c956a400cfe642d6ef0cf52",
38
+ "sha256": "a9aeeeef278f4e5357d4e49253d24f5f1c60df88cfcb623ccbd01481dc1503ce",
39
+ "change": "Relative imports only; implementation retained."
40
+ },
41
+ {
42
+ "path": "speedasr_core/joiner.py",
43
+ "origin": "egs/librispeech/ASR/zipformer/joiner.py",
44
+ "original_sha256": "2d924a5a00f7a2bb2340c1a991f8a71037bcfbc7e98cbabe26527ffa7d4030d4",
45
+ "sha256": "81faad6e0d07342d1abab28f2271201256f2139f5816b04296d6a8cbe0e2c887",
46
+ "change": "Relative imports only; implementation retained."
47
+ },
48
+ {
49
+ "path": "speedasr_core/model.py",
50
+ "origin": "egs/librispeech/ASR/zipformer/model.py",
51
+ "original_sha256": "1e516fec1ad65220470a7db69bf3d16e4736a0b9943ef8f5ad75bb962d7a3296",
52
+ "sha256": "46b8e181fb326e8cd82c92a4c0ffd0917eb1474a26bd5b60b39b48daac28e90f",
53
+ "change": "Keep AsrModel constructor and forward_encoder unchanged; remove training loss methods and training imports."
54
+ },
55
+ {
56
+ "path": "speedasr_core/beam_search.py",
57
+ "origin": "egs/librispeech/ASR/zipformer/beam_search.py",
58
+ "original_sha256": "ae6afa2654215175ba5be630c7c34ee885a42df4cc79b1abb743c707d716212a",
59
+ "sha256": "d2e0297d1626b14b834a9b357caa875a5b11ce8f0aabc475debb7d20dec3b8c1",
60
+ "change": "Extract Hypothesis/HypothesisList/get_hyps_shape/modified_beam_search unchanged; remove unrelated decoders and LM/training imports; retain annotation-only context names."
61
+ },
62
+ {
63
+ "path": "speedasr_core/utils.py",
64
+ "origin": "icefall/utils.py",
65
+ "original_sha256": "35beb5077b458872621094d7b176adf58fb6a7d4de81a60f033184052b6555d6",
66
+ "sha256": "682d0733f0e149be0ae9b62cb8de60d7baef65be4e64816001b6bb089b4e787b",
67
+ "change": "Extract only torch_autocast and make_pad_mask unchanged; remove unrelated training/data helpers."
68
+ }
69
+ ]
70
+ }
speedasr_core/LICENSE ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
5
+
6
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
+
8
+ 1. Definitions.
9
+
10
+ "License" shall mean the terms and conditions for use, reproduction,
11
+ and distribution as defined by Sections 1 through 9 of this document.
12
+
13
+ "Licensor" shall mean the copyright owner or entity authorized by
14
+ the copyright owner that is granting the License.
15
+
16
+ "Legal Entity" shall mean the union of the acting entity and all
17
+ other entities that control, are controlled by, or are under common
18
+ control with that entity. For the purposes of this definition,
19
+ "control" means (i) the power, direct or indirect, to cause the
20
+ direction or management of such entity, whether by contract or
21
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
22
+ outstanding shares, or (iii) beneficial ownership of such entity.
23
+
24
+ "You" (or "Your") shall mean an individual or Legal Entity
25
+ exercising permissions granted by this License.
26
+
27
+ "Source" form shall mean the preferred form for making modifications,
28
+ including but not limited to software source code, documentation
29
+ source, and configuration files.
30
+
31
+ "Object" form shall mean any form resulting from mechanical
32
+ transformation or translation of a Source form, including but
33
+ not limited to compiled object code, generated documentation,
34
+ and conversions to other media types.
35
+
36
+ "Work" shall mean the work of authorship, whether in Source or
37
+ Object form, made available under the License, as indicated by a
38
+ copyright notice that is included in or attached to the work
39
+ (an example is provided in the Appendix below).
40
+
41
+ "Derivative Works" shall mean any work, whether in Source or Object
42
+ form, that is based on (or derived from) the Work and for which the
43
+ editorial revisions, annotations, elaborations, or other modifications
44
+ represent, as a whole, an original work of authorship. For the purposes
45
+ of this License, Derivative Works shall not include works that remain
46
+ separable from, or merely link (or bind by name) to the interfaces of,
47
+ the Work and Derivative Works thereof.
48
+
49
+ "Contribution" shall mean any work of authorship, including
50
+ the original version of the Work and any modifications or additions
51
+ to that Work or Derivative Works thereof, that is intentionally
52
+ submitted to Licensor for inclusion in the Work by the copyright owner
53
+ or by an individual or Legal Entity authorized to submit on behalf of
54
+ the copyright owner. For the purposes of this definition, "submitted"
55
+ means any form of electronic, verbal, or written communication sent
56
+ to the Licensor or its representatives, including but not limited to
57
+ communication on electronic mailing lists, source code control systems,
58
+ and issue tracking systems that are managed by, or on behalf of, the
59
+ Licensor for the purpose of discussing and improving the Work, but
60
+ excluding communication that is conspicuously marked or otherwise
61
+ designated in writing by the copyright owner as "Not a Contribution."
62
+
63
+ "Contributor" shall mean Licensor and any individual or Legal Entity
64
+ on behalf of whom a Contribution has been received by Licensor and
65
+ subsequently incorporated within the Work.
66
+
67
+ 2. Grant of Copyright License. Subject to the terms and conditions of
68
+ this License, each Contributor hereby grants to You a perpetual,
69
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70
+ copyright license to reproduce, prepare Derivative Works of,
71
+ publicly display, publicly perform, sublicense, and distribute the
72
+ Work and such Derivative Works in Source or Object form.
73
+
74
+ 3. Grant of Patent License. Subject to the terms and conditions of
75
+ this License, each Contributor hereby grants to You a perpetual,
76
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77
+ (except as stated in this section) patent license to make, have made,
78
+ use, offer to sell, sell, import, and otherwise transfer the Work,
79
+ where such license applies only to those patent claims licensable
80
+ by such Contributor that are necessarily infringed by their
81
+ Contribution(s) alone or by combination of their Contribution(s)
82
+ with the Work to which such Contribution(s) was submitted. If You
83
+ institute patent litigation against any entity (including a
84
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
85
+ or a Contribution incorporated within the Work constitutes direct
86
+ or contributory patent infringement, then any patent licenses
87
+ granted to You under this License for that Work shall terminate
88
+ as of the date such litigation is filed.
89
+
90
+ 4. Redistribution. You may reproduce and distribute copies of the
91
+ Work or Derivative Works thereof in any medium, with or without
92
+ modifications, and in Source or Object form, provided that You
93
+ meet the following conditions:
94
+
95
+ (a) You must give any other recipients of the Work or
96
+ Derivative Works a copy of this License; and
97
+
98
+ (b) You must cause any modified files to carry prominent notices
99
+ stating that You changed the files; and
100
+
101
+ (c) You must retain, in the Source form of any Derivative Works
102
+ that You distribute, all copyright, patent, trademark, and
103
+ attribution notices from the Source form of the Work,
104
+ excluding those notices that do not pertain to any part of
105
+ the Derivative Works; and
106
+
107
+ (d) If the Work includes a "NOTICE" text file as part of its
108
+ distribution, then any Derivative Works that You distribute must
109
+ include a readable copy of the attribution notices contained
110
+ within such NOTICE file, excluding those notices that do not
111
+ pertain to any part of the Derivative Works, in at least one
112
+ of the following places: within a NOTICE text file distributed
113
+ as part of the Derivative Works; within the Source form or
114
+ documentation, if provided along with the Derivative Works; or,
115
+ within a display generated by the Derivative Works, if and
116
+ wherever such third-party notices normally appear. The contents
117
+ of the NOTICE file are for informational purposes only and
118
+ do not modify the License. You may add Your own attribution
119
+ notices within Derivative Works that You distribute, alongside
120
+ or as an addendum to the NOTICE text from the Work, provided
121
+ that such additional attribution notices cannot be construed
122
+ as modifying the License.
123
+
124
+ You may add Your own copyright statement to Your modifications and
125
+ may provide additional or different license terms and conditions
126
+ for use, reproduction, or distribution of Your modifications, or
127
+ for any such Derivative Works as a whole, provided Your use,
128
+ reproduction, and distribution of the Work otherwise complies with
129
+ the conditions stated in this License.
130
+
131
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
132
+ any Contribution intentionally submitted for inclusion in the Work
133
+ by You to the Licensor shall be under the terms and conditions of
134
+ this License, without any additional terms or conditions.
135
+ Notwithstanding the above, nothing herein shall supersede or modify
136
+ the terms of any separate license agreement you may have executed
137
+ with Licensor regarding such Contributions.
138
+
139
+ 6. Trademarks. This License does not grant permission to use the trade
140
+ names, trademarks, service marks, or product names of the Licensor,
141
+ except as required for reasonable and customary use in describing the
142
+ origin of the Work and reproducing the content of the NOTICE file.
143
+
144
+ 7. Disclaimer of Warranty. Unless required by applicable law or
145
+ agreed to in writing, Licensor provides the Work (and each
146
+ Contributor provides its Contributions) on an "AS IS" BASIS,
147
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148
+ implied, including, without limitation, any warranties or conditions
149
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150
+ PARTICULAR PURPOSE. You are solely responsible for determining the
151
+ appropriateness of using or redistributing the Work and assume any
152
+ risks associated with Your exercise of permissions under this License.
153
+
154
+ 8. Limitation of Liability. In no event and under no legal theory,
155
+ whether in tort (including negligence), contract, or otherwise,
156
+ unless required by applicable law (such as deliberate and grossly
157
+ negligent acts) or agreed to in writing, shall any Contributor be
158
+ liable to You for damages, including any direct, indirect, special,
159
+ incidental, or consequential damages of any character arising as a
160
+ result of this License or out of the use or inability to use the
161
+ Work (including but not limited to damages for loss of goodwill,
162
+ work stoppage, computer failure or malfunction, or any and all
163
+ other commercial damages or losses), even if such Contributor
164
+ has been advised of the possibility of such damages.
165
+
166
+ 9. Accepting Warranty or Additional Liability. While redistributing
167
+ the Work or Derivative Works thereof, You may choose to offer,
168
+ and charge a fee for, acceptance of support, warranty, indemnity,
169
+ or other liability obligations and/or rights consistent with this
170
+ License. However, in accepting such obligations, You may act only
171
+ on Your own behalf and on Your sole responsibility, not on behalf
172
+ of any other Contributor, and only if You agree to indemnify,
173
+ defend, and hold each Contributor harmless for any liability
174
+ incurred by, or claims asserted against, such Contributor by reason
175
+ of your accepting any such warranty or additional liability.
176
+
177
+ END OF TERMS AND CONDITIONS
178
+
179
+ APPENDIX: How to apply the Apache License to your work.
180
+
181
+ To apply the Apache License to your work, attach the following
182
+ boilerplate notice, with the fields enclosed by brackets "[]"
183
+ replaced with your own identifying information. (Don't include
184
+ the brackets!) The text should be enclosed in the appropriate
185
+ comment syntax for the file format. We also recommend that a
186
+ file or class name and description of purpose be included on the
187
+ same "printed page" as the copyright notice for easier
188
+ identification within third-party archives.
189
+
190
+ Copyright [yyyy] [name of copyright owner]
191
+
192
+ Licensed under the Apache License, Version 2.0 (the "License");
193
+ you may not use this file except in compliance with the License.
194
+ You may obtain a copy of the License at
195
+
196
+ http://www.apache.org/licenses/LICENSE-2.0
197
+
198
+ Unless required by applicable law or agreed to in writing, software
199
+ distributed under the License is distributed on an "AS IS" BASIS,
200
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201
+ See the License for the specific language governing permissions and
202
+ limitations under the License.
speedasr_core/__init__.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ """Vendored Apache-2.0 Icefall inference components; see LICENSE.
2
+
3
+ No external Icefall checkout is required.
4
+ """
speedasr_core/beam_search.py ADDED
@@ -0,0 +1,423 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang
2
+ # Xiaoyu Yang)
3
+ #
4
+ # See ../../../../LICENSE for clarification regarding multiple authors
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+
18
+ from __future__ import annotations
19
+
20
+ import warnings
21
+ from dataclasses import dataclass, field
22
+ from typing import Any, Dict, List, Optional, Tuple, Union
23
+
24
+ import k2
25
+ import torch
26
+ from torch import nn
27
+
28
+ # Optional context/LM types are annotations only; SpeedASR uses no context graph.
29
+ ContextGraph = ContextState = NgramLmStateCost = Any
30
+
31
+ @dataclass
32
+ class DecodingResults:
33
+ hyps: List[List[int]]
34
+ timestamps: List[List[int]]
35
+
36
+
37
+ @dataclass
38
+ class Hypothesis:
39
+ # The predicted tokens so far.
40
+ # Newly predicted tokens are appended to `ys`.
41
+ ys: List[int]
42
+
43
+ # The log prob of ys.
44
+ # It contains only one entry.
45
+ log_prob: torch.Tensor
46
+
47
+ ac_probs: Optional[List[float]] = None
48
+
49
+ # timestamp[i] is the frame index after subsampling
50
+ # on which ys[i] is decoded
51
+ timestamp: List[int] = field(default_factory=list)
52
+
53
+ # the lm score for next token given the current ys
54
+ lm_score: Optional[torch.Tensor] = None
55
+
56
+ # the RNNLM states (h and c in LSTM)
57
+ state: Optional[Tuple[torch.Tensor, torch.Tensor]] = None
58
+
59
+ # N-gram LM state
60
+ state_cost: Optional[NgramLmStateCost] = None
61
+
62
+ # Context graph state
63
+ context_state: Optional[ContextState] = None
64
+
65
+ num_tailing_blanks: int = 0
66
+
67
+ @property
68
+ def key(self) -> str:
69
+ """Return a string representation of self.ys"""
70
+ return "_".join(map(str, self.ys))
71
+
72
+
73
+ class HypothesisList(object):
74
+ def __init__(self, data: Optional[Dict[str, Hypothesis]] = None) -> None:
75
+ """
76
+ Args:
77
+ data:
78
+ A dict of Hypotheses. Its key is its `value.key`.
79
+ """
80
+ if data is None:
81
+ self._data = {}
82
+ else:
83
+ self._data = data
84
+
85
+ @property
86
+ def data(self) -> Dict[str, Hypothesis]:
87
+ return self._data
88
+
89
+ def add(self, hyp: Hypothesis) -> None:
90
+ """Add a Hypothesis to `self`.
91
+
92
+ If `hyp` already exists in `self`, its probability is updated using
93
+ `log-sum-exp` with the existed one.
94
+
95
+ Args:
96
+ hyp:
97
+ The hypothesis to be added.
98
+ """
99
+ key = hyp.key
100
+ if key in self:
101
+ old_hyp = self._data[key] # shallow copy
102
+ torch.logaddexp(old_hyp.log_prob, hyp.log_prob, out=old_hyp.log_prob)
103
+ else:
104
+ self._data[key] = hyp
105
+
106
+ def get_most_probable(self, length_norm: bool = False) -> Hypothesis:
107
+ """Get the most probable hypothesis, i.e., the one with
108
+ the largest `log_prob`.
109
+
110
+ Args:
111
+ length_norm:
112
+ If True, the `log_prob` of a hypothesis is normalized by the
113
+ number of tokens in it.
114
+ Returns:
115
+ Return the hypothesis that has the largest `log_prob`.
116
+ """
117
+ if length_norm:
118
+ return max(self._data.values(), key=lambda hyp: hyp.log_prob / len(hyp.ys))
119
+ else:
120
+ return max(self._data.values(), key=lambda hyp: hyp.log_prob)
121
+
122
+ def remove(self, hyp: Hypothesis) -> None:
123
+ """Remove a given hypothesis.
124
+
125
+ Caution:
126
+ `self` is modified **in-place**.
127
+
128
+ Args:
129
+ hyp:
130
+ The hypothesis to be removed from `self`.
131
+ Note: It must be contained in `self`. Otherwise,
132
+ an exception is raised.
133
+ """
134
+ key = hyp.key
135
+ assert key in self, f"{key} does not exist"
136
+ del self._data[key]
137
+
138
+ def filter(self, threshold: torch.Tensor) -> "HypothesisList":
139
+ """Remove all Hypotheses whose log_prob is less than threshold.
140
+
141
+ Caution:
142
+ `self` is not modified. Instead, a new HypothesisList is returned.
143
+
144
+ Returns:
145
+ Return a new HypothesisList containing all hypotheses from `self`
146
+ with `log_prob` being greater than the given `threshold`.
147
+ """
148
+ ans = HypothesisList()
149
+ for _, hyp in self._data.items():
150
+ if hyp.log_prob > threshold:
151
+ ans.add(hyp) # shallow copy
152
+ return ans
153
+
154
+ def topk(self, k: int, length_norm: bool = False) -> "HypothesisList":
155
+ """Return the top-k hypothesis.
156
+
157
+ Args:
158
+ length_norm:
159
+ If True, the `log_prob` of a hypothesis is normalized by the
160
+ number of tokens in it.
161
+ """
162
+ hyps = list(self._data.items())
163
+
164
+ if length_norm:
165
+ hyps = sorted(
166
+ hyps, key=lambda h: h[1].log_prob / len(h[1].ys), reverse=True
167
+ )[:k]
168
+ else:
169
+ hyps = sorted(hyps, key=lambda h: h[1].log_prob, reverse=True)[:k]
170
+
171
+ ans = HypothesisList(dict(hyps))
172
+ return ans
173
+
174
+ def __contains__(self, key: str):
175
+ return key in self._data
176
+
177
+ def __iter__(self):
178
+ return iter(self._data.values())
179
+
180
+ def __len__(self) -> int:
181
+ return len(self._data)
182
+
183
+ def __str__(self) -> str:
184
+ s = []
185
+ for key in self:
186
+ s.append(key)
187
+ return ", ".join(s)
188
+
189
+
190
+ def get_hyps_shape(hyps: List[HypothesisList]) -> k2.RaggedShape:
191
+ """Return a ragged shape with axes [utt][num_hyps].
192
+
193
+ Args:
194
+ hyps:
195
+ len(hyps) == batch_size. It contains the current hypothesis for
196
+ each utterance in the batch.
197
+ Returns:
198
+ Return a ragged shape with 2 axes [utt][num_hyps]. Note that
199
+ the shape is on CPU.
200
+ """
201
+ num_hyps = [len(h) for h in hyps]
202
+
203
+ # torch.cumsum() is inclusive sum, so we put a 0 at the beginning
204
+ # to get exclusive sum later.
205
+ num_hyps.insert(0, 0)
206
+
207
+ num_hyps = torch.tensor(num_hyps)
208
+ row_splits = torch.cumsum(num_hyps, dim=0, dtype=torch.int32)
209
+ ans = k2.ragged.create_ragged_shape2(
210
+ row_splits=row_splits, cached_tot_size=row_splits[-1].item()
211
+ )
212
+ return ans
213
+
214
+
215
+
216
+ def modified_beam_search(
217
+ model: nn.Module,
218
+ encoder_out: torch.Tensor,
219
+ encoder_out_lens: torch.Tensor,
220
+ context_graph: Optional[ContextGraph] = None,
221
+ beam: int = 4,
222
+ temperature: float = 1.0,
223
+ blank_penalty: float = 0.0,
224
+ return_timestamps: bool = False,
225
+ ) -> Union[List[List[int]], DecodingResults]:
226
+ """Beam search in batch mode with --max-sym-per-frame=1 being hardcoded.
227
+
228
+ Args:
229
+ model:
230
+ The transducer model.
231
+ encoder_out:
232
+ Output from the encoder. Its shape is (N, T, C).
233
+ encoder_out_lens:
234
+ A 1-D tensor of shape (N,), containing number of valid frames in
235
+ encoder_out before padding.
236
+ beam:
237
+ Number of active paths during the beam search.
238
+ temperature:
239
+ Softmax temperature.
240
+ return_timestamps:
241
+ Whether to return timestamps.
242
+ Returns:
243
+ If return_timestamps is False, return the decoded result.
244
+ Else, return a DecodingResults object containing
245
+ decoded result and corresponding timestamps.
246
+ """
247
+ assert encoder_out.ndim == 3, encoder_out.shape
248
+ assert encoder_out.size(0) >= 1, encoder_out.size(0)
249
+
250
+ packed_encoder_out = torch.nn.utils.rnn.pack_padded_sequence(
251
+ input=encoder_out,
252
+ lengths=encoder_out_lens.cpu(),
253
+ batch_first=True,
254
+ enforce_sorted=False,
255
+ )
256
+
257
+ blank_id = model.decoder.blank_id
258
+ unk_id = getattr(model, "unk_id", blank_id)
259
+ context_size = model.decoder.context_size
260
+ device = next(model.parameters()).device
261
+
262
+ batch_size_list = packed_encoder_out.batch_sizes.tolist()
263
+ N = encoder_out.size(0)
264
+ assert torch.all(encoder_out_lens > 0), encoder_out_lens
265
+ assert N == batch_size_list[0], (N, batch_size_list)
266
+
267
+ B = [HypothesisList() for _ in range(N)]
268
+ for i in range(N):
269
+ B[i].add(
270
+ Hypothesis(
271
+ ys=[-1] * (context_size - 1) + [blank_id],
272
+ log_prob=torch.zeros(1, dtype=torch.float32, device=device),
273
+ context_state=None if context_graph is None else context_graph.root,
274
+ timestamp=[],
275
+ )
276
+ )
277
+
278
+ encoder_out = model.joiner.encoder_proj(packed_encoder_out.data)
279
+
280
+ offset = 0
281
+ finalized_B = []
282
+ for t, batch_size in enumerate(batch_size_list):
283
+ start = offset
284
+ end = offset + batch_size
285
+ current_encoder_out = encoder_out.data[start:end]
286
+ current_encoder_out = current_encoder_out.unsqueeze(1).unsqueeze(1)
287
+ # current_encoder_out's shape is (batch_size, 1, 1, encoder_out_dim)
288
+ offset = end
289
+
290
+ finalized_B = B[batch_size:] + finalized_B
291
+ B = B[:batch_size]
292
+
293
+ hyps_shape = get_hyps_shape(B).to(device)
294
+
295
+ A = [list(b) for b in B]
296
+
297
+ B = [HypothesisList() for _ in range(batch_size)]
298
+
299
+ ys_log_probs = torch.cat(
300
+ [hyp.log_prob.reshape(1, 1) for hyps in A for hyp in hyps]
301
+ ) # (num_hyps, 1)
302
+
303
+ decoder_input = torch.tensor(
304
+ [hyp.ys[-context_size:] for hyps in A for hyp in hyps],
305
+ device=device,
306
+ dtype=torch.int64,
307
+ ) # (num_hyps, context_size)
308
+
309
+ decoder_out = model.decoder(decoder_input, need_pad=False).unsqueeze(1)
310
+ decoder_out = model.joiner.decoder_proj(decoder_out)
311
+ # decoder_out is of shape (num_hyps, 1, 1, joiner_dim)
312
+
313
+ # Note: For torch 1.7.1 and below, it requires a torch.int64 tensor
314
+ # as index, so we use `to(torch.int64)` below.
315
+ current_encoder_out = torch.index_select(
316
+ current_encoder_out,
317
+ dim=0,
318
+ index=hyps_shape.row_ids(1).to(torch.int64),
319
+ ) # (num_hyps, 1, 1, encoder_out_dim)
320
+
321
+ logits = model.joiner(
322
+ current_encoder_out,
323
+ decoder_out,
324
+ project_input=False,
325
+ ) # (num_hyps, 1, 1, vocab_size)
326
+
327
+ logits = logits.squeeze(1).squeeze(1) # (num_hyps, vocab_size)
328
+
329
+ if blank_penalty != 0:
330
+ logits[:, 0] -= blank_penalty
331
+
332
+ log_probs = (logits / temperature).log_softmax(dim=-1) # (num_hyps, vocab_size)
333
+
334
+ log_probs.add_(ys_log_probs)
335
+
336
+ vocab_size = log_probs.size(-1)
337
+
338
+ log_probs = log_probs.reshape(-1)
339
+
340
+ row_splits = hyps_shape.row_splits(1) * vocab_size
341
+ log_probs_shape = k2.ragged.create_ragged_shape2(
342
+ row_splits=row_splits, cached_tot_size=log_probs.numel()
343
+ )
344
+ ragged_log_probs = k2.RaggedTensor(shape=log_probs_shape, value=log_probs)
345
+
346
+ for i in range(batch_size):
347
+ topk_log_probs, topk_indexes = ragged_log_probs[i].topk(beam)
348
+
349
+ with warnings.catch_warnings():
350
+ warnings.simplefilter("ignore")
351
+ topk_hyp_indexes = (topk_indexes // vocab_size).tolist()
352
+ topk_token_indexes = (topk_indexes % vocab_size).tolist()
353
+
354
+ for k in range(len(topk_hyp_indexes)):
355
+ hyp_idx = topk_hyp_indexes[k]
356
+ hyp = A[i][hyp_idx]
357
+ new_ys = hyp.ys[:]
358
+ new_token = topk_token_indexes[k]
359
+ new_timestamp = hyp.timestamp[:]
360
+ context_score = 0
361
+ new_context_state = None if context_graph is None else hyp.context_state
362
+ if new_token not in (blank_id, unk_id):
363
+ new_ys.append(new_token)
364
+ new_timestamp.append(t)
365
+ if context_graph is not None:
366
+ (
367
+ context_score,
368
+ new_context_state,
369
+ _,
370
+ ) = context_graph.forward_one_step(
371
+ hyp.context_state, new_token, strict_mode=False
372
+ )
373
+
374
+ new_log_prob = topk_log_probs[k] + context_score
375
+
376
+ new_hyp = Hypothesis(
377
+ ys=new_ys,
378
+ log_prob=new_log_prob,
379
+ timestamp=new_timestamp,
380
+ context_state=new_context_state,
381
+ )
382
+ B[i].add(new_hyp)
383
+
384
+ B = B + finalized_B
385
+
386
+ # finalize context_state, if the matched contexts do not reach final state
387
+ # we need to add the score on the corresponding backoff arc
388
+ if context_graph is not None:
389
+ finalized_B = [HypothesisList() for _ in range(len(B))]
390
+ for i, hyps in enumerate(B):
391
+ for hyp in list(hyps):
392
+ context_score, new_context_state = context_graph.finalize(
393
+ hyp.context_state
394
+ )
395
+ finalized_B[i].add(
396
+ Hypothesis(
397
+ ys=hyp.ys,
398
+ log_prob=hyp.log_prob + context_score,
399
+ timestamp=hyp.timestamp,
400
+ context_state=new_context_state,
401
+ )
402
+ )
403
+ B = finalized_B
404
+
405
+ best_hyps = [b.get_most_probable(length_norm=True) for b in B]
406
+
407
+ sorted_ans = [h.ys[context_size:] for h in best_hyps]
408
+ sorted_timestamps = [h.timestamp for h in best_hyps]
409
+ ans = []
410
+ ans_timestamps = []
411
+ unsorted_indices = packed_encoder_out.unsorted_indices.tolist()
412
+ for i in range(N):
413
+ ans.append(sorted_ans[unsorted_indices[i]])
414
+ ans_timestamps.append(sorted_timestamps[unsorted_indices[i]])
415
+
416
+ if not return_timestamps:
417
+ return ans
418
+ else:
419
+ return DecodingResults(
420
+ hyps=ans,
421
+ timestamps=ans_timestamps,
422
+ )
423
+
speedasr_core/decoder.py ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang)
2
+ #
3
+ # See ../../../../LICENSE for clarification regarding multiple authors
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+
17
+ import torch
18
+ import torch.nn as nn
19
+ import torch.nn.functional as F
20
+ from .scaling import Balancer
21
+
22
+
23
+ class Decoder(nn.Module):
24
+ """This class modifies the stateless decoder from the following paper:
25
+
26
+ RNN-transducer with stateless prediction network
27
+ https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=9054419
28
+
29
+ It removes the recurrent connection from the decoder, i.e., the prediction
30
+ network. Different from the above paper, it adds an extra Conv1d
31
+ right after the embedding layer.
32
+
33
+ TODO: Implement https://arxiv.org/pdf/2109.07513.pdf
34
+ """
35
+
36
+ def __init__(
37
+ self,
38
+ vocab_size: int,
39
+ decoder_dim: int,
40
+ blank_id: int,
41
+ context_size: int,
42
+ ):
43
+ """
44
+ Args:
45
+ vocab_size:
46
+ Number of tokens of the modeling unit including blank.
47
+ decoder_dim:
48
+ Dimension of the input embedding, and of the decoder output.
49
+ blank_id:
50
+ The ID of the blank symbol.
51
+ context_size:
52
+ Number of previous words to use to predict the next word.
53
+ 1 means bigram; 2 means trigram. n means (n+1)-gram.
54
+ """
55
+ super().__init__()
56
+
57
+ self.embedding = nn.Embedding(
58
+ num_embeddings=vocab_size,
59
+ embedding_dim=decoder_dim,
60
+ )
61
+ # the balancers are to avoid any drift in the magnitude of the
62
+ # embeddings, which would interact badly with parameter averaging.
63
+ self.balancer = Balancer(
64
+ decoder_dim,
65
+ channel_dim=-1,
66
+ min_positive=0.0,
67
+ max_positive=1.0,
68
+ min_abs=0.5,
69
+ max_abs=1.0,
70
+ prob=0.05,
71
+ )
72
+
73
+ self.blank_id = blank_id
74
+
75
+ assert context_size >= 1, context_size
76
+ self.context_size = context_size
77
+ self.vocab_size = vocab_size
78
+
79
+ if context_size > 1:
80
+ self.conv = nn.Conv1d(
81
+ in_channels=decoder_dim,
82
+ out_channels=decoder_dim,
83
+ kernel_size=context_size,
84
+ padding=0,
85
+ groups=decoder_dim // 4, # group size == 4
86
+ bias=False,
87
+ )
88
+ self.balancer2 = Balancer(
89
+ decoder_dim,
90
+ channel_dim=-1,
91
+ min_positive=0.0,
92
+ max_positive=1.0,
93
+ min_abs=0.5,
94
+ max_abs=1.0,
95
+ prob=0.05,
96
+ )
97
+ else:
98
+ # To avoid `RuntimeError: Module 'Decoder' has no attribute 'conv'`
99
+ # when inference with torch.jit.script and context_size == 1
100
+ self.conv = nn.Identity()
101
+ self.balancer2 = nn.Identity()
102
+
103
+ def forward(self, y: torch.Tensor, need_pad: bool = True) -> torch.Tensor:
104
+ """
105
+ Args:
106
+ y:
107
+ A 2-D tensor of shape (N, U).
108
+ need_pad:
109
+ True to left pad the input. Should be True during training.
110
+ False to not pad the input. Should be False during inference.
111
+ Returns:
112
+ Return a tensor of shape (N, U, decoder_dim).
113
+ """
114
+ y = y.to(torch.int64)
115
+ # this stuff about clamp() is a temporary fix for a mismatch
116
+ # at utterance start, we use negative ids in beam_search.py
117
+ embedding_out = self.embedding(y.clamp(min=0)) * (y >= 0).unsqueeze(-1)
118
+
119
+ embedding_out = self.balancer(embedding_out)
120
+
121
+ if self.context_size > 1:
122
+ embedding_out = embedding_out.permute(0, 2, 1)
123
+ if need_pad is True:
124
+ embedding_out = F.pad(embedding_out, pad=(self.context_size - 1, 0))
125
+ else:
126
+ # During inference time, there is no need to do extra padding
127
+ # as we only need one output
128
+ assert embedding_out.size(-1) == self.context_size
129
+ embedding_out = self.conv(embedding_out)
130
+ embedding_out = embedding_out.permute(0, 2, 1)
131
+ embedding_out = F.relu(embedding_out)
132
+ embedding_out = self.balancer2(embedding_out)
133
+
134
+ return embedding_out
speedasr_core/encoder_interface.py ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang)
2
+ #
3
+ # See ../../../../LICENSE for clarification regarding multiple authors
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+
17
+ from typing import Tuple
18
+
19
+ import torch
20
+ import torch.nn as nn
21
+
22
+
23
+ class EncoderInterface(nn.Module):
24
+ def forward(
25
+ self, x: torch.Tensor, x_lens: torch.Tensor
26
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
27
+ """
28
+ Args:
29
+ x:
30
+ A tensor of shape (batch_size, input_seq_len, num_features)
31
+ containing the input features.
32
+ x_lens:
33
+ A tensor of shape (batch_size,) containing the number of frames
34
+ in `x` before padding.
35
+ Returns:
36
+ Return a tuple containing two tensors:
37
+ - encoder_out, a tensor of (batch_size, out_seq_len, output_dim)
38
+ containing unnormalized probabilities, i.e., the output of a
39
+ linear layer.
40
+ - encoder_out_lens, a tensor of shape (batch_size,) containing
41
+ the number of frames in `encoder_out` before padding.
42
+ """
43
+ raise NotImplementedError("Please implement it in a subclass")
speedasr_core/joiner.py ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang)
2
+ #
3
+ # See ../../../../LICENSE for clarification regarding multiple authors
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+
17
+ import torch
18
+ import torch.nn as nn
19
+ from .scaling import ScaledLinear
20
+
21
+
22
+ class Joiner(nn.Module):
23
+ def __init__(
24
+ self,
25
+ encoder_dim: int,
26
+ decoder_dim: int,
27
+ joiner_dim: int,
28
+ vocab_size: int,
29
+ ):
30
+ super().__init__()
31
+
32
+ self.encoder_proj = ScaledLinear(encoder_dim, joiner_dim, initial_scale=0.25)
33
+ self.decoder_proj = ScaledLinear(decoder_dim, joiner_dim, initial_scale=0.25)
34
+ self.output_linear = nn.Linear(joiner_dim, vocab_size)
35
+
36
+ def forward(
37
+ self,
38
+ encoder_out: torch.Tensor,
39
+ decoder_out: torch.Tensor,
40
+ project_input: bool = True,
41
+ ) -> torch.Tensor:
42
+ """
43
+ Args:
44
+ encoder_out:
45
+ Output from the encoder. Its shape is (N, T, s_range, C).
46
+ decoder_out:
47
+ Output from the decoder. Its shape is (N, T, s_range, C).
48
+ project_input:
49
+ If true, apply input projections encoder_proj and decoder_proj.
50
+ If this is false, it is the user's responsibility to do this
51
+ manually.
52
+ Returns:
53
+ Return a tensor of shape (N, T, s_range, C).
54
+ """
55
+ assert encoder_out.ndim == decoder_out.ndim, (
56
+ encoder_out.shape,
57
+ decoder_out.shape,
58
+ )
59
+
60
+ if project_input:
61
+ logit = self.encoder_proj(encoder_out) + self.decoder_proj(decoder_out)
62
+ else:
63
+ logit = encoder_out + decoder_out
64
+
65
+ logit = self.output_linear(torch.tanh(logit))
66
+
67
+ return logit
speedasr_core/model.py ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2021-2023 Xiaomi Corp. (authors: Fangjun Kuang,
2
+ # Wei Kang,
3
+ # Zengwei Yao)
4
+ #
5
+ # See ../../../../LICENSE for clarification regarding multiple authors
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+
19
+ from typing import Optional, Tuple
20
+
21
+ import torch
22
+ from torch import nn
23
+ from .encoder_interface import EncoderInterface
24
+ from .scaling import ScaledLinear
25
+ from .utils import make_pad_mask
26
+
27
+
28
+ class AsrModel(nn.Module):
29
+ def __init__(
30
+ self,
31
+ encoder_embed: nn.Module,
32
+ encoder: EncoderInterface,
33
+ decoder: Optional[nn.Module] = None,
34
+ joiner: Optional[nn.Module] = None,
35
+ attention_decoder: Optional[nn.Module] = None,
36
+ encoder_dim: int = 384,
37
+ decoder_dim: int = 512,
38
+ vocab_size: int = 500,
39
+ use_transducer: bool = True,
40
+ use_ctc: bool = False,
41
+ use_attention_decoder: bool = False,
42
+ ):
43
+ """A joint CTC & Transducer ASR model.
44
+
45
+ - Connectionist temporal classification: labelling unsegmented sequence data with recurrent neural networks (http://imagine.enpc.fr/~obozinsg/teaching/mva_gm/papers/ctc.pdf)
46
+ - Sequence Transduction with Recurrent Neural Networks (https://arxiv.org/pdf/1211.3711.pdf)
47
+ - Pruned RNN-T for fast, memory-efficient ASR training (https://arxiv.org/pdf/2206.13236.pdf)
48
+
49
+ Args:
50
+ encoder_embed:
51
+ It is a Convolutional 2D subsampling module. It converts
52
+ an input of shape (N, T, idim) to an output of of shape
53
+ (N, T', odim), where T' = (T-3)//2-2 = (T-7)//2.
54
+ encoder:
55
+ It is the transcription network in the paper. Its accepts
56
+ two inputs: `x` of (N, T, encoder_dim) and `x_lens` of shape (N,).
57
+ It returns two tensors: `logits` of shape (N, T, encoder_dim) and
58
+ `logit_lens` of shape (N,).
59
+ decoder:
60
+ It is the prediction network in the paper. Its input shape
61
+ is (N, U) and its output shape is (N, U, decoder_dim).
62
+ It should contain one attribute: `blank_id`.
63
+ It is used when use_transducer is True.
64
+ joiner:
65
+ It has two inputs with shapes: (N, T, encoder_dim) and (N, U, decoder_dim).
66
+ Its output shape is (N, T, U, vocab_size). Note that its output contains
67
+ unnormalized probs, i.e., not processed by log-softmax.
68
+ It is used when use_transducer is True.
69
+ use_transducer:
70
+ Whether use transducer head. Default: True.
71
+ use_ctc:
72
+ Whether use CTC head. Default: False.
73
+ use_attention_decoder:
74
+ Whether use attention-decoder head. Default: False.
75
+ """
76
+ super().__init__()
77
+
78
+ assert (
79
+ use_transducer or use_ctc
80
+ ), f"At least one of them should be True, but got use_transducer={use_transducer}, use_ctc={use_ctc}"
81
+
82
+ assert isinstance(encoder, EncoderInterface), type(encoder)
83
+
84
+ self.encoder_embed = encoder_embed
85
+ self.encoder = encoder
86
+
87
+ self.use_transducer = use_transducer
88
+ if use_transducer:
89
+ # Modules for Transducer head
90
+ assert decoder is not None
91
+ assert hasattr(decoder, "blank_id")
92
+ assert joiner is not None
93
+
94
+ self.decoder = decoder
95
+ self.joiner = joiner
96
+
97
+ self.simple_am_proj = ScaledLinear(
98
+ encoder_dim, vocab_size, initial_scale=0.25
99
+ )
100
+ self.simple_lm_proj = ScaledLinear(
101
+ decoder_dim, vocab_size, initial_scale=0.25
102
+ )
103
+ else:
104
+ assert decoder is None
105
+ assert joiner is None
106
+
107
+ self.use_ctc = use_ctc
108
+ if use_ctc:
109
+ # Modules for CTC head
110
+ self.ctc_output = nn.Sequential(
111
+ nn.Dropout(p=0.1),
112
+ nn.Linear(encoder_dim, vocab_size),
113
+ nn.LogSoftmax(dim=-1),
114
+ )
115
+
116
+ self.use_attention_decoder = use_attention_decoder
117
+ if use_attention_decoder:
118
+ self.attention_decoder = attention_decoder
119
+ else:
120
+ assert attention_decoder is None
121
+
122
+ def forward_encoder(
123
+ self, x: torch.Tensor, x_lens: torch.Tensor
124
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
125
+ """Compute encoder outputs.
126
+ Args:
127
+ x:
128
+ A 3-D tensor of shape (N, T, C).
129
+ x_lens:
130
+ A 1-D tensor of shape (N,). It contains the number of frames in `x`
131
+ before padding.
132
+
133
+ Returns:
134
+ encoder_out:
135
+ Encoder output, of shape (N, T, C).
136
+ encoder_out_lens:
137
+ Encoder output lengths, of shape (N,).
138
+ """
139
+ # logging.info(f"Memory allocated at entry: {torch.cuda.memory_allocated() // 1000000}M")
140
+ x, x_lens = self.encoder_embed(x, x_lens)
141
+ # logging.info(f"Memory allocated after encoder_embed: {torch.cuda.memory_allocated() // 1000000}M")
142
+
143
+ src_key_padding_mask = make_pad_mask(x_lens)
144
+ x = x.permute(1, 0, 2) # (N, T, C) -> (T, N, C)
145
+
146
+ encoder_out, encoder_out_lens = self.encoder(x, x_lens, src_key_padding_mask)
147
+
148
+ encoder_out = encoder_out.permute(1, 0, 2) # (T, N, C) ->(N, T, C)
149
+ assert torch.all(encoder_out_lens > 0), (x_lens, encoder_out_lens)
150
+
151
+ return encoder_out, encoder_out_lens
152
+
speedasr_core/scaling.py ADDED
@@ -0,0 +1,1913 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022-2023 Xiaomi Corp. (authors: Daniel Povey)
2
+ #
3
+ # See ../../../../LICENSE for clarification regarding multiple authors
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+
17
+
18
+ import logging
19
+ import math
20
+ import random
21
+ from typing import Optional, Tuple, Union
22
+
23
+ import k2
24
+ import torch
25
+ import torch.nn as nn
26
+ from torch import Tensor
27
+ from torch.cuda.amp import custom_bwd, custom_fwd
28
+
29
+ from .utils import torch_autocast
30
+
31
+
32
+ def logaddexp_onnx(x: Tensor, y: Tensor) -> Tensor:
33
+ max_value = torch.max(x, y)
34
+ diff = torch.abs(x - y)
35
+ return max_value + torch.log1p(torch.exp(-diff))
36
+
37
+
38
+ # RuntimeError: Exporting the operator logaddexp to ONNX opset version
39
+ # 14 is not supported. Please feel free to request support or submit
40
+ # a pull request on PyTorch GitHub.
41
+ #
42
+ # The following function is to solve the above error when exporting
43
+ # models to ONNX via torch.jit.trace()
44
+ def logaddexp(x: Tensor, y: Tensor) -> Tensor:
45
+ # Caution(fangjun): Put torch.jit.is_scripting() before
46
+ # torch.onnx.is_in_onnx_export();
47
+ # otherwise, it will cause errors for torch.jit.script().
48
+ #
49
+ # torch.logaddexp() works for both torch.jit.script() and
50
+ # torch.jit.trace() but it causes errors for ONNX export.
51
+ #
52
+ if torch.jit.is_scripting():
53
+ # Note: We cannot use torch.jit.is_tracing() here as it also
54
+ # matches torch.onnx.export().
55
+ return torch.logaddexp(x, y)
56
+ elif torch.onnx.is_in_onnx_export():
57
+ return logaddexp_onnx(x, y)
58
+ else:
59
+ # for torch.jit.trace()
60
+ return torch.logaddexp(x, y)
61
+
62
+
63
+ class PiecewiseLinear(object):
64
+ """
65
+ Piecewise linear function, from float to float, specified as nonempty list of (x,y) pairs with
66
+ the x values in order. x values <[initial x] or >[final x] are map to [initial y], [final y]
67
+ respectively.
68
+ """
69
+
70
+ def __init__(self, *args):
71
+ assert len(args) >= 1, len(args)
72
+ if len(args) == 1 and isinstance(args[0], PiecewiseLinear):
73
+ self.pairs = list(args[0].pairs)
74
+ else:
75
+ self.pairs = [(float(x), float(y)) for x, y in args]
76
+ for x, y in self.pairs:
77
+ assert isinstance(x, (float, int)), type(x)
78
+ assert isinstance(y, (float, int)), type(y)
79
+
80
+ for i in range(len(self.pairs) - 1):
81
+ assert self.pairs[i + 1][0] > self.pairs[i][0], (
82
+ i,
83
+ self.pairs[i],
84
+ self.pairs[i + 1],
85
+ )
86
+
87
+ def __str__(self):
88
+ # e.g. 'PiecewiseLinear((0., 10.), (100., 0.))'
89
+ return f"PiecewiseLinear({str(self.pairs)[1:-1]})"
90
+
91
+ def __call__(self, x):
92
+ if x <= self.pairs[0][0]:
93
+ return self.pairs[0][1]
94
+ elif x >= self.pairs[-1][0]:
95
+ return self.pairs[-1][1]
96
+ else:
97
+ cur_x, cur_y = self.pairs[0]
98
+ for i in range(1, len(self.pairs)):
99
+ next_x, next_y = self.pairs[i]
100
+ if x >= cur_x and x <= next_x:
101
+ return cur_y + (next_y - cur_y) * (x - cur_x) / (next_x - cur_x)
102
+ cur_x, cur_y = next_x, next_y
103
+ assert False
104
+
105
+ def __mul__(self, alpha):
106
+ return PiecewiseLinear(*[(x, y * alpha) for x, y in self.pairs])
107
+
108
+ def __add__(self, x):
109
+ if isinstance(x, (float, int)):
110
+ return PiecewiseLinear(*[(p[0], p[1] + x) for p in self.pairs])
111
+ s, x = self.get_common_basis(x)
112
+ return PiecewiseLinear(
113
+ *[(sp[0], sp[1] + xp[1]) for sp, xp in zip(s.pairs, x.pairs)]
114
+ )
115
+
116
+ def max(self, x):
117
+ if isinstance(x, (float, int)):
118
+ x = PiecewiseLinear((0, x))
119
+ s, x = self.get_common_basis(x, include_crossings=True)
120
+ return PiecewiseLinear(
121
+ *[(sp[0], max(sp[1], xp[1])) for sp, xp in zip(s.pairs, x.pairs)]
122
+ )
123
+
124
+ def min(self, x):
125
+ if isinstance(x, float) or isinstance(x, int):
126
+ x = PiecewiseLinear((0, x))
127
+ s, x = self.get_common_basis(x, include_crossings=True)
128
+ return PiecewiseLinear(
129
+ *[(sp[0], min(sp[1], xp[1])) for sp, xp in zip(s.pairs, x.pairs)]
130
+ )
131
+
132
+ def __eq__(self, other):
133
+ return self.pairs == other.pairs
134
+
135
+ def get_common_basis(self, p: "PiecewiseLinear", include_crossings: bool = False):
136
+ """
137
+ Returns (self_mod, p_mod) which are equivalent piecewise linear
138
+ functions to self and p, but with the same x values.
139
+
140
+ p: the other piecewise linear function
141
+ include_crossings: if true, include in the x values positions
142
+ where the functions indicate by this and p cross.
143
+ """
144
+ assert isinstance(p, PiecewiseLinear), type(p)
145
+
146
+ # get sorted x-values without repetition.
147
+ x_vals = sorted(set([x for x, _ in self.pairs] + [x for x, _ in p.pairs]))
148
+ y_vals1 = [self(x) for x in x_vals]
149
+ y_vals2 = [p(x) for x in x_vals]
150
+
151
+ if include_crossings:
152
+ extra_x_vals = []
153
+ for i in range(len(x_vals) - 1):
154
+ if (y_vals1[i] > y_vals2[i]) != (y_vals1[i + 1] > y_vals2[i + 1]):
155
+ # if the two lines in this subsegment potentially cross each other..
156
+ diff_cur = abs(y_vals1[i] - y_vals2[i])
157
+ diff_next = abs(y_vals1[i + 1] - y_vals2[i + 1])
158
+ # `pos`, between 0 and 1, gives the relative x position,
159
+ # with 0 being x_vals[i] and 1 being x_vals[i+1].
160
+ pos = diff_cur / (diff_cur + diff_next)
161
+ extra_x_val = x_vals[i] + pos * (x_vals[i + 1] - x_vals[i])
162
+ extra_x_vals.append(extra_x_val)
163
+ if len(extra_x_vals) > 0:
164
+ x_vals = sorted(set(x_vals + extra_x_vals))
165
+
166
+ y_vals1 = [self(x) for x in x_vals]
167
+ y_vals2 = [p(x) for x in x_vals]
168
+
169
+ return (
170
+ PiecewiseLinear(*zip(x_vals, y_vals1)),
171
+ PiecewiseLinear(*zip(x_vals, y_vals2)),
172
+ )
173
+
174
+
175
+ class ScheduledFloat(torch.nn.Module):
176
+ """
177
+ This object is a torch.nn.Module only because we want it to show up in [top_level module].modules();
178
+ it does not have a working forward() function. You are supposed to cast it to float, as
179
+ in, float(parent_module.whatever), and use it as something like a dropout prob.
180
+
181
+ It is a floating point value whose value changes depending on the batch count of the
182
+ training loop. It is a piecewise linear function where you specify the (x,y) pairs
183
+ in sorted order on x; x corresponds to the batch index. For batch-index values before the
184
+ first x or after the last x, we just use the first or last y value.
185
+
186
+ Example:
187
+ self.dropout = ScheduledFloat((0.0, 0.2), (4000.0, 0.0), default=0.0)
188
+
189
+ `default` is used when self.batch_count is not set or not in training mode or in
190
+ torch.jit scripting mode.
191
+ """
192
+
193
+ def __init__(self, *args, default: float = 0.0):
194
+ super().__init__()
195
+ # self.batch_count and self.name will be written to in the training loop.
196
+ self.batch_count = None
197
+ self.name = None
198
+ self.default = default
199
+ self.schedule = PiecewiseLinear(*args)
200
+
201
+ def extra_repr(self) -> str:
202
+ return (
203
+ f"batch_count={self.batch_count}, schedule={str(self.schedule.pairs[1:-1])}"
204
+ )
205
+
206
+ def __float__(self):
207
+ batch_count = self.batch_count
208
+ if (
209
+ batch_count is None
210
+ or not self.training
211
+ or torch.jit.is_scripting()
212
+ or torch.jit.is_tracing()
213
+ ):
214
+ return float(self.default)
215
+ else:
216
+ ans = self.schedule(self.batch_count)
217
+ if random.random() < 0.0002:
218
+ logging.info(
219
+ f"ScheduledFloat: name={self.name}, batch_count={self.batch_count}, ans={ans}"
220
+ )
221
+ return ans
222
+
223
+ def __add__(self, x):
224
+ if isinstance(x, float) or isinstance(x, int):
225
+ return ScheduledFloat(self.schedule + x, default=self.default)
226
+ else:
227
+ return ScheduledFloat(
228
+ self.schedule + x.schedule, default=self.default + x.default
229
+ )
230
+
231
+ def max(self, x):
232
+ if isinstance(x, float) or isinstance(x, int):
233
+ return ScheduledFloat(self.schedule.max(x), default=self.default)
234
+ else:
235
+ return ScheduledFloat(
236
+ self.schedule.max(x.schedule), default=max(self.default, x.default)
237
+ )
238
+
239
+
240
+ FloatLike = Union[float, ScheduledFloat]
241
+
242
+
243
+ def random_cast_to_half(x: Tensor, min_abs: float = 5.0e-06) -> Tensor:
244
+ """
245
+ A randomized way of casting a floating point value to half precision.
246
+ """
247
+ if x.dtype == torch.float16:
248
+ return x
249
+ x_abs = x.abs()
250
+ is_too_small = x_abs < min_abs
251
+ # for elements where is_too_small is true, random_val will contain +-min_abs with
252
+ # probability (x.abs() / min_abs), and 0.0 otherwise. [so this preserves expectations,
253
+ # for those elements].
254
+ random_val = min_abs * x.sign() * (torch.rand_like(x) * min_abs < x_abs)
255
+ return torch.where(is_too_small, random_val, x).to(torch.float16)
256
+
257
+
258
+ class CutoffEstimator:
259
+ """
260
+ Estimates cutoffs of an arbitrary numerical quantity such that a specified
261
+ proportion of items will be above the cutoff on average.
262
+
263
+ p is the proportion of items that should be above the cutoff.
264
+ """
265
+
266
+ def __init__(self, p: float):
267
+ self.p = p
268
+ # total count of items
269
+ self.count = 0
270
+ # total count of items that were above the cutoff
271
+ self.count_above = 0
272
+ # initial cutoff value
273
+ self.cutoff = 0
274
+
275
+ def __call__(self, x: float) -> bool:
276
+ """
277
+ Returns true if x is above the cutoff.
278
+ """
279
+ ans = x > self.cutoff
280
+ self.count += 1
281
+ if ans:
282
+ self.count_above += 1
283
+ cur_p = self.count_above / self.count
284
+ delta_p = cur_p - self.p
285
+ if (delta_p > 0) == ans:
286
+ q = abs(delta_p)
287
+ self.cutoff = x * q + self.cutoff * (1 - q)
288
+ return ans
289
+
290
+
291
+ class SoftmaxFunction(torch.autograd.Function):
292
+ """
293
+ Tries to handle half-precision derivatives in a randomized way that should
294
+ be more accurate for training than the default behavior.
295
+ """
296
+
297
+ @staticmethod
298
+ def forward(ctx, x: Tensor, dim: int):
299
+ ans = x.softmax(dim=dim)
300
+ # if x dtype is float16, x.softmax() returns a float32 because
301
+ # (presumably) that op does not support float16, and autocast
302
+ # is enabled.
303
+ if torch.is_autocast_enabled():
304
+ ans = ans.to(torch.get_autocast_gpu_dtype())
305
+ ctx.save_for_backward(ans)
306
+ ctx.x_dtype = x.dtype
307
+ ctx.dim = dim
308
+ return ans
309
+
310
+ @staticmethod
311
+ def backward(ctx, ans_grad: Tensor):
312
+ (ans,) = ctx.saved_tensors
313
+ with torch_autocast(enabled=False):
314
+ ans_grad = ans_grad.to(torch.float32)
315
+ ans = ans.to(torch.float32)
316
+ x_grad = ans_grad * ans
317
+ x_grad = x_grad - ans * x_grad.sum(dim=ctx.dim, keepdim=True)
318
+ return x_grad, None
319
+
320
+
321
+ def softmax(x: Tensor, dim: int):
322
+ if not x.requires_grad or torch.jit.is_scripting() or torch.jit.is_tracing():
323
+ return x.softmax(dim=dim)
324
+
325
+ return SoftmaxFunction.apply(x, dim)
326
+
327
+
328
+ class MaxEigLimiterFunction(torch.autograd.Function):
329
+ @staticmethod
330
+ def forward(
331
+ ctx,
332
+ x: Tensor,
333
+ coeffs: Tensor,
334
+ direction: Tensor,
335
+ channel_dim: int,
336
+ grad_scale: float,
337
+ ) -> Tensor:
338
+ ctx.channel_dim = channel_dim
339
+ ctx.grad_scale = grad_scale
340
+ ctx.save_for_backward(x.detach(), coeffs.detach(), direction.detach())
341
+ return x
342
+
343
+ @staticmethod
344
+ def backward(ctx, x_grad, *args):
345
+ with torch.enable_grad():
346
+ (x_orig, coeffs, new_direction) = ctx.saved_tensors
347
+ x_orig.requires_grad = True
348
+ num_channels = x_orig.shape[ctx.channel_dim]
349
+ x = x_orig.transpose(ctx.channel_dim, -1).reshape(-1, num_channels)
350
+ new_direction.requires_grad = False
351
+ x = x - x.mean(dim=0)
352
+ x_var = (x**2).mean()
353
+ x_residual = x - coeffs * new_direction
354
+ x_residual_var = (x_residual**2).mean()
355
+ # `variance_proportion` is the proportion of the variance accounted for
356
+ # by the top eigen-direction. This is to be minimized.
357
+ variance_proportion = (x_var - x_residual_var) / (x_var + 1.0e-20)
358
+ variance_proportion.backward()
359
+ x_orig_grad = x_orig.grad
360
+ x_extra_grad = (
361
+ x_orig.grad
362
+ * ctx.grad_scale
363
+ * x_grad.norm()
364
+ / (x_orig_grad.norm() + 1.0e-20)
365
+ )
366
+ return x_grad + x_extra_grad.detach(), None, None, None, None
367
+
368
+
369
+ class BiasNormFunction(torch.autograd.Function):
370
+ # This computes:
371
+ # scales = (torch.mean((x - bias) ** 2, keepdim=True)) ** -0.5 * log_scale.exp()
372
+ # return x * scales
373
+ # (after unsqueezing the bias), but it does it in a memory-efficient way so that
374
+ # it can just store the returned value (chances are, this will also be needed for
375
+ # some other reason, related to the next operation, so we can save memory).
376
+ @staticmethod
377
+ def forward(
378
+ ctx,
379
+ x: Tensor,
380
+ bias: Tensor,
381
+ log_scale: Tensor,
382
+ channel_dim: int,
383
+ store_output_for_backprop: bool,
384
+ ) -> Tensor:
385
+ assert bias.ndim == 1
386
+ if channel_dim < 0:
387
+ channel_dim = channel_dim + x.ndim
388
+ ctx.store_output_for_backprop = store_output_for_backprop
389
+ ctx.channel_dim = channel_dim
390
+ for _ in range(channel_dim + 1, x.ndim):
391
+ bias = bias.unsqueeze(-1)
392
+ scales = (
393
+ torch.mean((x - bias) ** 2, dim=channel_dim, keepdim=True) ** -0.5
394
+ ) * log_scale.exp()
395
+ ans = x * scales
396
+ ctx.save_for_backward(
397
+ ans.detach() if store_output_for_backprop else x,
398
+ scales.detach(),
399
+ bias.detach(),
400
+ log_scale.detach(),
401
+ )
402
+ return ans
403
+
404
+ @staticmethod
405
+ def backward(ctx, ans_grad: Tensor) -> Tensor:
406
+ ans_or_x, scales, bias, log_scale = ctx.saved_tensors
407
+ if ctx.store_output_for_backprop:
408
+ x = ans_or_x / scales
409
+ else:
410
+ x = ans_or_x
411
+ x = x.detach()
412
+ x.requires_grad = True
413
+ bias.requires_grad = True
414
+ log_scale.requires_grad = True
415
+ with torch.enable_grad():
416
+ # recompute scales from x, bias and log_scale.
417
+ scales = (
418
+ torch.mean((x - bias) ** 2, dim=ctx.channel_dim, keepdim=True) ** -0.5
419
+ ) * log_scale.exp()
420
+ ans = x * scales
421
+ ans.backward(gradient=ans_grad)
422
+ return x.grad, bias.grad.flatten(), log_scale.grad, None, None
423
+
424
+
425
+ class BiasNorm(torch.nn.Module):
426
+ """
427
+ This is intended to be a simpler, and hopefully cheaper, replacement for
428
+ LayerNorm. The observation this is based on, is that Transformer-type
429
+ networks, especially with pre-norm, sometimes seem to set one of the
430
+ feature dimensions to a large constant value (e.g. 50), which "defeats"
431
+ the LayerNorm because the output magnitude is then not strongly dependent
432
+ on the other (useful) features. Presumably the weight and bias of the
433
+ LayerNorm are required to allow it to do this.
434
+
435
+ Instead, we give the BiasNorm a trainable bias that it can use when
436
+ computing the scale for normalization. We also give it a (scalar)
437
+ trainable scale on the output.
438
+
439
+
440
+ Args:
441
+ num_channels: the number of channels, e.g. 512.
442
+ channel_dim: the axis/dimension corresponding to the channel,
443
+ interpreted as an offset from the input's ndim if negative.
444
+ This is NOT the num_channels; it should typically be one of
445
+ {-2, -1, 0, 1, 2, 3}.
446
+ log_scale: the initial log-scale that we multiply the output by; this
447
+ is learnable.
448
+ log_scale_min: FloatLike, minimum allowed value of log_scale
449
+ log_scale_max: FloatLike, maximum allowed value of log_scale
450
+ store_output_for_backprop: only possibly affects memory use; recommend
451
+ to set to True if you think the output of this module is more likely
452
+ than the input of this module to be required to be stored for the
453
+ backprop.
454
+ """
455
+
456
+ def __init__(
457
+ self,
458
+ num_channels: int,
459
+ channel_dim: int = -1, # CAUTION: see documentation.
460
+ log_scale: float = 1.0,
461
+ log_scale_min: float = -1.5,
462
+ log_scale_max: float = 1.5,
463
+ store_output_for_backprop: bool = False,
464
+ ) -> None:
465
+ super(BiasNorm, self).__init__()
466
+ self.num_channels = num_channels
467
+ self.channel_dim = channel_dim
468
+ self.log_scale = nn.Parameter(torch.tensor(log_scale))
469
+ self.bias = nn.Parameter(torch.empty(num_channels).normal_(mean=0, std=1e-4))
470
+
471
+ self.log_scale_min = log_scale_min
472
+ self.log_scale_max = log_scale_max
473
+
474
+ self.store_output_for_backprop = store_output_for_backprop
475
+
476
+ def forward(self, x: Tensor) -> Tensor:
477
+ assert x.shape[self.channel_dim] == self.num_channels
478
+
479
+ if torch.jit.is_scripting() or torch.jit.is_tracing():
480
+ channel_dim = self.channel_dim
481
+ if channel_dim < 0:
482
+ channel_dim += x.ndim
483
+ bias = self.bias
484
+ for _ in range(channel_dim + 1, x.ndim):
485
+ bias = bias.unsqueeze(-1)
486
+ scales = (
487
+ torch.mean((x - bias) ** 2, dim=channel_dim, keepdim=True) ** -0.5
488
+ ) * self.log_scale.exp()
489
+ return x * scales
490
+
491
+ log_scale = limit_param_value(
492
+ self.log_scale,
493
+ min=float(self.log_scale_min),
494
+ max=float(self.log_scale_max),
495
+ training=self.training,
496
+ )
497
+
498
+ return BiasNormFunction.apply(
499
+ x, self.bias, log_scale, self.channel_dim, self.store_output_for_backprop
500
+ )
501
+
502
+
503
+ def ScaledLinear(*args, initial_scale: float = 1.0, **kwargs) -> nn.Linear:
504
+ """
505
+ Behaves like a constructor of a modified version of nn.Linear
506
+ that gives an easy way to set the default initial parameter scale.
507
+
508
+ Args:
509
+ Accepts the standard args and kwargs that nn.Linear accepts
510
+ e.g. in_features, out_features, bias=False.
511
+
512
+ initial_scale: you can override this if you want to increase
513
+ or decrease the initial magnitude of the module's output
514
+ (affects the initialization of weight_scale and bias_scale).
515
+ Another option, if you want to do something like this, is
516
+ to re-initialize the parameters.
517
+ """
518
+ ans = nn.Linear(*args, **kwargs)
519
+ with torch.no_grad():
520
+ ans.weight[:] *= initial_scale
521
+ if ans.bias is not None:
522
+ torch.nn.init.uniform_(ans.bias, -0.1 * initial_scale, 0.1 * initial_scale)
523
+ return ans
524
+
525
+
526
+ def ScaledConv1d(*args, initial_scale: float = 1.0, **kwargs) -> nn.Conv1d:
527
+ """
528
+ Behaves like a constructor of a modified version of nn.Conv1d
529
+ that gives an easy way to set the default initial parameter scale.
530
+
531
+ Args:
532
+ Accepts the standard args and kwargs that nn.Linear accepts
533
+ e.g. in_features, out_features, bias=False.
534
+
535
+ initial_scale: you can override this if you want to increase
536
+ or decrease the initial magnitude of the module's output
537
+ (affects the initialization of weight_scale and bias_scale).
538
+ Another option, if you want to do something like this, is
539
+ to re-initialize the parameters.
540
+ """
541
+ ans = nn.Conv1d(*args, **kwargs)
542
+ with torch.no_grad():
543
+ ans.weight[:] *= initial_scale
544
+ if ans.bias is not None:
545
+ torch.nn.init.uniform_(ans.bias, -0.1 * initial_scale, 0.1 * initial_scale)
546
+ return ans
547
+
548
+
549
+ def ScaledConv2d(*args, initial_scale: float = 1.0, **kwargs) -> nn.Conv2d:
550
+ """
551
+ Behaves like a constructor of a modified version of nn.Conv2d
552
+ that gives an easy way to set the default initial parameter scale.
553
+
554
+ Args:
555
+ Accepts the standard args and kwargs that nn.Linear accepts
556
+ e.g. in_features, out_features, bias=False, but:
557
+ NO PADDING-RELATED ARGS.
558
+
559
+ initial_scale: you can override this if you want to increase
560
+ or decrease the initial magnitude of the module's output
561
+ (affects the initialization of weight_scale and bias_scale).
562
+ Another option, if you want to do something like this, is
563
+ to re-initialize the parameters.
564
+ """
565
+ ans = nn.Conv2d(*args, **kwargs)
566
+ with torch.no_grad():
567
+ ans.weight[:] *= initial_scale
568
+ if ans.bias is not None:
569
+ torch.nn.init.uniform_(ans.bias, -0.1 * initial_scale, 0.1 * initial_scale)
570
+ return ans
571
+
572
+
573
+ class ChunkCausalDepthwiseConv1d(torch.nn.Module):
574
+ """
575
+ Behaves like a depthwise 1d convolution, except that it is causal in
576
+ a chunkwise way, as if we had a block-triangular attention mask.
577
+ The chunk size is provided at test time (it should probably be
578
+ kept in sync with the attention mask).
579
+
580
+ This has a little more than twice the parameters of a conventional
581
+ depthwise conv1d module: we implement it by having one
582
+ depthwise convolution, of half the width, that is causal (via
583
+ right-padding); and one depthwise convolution that is applied only
584
+ within chunks, that we multiply by a scaling factor which depends
585
+ on the position within the chunk.
586
+
587
+ Args:
588
+ Accepts the standard args and kwargs that nn.Linear accepts
589
+ e.g. in_features, out_features, bias=False.
590
+
591
+ initial_scale: you can override this if you want to increase
592
+ or decrease the initial magnitude of the module's output
593
+ (affects the initialization of weight_scale and bias_scale).
594
+ Another option, if you want to do something like this, is
595
+ to re-initialize the parameters.
596
+ """
597
+
598
+ def __init__(
599
+ self,
600
+ channels: int,
601
+ kernel_size: int,
602
+ initial_scale: float = 1.0,
603
+ bias: bool = True,
604
+ ):
605
+ super().__init__()
606
+ assert kernel_size % 2 == 1
607
+
608
+ half_kernel_size = (kernel_size + 1) // 2
609
+ # will pad manually, on one side.
610
+ self.causal_conv = nn.Conv1d(
611
+ in_channels=channels,
612
+ out_channels=channels,
613
+ groups=channels,
614
+ kernel_size=half_kernel_size,
615
+ padding=0,
616
+ bias=True,
617
+ )
618
+
619
+ self.chunkwise_conv = nn.Conv1d(
620
+ in_channels=channels,
621
+ out_channels=channels,
622
+ groups=channels,
623
+ kernel_size=kernel_size,
624
+ padding=kernel_size // 2,
625
+ bias=bias,
626
+ )
627
+
628
+ # first row is correction factors added to the scale near the left edge of the chunk,
629
+ # second row is correction factors added to the scale near the right edge of the chunk,
630
+ # both of these are added to a default scale of 1.0.
631
+ self.chunkwise_conv_scale = nn.Parameter(torch.zeros(2, channels, kernel_size))
632
+ self.kernel_size = kernel_size
633
+
634
+ with torch.no_grad():
635
+ self.causal_conv.weight[:] *= initial_scale
636
+ self.chunkwise_conv.weight[:] *= initial_scale
637
+ if bias:
638
+ torch.nn.init.uniform_(
639
+ self.causal_conv.bias, -0.1 * initial_scale, 0.1 * initial_scale
640
+ )
641
+
642
+ def forward(self, x: Tensor, chunk_size: int = -1) -> Tensor:
643
+ """Forward function.
644
+
645
+ Args:
646
+ x: a Tensor of shape (batch_size, channels, seq_len)
647
+ chunk_size: the chunk size, in frames; does not have to divide seq_len exactly.
648
+ """
649
+ (batch_size, num_channels, seq_len) = x.shape
650
+
651
+ # half_kernel_size = self.kernel_size + 1 // 2
652
+ # left_pad is half_kernel_size - 1 where half_kernel_size is the size used
653
+ # in the causal conv. It's the amount by which we must pad on the left,
654
+ # to make the convolution causal.
655
+ left_pad = self.kernel_size // 2
656
+
657
+ if chunk_size < 0 or chunk_size > seq_len:
658
+ chunk_size = seq_len
659
+ right_pad = -seq_len % chunk_size
660
+
661
+ x = torch.nn.functional.pad(x, (left_pad, right_pad))
662
+
663
+ x_causal = self.causal_conv(x[..., : left_pad + seq_len])
664
+ assert x_causal.shape == (batch_size, num_channels, seq_len)
665
+
666
+ x_chunk = x[..., left_pad:]
667
+ num_chunks = x_chunk.shape[2] // chunk_size
668
+ x_chunk = x_chunk.reshape(batch_size, num_channels, num_chunks, chunk_size)
669
+ x_chunk = x_chunk.permute(0, 2, 1, 3).reshape(
670
+ batch_size * num_chunks, num_channels, chunk_size
671
+ )
672
+ x_chunk = self.chunkwise_conv(x_chunk) # does not change shape
673
+
674
+ chunk_scale = self._get_chunk_scale(chunk_size)
675
+
676
+ x_chunk = x_chunk * chunk_scale
677
+ x_chunk = x_chunk.reshape(
678
+ batch_size, num_chunks, num_channels, chunk_size
679
+ ).permute(0, 2, 1, 3)
680
+ x_chunk = x_chunk.reshape(batch_size, num_channels, num_chunks * chunk_size)[
681
+ ..., :seq_len
682
+ ]
683
+
684
+ return x_chunk + x_causal
685
+
686
+ def _get_chunk_scale(self, chunk_size: int):
687
+ """Returns tensor of shape (num_channels, chunk_size) that will be used to
688
+ scale the output of self.chunkwise_conv."""
689
+ left_edge = self.chunkwise_conv_scale[0]
690
+ right_edge = self.chunkwise_conv_scale[1]
691
+ if chunk_size < self.kernel_size:
692
+ left_edge = left_edge[:, :chunk_size]
693
+ right_edge = right_edge[:, -chunk_size:]
694
+ else:
695
+ t = chunk_size - self.kernel_size
696
+ channels = left_edge.shape[0]
697
+ pad = torch.zeros(
698
+ channels, t, device=left_edge.device, dtype=left_edge.dtype
699
+ )
700
+ left_edge = torch.cat((left_edge, pad), dim=-1)
701
+ right_edge = torch.cat((pad, right_edge), dim=-1)
702
+ return 1.0 + (left_edge + right_edge)
703
+
704
+ def streaming_forward(
705
+ self,
706
+ x: Tensor,
707
+ cache: Tensor,
708
+ ) -> Tuple[Tensor, Tensor]:
709
+ """Streaming Forward function.
710
+
711
+ Args:
712
+ x: a Tensor of shape (batch_size, channels, seq_len)
713
+ cache: cached left context of shape (batch_size, channels, left_pad)
714
+ """
715
+ (batch_size, num_channels, seq_len) = x.shape
716
+
717
+ # left_pad is half_kernel_size - 1 where half_kernel_size is the size used
718
+ # in the causal conv. It's the amount by which we must pad on the left,
719
+ # to make the convolution causal.
720
+ left_pad = self.kernel_size // 2
721
+
722
+ # Pad cache
723
+ assert cache.shape[-1] == left_pad, (cache.shape[-1], left_pad)
724
+ x = torch.cat([cache, x], dim=2)
725
+ # Update cache
726
+ cache = x[..., -left_pad:]
727
+
728
+ x_causal = self.causal_conv(x)
729
+ assert x_causal.shape == (batch_size, num_channels, seq_len)
730
+
731
+ x_chunk = x[..., left_pad:]
732
+ x_chunk = self.chunkwise_conv(x_chunk) # does not change shape
733
+
734
+ chunk_scale = self._get_chunk_scale(chunk_size=seq_len)
735
+ x_chunk = x_chunk * chunk_scale
736
+
737
+ return x_chunk + x_causal, cache
738
+
739
+
740
+ class BalancerFunction(torch.autograd.Function):
741
+ @staticmethod
742
+ def forward(
743
+ ctx,
744
+ x: Tensor,
745
+ min_mean: float,
746
+ max_mean: float,
747
+ min_rms: float,
748
+ max_rms: float,
749
+ grad_scale: float,
750
+ channel_dim: int,
751
+ ) -> Tensor:
752
+ if channel_dim < 0:
753
+ channel_dim += x.ndim
754
+ ctx.channel_dim = channel_dim
755
+ ctx.save_for_backward(x)
756
+ ctx.config = (min_mean, max_mean, min_rms, max_rms, grad_scale, channel_dim)
757
+ return x
758
+
759
+ @staticmethod
760
+ def backward(ctx, x_grad: Tensor) -> Tuple[Tensor, None, None, None, None, None]:
761
+ (x,) = ctx.saved_tensors
762
+ (min_mean, max_mean, min_rms, max_rms, grad_scale, channel_dim) = ctx.config
763
+
764
+ try:
765
+ with torch.enable_grad():
766
+ with torch_autocast(enabled=False):
767
+ x = x.to(torch.float32)
768
+ x = x.detach()
769
+ x.requires_grad = True
770
+ mean_dims = [i for i in range(x.ndim) if i != channel_dim]
771
+ uncentered_var = (x**2).mean(dim=mean_dims, keepdim=True)
772
+ mean = x.mean(dim=mean_dims, keepdim=True)
773
+ stddev = (uncentered_var - (mean * mean)).clamp(min=1.0e-20).sqrt()
774
+ rms = uncentered_var.clamp(min=1.0e-20).sqrt()
775
+
776
+ m = mean / stddev
777
+ # part of loss that relates to mean / stddev
778
+ m_loss = (m - m.clamp(min=min_mean, max=max_mean)).abs()
779
+
780
+ # put a much larger scale on the RMS-max-limit loss, so that if both it and the
781
+ # m_loss are violated we fix the RMS loss first.
782
+ rms_clamped = rms.clamp(min=min_rms, max=max_rms)
783
+ r_loss = (rms_clamped / rms).log().abs()
784
+
785
+ loss = m_loss + r_loss
786
+
787
+ loss.backward(gradient=torch.ones_like(loss))
788
+ loss_grad = x.grad
789
+ loss_grad_rms = (
790
+ (loss_grad**2)
791
+ .mean(dim=mean_dims, keepdim=True)
792
+ .sqrt()
793
+ .clamp(min=1.0e-20)
794
+ )
795
+
796
+ loss_grad = loss_grad * (grad_scale / loss_grad_rms)
797
+
798
+ x_grad_float = x_grad.to(torch.float32)
799
+ # scale each element of loss_grad by the absolute value of the corresponding
800
+ # element of x_grad, which we view as a noisy estimate of its magnitude for that
801
+ # (frame and dimension). later we can consider factored versions.
802
+ x_grad_mod = x_grad_float + (x_grad_float.abs() * loss_grad)
803
+ x_grad = x_grad_mod.to(x_grad.dtype)
804
+ except Exception as e:
805
+ logging.info(
806
+ f"Caught exception in Balancer backward: {e}, size={list(x_grad.shape)}, will continue."
807
+ )
808
+
809
+ return x_grad, None, None, None, None, None, None
810
+
811
+
812
+ class Balancer(torch.nn.Module):
813
+ """
814
+ Modifies the backpropped derivatives of a function to try to encourage, for
815
+ each channel, that it is positive at least a proportion `threshold` of the
816
+ time. It does this by multiplying negative derivative values by up to
817
+ (1+max_factor), and positive derivative values by up to (1-max_factor),
818
+ interpolated from 1 at the threshold to those extremal values when none
819
+ of the inputs are positive.
820
+
821
+ Args:
822
+ num_channels: the number of channels
823
+ channel_dim: the dimension/axis corresponding to the channel, e.g.
824
+ -1, 0, 1, 2; will be interpreted as an offset from x.ndim if negative.
825
+ min_positive: the minimum, per channel, of the proportion of the time
826
+ that (x > 0), below which we start to modify the derivatives.
827
+ max_positive: the maximum, per channel, of the proportion of the time
828
+ that (x > 0), above which we start to modify the derivatives.
829
+ scale_gain_factor: determines the 'gain' with which we increase the
830
+ change in gradient once the constraints on min_abs and max_abs
831
+ are violated.
832
+ min_abs: the minimum average-absolute-value difference from the mean
833
+ value per channel, which we allow, before we start to modify
834
+ the derivatives to prevent this.
835
+ max_abs: the maximum average-absolute-value difference from the mean
836
+ value per channel, which we allow, before we start to modify
837
+ the derivatives to prevent this.
838
+ prob: determines the minimum probability with which we modify the
839
+ gradients for the {min,max}_positive and {min,max}_abs constraints,
840
+ on each forward(). This is done randomly to prevent all layers
841
+ from doing it at the same time.
842
+ """
843
+
844
+ def __init__(
845
+ self,
846
+ num_channels: int,
847
+ channel_dim: int,
848
+ min_positive: FloatLike = 0.05,
849
+ max_positive: FloatLike = 0.95,
850
+ min_abs: FloatLike = 0.2,
851
+ max_abs: FloatLike = 100.0,
852
+ grad_scale: FloatLike = 0.04,
853
+ prob: Optional[FloatLike] = None,
854
+ ):
855
+ super().__init__()
856
+
857
+ if prob is None:
858
+ prob = ScheduledFloat((0.0, 0.5), (8000.0, 0.125), default=0.4)
859
+ self.prob = prob
860
+ # 5% of the time we will return and do nothing because memory usage is
861
+ # too high.
862
+ self.mem_cutoff = CutoffEstimator(0.05)
863
+
864
+ # actually self.num_channels is no longer needed except for an assertion.
865
+ self.num_channels = num_channels
866
+ self.channel_dim = channel_dim
867
+ self.min_positive = min_positive
868
+ self.max_positive = max_positive
869
+ self.min_abs = min_abs
870
+ self.max_abs = max_abs
871
+ self.grad_scale = grad_scale
872
+
873
+ def forward(self, x: Tensor) -> Tensor:
874
+ if (
875
+ torch.jit.is_scripting()
876
+ or not x.requires_grad
877
+ or (x.is_cuda and self.mem_cutoff(torch.cuda.memory_allocated()))
878
+ ):
879
+ return _no_op(x)
880
+
881
+ prob = float(self.prob)
882
+ if random.random() < prob:
883
+ # The following inner-functions convert from the way we historically specified
884
+ # these limitations, as limits on the absolute value and the proportion of positive
885
+ # values, to limits on the RMS value and the (mean / stddev).
886
+ def _abs_to_rms(x):
887
+ # for normally distributed data, if the expected absolute value is x, the
888
+ # expected rms value will be sqrt(pi/2) * x.
889
+ return 1.25331413732 * x
890
+
891
+ def _proportion_positive_to_mean(x):
892
+ def _atanh(x):
893
+ eps = 1.0e-10
894
+ # eps is to prevent crashes if x is exactly 0 or 1.
895
+ # we'll just end up returning a fairly large value.
896
+ return (math.log(1 + x + eps) - math.log(1 - x + eps)) / 2.0
897
+
898
+ def _approx_inverse_erf(x):
899
+ # 1 / (sqrt(pi) * ln(2)),
900
+ # see https://math.stackexchange.com/questions/321569/approximating-the-error-function-erf-by-analytical-functions
901
+ # this approximation is extremely crude and gets progressively worse for
902
+ # x very close to -1 or +1, but we mostly care about the "middle" region
903
+ # e.g. _approx_inverse_erf(0.05) = 0.0407316414078772,
904
+ # and math.erf(0.0407316414078772) = 0.045935330944660666,
905
+ # which is pretty close to 0.05.
906
+ return 0.8139535143 * _atanh(x)
907
+
908
+ # first convert x from the range 0..1 to the range -1..1 which the error
909
+ # function returns
910
+ x = -1 + (2 * x)
911
+ return _approx_inverse_erf(x)
912
+
913
+ min_mean = _proportion_positive_to_mean(float(self.min_positive))
914
+ max_mean = _proportion_positive_to_mean(float(self.max_positive))
915
+ min_rms = _abs_to_rms(float(self.min_abs))
916
+ max_rms = _abs_to_rms(float(self.max_abs))
917
+ grad_scale = float(self.grad_scale)
918
+
919
+ assert x.shape[self.channel_dim] == self.num_channels
920
+
921
+ return BalancerFunction.apply(
922
+ x, min_mean, max_mean, min_rms, max_rms, grad_scale, self.channel_dim
923
+ )
924
+ else:
925
+ return _no_op(x)
926
+
927
+
928
+ def penalize_abs_values_gt(
929
+ x: Tensor, limit: float, penalty: float, name: str = None
930
+ ) -> Tensor:
931
+ """
932
+ Returns x unmodified, but in backprop will put a penalty for the excess of
933
+ the absolute values of elements of x over the limit "limit". E.g. if
934
+ limit == 10.0, then if x has any values over 10 it will get a penalty.
935
+
936
+ Caution: the value of this penalty will be affected by grad scaling used
937
+ in automatic mixed precision training. For this reasons we use this,
938
+ it shouldn't really matter, or may even be helpful; we just use this
939
+ to disallow really implausible values of scores to be given to softmax.
940
+
941
+ The name is for randomly printed debug info.
942
+ """
943
+ x_sign = x.sign()
944
+ over_limit = (x.abs() - limit) > 0
945
+ # The following is a memory efficient way to penalize the absolute values of
946
+ # x that's over the limit. (The memory efficiency comes when you think
947
+ # about which items torch needs to cache for the autograd, and which ones it
948
+ # can throw away). The numerical value of aux_loss as computed here will
949
+ # actually be larger than it should be, by limit * over_limit.sum(), but it
950
+ # has the same derivative as the real aux_loss which is penalty * (x.abs() -
951
+ # limit).relu().
952
+ aux_loss = penalty * ((x_sign * over_limit).to(torch.int8) * x)
953
+ # note: we don't do sum() here on aux)_loss, but it's as if we had done
954
+ # sum() due to how with_loss() works.
955
+ x = with_loss(x, aux_loss, name)
956
+ # you must use x for something, or this will be ineffective.
957
+ return x
958
+
959
+
960
+ def _diag(x: Tensor): # like .diag(), but works for tensors with 3 dims.
961
+ if x.ndim == 2:
962
+ return x.diag()
963
+ else:
964
+ (batch, dim, dim) = x.shape
965
+ x = x.reshape(batch, dim * dim)
966
+ x = x[:, :: dim + 1]
967
+ assert x.shape == (batch, dim)
968
+ return x
969
+
970
+
971
+ def _whitening_metric(x: Tensor, num_groups: int):
972
+ """
973
+ Computes the "whitening metric", a value which will be 1.0 if all the eigenvalues of
974
+ of the centered feature covariance are the same within each group's covariance matrix
975
+ and also between groups.
976
+ Args:
977
+ x: a Tensor of shape (*, num_channels)
978
+ num_groups: the number of groups of channels, a number >=1 that divides num_channels
979
+ Returns:
980
+ Returns a scalar Tensor that will be 1.0 if the data is "perfectly white" and
981
+ greater than 1.0 otherwise.
982
+ """
983
+ assert x.dtype != torch.float16
984
+ x = x.reshape(-1, x.shape[-1])
985
+ (num_frames, num_channels) = x.shape
986
+ assert num_channels % num_groups == 0
987
+ channels_per_group = num_channels // num_groups
988
+ x = x.reshape(num_frames, num_groups, channels_per_group).transpose(0, 1)
989
+ # x now has shape (num_groups, num_frames, channels_per_group)
990
+ # subtract the mean so we use the centered, not uncentered, covariance.
991
+ # My experience has been that when we "mess with the gradients" like this,
992
+ # it's better not do anything that tries to move the mean around, because
993
+ # that can easily cause instability.
994
+ x = x - x.mean(dim=1, keepdim=True)
995
+ # x_covar: (num_groups, channels_per_group, channels_per_group)
996
+ x_covar = torch.matmul(x.transpose(1, 2), x)
997
+ x_covar_mean_diag = _diag(x_covar).mean()
998
+ # the following expression is what we'd get if we took the matrix product
999
+ # of each covariance and measured the mean of its trace, i.e.
1000
+ # the same as _diag(torch.matmul(x_covar, x_covar)).mean().
1001
+ x_covarsq_mean_diag = (x_covar**2).sum() / (num_groups * channels_per_group)
1002
+ # this metric will be >= 1.0; the larger it is, the less 'white' the data was.
1003
+ metric = x_covarsq_mean_diag / (x_covar_mean_diag**2 + 1.0e-20)
1004
+ return metric
1005
+
1006
+
1007
+ class WhiteningPenaltyFunction(torch.autograd.Function):
1008
+ @staticmethod
1009
+ def forward(ctx, x: Tensor, module: nn.Module) -> Tensor:
1010
+ ctx.save_for_backward(x)
1011
+ ctx.module = module
1012
+ return x
1013
+
1014
+ @staticmethod
1015
+ def backward(ctx, x_grad: Tensor):
1016
+ (x_orig,) = ctx.saved_tensors
1017
+ w = ctx.module
1018
+
1019
+ try:
1020
+ with torch.enable_grad():
1021
+ with torch_autocast(enabled=False):
1022
+ x_detached = x_orig.to(torch.float32).detach()
1023
+ x_detached.requires_grad = True
1024
+
1025
+ metric = _whitening_metric(x_detached, w.num_groups)
1026
+
1027
+ if random.random() < 0.005 or __name__ == "__main__":
1028
+ logging.info(
1029
+ f"Whitening: name={w.name}, num_groups={w.num_groups}, num_channels={x_orig.shape[-1]}, "
1030
+ f"metric={metric.item():.2f} vs. limit={float(w.whitening_limit)}"
1031
+ )
1032
+
1033
+ if metric < float(w.whitening_limit):
1034
+ w.prob = w.min_prob
1035
+ return x_grad, None
1036
+ else:
1037
+ w.prob = w.max_prob
1038
+ metric.backward()
1039
+ penalty_grad = x_detached.grad
1040
+ scale = float(w.grad_scale) * (
1041
+ x_grad.to(torch.float32).norm()
1042
+ / (penalty_grad.norm() + 1.0e-20)
1043
+ )
1044
+ penalty_grad = penalty_grad * scale
1045
+ return x_grad + penalty_grad.to(x_grad.dtype), None
1046
+ except Exception as e:
1047
+ logging.info(
1048
+ f"Caught exception in Whiten backward: {e}, size={list(x_grad.shape)}, will continue."
1049
+ )
1050
+ return x_grad, None
1051
+
1052
+
1053
+ class Whiten(nn.Module):
1054
+ def __init__(
1055
+ self,
1056
+ num_groups: int,
1057
+ whitening_limit: FloatLike,
1058
+ prob: Union[float, Tuple[float, float]],
1059
+ grad_scale: FloatLike,
1060
+ ):
1061
+ """
1062
+ Args:
1063
+ num_groups: the number of groups to divide the channel dim into before
1064
+ whitening. We will attempt to make the feature covariance
1065
+ within each group, after mean subtraction, as "white" as possible,
1066
+ while having the same trace across all groups.
1067
+ whitening_limit: a value greater than 1.0, that dictates how much
1068
+ freedom we have to violate the constraints. 1.0 would mean perfectly
1069
+ white, with exactly the same trace across groups; larger values
1070
+ give more freedom. E.g. 2.0.
1071
+ prob: the probability with which we apply the gradient modification
1072
+ (also affects the grad scale). May be supplied as a float,
1073
+ or as a pair (min_prob, max_prob)
1074
+
1075
+ grad_scale: determines the scale on the gradient term from this object,
1076
+ relative to the rest of the gradient on the attention weights.
1077
+ E.g. 0.02 (you may want to use smaller values than this if prob is large)
1078
+ """
1079
+ super(Whiten, self).__init__()
1080
+ assert num_groups >= 1
1081
+ assert float(whitening_limit) >= 1
1082
+ assert float(grad_scale) >= 0
1083
+ self.num_groups = num_groups
1084
+ self.whitening_limit = whitening_limit
1085
+ self.grad_scale = grad_scale
1086
+
1087
+ if isinstance(prob, float):
1088
+ prob = (prob, prob)
1089
+ (self.min_prob, self.max_prob) = prob
1090
+ assert 0 < self.min_prob <= self.max_prob <= 1
1091
+ self.prob = self.max_prob
1092
+ self.name = None # will be set in training loop
1093
+
1094
+ def forward(self, x: Tensor) -> Tensor:
1095
+ """
1096
+ In the forward pass, this function just returns the input unmodified.
1097
+ In the backward pass, it will modify the gradients to ensure that the
1098
+ distribution in each group has close to (lambda times I) as the covariance
1099
+ after mean subtraction, with the same lambda across groups.
1100
+ For whitening_limit > 1, there will be more freedom to violate this
1101
+ constraint.
1102
+
1103
+ Args:
1104
+ x: the input of shape (*, num_channels)
1105
+
1106
+ Returns:
1107
+ x, unmodified. You should make sure
1108
+ you use the returned value, or the graph will be freed
1109
+ and nothing will happen in backprop.
1110
+ """
1111
+ grad_scale = float(self.grad_scale)
1112
+ if not x.requires_grad or random.random() > self.prob or grad_scale == 0:
1113
+ return _no_op(x)
1114
+ else:
1115
+ return WhiteningPenaltyFunction.apply(x, self)
1116
+
1117
+
1118
+ class WithLoss(torch.autograd.Function):
1119
+ @staticmethod
1120
+ def forward(ctx, x: Tensor, y: Tensor, name: str):
1121
+ ctx.y_shape = y.shape
1122
+ if random.random() < 0.002 and name is not None:
1123
+ loss_sum = y.sum().item()
1124
+ logging.info(f"WithLoss: name={name}, loss-sum={loss_sum:.3e}")
1125
+ return x
1126
+
1127
+ @staticmethod
1128
+ def backward(ctx, ans_grad: Tensor):
1129
+ return (
1130
+ ans_grad,
1131
+ torch.ones(ctx.y_shape, dtype=ans_grad.dtype, device=ans_grad.device),
1132
+ None,
1133
+ )
1134
+
1135
+
1136
+ def with_loss(x, y, name):
1137
+ # returns x but adds y.sum() to the loss function.
1138
+ return WithLoss.apply(x, y, name)
1139
+
1140
+
1141
+ class ScaleGradFunction(torch.autograd.Function):
1142
+ @staticmethod
1143
+ def forward(ctx, x: Tensor, alpha: float) -> Tensor:
1144
+ ctx.alpha = alpha
1145
+ return x
1146
+
1147
+ @staticmethod
1148
+ def backward(ctx, grad: Tensor):
1149
+ return grad * ctx.alpha, None
1150
+
1151
+
1152
+ def scale_grad(x: Tensor, alpha: float):
1153
+ return ScaleGradFunction.apply(x, alpha)
1154
+
1155
+
1156
+ class ScaleGrad(nn.Module):
1157
+ def __init__(self, alpha: float):
1158
+ super().__init__()
1159
+ self.alpha = alpha
1160
+
1161
+ def forward(self, x: Tensor) -> Tensor:
1162
+ if torch.jit.is_scripting() or torch.jit.is_tracing() or not self.training:
1163
+ return x
1164
+ return scale_grad(x, self.alpha)
1165
+
1166
+
1167
+ class LimitParamValue(torch.autograd.Function):
1168
+ @staticmethod
1169
+ def forward(ctx, x: Tensor, min: float, max: float):
1170
+ ctx.save_for_backward(x)
1171
+ assert max >= min
1172
+ ctx.min = min
1173
+ ctx.max = max
1174
+ return x
1175
+
1176
+ @staticmethod
1177
+ def backward(ctx, x_grad: Tensor):
1178
+ (x,) = ctx.saved_tensors
1179
+ # where x < ctx.min, ensure all grads are negative (this will tend to make
1180
+ # x more positive).
1181
+ x_grad = x_grad * torch.where(
1182
+ torch.logical_and(x_grad > 0, x < ctx.min), -1.0, 1.0
1183
+ )
1184
+ # where x > ctx.max, ensure all grads are positive (this will tend to make
1185
+ # x more negative).
1186
+ x_grad *= torch.where(torch.logical_and(x_grad < 0, x > ctx.max), -1.0, 1.0)
1187
+ return x_grad, None, None
1188
+
1189
+
1190
+ def limit_param_value(
1191
+ x: Tensor, min: float, max: float, prob: float = 0.6, training: bool = True
1192
+ ):
1193
+ # You apply this to (typically) an nn.Parameter during training to ensure that its
1194
+ # (elements mostly) stays within a supplied range. This is done by modifying the
1195
+ # gradients in backprop.
1196
+ # It's not necessary to do this on every batch: do it only some of the time,
1197
+ # to save a little time.
1198
+ if training and random.random() < prob:
1199
+ return LimitParamValue.apply(x, min, max)
1200
+ else:
1201
+ return x
1202
+
1203
+
1204
+ def _no_op(x: Tensor) -> Tensor:
1205
+ if torch.jit.is_scripting() or torch.jit.is_tracing():
1206
+ return x
1207
+ else:
1208
+ # a no-op function that will have a node in the autograd graph,
1209
+ # to avoid certain bugs relating to backward hooks
1210
+ return x.chunk(1, dim=-1)[0]
1211
+
1212
+
1213
+ class Identity(torch.nn.Module):
1214
+ def __init__(self):
1215
+ super(Identity, self).__init__()
1216
+
1217
+ def forward(self, x):
1218
+ return _no_op(x)
1219
+
1220
+
1221
+ class DoubleSwishFunction(torch.autograd.Function):
1222
+ """
1223
+ double_swish(x) = x * torch.sigmoid(x-1)
1224
+
1225
+ This is a definition, originally motivated by its close numerical
1226
+ similarity to swish(swish(x)), where swish(x) = x * sigmoid(x).
1227
+
1228
+ Memory-efficient derivative computation:
1229
+ double_swish(x) = x * s, where s(x) = torch.sigmoid(x-1)
1230
+ double_swish'(x) = d/dx double_swish(x) = x * s'(x) + x' * s(x) = x * s'(x) + s(x).
1231
+ Now, s'(x) = s(x) * (1-s(x)).
1232
+ double_swish'(x) = x * s'(x) + s(x).
1233
+ = x * s(x) * (1-s(x)) + s(x).
1234
+ = double_swish(x) * (1-s(x)) + s(x)
1235
+ ... so we just need to remember s(x) but not x itself.
1236
+ """
1237
+
1238
+ @staticmethod
1239
+ def forward(ctx, x: Tensor) -> Tensor:
1240
+ requires_grad = x.requires_grad
1241
+ if x.dtype == torch.float16 or x.dtype == torch.bfloat16:
1242
+ x = x.to(torch.float32)
1243
+
1244
+ s = torch.sigmoid(x - 1.0)
1245
+ y = x * s
1246
+
1247
+ if requires_grad:
1248
+ deriv = y * (1 - s) + s
1249
+
1250
+ # notes on derivative of x * sigmoid(x - 1):
1251
+ # https://www.wolframalpha.com/input?i=d%2Fdx+%28x+*+sigmoid%28x-1%29%29
1252
+ # min \simeq -0.043638. Take floor as -0.044 so it's a lower bund
1253
+ # max \simeq 1.1990. Take ceil to be 1.2 so it's an upper bound.
1254
+ # the combination of "+ torch.rand_like(deriv)" and casting to torch.uint8 (which
1255
+ # floors), should be expectation-preserving.
1256
+ floor = -0.044
1257
+ ceil = 1.2
1258
+ d_scaled = (deriv - floor) * (255.0 / (ceil - floor)) + torch.rand_like(
1259
+ deriv
1260
+ )
1261
+ if __name__ == "__main__":
1262
+ # for self-testing only.
1263
+ assert d_scaled.min() >= 0.0
1264
+ assert d_scaled.max() < 256.0
1265
+ d_int = d_scaled.to(torch.uint8)
1266
+ ctx.save_for_backward(d_int)
1267
+ if x.dtype == torch.float16 or torch.is_autocast_enabled():
1268
+ y = y.to(torch.float16)
1269
+ return y
1270
+
1271
+ @staticmethod
1272
+ def backward(ctx, y_grad: Tensor) -> Tensor:
1273
+ (d,) = ctx.saved_tensors
1274
+ # the same constants as used in forward pass.
1275
+ floor = -0.043637
1276
+ ceil = 1.2
1277
+
1278
+ d = d * ((ceil - floor) / 255.0) + floor
1279
+ return y_grad * d
1280
+
1281
+
1282
+ class DoubleSwish(torch.nn.Module):
1283
+ def __init__(self):
1284
+ super().__init__()
1285
+
1286
+ def forward(self, x: Tensor) -> Tensor:
1287
+ """Return double-swish activation function which is an approximation to Swish(Swish(x)),
1288
+ that we approximate closely with x * sigmoid(x-1).
1289
+ """
1290
+ if torch.jit.is_scripting() or torch.jit.is_tracing():
1291
+ return x * torch.sigmoid(x - 1.0)
1292
+ return DoubleSwishFunction.apply(x)
1293
+
1294
+
1295
+ # Dropout2 is just like normal dropout, except it supports schedules on the dropout rates.
1296
+ class Dropout2(nn.Module):
1297
+ def __init__(self, p: FloatLike):
1298
+ super().__init__()
1299
+ self.p = p
1300
+
1301
+ def forward(self, x: Tensor) -> Tensor:
1302
+ return torch.nn.functional.dropout(x, p=float(self.p), training=self.training)
1303
+
1304
+
1305
+ class MulForDropout3(torch.autograd.Function):
1306
+ # returns (x * y * alpha) where alpha is a float and y doesn't require
1307
+ # grad and is zero-or-one.
1308
+ @staticmethod
1309
+ @custom_fwd
1310
+ def forward(ctx, x, y, alpha):
1311
+ assert not y.requires_grad
1312
+ ans = x * y * alpha
1313
+ ctx.save_for_backward(ans)
1314
+ ctx.alpha = alpha
1315
+ return ans
1316
+
1317
+ @staticmethod
1318
+ @custom_bwd
1319
+ def backward(ctx, ans_grad):
1320
+ (ans,) = ctx.saved_tensors
1321
+ x_grad = ctx.alpha * ans_grad * (ans != 0)
1322
+ return x_grad, None, None
1323
+
1324
+
1325
+ # Dropout3 is just like normal dropout, except it supports schedules on the dropout rates,
1326
+ # and it lets you choose one dimension to share the dropout mask over
1327
+ class Dropout3(nn.Module):
1328
+ def __init__(self, p: FloatLike, shared_dim: int):
1329
+ super().__init__()
1330
+ self.p = p
1331
+ self.shared_dim = shared_dim
1332
+
1333
+ def forward(self, x: Tensor) -> Tensor:
1334
+ p = float(self.p)
1335
+ if not self.training or p == 0:
1336
+ return _no_op(x)
1337
+ scale = 1.0 / (1 - p)
1338
+ rand_shape = list(x.shape)
1339
+ rand_shape[self.shared_dim] = 1
1340
+ mask = torch.rand(*rand_shape, device=x.device) > p
1341
+ ans = MulForDropout3.apply(x, mask, scale)
1342
+ return ans
1343
+
1344
+
1345
+ class SwooshLFunction(torch.autograd.Function):
1346
+ """
1347
+ swoosh_l(x) = log(1 + exp(x-4)) - 0.08*x - 0.035
1348
+ """
1349
+
1350
+ @staticmethod
1351
+ def forward(ctx, x: Tensor) -> Tensor:
1352
+ requires_grad = x.requires_grad
1353
+ if x.dtype == torch.float16 or x.dtype == torch.bfloat16:
1354
+ x = x.to(torch.float32)
1355
+
1356
+ zero = torch.tensor(0.0, dtype=x.dtype, device=x.device)
1357
+
1358
+ coeff = -0.08
1359
+
1360
+ with torch_autocast(enabled=False):
1361
+ with torch.enable_grad():
1362
+ x = x.detach()
1363
+ x.requires_grad = True
1364
+ y = torch.logaddexp(zero, x - 4.0) + coeff * x - 0.035
1365
+
1366
+ if not requires_grad:
1367
+ return y
1368
+
1369
+ y.backward(gradient=torch.ones_like(y))
1370
+
1371
+ grad = x.grad
1372
+ floor = coeff
1373
+ ceil = 1.0 + coeff + 0.005
1374
+
1375
+ d_scaled = (grad - floor) * (255.0 / (ceil - floor)) + torch.rand_like(
1376
+ grad
1377
+ )
1378
+ if __name__ == "__main__":
1379
+ # for self-testing only.
1380
+ assert d_scaled.min() >= 0.0
1381
+ assert d_scaled.max() < 256.0
1382
+
1383
+ d_int = d_scaled.to(torch.uint8)
1384
+ ctx.save_for_backward(d_int)
1385
+ if x.dtype == torch.float16 or torch.is_autocast_enabled():
1386
+ y = y.to(torch.get_autocast_gpu_dtype())
1387
+ return y
1388
+
1389
+ @staticmethod
1390
+ def backward(ctx, y_grad: Tensor) -> Tensor:
1391
+ (d,) = ctx.saved_tensors
1392
+ # the same constants as used in forward pass.
1393
+
1394
+ coeff = -0.08
1395
+ floor = coeff
1396
+ ceil = 1.0 + coeff + 0.005
1397
+ d = d * ((ceil - floor) / 255.0) + floor
1398
+ return y_grad * d
1399
+
1400
+
1401
+ class SwooshL(torch.nn.Module):
1402
+ def forward(self, x: Tensor) -> Tensor:
1403
+ """Return Swoosh-L activation."""
1404
+ if torch.jit.is_scripting() or torch.jit.is_tracing():
1405
+ zero = torch.tensor(0.0, dtype=x.dtype, device=x.device)
1406
+ return logaddexp(zero, x - 4.0) - 0.08 * x - 0.035
1407
+ if not x.requires_grad:
1408
+ return k2.swoosh_l_forward(x).to(x.dtype)
1409
+ else:
1410
+ return k2.swoosh_l(x).to(x.dtype)
1411
+ # return SwooshLFunction.apply(x)
1412
+
1413
+
1414
+ class SwooshLOnnx(torch.nn.Module):
1415
+ def forward(self, x: Tensor) -> Tensor:
1416
+ """Return Swoosh-L activation."""
1417
+ zero = torch.tensor(0.0, dtype=x.dtype, device=x.device)
1418
+ return logaddexp_onnx(zero, x - 4.0) - 0.08 * x - 0.035
1419
+
1420
+
1421
+ class SwooshRFunction(torch.autograd.Function):
1422
+ """
1423
+ swoosh_r(x) = log(1 + exp(x-1)) - 0.08*x - 0.313261687
1424
+
1425
+ derivatives are between -0.08 and 0.92.
1426
+ """
1427
+
1428
+ @staticmethod
1429
+ def forward(ctx, x: Tensor) -> Tensor:
1430
+ requires_grad = x.requires_grad
1431
+
1432
+ if x.dtype == torch.float16 or x.dtype == torch.bfloat16:
1433
+ x = x.to(torch.float32)
1434
+
1435
+ zero = torch.tensor(0.0, dtype=x.dtype, device=x.device)
1436
+
1437
+ with torch_autocast(enabled=False):
1438
+ with torch.enable_grad():
1439
+ x = x.detach()
1440
+ x.requires_grad = True
1441
+ y = torch.logaddexp(zero, x - 1.0) - 0.08 * x - 0.313261687
1442
+
1443
+ if not requires_grad:
1444
+ return y
1445
+ y.backward(gradient=torch.ones_like(y))
1446
+
1447
+ grad = x.grad
1448
+ floor = -0.08
1449
+ ceil = 0.925
1450
+
1451
+ d_scaled = (grad - floor) * (255.0 / (ceil - floor)) + torch.rand_like(
1452
+ grad
1453
+ )
1454
+ if __name__ == "__main__":
1455
+ # for self-testing only.
1456
+ assert d_scaled.min() >= 0.0
1457
+ assert d_scaled.max() < 256.0
1458
+
1459
+ d_int = d_scaled.to(torch.uint8)
1460
+ ctx.save_for_backward(d_int)
1461
+ if x.dtype == torch.float16 or torch.is_autocast_enabled():
1462
+ y = y.to(torch.get_autocast_gpu_dtype())
1463
+ return y
1464
+
1465
+ @staticmethod
1466
+ def backward(ctx, y_grad: Tensor) -> Tensor:
1467
+ (d,) = ctx.saved_tensors
1468
+ # the same constants as used in forward pass.
1469
+ floor = -0.08
1470
+ ceil = 0.925
1471
+ d = d * ((ceil - floor) / 255.0) + floor
1472
+ return y_grad * d
1473
+
1474
+
1475
+ class SwooshR(torch.nn.Module):
1476
+ def forward(self, x: Tensor) -> Tensor:
1477
+ """Return Swoosh-R activation."""
1478
+ if torch.jit.is_scripting() or torch.jit.is_tracing():
1479
+ zero = torch.tensor(0.0, dtype=x.dtype, device=x.device)
1480
+ return logaddexp(zero, x - 1.0) - 0.08 * x - 0.313261687
1481
+ if not x.requires_grad:
1482
+ return k2.swoosh_r_forward(x).to(x.dtype)
1483
+ else:
1484
+ return k2.swoosh_r(x).to(x.dtype)
1485
+ # return SwooshRFunction.apply(x)
1486
+
1487
+
1488
+ class SwooshROnnx(torch.nn.Module):
1489
+ def forward(self, x: Tensor) -> Tensor:
1490
+ """Return Swoosh-R activation."""
1491
+ zero = torch.tensor(0.0, dtype=x.dtype, device=x.device)
1492
+ return logaddexp_onnx(zero, x - 1.0) - 0.08 * x - 0.313261687
1493
+
1494
+
1495
+ # simple version of SwooshL that does not redefine the backprop, used in
1496
+ # ActivationDropoutAndLinearFunction.
1497
+ def SwooshLForward(x: Tensor):
1498
+ x_offset = x - 4.0
1499
+ log_sum = (1.0 + x_offset.exp()).log().to(x.dtype)
1500
+ log_sum = torch.where(log_sum == float("inf"), x_offset, log_sum)
1501
+ return log_sum - 0.08 * x - 0.035
1502
+
1503
+
1504
+ # simple version of SwooshR that does not redefine the backprop, used in
1505
+ # ActivationDropoutAndLinearFunction.
1506
+ def SwooshRForward(x: Tensor):
1507
+ x_offset = x - 1.0
1508
+ log_sum = (1.0 + x_offset.exp()).log().to(x.dtype)
1509
+ log_sum = torch.where(log_sum == float("inf"), x_offset, log_sum)
1510
+ return log_sum - 0.08 * x - 0.313261687
1511
+
1512
+
1513
+ class ActivationDropoutAndLinearFunction(torch.autograd.Function):
1514
+ @staticmethod
1515
+ @custom_fwd
1516
+ def forward(
1517
+ ctx,
1518
+ x: Tensor,
1519
+ weight: Tensor,
1520
+ bias: Optional[Tensor],
1521
+ activation: str,
1522
+ dropout_p: float,
1523
+ dropout_shared_dim: Optional[int],
1524
+ ):
1525
+ if dropout_p != 0.0:
1526
+ dropout_shape = list(x.shape)
1527
+ if dropout_shared_dim is not None:
1528
+ dropout_shape[dropout_shared_dim] = 1
1529
+ # else it won't be very memory efficient.
1530
+ dropout_mask = (1.0 / (1.0 - dropout_p)) * (
1531
+ torch.rand(*dropout_shape, device=x.device, dtype=x.dtype) > dropout_p
1532
+ )
1533
+ else:
1534
+ dropout_mask = None
1535
+
1536
+ ctx.save_for_backward(x, weight, bias, dropout_mask)
1537
+
1538
+ ctx.activation = activation
1539
+
1540
+ forward_activation_dict = {
1541
+ "SwooshL": k2.swoosh_l_forward,
1542
+ "SwooshR": k2.swoosh_r_forward,
1543
+ }
1544
+ # it will raise a KeyError if this fails. This will be an error. We let it
1545
+ # propagate to the user.
1546
+ activation_func = forward_activation_dict[activation]
1547
+ x = activation_func(x)
1548
+ if dropout_mask is not None:
1549
+ x = x * dropout_mask
1550
+ x = torch.nn.functional.linear(x, weight, bias)
1551
+ return x
1552
+
1553
+ @staticmethod
1554
+ @custom_bwd
1555
+ def backward(ctx, ans_grad: Tensor):
1556
+ saved = ctx.saved_tensors
1557
+ (x, weight, bias, dropout_mask) = saved
1558
+
1559
+ forward_and_deriv_activation_dict = {
1560
+ "SwooshL": k2.swoosh_l_forward_and_deriv,
1561
+ "SwooshR": k2.swoosh_r_forward_and_deriv,
1562
+ }
1563
+ # the following lines a KeyError if the activation is unrecognized.
1564
+ # This will be an error. We let it propagate to the user.
1565
+ func = forward_and_deriv_activation_dict[ctx.activation]
1566
+
1567
+ y, func_deriv = func(x)
1568
+ if dropout_mask is not None:
1569
+ y = y * dropout_mask
1570
+ # now compute derivative of y w.r.t. weight and bias..
1571
+ # y: (..., in_channels), ans_grad: (..., out_channels),
1572
+ (out_channels, in_channels) = weight.shape
1573
+
1574
+ in_channels = y.shape[-1]
1575
+ g = ans_grad.reshape(-1, out_channels)
1576
+ weight_deriv = torch.matmul(g.t(), y.reshape(-1, in_channels))
1577
+ y_deriv = torch.matmul(ans_grad, weight)
1578
+ bias_deriv = None if bias is None else g.sum(dim=0)
1579
+ x_deriv = y_deriv * func_deriv
1580
+ if dropout_mask is not None:
1581
+ # order versus func_deriv does not matter
1582
+ x_deriv = x_deriv * dropout_mask
1583
+
1584
+ return x_deriv, weight_deriv, bias_deriv, None, None, None
1585
+
1586
+
1587
+ class ActivationDropoutAndLinear(torch.nn.Module):
1588
+ """
1589
+ This merges an activation function followed by dropout and then a nn.Linear module;
1590
+ it does so in a memory efficient way so that it only stores the input to the whole
1591
+ module. If activation == SwooshL and dropout_shared_dim != None, this will be
1592
+ equivalent to:
1593
+ nn.Sequential(SwooshL(),
1594
+ Dropout3(dropout_p, shared_dim=dropout_shared_dim),
1595
+ ScaledLinear(in_channels, out_channels, bias=bias,
1596
+ initial_scale=initial_scale))
1597
+ If dropout_shared_dim is None, the dropout would be equivalent to
1598
+ Dropout2(dropout_p). Note: Dropout3 will be more memory efficient as the dropout
1599
+ mask is smaller.
1600
+
1601
+ Args:
1602
+ in_channels: number of input channels, e.g. 256
1603
+ out_channels: number of output channels, e.g. 256
1604
+ bias: if true, have a bias
1605
+ activation: the activation function, for now just support SwooshL.
1606
+ dropout_p: the dropout probability or schedule (happens after nonlinearity).
1607
+ dropout_shared_dim: the dimension, if any, across which the dropout mask is
1608
+ shared (e.g. the time dimension). If None, this may be less memory
1609
+ efficient if there are modules before this one that cache the input
1610
+ for their backprop (e.g. Balancer or Whiten).
1611
+ """
1612
+
1613
+ def __init__(
1614
+ self,
1615
+ in_channels: int,
1616
+ out_channels: int,
1617
+ bias: bool = True,
1618
+ activation: str = "SwooshL",
1619
+ dropout_p: FloatLike = 0.0,
1620
+ dropout_shared_dim: Optional[int] = -1,
1621
+ initial_scale: float = 1.0,
1622
+ ):
1623
+ super().__init__()
1624
+ # create a temporary module of nn.Linear that we'll steal the
1625
+ # weights and bias from
1626
+ l = ScaledLinear(
1627
+ in_channels, out_channels, bias=bias, initial_scale=initial_scale
1628
+ )
1629
+
1630
+ self.weight = l.weight
1631
+ # register_parameter properly handles making it a parameter when l.bias
1632
+ # is None. I think there is some reason for doing it this way rather
1633
+ # than just setting it to None but I don't know what it is, maybe
1634
+ # something to do with exporting the module..
1635
+ self.register_parameter("bias", l.bias)
1636
+
1637
+ self.activation = activation
1638
+ self.dropout_p = dropout_p
1639
+ self.dropout_shared_dim = dropout_shared_dim
1640
+
1641
+ def forward(self, x: Tensor):
1642
+ if not self.training or torch.jit.is_scripting() or torch.jit.is_tracing():
1643
+ if self.activation == "SwooshL":
1644
+ x = SwooshLForward(x)
1645
+ elif self.activation == "SwooshR":
1646
+ x = SwooshRForward(x)
1647
+ else:
1648
+ assert False, self.activation
1649
+ return torch.nn.functional.linear(x, self.weight, self.bias)
1650
+
1651
+ return ActivationDropoutAndLinearFunction.apply(
1652
+ x,
1653
+ self.weight,
1654
+ self.bias,
1655
+ self.activation,
1656
+ float(self.dropout_p),
1657
+ self.dropout_shared_dim,
1658
+ )
1659
+
1660
+
1661
+ def convert_num_channels(x: Tensor, num_channels: int) -> Tensor:
1662
+ if num_channels <= x.shape[-1]:
1663
+ return x[..., :num_channels]
1664
+ else:
1665
+ shape = list(x.shape)
1666
+ shape[-1] = num_channels - shape[-1]
1667
+ zeros = torch.zeros(shape, dtype=x.dtype, device=x.device)
1668
+ return torch.cat((x, zeros), dim=-1)
1669
+
1670
+
1671
+ def _test_whiten():
1672
+ for proportion in [0.1, 0.5, 10.0]:
1673
+ logging.info(f"_test_whiten(): proportion = {proportion}")
1674
+ x = torch.randn(100, 128)
1675
+ direction = torch.randn(128)
1676
+ coeffs = torch.randn(100, 1)
1677
+ x += proportion * direction * coeffs
1678
+
1679
+ x.requires_grad = True
1680
+
1681
+ m = Whiten(
1682
+ 1, 5.0, prob=1.0, grad_scale=0.1 # num_groups # whitening_limit,
1683
+ ) # grad_scale
1684
+
1685
+ for _ in range(4):
1686
+ y = m(x)
1687
+
1688
+ y_grad = torch.randn_like(x)
1689
+ y.backward(gradient=y_grad)
1690
+
1691
+ if proportion < 0.2:
1692
+ assert torch.allclose(x.grad, y_grad)
1693
+ elif proportion > 1.0:
1694
+ assert not torch.allclose(x.grad, y_grad)
1695
+
1696
+
1697
+ def _test_balancer_sign():
1698
+ probs = torch.arange(0, 1, 0.01)
1699
+ N = 1000
1700
+ x = 1.0 * ((2.0 * (torch.rand(probs.numel(), N) < probs.unsqueeze(-1))) - 1.0)
1701
+ x = x.detach()
1702
+ x.requires_grad = True
1703
+ m = Balancer(
1704
+ probs.numel(),
1705
+ channel_dim=0,
1706
+ min_positive=0.05,
1707
+ max_positive=0.95,
1708
+ min_abs=0.0,
1709
+ prob=1.0,
1710
+ )
1711
+
1712
+ y_grad = torch.sign(torch.randn(probs.numel(), N))
1713
+
1714
+ y = m(x)
1715
+ y.backward(gradient=y_grad)
1716
+ print("_test_balancer_sign: x = ", x)
1717
+ print("_test_balancer_sign: y grad = ", y_grad)
1718
+ print("_test_balancer_sign: x grad = ", x.grad)
1719
+
1720
+
1721
+ def _test_balancer_magnitude():
1722
+ magnitudes = torch.arange(0, 1, 0.01)
1723
+ N = 1000
1724
+ x = torch.sign(torch.randn(magnitudes.numel(), N)) * magnitudes.unsqueeze(-1)
1725
+ x = x.detach()
1726
+ x.requires_grad = True
1727
+ m = Balancer(
1728
+ magnitudes.numel(),
1729
+ channel_dim=0,
1730
+ min_positive=0.0,
1731
+ max_positive=1.0,
1732
+ min_abs=0.2,
1733
+ max_abs=0.7,
1734
+ prob=1.0,
1735
+ )
1736
+
1737
+ y_grad = torch.sign(torch.randn(magnitudes.numel(), N))
1738
+
1739
+ y = m(x)
1740
+ y.backward(gradient=y_grad)
1741
+ print("_test_balancer_magnitude: x = ", x)
1742
+ print("_test_balancer_magnitude: y grad = ", y_grad)
1743
+ print("_test_balancer_magnitude: x grad = ", x.grad)
1744
+
1745
+
1746
+ def _test_double_swish_deriv():
1747
+ x = torch.randn(10, 12, dtype=torch.double) * 3.0
1748
+ x.requires_grad = True
1749
+ m = DoubleSwish()
1750
+
1751
+ tol = (1.2 - (-0.043637)) / 255.0
1752
+ torch.autograd.gradcheck(m, x, atol=tol)
1753
+
1754
+ # for self-test.
1755
+ x = torch.randn(1000, 1000, dtype=torch.double) * 3.0
1756
+ x.requires_grad = True
1757
+ y = m(x)
1758
+
1759
+
1760
+ def _test_swooshl_deriv():
1761
+ x = torch.randn(10, 12, dtype=torch.double) * 3.0
1762
+ x.requires_grad = True
1763
+ m = SwooshL()
1764
+
1765
+ tol = 1.0 / 255.0
1766
+ torch.autograd.gradcheck(m, x, atol=tol, eps=0.01)
1767
+
1768
+ # for self-test.
1769
+ x = torch.randn(1000, 1000, dtype=torch.double) * 3.0
1770
+ x.requires_grad = True
1771
+ y = m(x)
1772
+
1773
+
1774
+ def _test_swooshr_deriv():
1775
+ x = torch.randn(10, 12, dtype=torch.double) * 3.0
1776
+ x.requires_grad = True
1777
+ m = SwooshR()
1778
+
1779
+ tol = 1.0 / 255.0
1780
+ torch.autograd.gradcheck(m, x, atol=tol, eps=0.01)
1781
+
1782
+ # for self-test.
1783
+ x = torch.randn(1000, 1000, dtype=torch.double) * 3.0
1784
+ x.requires_grad = True
1785
+ y = m(x)
1786
+
1787
+
1788
+ def _test_softmax():
1789
+ a = torch.randn(2, 10, dtype=torch.float64)
1790
+ b = a.clone()
1791
+ a.requires_grad = True
1792
+ b.requires_grad = True
1793
+ a.softmax(dim=1)[:, 0].sum().backward()
1794
+ print("a grad = ", a.grad)
1795
+ softmax(b, dim=1)[:, 0].sum().backward()
1796
+ print("b grad = ", b.grad)
1797
+ assert torch.allclose(a.grad, b.grad)
1798
+
1799
+
1800
+ def _test_piecewise_linear():
1801
+ p = PiecewiseLinear((0, 10.0))
1802
+ for x in [-100, 0, 100]:
1803
+ assert p(x) == 10.0
1804
+ p = PiecewiseLinear((0, 10.0), (1, 0.0))
1805
+ for x, y in [(-100, 10.0), (0, 10.0), (0.5, 5.0), (1, 0.0), (2, 0.0)]:
1806
+ print("x, y = ", x, y)
1807
+ assert p(x) == y, (x, p(x), y)
1808
+
1809
+ q = PiecewiseLinear((0.5, 15.0), (0.6, 1.0))
1810
+ x_vals = [-1.0, 0.0, 0.1, 0.2, 0.5, 0.6, 0.7, 0.9, 1.0, 2.0]
1811
+ pq = p.max(q)
1812
+ for x in x_vals:
1813
+ y1 = max(p(x), q(x))
1814
+ y2 = pq(x)
1815
+ assert abs(y1 - y2) < 0.001
1816
+ pq = p.min(q)
1817
+ for x in x_vals:
1818
+ y1 = min(p(x), q(x))
1819
+ y2 = pq(x)
1820
+ assert abs(y1 - y2) < 0.001
1821
+ pq = p + q
1822
+ for x in x_vals:
1823
+ y1 = p(x) + q(x)
1824
+ y2 = pq(x)
1825
+ assert abs(y1 - y2) < 0.001
1826
+
1827
+
1828
+ def _test_activation_dropout_and_linear():
1829
+ in_channels = 20
1830
+ out_channels = 30
1831
+
1832
+ for bias in [True, False]:
1833
+ # actually we don't test for dropout_p != 0.0 because forward functions will give
1834
+ # different answers. This is because we are using the k2 implementation of
1835
+ # swoosh_l an swoosh_r inside SwooshL() and SwooshR(), and they call randn()
1836
+ # internally, messing up the random state.
1837
+ for dropout_p in [0.0]:
1838
+ for activation in ["SwooshL", "SwooshR"]:
1839
+ m1 = nn.Sequential(
1840
+ SwooshL() if activation == "SwooshL" else SwooshR(),
1841
+ Dropout3(p=dropout_p, shared_dim=-1),
1842
+ ScaledLinear(
1843
+ in_channels, out_channels, bias=bias, initial_scale=0.5
1844
+ ),
1845
+ )
1846
+ m2 = ActivationDropoutAndLinear(
1847
+ in_channels,
1848
+ out_channels,
1849
+ bias=bias,
1850
+ initial_scale=0.5,
1851
+ activation=activation,
1852
+ dropout_p=dropout_p,
1853
+ )
1854
+ with torch.no_grad():
1855
+ m2.weight[:] = m1[2].weight
1856
+ if bias:
1857
+ m2.bias[:] = m1[2].bias
1858
+ # make sure forward gives same result.
1859
+ x1 = torch.randn(10, in_channels)
1860
+ x1.requires_grad = True
1861
+
1862
+ # TEMP.
1863
+ assert torch.allclose(
1864
+ SwooshRFunction.apply(x1), SwooshRForward(x1), atol=1.0e-03
1865
+ )
1866
+
1867
+ x2 = x1.clone().detach()
1868
+ x2.requires_grad = True
1869
+ seed = 10
1870
+ torch.manual_seed(seed)
1871
+ y1 = m1(x1)
1872
+ y_grad = torch.randn_like(y1)
1873
+ y1.backward(gradient=y_grad)
1874
+ torch.manual_seed(seed)
1875
+ y2 = m2(x2)
1876
+ y2.backward(gradient=y_grad)
1877
+
1878
+ print(
1879
+ f"bias = {bias}, dropout_p = {dropout_p}, activation = {activation}"
1880
+ )
1881
+ print("y1 = ", y1)
1882
+ print("y2 = ", y2)
1883
+ assert torch.allclose(y1, y2, atol=0.02)
1884
+ assert torch.allclose(m1[2].weight.grad, m2.weight.grad, atol=1.0e-05)
1885
+ if bias:
1886
+ assert torch.allclose(m1[2].bias.grad, m2.bias.grad, atol=1.0e-05)
1887
+ print("x1.grad = ", x1.grad)
1888
+ print("x2.grad = ", x2.grad)
1889
+
1890
+ def isclose(a, b):
1891
+ # return true if cosine similarity is > 0.9.
1892
+ return (a * b).sum() > 0.9 * (
1893
+ (a**2).sum() * (b**2).sum()
1894
+ ).sqrt()
1895
+
1896
+ # the SwooshL() implementation has a noisy gradient due to 1-byte
1897
+ # storage of it.
1898
+ assert isclose(x1.grad, x2.grad)
1899
+
1900
+
1901
+ if __name__ == "__main__":
1902
+ logging.getLogger().setLevel(logging.INFO)
1903
+ torch.set_num_threads(1)
1904
+ torch.set_num_interop_threads(1)
1905
+ _test_piecewise_linear()
1906
+ _test_softmax()
1907
+ _test_whiten()
1908
+ _test_balancer_sign()
1909
+ _test_balancer_magnitude()
1910
+ _test_double_swish_deriv()
1911
+ _test_swooshr_deriv()
1912
+ _test_swooshl_deriv()
1913
+ _test_activation_dropout_and_linear()
speedasr_core/subsampling.py ADDED
@@ -0,0 +1,406 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ # Copyright 2023 Xiaomi Corp. (authors: Daniel Povey,
3
+ # Zengwei Yao)
4
+ #
5
+ # See ../../../../LICENSE for clarification regarding multiple authors
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+
19
+ import warnings
20
+ from typing import Tuple
21
+
22
+ import torch
23
+ from .scaling import (
24
+ Balancer,
25
+ BiasNorm,
26
+ Dropout3,
27
+ FloatLike,
28
+ Optional,
29
+ ScaledConv2d,
30
+ ScaleGrad,
31
+ ScheduledFloat,
32
+ SwooshL,
33
+ SwooshR,
34
+ Whiten,
35
+ )
36
+ from torch import Tensor, nn
37
+
38
+
39
+ class ConvNeXt(nn.Module):
40
+ """
41
+ Our interpretation of the ConvNeXt module as used in https://arxiv.org/pdf/2206.14747.pdf
42
+ """
43
+
44
+ def __init__(
45
+ self,
46
+ channels: int,
47
+ hidden_ratio: int = 3,
48
+ kernel_size: Tuple[int, int] = (7, 7),
49
+ layerdrop_rate: FloatLike = None,
50
+ ):
51
+ super().__init__()
52
+ self.padding = ((kernel_size[0] - 1) // 2, (kernel_size[1] - 1) // 2)
53
+ hidden_channels = channels * hidden_ratio
54
+ if layerdrop_rate is None:
55
+ layerdrop_rate = ScheduledFloat((0.0, 0.2), (20000.0, 0.015))
56
+ self.layerdrop_rate = layerdrop_rate
57
+
58
+ self.depthwise_conv = nn.Conv2d(
59
+ in_channels=channels,
60
+ out_channels=channels,
61
+ groups=channels,
62
+ kernel_size=kernel_size,
63
+ padding=self.padding,
64
+ )
65
+
66
+ self.pointwise_conv1 = nn.Conv2d(
67
+ in_channels=channels, out_channels=hidden_channels, kernel_size=1
68
+ )
69
+
70
+ self.hidden_balancer = Balancer(
71
+ hidden_channels,
72
+ channel_dim=1,
73
+ min_positive=0.3,
74
+ max_positive=1.0,
75
+ min_abs=0.75,
76
+ max_abs=5.0,
77
+ )
78
+
79
+ self.activation = SwooshL()
80
+ self.pointwise_conv2 = ScaledConv2d(
81
+ in_channels=hidden_channels,
82
+ out_channels=channels,
83
+ kernel_size=1,
84
+ initial_scale=0.01,
85
+ )
86
+
87
+ self.out_balancer = Balancer(
88
+ channels,
89
+ channel_dim=1,
90
+ min_positive=0.4,
91
+ max_positive=0.6,
92
+ min_abs=1.0,
93
+ max_abs=6.0,
94
+ )
95
+ self.out_whiten = Whiten(
96
+ num_groups=1,
97
+ whitening_limit=5.0,
98
+ prob=(0.025, 0.25),
99
+ grad_scale=0.01,
100
+ )
101
+
102
+ def forward(self, x: Tensor) -> Tensor:
103
+ if torch.jit.is_scripting() or torch.jit.is_tracing() or not self.training:
104
+ return self.forward_internal(x)
105
+ layerdrop_rate = float(self.layerdrop_rate)
106
+
107
+ if layerdrop_rate != 0.0:
108
+ batch_size = x.shape[0]
109
+ mask = (
110
+ torch.rand((batch_size, 1, 1, 1), dtype=x.dtype, device=x.device)
111
+ > layerdrop_rate
112
+ )
113
+ else:
114
+ mask = None
115
+ # turns out this caching idea does not work with --world-size > 1
116
+ # return caching_eval(self.forward_internal, x, mask)
117
+ return self.forward_internal(x, mask)
118
+
119
+ def forward_internal(
120
+ self, x: Tensor, layer_skip_mask: Optional[Tensor] = None
121
+ ) -> Tensor:
122
+ """
123
+ x layout: (N, C, H, W), i.e. (batch_size, num_channels, num_frames, num_freqs)
124
+
125
+ The returned value has the same shape as x.
126
+ """
127
+ bypass = x
128
+ x = self.depthwise_conv(x)
129
+ x = self.pointwise_conv1(x)
130
+ x = self.hidden_balancer(x)
131
+ x = self.activation(x)
132
+ x = self.pointwise_conv2(x)
133
+
134
+ if layer_skip_mask is not None:
135
+ x = x * layer_skip_mask
136
+
137
+ x = bypass + x
138
+ x = self.out_balancer(x)
139
+
140
+ if x.requires_grad:
141
+ x = x.transpose(1, 3) # (N, W, H, C); need channel dim to be last
142
+ x = self.out_whiten(x)
143
+ x = x.transpose(1, 3) # (N, C, H, W)
144
+
145
+ return x
146
+
147
+ def streaming_forward(
148
+ self,
149
+ x: Tensor,
150
+ cached_left_pad: Tensor,
151
+ ) -> Tuple[Tensor, Tensor]:
152
+ """
153
+ Args:
154
+ x layout: (N, C, H, W), i.e. (batch_size, num_channels, num_frames, num_freqs)
155
+ cached_left_pad: (batch_size, num_channels, left_pad, num_freqs)
156
+
157
+ Returns:
158
+ - The returned value has the same shape as x.
159
+ - Updated cached_left_pad.
160
+ """
161
+ padding = self.padding
162
+
163
+ # The length without right padding for depth-wise conv
164
+ T = x.size(2) - padding[0]
165
+
166
+ bypass = x[:, :, :T, :]
167
+
168
+ # Pad left side
169
+ assert cached_left_pad.size(2) == padding[0], (
170
+ cached_left_pad.size(2),
171
+ padding[0],
172
+ )
173
+ x = torch.cat([cached_left_pad, x], dim=2)
174
+ # Update cached left padding
175
+ cached_left_pad = x[:, :, T : padding[0] + T, :]
176
+
177
+ # depthwise_conv
178
+ x = torch.nn.functional.conv2d(
179
+ x,
180
+ weight=self.depthwise_conv.weight,
181
+ bias=self.depthwise_conv.bias,
182
+ padding=(0, padding[1]),
183
+ groups=self.depthwise_conv.groups,
184
+ )
185
+ x = self.pointwise_conv1(x)
186
+ x = self.hidden_balancer(x)
187
+ x = self.activation(x)
188
+ x = self.pointwise_conv2(x)
189
+
190
+ x = bypass + x
191
+ return x, cached_left_pad
192
+
193
+
194
+ class Conv2dSubsampling(nn.Module):
195
+ """Convolutional 2D subsampling (to 1/2 length).
196
+
197
+ Convert an input of shape (N, T, idim) to an output
198
+ with shape (N, T', odim), where
199
+ T' = (T-3)//2 - 2 == (T-7)//2
200
+
201
+ It is based on
202
+ https://github.com/espnet/espnet/blob/master/espnet/nets/pytorch_backend/transformer/subsampling.py # noqa
203
+ """
204
+
205
+ def __init__(
206
+ self,
207
+ in_channels: int,
208
+ out_channels: int,
209
+ layer1_channels: int = 8,
210
+ layer2_channels: int = 32,
211
+ layer3_channels: int = 128,
212
+ dropout: FloatLike = 0.1,
213
+ ) -> None:
214
+ """
215
+ Args:
216
+ in_channels:
217
+ Number of channels in. The input shape is (N, T, in_channels).
218
+ Caution: It requires: T >=7, in_channels >=7
219
+ out_channels
220
+ Output dim. The output shape is (N, (T-3)//2, out_channels)
221
+ layer1_channels:
222
+ Number of channels in layer1
223
+ layer1_channels:
224
+ Number of channels in layer2
225
+ bottleneck:
226
+ bottleneck dimension for 1d squeeze-excite
227
+ """
228
+ assert in_channels >= 7
229
+ super().__init__()
230
+
231
+ # The ScaleGrad module is there to prevent the gradients
232
+ # w.r.t. the weight or bias of the first Conv2d module in self.conv from
233
+ # exceeding the range of fp16 when using automatic mixed precision (amp)
234
+ # training. (The second one is necessary to stop its bias from getting
235
+ # a too-large gradient).
236
+
237
+ self.conv = nn.Sequential(
238
+ nn.Conv2d(
239
+ in_channels=1,
240
+ out_channels=layer1_channels,
241
+ kernel_size=3,
242
+ padding=(0, 1), # (time, freq)
243
+ ),
244
+ ScaleGrad(0.2),
245
+ Balancer(layer1_channels, channel_dim=1, max_abs=1.0),
246
+ SwooshR(),
247
+ nn.Conv2d(
248
+ in_channels=layer1_channels,
249
+ out_channels=layer2_channels,
250
+ kernel_size=3,
251
+ stride=2,
252
+ padding=0,
253
+ ),
254
+ Balancer(layer2_channels, channel_dim=1, max_abs=4.0),
255
+ SwooshR(),
256
+ nn.Conv2d(
257
+ in_channels=layer2_channels,
258
+ out_channels=layer3_channels,
259
+ kernel_size=3,
260
+ stride=(1, 2), # (time, freq)
261
+ ),
262
+ Balancer(layer3_channels, channel_dim=1, max_abs=4.0),
263
+ SwooshR(),
264
+ )
265
+
266
+ # just one convnext layer
267
+ self.convnext = ConvNeXt(layer3_channels, kernel_size=(7, 7))
268
+
269
+ # (in_channels-3)//4
270
+ self.out_width = (((in_channels - 1) // 2) - 1) // 2
271
+ self.layer3_channels = layer3_channels
272
+
273
+ self.out = nn.Linear(self.out_width * layer3_channels, out_channels)
274
+ # use a larger than normal grad_scale on this whitening module; there is
275
+ # only one such module, so there is not a concern about adding together
276
+ # many copies of this extra gradient term.
277
+ self.out_whiten = Whiten(
278
+ num_groups=1,
279
+ whitening_limit=ScheduledFloat((0.0, 4.0), (20000.0, 8.0), default=4.0),
280
+ prob=(0.025, 0.25),
281
+ grad_scale=0.02,
282
+ )
283
+
284
+ # max_log_eps=0.0 is to prevent both eps and the output of self.out from
285
+ # getting large, there is an unnecessary degree of freedom.
286
+ self.out_norm = BiasNorm(out_channels)
287
+ self.dropout = Dropout3(dropout, shared_dim=1)
288
+
289
+ def forward(
290
+ self, x: torch.Tensor, x_lens: torch.Tensor
291
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
292
+ """Subsample x.
293
+
294
+ Args:
295
+ x:
296
+ Its shape is (N, T, idim).
297
+ x_lens:
298
+ A tensor of shape (batch_size,) containing the number of frames in
299
+
300
+ Returns:
301
+ - a tensor of shape (N, (T-7)//2, odim)
302
+ - output lengths, of shape (batch_size,)
303
+ """
304
+ # On entry, x is (N, T, idim)
305
+ x = x.unsqueeze(1) # (N, T, idim) -> (N, 1, T, idim) i.e., (N, C, H, W)
306
+ # scaling x by 0.1 allows us to use a larger grad-scale in fp16 "amp" (automatic mixed precision)
307
+ # training, since the weights in the first convolution are otherwise the limiting factor for getting infinite
308
+ # gradients.
309
+ x = self.conv(x)
310
+ x = self.convnext(x)
311
+
312
+ # Now x is of shape (N, odim, (T-7)//2, (idim-3)//4)
313
+ b, c, t, f = x.size()
314
+
315
+ x = x.transpose(1, 2).reshape(b, t, c * f)
316
+ # now x: (N, (T-7)//2, out_width * layer3_channels))
317
+
318
+ x = self.out(x)
319
+ # Now x is of shape (N, (T-7)//2, odim)
320
+ x = self.out_whiten(x)
321
+ x = self.out_norm(x)
322
+ x = self.dropout(x)
323
+
324
+ if torch.jit.is_scripting() or torch.jit.is_tracing():
325
+ x_lens = (x_lens - 7) // 2
326
+ else:
327
+ with warnings.catch_warnings():
328
+ warnings.simplefilter("ignore")
329
+ x_lens = (x_lens - 7) // 2
330
+ assert x.size(1) == x_lens.max().item(), (x.size(1), x_lens.max())
331
+
332
+ return x, x_lens
333
+
334
+ def streaming_forward(
335
+ self,
336
+ x: torch.Tensor,
337
+ x_lens: torch.Tensor,
338
+ cached_left_pad: Tensor,
339
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
340
+ """Subsample x.
341
+
342
+ Args:
343
+ x:
344
+ Its shape is (N, T, idim).
345
+ x_lens:
346
+ A tensor of shape (batch_size,) containing the number of frames in
347
+
348
+ Returns:
349
+ - a tensor of shape (N, (T-7)//2, odim)
350
+ - output lengths, of shape (batch_size,)
351
+ - updated cache
352
+ """
353
+ # On entry, x is (N, T, idim)
354
+ x = x.unsqueeze(1) # (N, T, idim) -> (N, 1, T, idim) i.e., (N, C, H, W)
355
+
356
+ # T' = (T-7)//2
357
+ x = self.conv(x)
358
+
359
+ # T' = (T-7)//2-3
360
+ x, cached_left_pad = self.convnext.streaming_forward(
361
+ x, cached_left_pad=cached_left_pad
362
+ )
363
+
364
+ # Now x is of shape (N, odim, T', ((idim-1)//2 - 1)//2)
365
+ b, c, t, f = x.size()
366
+
367
+ x = x.transpose(1, 2).reshape(b, t, c * f)
368
+ # now x: (N, T', out_width * layer3_channels))
369
+
370
+ x = self.out(x)
371
+ # Now x is of shape (N, T', odim)
372
+ x = self.out_norm(x)
373
+
374
+ if torch.jit.is_scripting() or torch.jit.is_tracing():
375
+ assert self.convnext.padding[0] == 3
376
+ # The ConvNeXt module needs 3 frames of right padding after subsampling
377
+ x_lens = (x_lens - 7) // 2 - 3
378
+ else:
379
+ with warnings.catch_warnings():
380
+ warnings.simplefilter("ignore")
381
+ # The ConvNeXt module needs 3 frames of right padding after subsampling
382
+ assert self.convnext.padding[0] == 3
383
+ x_lens = (x_lens - 7) // 2 - 3
384
+
385
+ assert x.size(1) == x_lens.max().item(), (x.shape, x_lens.max())
386
+
387
+ return x, x_lens, cached_left_pad
388
+
389
+ @torch.jit.export
390
+ def get_init_states(
391
+ self,
392
+ batch_size: int = 1,
393
+ device: torch.device = torch.device("cpu"),
394
+ ) -> Tensor:
395
+ """Get initial states for Conv2dSubsampling module.
396
+ It is the cached left padding for ConvNeXt module,
397
+ of shape (batch_size, num_channels, left_pad, num_freqs)
398
+ """
399
+ left_pad = self.convnext.padding[0]
400
+ freq = self.out_width
401
+ channels = self.layer3_channels
402
+ cached_embed_left_pad = torch.zeros(batch_size, channels, left_pad, freq).to(
403
+ device
404
+ )
405
+
406
+ return cached_embed_left_pad
speedasr_core/utils.py ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang,
2
+ # Mingshuang Luo,
3
+ # Zengwei Yao)
4
+ # 2023 Johns Hopkins University (authors: Amir Hussein)
5
+ #
6
+ # See ../../LICENSE for clarification regarding multiple authors
7
+ #
8
+ # Licensed under the Apache License, Version 2.0 (the "License");
9
+ # you may not use this file except in compliance with the License.
10
+ # You may obtain a copy of the License at
11
+ #
12
+ # http://www.apache.org/licenses/LICENSE-2.0
13
+ #
14
+ # Unless required by applicable law or agreed to in writing, software
15
+ # distributed under the License is distributed on an "AS IS" BASIS,
16
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+ # See the License for the specific language governing permissions and
18
+ # limitations under the License.
19
+
20
+
21
+ from contextlib import contextmanager
22
+ import warnings
23
+
24
+ import torch
25
+ from packaging import version
26
+
27
+ TORCH_VERSION = version.parse(torch.__version__.split("+")[0])
28
+
29
+
30
+ @contextmanager
31
+ def torch_autocast(device_type="cuda", **kwargs):
32
+ """
33
+ To fix the following warnings:
34
+ /icefall/egs/librispeech/ASR/zipformer/model.py:323:
35
+ FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated.
36
+ Please use `torch.amp.autocast('cuda', args...)` instead.
37
+ with torch.cuda.amp.autocast(enabled=False):
38
+ """
39
+ if TORCH_VERSION >= version.parse("2.3.0"):
40
+ # Use new unified API
41
+ with torch.amp.autocast(device_type=device_type, **kwargs):
42
+ yield
43
+ else:
44
+ # Suppress deprecation warning and use old CUDA-specific autocast
45
+ with warnings.catch_warnings():
46
+ warnings.simplefilter("ignore", category=FutureWarning)
47
+ with torch.cuda.amp.autocast(**kwargs):
48
+ yield
49
+
50
+
51
+
52
+
53
+ def make_pad_mask(
54
+ lengths: torch.Tensor,
55
+ max_len: int = 0,
56
+ pad_left: bool = False,
57
+ ) -> torch.Tensor:
58
+ """
59
+ Args:
60
+ lengths:
61
+ A 1-D tensor containing sentence lengths.
62
+ max_len:
63
+ The length of masks.
64
+ pad_left:
65
+ If ``False`` (default), padding is on the right.
66
+ If ``True``, padding is on the left.
67
+ Returns:
68
+ Return a 2-D bool tensor, where masked positions
69
+ are filled with `True` and non-masked positions are
70
+ filled with `False`.
71
+
72
+ >>> lengths = torch.tensor([1, 3, 2, 5])
73
+ >>> make_pad_mask(lengths)
74
+ tensor([[False, True, True, True, True],
75
+ [False, False, False, True, True],
76
+ [False, False, True, True, True],
77
+ [False, False, False, False, False]])
78
+ """
79
+ assert lengths.ndim == 1, lengths.ndim
80
+ max_len = max(max_len, lengths.max())
81
+ n = lengths.size(0)
82
+ seq_range = torch.arange(0, max_len, device=lengths.device)
83
+ expanded_lengths = seq_range.unsqueeze(0).expand(n, max_len)
84
+
85
+ if pad_left:
86
+ mask = expanded_lengths < (max_len - lengths).unsqueeze(1)
87
+ else:
88
+ mask = expanded_lengths >= lengths.unsqueeze(-1)
89
+
90
+ return mask
91
+
92
+