Publish SpeedASR-v1-65M weights, portable inference and Open ASR evaluation package
Browse filesOriginal epoch-30-avg-9 checkpoint and tokenizer; FP32 offline beam-4 inference; public results and validation; pinned Docker/Jobs submission adapter. Existing Apache 2.0 LICENSE preserved.
This view is limited to 50 files because it contains too many changes. See raw diff
- .dockerignore +13 -0
- .gitattributes +3 -0
- README.md +190 -0
- artifact_manifest.json +269 -0
- benchmark_results/adapter_validation.json +54 -0
- benchmark_results/dependency_validation.json +100 -0
- benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_ArtificialAnalysis-Earnings22-Cleaned-AA-chunked_earnings22_cleaned_aa_chunked_test.jsonl +3 -0
- benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_VoiceArena-Monsoon_en_IN_test_monsoon_en_in_test.jsonl +0 -0
- benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_ami_cleaned_test.jsonl +0 -0
- benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_gigaspeech_cleaned_test.jsonl +3 -0
- benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_spgispeech_test.jsonl +3 -0
- benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_clean_test.jsonl +0 -0
- benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_test.jsonl +0 -0
- benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_voxpopuli_cleaned_aa_test.jsonl +0 -0
- benchmark_results/local_3090ti_20261119/REPRODUCIBILITY.txt +12 -0
- benchmark_results/local_3090ti_20261119/current_scores.json +170 -0
- benchmark_results/local_3090ti_20261119/export_manifest.json +62 -0
- benchmark_results/local_3090ti_20261119/independent_validation.json +162 -0
- benchmark_results/local_3090ti_20261119/input_provenance.json +16 -0
- benchmark_results/local_3090ti_20261119/run_config.json +72 -0
- benchmark_results/portable_validation.json +228 -0
- inference.py +407 -0
- leaderboard/Dockerfile +32 -0
- leaderboard/cloud_submission.txt +191 -0
- leaderboard/dockerignore.txt +13 -0
- leaderboard/model_metadata.json +53 -0
- leaderboard/requirements.txt +12 -0
- leaderboard/run_eval.py +264 -0
- leaderboard/space_app.py +26 -0
- leaderboard/space_card.txt +15 -0
- leaderboard/submit_jobs.sh +95 -0
- model_config.json +59 -0
- onnx/decoder-epoch-99-avg-1.onnx +3 -0
- onnx/encoder-epoch-99-avg-1.onnx +3 -0
- onnx/joiner-epoch-99-avg-1.onnx +3 -0
- pretrained.pt +3 -0
- requirements-core.txt +5 -0
- requirements-linux-cu124.txt +7 -0
- requirements.txt +3 -0
- runtime_source_manifest.json +70 -0
- speedasr_core/LICENSE +202 -0
- speedasr_core/__init__.py +4 -0
- speedasr_core/beam_search.py +423 -0
- speedasr_core/decoder.py +134 -0
- speedasr_core/encoder_interface.py +43 -0
- speedasr_core/joiner.py +67 -0
- speedasr_core/model.py +152 -0
- speedasr_core/scaling.py +1913 -0
- speedasr_core/subsampling.py +406 -0
- speedasr_core/utils.py +92 -0
.dockerignore
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.git
|
| 2 |
+
*.pt
|
| 3 |
+
*.onnx
|
| 4 |
+
*.safetensors
|
| 5 |
+
benchmarks
|
| 6 |
+
results
|
| 7 |
+
__pycache__
|
| 8 |
+
*.pyc
|
| 9 |
+
.env
|
| 10 |
+
*token*
|
| 11 |
+
!tokens.txt
|
| 12 |
+
|
| 13 |
+
benchmark_results
|
.gitattributes
CHANGED
|
@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_ArtificialAnalysis-Earnings22-Cleaned-AA-chunked_earnings22_cleaned_aa_chunked_test.jsonl filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_gigaspeech_cleaned_test.jsonl filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_spgispeech_test.jsonl filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -1,3 +1,193 @@
|
|
| 1 |
---
|
| 2 |
license: apache-2.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
license: apache-2.0
|
| 3 |
+
language:
|
| 4 |
+
- en
|
| 5 |
+
pipeline_tag: automatic-speech-recognition
|
| 6 |
+
library_name: pytorch
|
| 7 |
+
tags:
|
| 8 |
+
- icefall
|
| 9 |
+
- zipformer
|
| 10 |
+
- transducer
|
| 11 |
+
- offline-asr
|
| 12 |
+
- english
|
| 13 |
+
metrics:
|
| 14 |
+
- wer
|
| 15 |
---
|
| 16 |
+
|
| 17 |
+
# SpeedASR-v1-65M
|
| 18 |
+
|
| 19 |
+
SpeedASR-v1-65M is an **offline English Zipformer transducer** with
|
| 20 |
+
**65,551,059 parameters (65.55M)**. This release contains the original FP32
|
| 21 |
+
checkpoint, its 500-entry token vocabulary, portable batched inference code,
|
| 22 |
+
and public Open ASR Leaderboard evaluation artifacts.
|
| 23 |
+
|
| 24 |
+
Original encoder/decoder/joiner ONNX exports are also retained in [onnx](onnx).
|
| 25 |
+
Their `epoch-99` filenames are export labels around the epoch-30-avg-9 checkpoint
|
| 26 |
+
alias, not a claim that this is an epoch-99 model. The reported scores and cloud
|
| 27 |
+
adapter use PyTorch; these ONNX files have not been evaluated for numerical
|
| 28 |
+
parity by this release's checks.
|
| 29 |
+
|
| 30 |
+
The non-causal encoder processes complete recordings. This release does not
|
| 31 |
+
provide a native streaming recognizer or claim a measured phone/NPU benchmark.
|
| 32 |
+
|
| 33 |
+
| Property | Value |
|
| 34 |
+
|---|---|
|
| 35 |
+
| Language | English |
|
| 36 |
+
| Encoder | Zipformer2, six stacks / 16 encoder layers; non-causal |
|
| 37 |
+
| Prediction network | Transducer, context size 3, dimension 512 |
|
| 38 |
+
| Joiner | Dimension 512 |
|
| 39 |
+
| Parameters | 65,551,059 total |
|
| 40 |
+
| Audio | 16 kHz, mono, floating-point samples |
|
| 41 |
+
| Features | 80-bin kaldifeat Fbank, dither 0, `snip_edges=False`, `high_freq=-400` |
|
| 42 |
+
| Decoding | Modified beam search, beam size 4, FP32 |
|
| 43 |
+
| Text formatting | VoiceA casing/punctuation tags restored; no benchmark WER normalization inside inference |
|
| 44 |
+
| External language model | None |
|
| 45 |
+
| Checkpoint | `pretrained.pt`, 262,613,934 bytes; epoch-30-avg-9 |
|
| 46 |
+
|
| 47 |
+
The complete frozen construction and inference settings are in
|
| 48 |
+
[model_config.json](model_config.json). The repository's existing Apache 2.0
|
| 49 |
+
license is preserved. Vendored Icefall components retain their upstream notices
|
| 50 |
+
and [Apache 2.0 license](speedasr_core/LICENSE).
|
| 51 |
+
|
| 52 |
+
## Inference
|
| 53 |
+
|
| 54 |
+
The pinned dependency set targets **Linux x86-64, Python 3.10, CUDA 12.4 and
|
| 55 |
+
PyTorch 2.6.0**. The k2 and kaldifeat wheels must match the PyTorch/CUDA ABI.
|
| 56 |
+
|
| 57 |
+
```bash
|
| 58 |
+
python3.10 -m venv .venv
|
| 59 |
+
source .venv/bin/activate
|
| 60 |
+
python -m pip install huggingface_hub==1.18.0
|
| 61 |
+
hf download SpeedAudio261119/SpeedASR-v1-65M --local-dir SpeedASR-v1-65M
|
| 62 |
+
cd SpeedASR-v1-65M
|
| 63 |
+
python -m pip install -r requirements-linux-cu124.txt
|
| 64 |
+
python inference.py /path/to/16khz.wav --device cuda:0
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
From a downloaded copy of this repository:
|
| 68 |
+
|
| 69 |
+
```python
|
| 70 |
+
import soundfile as sf
|
| 71 |
+
from inference import SpeedASR
|
| 72 |
+
|
| 73 |
+
audio, sample_rate = sf.read("sample.wav", dtype="float32")
|
| 74 |
+
model = SpeedASR(device="cuda:0")
|
| 75 |
+
text = model.transcribe(audio, sample_rate=sample_rate)
|
| 76 |
+
texts = model.transcribe_batch([audio, audio], sample_rate=sample_rate)
|
| 77 |
+
print(text)
|
| 78 |
+
```
|
| 79 |
+
|
| 80 |
+
Inputs must be mono and 16 kHz; resample other audio before calling the API.
|
| 81 |
+
The WAV CLI uses the first channel, matching the benchmark frontend.
|
| 82 |
+
Inputs shorter than one second are padded; inputs longer than 30 seconds are
|
| 83 |
+
decoded in consecutive 30-second pieces and joined without overlap. The same
|
| 84 |
+
policy and decoder settings apply to all datasets.
|
| 85 |
+
|
| 86 |
+
The loader checks the checkpoint/token hashes, loads the state dictionary
|
| 87 |
+
strictly, and verifies the parameter count. `SpeedASR.from_pretrained` can
|
| 88 |
+
download weights, tokens and configuration for an explicitly pinned Hub commit;
|
| 89 |
+
the local inference module and `speedasr_core` must already be available.
|
| 90 |
+
|
| 91 |
+
## Public benchmark results
|
| 92 |
+
|
| 93 |
+
The following **local measurements** were completed on **19 November 2026**
|
| 94 |
+
using an **NVIDIA GeForce RTX 3090 Ti, FP32**, with the official scorer at
|
| 95 |
+
[`4a113dd90c9ffd011d7d3d105da76980d511d029`](https://github.com/huggingface/open_asr_leaderboard/tree/4a113dd90c9ffd011d7d3d105da76980d511d029).
|
| 96 |
+
They are not official H200/cloud results or an official leaderboard ranking.
|
| 97 |
+
|
| 98 |
+
| Public testset | Clips | WER (%) |
|
| 99 |
+
|---|---:|---:|
|
| 100 |
+
| AMI Cleaned | 7,805 | 9.59 |
|
| 101 |
+
| GigaSpeech Cleaned | 18,768 | 8.07 |
|
| 102 |
+
| VoxPopuli Cleaned AA | 628 | 3.27 |
|
| 103 |
+
| Earnings22 Cleaned AA Chunked | 341 | 6.78 |
|
| 104 |
+
| SPGISpeech | 39,341 | 3.89 |
|
| 105 |
+
| Voice Arena Monsoon English | 2,102 | 4.44 |
|
| 106 |
+
| URGENT2024 | 1,000 | 12.29 |
|
| 107 |
+
| **Seven-set arithmetic mean** | **69,985** | **6.9043** |
|
| 108 |
+
| URGENT2024 Clean — optional, excluded from mean | 1,000 | 5.54 |
|
| 109 |
+
|
| 110 |
+
Earnings22 predictions are reconstructed and scored as **six parent calls**,
|
| 111 |
+
not 341 independent references. The two private score groups are unavailable
|
| 112 |
+
locally and are excluded from this public average.
|
| 113 |
+
|
| 114 |
+
Seven-set pooled local throughput was **192.6293 RTFx**, using up to 2,048
|
| 115 |
+
streams, a 5,000 padded-audio-second budget, and 512 long-audio chunks per
|
| 116 |
+
internal batch. This is batched GPU throughput, not single-request latency or
|
| 117 |
+
on-device efficiency. Cloud throughput must be measured separately.
|
| 118 |
+
|
| 119 |
+
The [benchmark_results/local_3090ti_20261119](benchmark_results/local_3090ti_20261119)
|
| 120 |
+
directory includes all **70,985** raw public prediction rows, final scores,
|
| 121 |
+
dataset revisions, sanitized historical configuration, and verification data.
|
| 122 |
+
Only machine-local audio paths and model-id filenames were changed for release;
|
| 123 |
+
raw references/predictions, timing and chunk metadata were preserved.
|
| 124 |
+
[export_manifest.json](benchmark_results/local_3090ti_20261119/export_manifest.json)
|
| 125 |
+
records original and released hashes. It contains no audio corpus or private
|
| 126 |
+
testset data.
|
| 127 |
+
|
| 128 |
+
The portable runtime passed strict loading and exact prediction comparisons
|
| 129 |
+
against the original runner on 24 representative short/middle/long examples
|
| 130 |
+
covering all eight public sets, plus a 16-example identical-batch comparison.
|
| 131 |
+
See [portable_validation.json](benchmark_results/portable_validation.json).
|
| 132 |
+
This is a targeted portability check, not a rerun of all testsets with the new
|
| 133 |
+
cloud adapter. Different batch grouping can cause numerical differences.
|
| 134 |
+
|
| 135 |
+
The [adapter smoke check](benchmark_results/adapter_validation.json) also passed
|
| 136 |
+
official data preparation, raw manifest writing and scoring using four cached
|
| 137 |
+
AMI utterances and one complete 51-chunk Earnings22 parent call. Dataset transport
|
| 138 |
+
and metadata requests were replaced with local test fixtures for that check.
|
| 139 |
+
Docker was unavailable on the validation host, so a clean image build and an
|
| 140 |
+
actual H200 Job remain subsequent validation steps.
|
| 141 |
+
The combined pinned inference/evaluation dependencies passed a fresh
|
| 142 |
+
`pip --dry-run --ignore-installed` resolution; see
|
| 143 |
+
[dependency_validation.json](benchmark_results/dependency_validation.json).
|
| 144 |
+
|
| 145 |
+
## Open ASR cloud evaluation
|
| 146 |
+
|
| 147 |
+
[leaderboard/run_eval.py](leaderboard/run_eval.py) uses the official data loader
|
| 148 |
+
and manifest writer, retains raw references and predictions, preserves
|
| 149 |
+
Earnings22 parent/chunk metadata, and uses one scoring worker. It fixes the
|
| 150 |
+
decoder settings above across datasets and records synchronized inference timing
|
| 151 |
+
after synthetic warmup.
|
| 152 |
+
|
| 153 |
+
The [Dockerfile](leaderboard/Dockerfile),
|
| 154 |
+
[HF Jobs launcher](leaderboard/submit_jobs.sh), and
|
| 155 |
+
[submission instructions](leaderboard/cloud_submission.txt) prepare a Docker
|
| 156 |
+
Space and the seven public cloud evaluation jobs. Build with:
|
| 157 |
+
|
| 158 |
+
```bash
|
| 159 |
+
docker build -f leaderboard/Dockerfile -t speedasr-open-asr .
|
| 160 |
+
```
|
| 161 |
+
|
| 162 |
+
Use an immutable model commit SHA for submitted runs. H200 batch settings should
|
| 163 |
+
be checked for the maximum practical throughput; the included defaults reproduce
|
| 164 |
+
the local limits and are not claimed to saturate an H200. Uploading this package
|
| 165 |
+
does not itself build a Space, launch paid Jobs, submit a GitHub PR, evaluate
|
| 166 |
+
private sets or register an official leaderboard entry.
|
| 167 |
+
|
| 168 |
+
## Training data
|
| 169 |
+
|
| 170 |
+
The author confirms that **all seven default public benchmark testsets listed
|
| 171 |
+
above were excluded from training**. This is an author-provided statement, not
|
| 172 |
+
an independent overlap audit. The full training-dataset composition has not
|
| 173 |
+
been disclosed in this release. No exclusion claim is made here for unavailable
|
| 174 |
+
private datasets or the optional eighth testset.
|
| 175 |
+
|
| 176 |
+
## Artifact identity
|
| 177 |
+
|
| 178 |
+
```text
|
| 179 |
+
pretrained.pt SHA-256
|
| 180 |
+
b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01
|
| 181 |
+
|
| 182 |
+
tokens.txt SHA-256
|
| 183 |
+
a238b7a0fa67aa6be998ca13dd129d69c50639970583e32232dd149e5448210a
|
| 184 |
+
```
|
| 185 |
+
|
| 186 |
+
[runtime_source_manifest.json](runtime_source_manifest.json) records vendored
|
| 187 |
+
source origins and transformations. The upstream model-code revision is
|
| 188 |
+
`3f848bb6d0acc970c9b294a30ca0a04a7c9c78d1`. The release artifact manifest
|
| 189 |
+
records the files actually published.
|
| 190 |
+
|
| 191 |
+
[leaderboard/model_metadata.json](leaderboard/model_metadata.json) provides
|
| 192 |
+
structured submission metadata and parameter counts by component, including
|
| 193 |
+
auxiliary training projections retained in the unchanged checkpoint.
|
artifact_manifest.json
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_id": "SpeedAudio261119/SpeedASR-v1-65M",
|
| 3 |
+
"release_date": "2026-11-19",
|
| 4 |
+
"source_repository_parent_commit": "c2d6f53b50895c81f8e624789da1239edf050e99",
|
| 5 |
+
"note": "This manifest covers uploaded package files except itself. Existing Hub LICENSE and .gitattributes are preserved and are not overwritten.",
|
| 6 |
+
"validation": {
|
| 7 |
+
"checkpoint_and_token_hashes_match_original_benchmark": true,
|
| 8 |
+
"portable_original_prediction_cases": 24,
|
| 9 |
+
"identical_batch_cases": 16,
|
| 10 |
+
"portable_exact_prediction_parity": true,
|
| 11 |
+
"adapter_official_preparation_writer_scoring_smoke_passed": true,
|
| 12 |
+
"clean_docker_build_performed": false,
|
| 13 |
+
"official_cloud_or_private_evaluation_performed": false,
|
| 14 |
+
"credential_scan_passed": true,
|
| 15 |
+
"fresh_dependency_resolution_passed": true
|
| 16 |
+
},
|
| 17 |
+
"files": [
|
| 18 |
+
{
|
| 19 |
+
"path": ".dockerignore",
|
| 20 |
+
"bytes": 112,
|
| 21 |
+
"sha256": "f5cbe531d1fdf5f84517ca8b9014f38b99a6dbc2fd7d71093fc4d0e5648780fe"
|
| 22 |
+
},
|
| 23 |
+
{
|
| 24 |
+
"path": "README.md",
|
| 25 |
+
"bytes": 8566,
|
| 26 |
+
"sha256": "628abdfa3846a092ad369eaa3f01e18f8cc5a182c29b69e88197205d06f41bfc"
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
"path": "benchmark_results/adapter_validation.json",
|
| 30 |
+
"bytes": 1435,
|
| 31 |
+
"sha256": "3d1622c48c966aec8e7a8239407108232217f7d9b999b5c9a12cc9d6a29506b4"
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"path": "benchmark_results/dependency_validation.json",
|
| 35 |
+
"bytes": 2733,
|
| 36 |
+
"sha256": "7bd98fc8178d36342d48928ada975abfa42aae73f2ac2ed83f8569de0f4a8b97"
|
| 37 |
+
},
|
| 38 |
+
{
|
| 39 |
+
"path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_ArtificialAnalysis-Earnings22-Cleaned-AA-chunked_earnings22_cleaned_aa_chunked_test.jsonl",
|
| 40 |
+
"bytes": 11172360,
|
| 41 |
+
"sha256": "5010bb0dcc3024b9d3dda5956734044a0d35ccaf9bb3214271efb9f03cc6ea92"
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_VoiceArena-Monsoon_en_IN_test_monsoon_en_in_test.jsonl",
|
| 45 |
+
"bytes": 1922498,
|
| 46 |
+
"sha256": "fd947f47d55206b7e720a96e8ce2d704b5213c3e6709c83a35a1d8138df0611c"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_ami_cleaned_test.jsonl",
|
| 50 |
+
"bytes": 5349141,
|
| 51 |
+
"sha256": "73b4f9d3c03e519f70a5897ae20b5195a9733a1f9baa8b9ff0618c590d86fb14"
|
| 52 |
+
},
|
| 53 |
+
{
|
| 54 |
+
"path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_gigaspeech_cleaned_test.jsonl",
|
| 55 |
+
"bytes": 16540413,
|
| 56 |
+
"sha256": "e25623081b67f8c37db13966077e8f44eff25e47de61f96db2c5f2938ae1aaa0"
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_spgispeech_test.jsonl",
|
| 60 |
+
"bytes": 41084800,
|
| 61 |
+
"sha256": "ce7512cd10eff18d797cdc9efb167ada34b37c4beda1b5ba4bcde206ced0330b"
|
| 62 |
+
},
|
| 63 |
+
{
|
| 64 |
+
"path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_clean_test.jsonl",
|
| 65 |
+
"bytes": 799100,
|
| 66 |
+
"sha256": "bf1f804714c7ae77d742a24cb499a3ddb50934e03ca8ce8b91c0c92928a1321a"
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_test.jsonl",
|
| 70 |
+
"bytes": 786374,
|
| 71 |
+
"sha256": "30edc6b2c7dcf13c664682f5ca06fb7b2dace5bb92cde4e8710688729206a887"
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"path": "benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_voxpopuli_cleaned_aa_test.jsonl",
|
| 75 |
+
"bytes": 739478,
|
| 76 |
+
"sha256": "1bb76945bcb6f8b6fb89420e10615b1ab310160391894062d53f443b3ff79d51"
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"path": "benchmark_results/local_3090ti_20261119/REPRODUCIBILITY.txt",
|
| 80 |
+
"bytes": 1353,
|
| 81 |
+
"sha256": "a441630f8054b9901b65209b447a0128e07bb09a22319fccd7625a4d95487b44"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"path": "benchmark_results/local_3090ti_20261119/current_scores.json",
|
| 85 |
+
"bytes": 6358,
|
| 86 |
+
"sha256": "a3e165a0a543a881ba210ae448e0d77fd209a963fe9c4a1d17410023ae19ba99"
|
| 87 |
+
},
|
| 88 |
+
{
|
| 89 |
+
"path": "benchmark_results/local_3090ti_20261119/export_manifest.json",
|
| 90 |
+
"bytes": 4555,
|
| 91 |
+
"sha256": "b6a4c38ad79119ed90b898986d4342c5f0f3af170a46639d5d9b0b83ce1873a4"
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"path": "benchmark_results/local_3090ti_20261119/independent_validation.json",
|
| 95 |
+
"bytes": 6095,
|
| 96 |
+
"sha256": "8e1e46acf345955e3290b5087416f9650d4e98dc5cf2a5acc749e7966ff4e70e"
|
| 97 |
+
},
|
| 98 |
+
{
|
| 99 |
+
"path": "benchmark_results/local_3090ti_20261119/input_provenance.json",
|
| 100 |
+
"bytes": 1008,
|
| 101 |
+
"sha256": "6936df47c16498b02e11bbff5c0bd0bc7318323b89a9e5cdbb8dedb7547a1e56"
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"path": "benchmark_results/local_3090ti_20261119/run_config.json",
|
| 105 |
+
"bytes": 2160,
|
| 106 |
+
"sha256": "bed45bbfb0bdad8d9e00dba4c0b7e8e65d6598d728d08eb7ef5bbb7a61045c19"
|
| 107 |
+
},
|
| 108 |
+
{
|
| 109 |
+
"path": "benchmark_results/portable_validation.json",
|
| 110 |
+
"bytes": 16310,
|
| 111 |
+
"sha256": "b8c2677d20c34b75eaa83c3980e464da6ecfeebc594dea47c65db38aef3f1048"
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"path": "inference.py",
|
| 115 |
+
"bytes": 19227,
|
| 116 |
+
"sha256": "17ea7c209dc40b0540b3ccca49cef04e265211b38b3daf00333e1180abdaa39e"
|
| 117 |
+
},
|
| 118 |
+
{
|
| 119 |
+
"path": "leaderboard/Dockerfile",
|
| 120 |
+
"bytes": 1565,
|
| 121 |
+
"sha256": "147627537d4bb763aa4d86e9f55fe66bf31248b90a91e386fef5c6700f4541ff"
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"path": "leaderboard/cloud_submission.txt",
|
| 125 |
+
"bytes": 9805,
|
| 126 |
+
"sha256": "234c74d6688308cf54ab9accab5ee127acb06ed9db06fec786a22fe79dffb362"
|
| 127 |
+
},
|
| 128 |
+
{
|
| 129 |
+
"path": "leaderboard/dockerignore.txt",
|
| 130 |
+
"bytes": 112,
|
| 131 |
+
"sha256": "f5cbe531d1fdf5f84517ca8b9014f38b99a6dbc2fd7d71093fc4d0e5648780fe"
|
| 132 |
+
},
|
| 133 |
+
{
|
| 134 |
+
"path": "leaderboard/model_metadata.json",
|
| 135 |
+
"bytes": 1788,
|
| 136 |
+
"sha256": "8a496e931996e92a5ceb6a4210a8e7015f83a8577371ce43bdfa5d3a52ac10c6"
|
| 137 |
+
},
|
| 138 |
+
{
|
| 139 |
+
"path": "leaderboard/requirements.txt",
|
| 140 |
+
"bytes": 373,
|
| 141 |
+
"sha256": "5a169ebe8356c349bfa076617d5b744a1853adeebbe95a1513c5379871b8c912"
|
| 142 |
+
},
|
| 143 |
+
{
|
| 144 |
+
"path": "leaderboard/run_eval.py",
|
| 145 |
+
"bytes": 12739,
|
| 146 |
+
"sha256": "98aa19d45326ec8ceefd6707f096460e610fa9b886d8585996e4361c7b1bd56e"
|
| 147 |
+
},
|
| 148 |
+
{
|
| 149 |
+
"path": "leaderboard/space_app.py",
|
| 150 |
+
"bytes": 1198,
|
| 151 |
+
"sha256": "11b9fd43d0fde5aaf9974c5564563253f22a51a5ee6735bccc1a718b2f15f913"
|
| 152 |
+
},
|
| 153 |
+
{
|
| 154 |
+
"path": "leaderboard/space_card.txt",
|
| 155 |
+
"bytes": 404,
|
| 156 |
+
"sha256": "02482305533c92b68080a10dd04931e96dbdbdc1224f0b5293eac4568eeb7889"
|
| 157 |
+
},
|
| 158 |
+
{
|
| 159 |
+
"path": "leaderboard/submit_jobs.sh",
|
| 160 |
+
"bytes": 4050,
|
| 161 |
+
"sha256": "8ed64f80d5b1c68eb987faf05a603cd429287c9cae169cf932b55b52f4ae23ab"
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"path": "model_config.json",
|
| 165 |
+
"bytes": 1760,
|
| 166 |
+
"sha256": "e17381d31eb19a4d781c2b56f2060986c6bff02154f9f7e31edb57de11eb0e7f"
|
| 167 |
+
},
|
| 168 |
+
{
|
| 169 |
+
"path": "onnx/decoder-epoch-99-avg-1.onnx",
|
| 170 |
+
"bytes": 2101273,
|
| 171 |
+
"sha256": "1fc497358cc2f63d4dcd3f249da1e4f29a04315c5d7e4aa358cfaf17ea8d5db1"
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"path": "onnx/encoder-epoch-99-avg-1.onnx",
|
| 175 |
+
"bytes": 263111425,
|
| 176 |
+
"sha256": "768fedad168901e872e46477b347adc95cc304ed9ee60ee57c95b60916da42f0"
|
| 177 |
+
},
|
| 178 |
+
{
|
| 179 |
+
"path": "onnx/joiner-epoch-99-avg-1.onnx",
|
| 180 |
+
"bytes": 1026462,
|
| 181 |
+
"sha256": "c26540ac6d9ac00191436193bc22a8dd3ceae7bd3a83058aa2d18e1470bed5f4"
|
| 182 |
+
},
|
| 183 |
+
{
|
| 184 |
+
"path": "pretrained.pt",
|
| 185 |
+
"bytes": 262613934,
|
| 186 |
+
"sha256": "b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01"
|
| 187 |
+
},
|
| 188 |
+
{
|
| 189 |
+
"path": "requirements-core.txt",
|
| 190 |
+
"bytes": 160,
|
| 191 |
+
"sha256": "c0e62985bd29049fce9e9899be94a0edbd7127c85f9405245ac9456542aeaa44"
|
| 192 |
+
},
|
| 193 |
+
{
|
| 194 |
+
"path": "requirements-linux-cu124.txt",
|
| 195 |
+
"bytes": 735,
|
| 196 |
+
"sha256": "2512fc51922748928a8cb4c25a688ab1f3f43f4f2c4a9f2383ab47ee132dbe82"
|
| 197 |
+
},
|
| 198 |
+
{
|
| 199 |
+
"path": "requirements.txt",
|
| 200 |
+
"bytes": 166,
|
| 201 |
+
"sha256": "415a0461b2a9926ec77ef9c595f46c2244506489d79c2cfa56dba4a000e81c8c"
|
| 202 |
+
},
|
| 203 |
+
{
|
| 204 |
+
"path": "runtime_source_manifest.json",
|
| 205 |
+
"bytes": 3840,
|
| 206 |
+
"sha256": "0e7bfe39d7201c2a38629c3395262d7996ea10fb64e239e2e17f9ff0745449ed"
|
| 207 |
+
},
|
| 208 |
+
{
|
| 209 |
+
"path": "speedasr_core/LICENSE",
|
| 210 |
+
"bytes": 11358,
|
| 211 |
+
"sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"
|
| 212 |
+
},
|
| 213 |
+
{
|
| 214 |
+
"path": "speedasr_core/__init__.py",
|
| 215 |
+
"bytes": 113,
|
| 216 |
+
"sha256": "596ac4de65fae9944440e22cf00ec4629b80da70c2e9b309e0702e4686ac988e"
|
| 217 |
+
},
|
| 218 |
+
{
|
| 219 |
+
"path": "speedasr_core/beam_search.py",
|
| 220 |
+
"bytes": 13852,
|
| 221 |
+
"sha256": "d2e0297d1626b14b834a9b357caa875a5b11ce8f0aabc475debb7d20dec3b8c1"
|
| 222 |
+
},
|
| 223 |
+
{
|
| 224 |
+
"path": "speedasr_core/decoder.py",
|
| 225 |
+
"bytes": 4756,
|
| 226 |
+
"sha256": "a9aeeeef278f4e5357d4e49253d24f5f1c60df88cfcb623ccbd01481dc1503ce"
|
| 227 |
+
},
|
| 228 |
+
{
|
| 229 |
+
"path": "speedasr_core/encoder_interface.py",
|
| 230 |
+
"bytes": 1652,
|
| 231 |
+
"sha256": "fc70f530c33e2489435ff2cb4bb0dad8b9d6f58a248c425cbd6100c62f628569"
|
| 232 |
+
},
|
| 233 |
+
{
|
| 234 |
+
"path": "speedasr_core/joiner.py",
|
| 235 |
+
"bytes": 2214,
|
| 236 |
+
"sha256": "81faad6e0d07342d1abab28f2271201256f2139f5816b04296d6a8cbe0e2c887"
|
| 237 |
+
},
|
| 238 |
+
{
|
| 239 |
+
"path": "speedasr_core/model.py",
|
| 240 |
+
"bytes": 5938,
|
| 241 |
+
"sha256": "46b8e181fb326e8cd82c92a4c0ffd0917eb1474a26bd5b60b39b48daac28e90f"
|
| 242 |
+
},
|
| 243 |
+
{
|
| 244 |
+
"path": "speedasr_core/scaling.py",
|
| 245 |
+
"bytes": 70936,
|
| 246 |
+
"sha256": "679b1fe2655902ba13ef3f5cf7c924129a16681e3edc4343e766ff7086f5e6d7"
|
| 247 |
+
},
|
| 248 |
+
{
|
| 249 |
+
"path": "speedasr_core/subsampling.py",
|
| 250 |
+
"bytes": 13139,
|
| 251 |
+
"sha256": "dc33ac662d2508e191f4da5562f0c8d23444819b707624736429a85517db9145"
|
| 252 |
+
},
|
| 253 |
+
{
|
| 254 |
+
"path": "speedasr_core/utils.py",
|
| 255 |
+
"bytes": 3046,
|
| 256 |
+
"sha256": "682d0733f0e149be0ae9b62cb8de60d7baef65be4e64816001b6bb089b4e787b"
|
| 257 |
+
},
|
| 258 |
+
{
|
| 259 |
+
"path": "speedasr_core/zipformer.py",
|
| 260 |
+
"bytes": 93872,
|
| 261 |
+
"sha256": "5cea6da601de9f37de2e1db6c2dadcd931220b45341472efaa426a25130a5a36"
|
| 262 |
+
},
|
| 263 |
+
{
|
| 264 |
+
"path": "tokens.txt",
|
| 265 |
+
"bytes": 5180,
|
| 266 |
+
"sha256": "a238b7a0fa67aa6be998ca13dd129d69c50639970583e32232dd149e5448210a"
|
| 267 |
+
}
|
| 268 |
+
]
|
| 269 |
+
}
|
benchmark_results/adapter_validation.json
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"compile": true,
|
| 3 |
+
"docker_available": false,
|
| 4 |
+
"bash_syntax": "passed",
|
| 5 |
+
"tests": [
|
| 6 |
+
{
|
| 7 |
+
"dataset": "ami",
|
| 8 |
+
"rows": 4,
|
| 9 |
+
"raw_text_preserved": true,
|
| 10 |
+
"chunk_metadata_preserved": true,
|
| 11 |
+
"score": {
|
| 12 |
+
"wer": 38.46,
|
| 13 |
+
"ins": 0,
|
| 14 |
+
"del": 3,
|
| 15 |
+
"sub": 2,
|
| 16 |
+
"audio_length": 4.12,
|
| 17 |
+
"inference_time": 0.08191168299890705,
|
| 18 |
+
"rtfx": 50.2981
|
| 19 |
+
}
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"dataset": "earnings",
|
| 23 |
+
"rows": 51,
|
| 24 |
+
"raw_text_preserved": true,
|
| 25 |
+
"chunk_metadata_preserved": true,
|
| 26 |
+
"score": {
|
| 27 |
+
"wer": 6.86,
|
| 28 |
+
"ins": 9,
|
| 29 |
+
"del": 25,
|
| 30 |
+
"sub": 112,
|
| 31 |
+
"audio_length": 985.0013749999999,
|
| 32 |
+
"inference_time": 6.488993022998328,
|
| 33 |
+
"rtfx": 151.7957
|
| 34 |
+
}
|
| 35 |
+
}
|
| 36 |
+
],
|
| 37 |
+
"help": "passed",
|
| 38 |
+
"component_parameters": {
|
| 39 |
+
"encoder_embed": 611953,
|
| 40 |
+
"encoder": 63382150,
|
| 41 |
+
"decoder": 262144,
|
| 42 |
+
"joiner": 781812,
|
| 43 |
+
"simple_am_proj": 256500,
|
| 44 |
+
"simple_lm_proj": 256500
|
| 45 |
+
},
|
| 46 |
+
"inference_sha256": "17ea7c209dc40b0540b3ccca49cef04e265211b38b3daf00333e1180abdaa39e",
|
| 47 |
+
"run_eval_sha256": "98aa19d45326ec8ceefd6707f096460e610fa9b886d8585996e4361c7b1bd56e",
|
| 48 |
+
"cpu_smoke": {
|
| 49 |
+
"prediction": "Yeah",
|
| 50 |
+
"parameters": 65551059
|
| 51 |
+
},
|
| 52 |
+
"all_passed": true,
|
| 53 |
+
"validation_scope": "Real inference plus official prepare_data/writer/scorer; local cached audio transport and metadata API stubs; not an HF Jobs or clean Docker build."
|
| 54 |
+
}
|
benchmark_results/dependency_validation.json
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"exit_code": 0,
|
| 3 |
+
"fresh_resolution": true,
|
| 4 |
+
"installed_packages": false,
|
| 5 |
+
"docker_build_performed": false,
|
| 6 |
+
"packages": {
|
| 7 |
+
"k2": "1.24.4.dev20260625+cuda12.4.torch2.6.0",
|
| 8 |
+
"kaldifeat": "1.25.5.dev20260626+cuda12.4.torch2.6.0",
|
| 9 |
+
"torch": "2.6.0+cu124",
|
| 10 |
+
"torchaudio": "2.6.0+cu124",
|
| 11 |
+
"numpy": "1.24.4",
|
| 12 |
+
"soundfile": "0.13.1",
|
| 13 |
+
"huggingface_hub": "1.18.0",
|
| 14 |
+
"datasets": "3.6.0",
|
| 15 |
+
"fsspec": "2025.3.0",
|
| 16 |
+
"librosa": "0.10.2.post1",
|
| 17 |
+
"pandas": "2.2.3",
|
| 18 |
+
"pyarrow": "20.0.0",
|
| 19 |
+
"kaldialign": "0.12.0",
|
| 20 |
+
"num2words": "0.5.14",
|
| 21 |
+
"regex": "2024.11.6",
|
| 22 |
+
"tqdm": "4.66.4",
|
| 23 |
+
"nvidia-cublas-cu12": "12.4.5.8",
|
| 24 |
+
"nvidia-cuda-cupti-cu12": "12.4.127",
|
| 25 |
+
"nvidia-cuda-nvrtc-cu12": "12.4.127",
|
| 26 |
+
"nvidia-cuda-runtime-cu12": "12.4.127",
|
| 27 |
+
"nvidia-cudnn-cu12": "9.1.0.70",
|
| 28 |
+
"nvidia-cufft-cu12": "11.2.1.3",
|
| 29 |
+
"nvidia-curand-cu12": "10.3.5.147",
|
| 30 |
+
"nvidia-cusolver-cu12": "11.6.1.9",
|
| 31 |
+
"nvidia-cusparse-cu12": "12.3.1.170",
|
| 32 |
+
"nvidia-cusparselt-cu12": "0.6.2",
|
| 33 |
+
"nvidia-nccl-cu12": "2.21.5",
|
| 34 |
+
"nvidia-nvjitlink-cu12": "12.4.127",
|
| 35 |
+
"nvidia-nvtx-cu12": "12.4.127",
|
| 36 |
+
"sympy": "1.13.1",
|
| 37 |
+
"triton": "3.2.0",
|
| 38 |
+
"dill": "0.3.8",
|
| 39 |
+
"hf-xet": "1.8.0",
|
| 40 |
+
"httpx": "0.28.1",
|
| 41 |
+
"httpcore": "1.0.9",
|
| 42 |
+
"mpmath": "1.3.0",
|
| 43 |
+
"multiprocess": "0.70.16",
|
| 44 |
+
"typer": "0.25.1",
|
| 45 |
+
"packaging": "26.3",
|
| 46 |
+
"aiohttp": "3.14.5",
|
| 47 |
+
"async-timeout": "5.0.1",
|
| 48 |
+
"multidict": "7.1.0",
|
| 49 |
+
"yarl": "1.25.1",
|
| 50 |
+
"aiohappyeyeballs": "2.7.1",
|
| 51 |
+
"aiosignal": "1.4.0",
|
| 52 |
+
"annotated-doc": "0.0.6",
|
| 53 |
+
"attrs": "26.1.0",
|
| 54 |
+
"audioread": "3.1.0",
|
| 55 |
+
"cffi": "2.1.1",
|
| 56 |
+
"click": "8.5.1",
|
| 57 |
+
"decorator": "5.3.1",
|
| 58 |
+
"docopt": "0.6.2",
|
| 59 |
+
"filelock": "4.1.0",
|
| 60 |
+
"frozenlist": "1.8.0",
|
| 61 |
+
"h11": "0.16.0",
|
| 62 |
+
"idna": "3.20",
|
| 63 |
+
"joblib": "1.6.0",
|
| 64 |
+
"cloudpickle": "3.1.3",
|
| 65 |
+
"lazy-loader": "0.6",
|
| 66 |
+
"msgpack": "1.2.4",
|
| 67 |
+
"numba": "0.68.1",
|
| 68 |
+
"llvmlite": "0.50.0",
|
| 69 |
+
"pooch": "1.9.0",
|
| 70 |
+
"platformdirs": "4.12.4",
|
| 71 |
+
"propcache": "0.5.4",
|
| 72 |
+
"python-dateutil": "3.0.0",
|
| 73 |
+
"pytz": "2026.6",
|
| 74 |
+
"PyYAML": "6.0.3",
|
| 75 |
+
"requests": "2.34.2",
|
| 76 |
+
"charset-normalizer": "3.5.2",
|
| 77 |
+
"urllib3": "2.8.0",
|
| 78 |
+
"certifi": "2026.10.23",
|
| 79 |
+
"rich": "15.0.0",
|
| 80 |
+
"Pygments": "2.21.0",
|
| 81 |
+
"markdown-it-py": "4.3.0",
|
| 82 |
+
"mdurl": "0.1.2",
|
| 83 |
+
"scikit-learn": "1.7.2",
|
| 84 |
+
"scipy": "1.15.3",
|
| 85 |
+
"shellingham": "1.5.4",
|
| 86 |
+
"six": "1.17.0",
|
| 87 |
+
"soxr": "1.1.0",
|
| 88 |
+
"threadpoolctl": "3.8.0",
|
| 89 |
+
"typing_extensions": "4.16.0",
|
| 90 |
+
"tzdata": "2026.6",
|
| 91 |
+
"anyio": "4.16.0",
|
| 92 |
+
"exceptiongroup": "1.3.1",
|
| 93 |
+
"graphviz": "0.21",
|
| 94 |
+
"Jinja2": "3.1.6",
|
| 95 |
+
"MarkupSafe": "3.0.4",
|
| 96 |
+
"networkx": "3.4.2",
|
| 97 |
+
"pycparser": "3.11",
|
| 98 |
+
"xxhash": "4.1.0"
|
| 99 |
+
}
|
| 100 |
+
}
|
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_ArtificialAnalysis-Earnings22-Cleaned-AA-chunked_earnings22_cleaned_aa_chunked_test.jsonl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5010bb0dcc3024b9d3dda5956734044a0d35ccaf9bb3214271efb9f03cc6ea92
|
| 3 |
+
size 11172360
|
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_VoiceArena-Monsoon_en_IN_test_monsoon_en_in_test.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_ami_cleaned_test.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_gigaspeech_cleaned_test.jsonl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e25623081b67f8c37db13966077e8f44eff25e47de61f96db2c5f2938ae1aaa0
|
| 3 |
+
size 16540413
|
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_spgispeech_test.jsonl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ce7512cd10eff18d797cdc9efb167ada34b37c4beda1b5ba4bcde206ced0330b
|
| 3 |
+
size 41084800
|
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_clean_test.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_test.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
benchmark_results/local_3090ti_20261119/MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_voxpopuli_cleaned_aa_test.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
benchmark_results/local_3090ti_20261119/REPRODUCIBILITY.txt
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
This folder contains the completed local RTX 3090 Ti evaluation dated 19 November 2026.
|
| 2 |
+
It is not an official cloud/H200 run and contains no private testset scores.
|
| 3 |
+
|
| 4 |
+
The checkpoint and token SHA256 hashes match the released root files.
|
| 5 |
+
Inference: FP32, causal=False, context_size=3, modified beam search, beam=4, VoiceA formatting, minimum1s input,30s internal chunks.
|
| 6 |
+
Batch configuration: up to2048 streams,5000 padded audio seconds,512 long-audio chunks,16PyTorch threads and8audio workers.
|
| 7 |
+
Final official scoring used one worker at repository commit4a113dd90c9ffd011d7d3d105da76980d511d029 after the original four-worker scorer stalled. Saved inference was not rerun.
|
| 8 |
+
|
| 9 |
+
Filenames were relabelled with the published Hub model id. Machine-specific audio paths were replaced by stable publicsource identifiers. Raw text, predictions, durations, inference timing and Earnings22 chunk metadata were preserved.
|
| 10 |
+
export_manifest.json maps original to released outputs and provides both hash sets. independent_validation.json records validation of the original outputs. current_scores.json is the authoritative final score summary. run_config.json is sanitized historical provenance, not an executable cloud command.
|
| 11 |
+
|
| 12 |
+
Default macro WER uses7publicsets; URGENT2024Clean is optional and excluded. Full-ranking averages may include private sets unavailable here.
|
benchmark_results/local_3090ti_20261119/current_scores.json
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"average_note": "Public seven-set macro-average only; the official default ranking also selects private score groups.",
|
| 3 |
+
"checked_at_utc": "2026-11-19T03:54:19.389175+00:00",
|
| 4 |
+
"complete": true,
|
| 5 |
+
"excluded_from_public_average": [
|
| 6 |
+
"urgent2024_clean"
|
| 7 |
+
],
|
| 8 |
+
"model_file_prefix": "MODEL_SpeedAudio261119-SpeedASR-v1-65M",
|
| 9 |
+
"official_commit": "4a113dd90c9ffd011d7d3d105da76980d511d029",
|
| 10 |
+
"official_default_ranking_average": null,
|
| 11 |
+
"private_results_available": false,
|
| 12 |
+
"public_default_average_wer": 6.904285714285714,
|
| 13 |
+
"public_default_datasets": [
|
| 14 |
+
"ami_cleaned",
|
| 15 |
+
"gigaspeech_cleaned",
|
| 16 |
+
"voxpopuli_cleaned_aa",
|
| 17 |
+
"earnings22_cleaned_aa_chunked",
|
| 18 |
+
"spgispeech",
|
| 19 |
+
"monsoon_en_in",
|
| 20 |
+
"urgent2024"
|
| 21 |
+
],
|
| 22 |
+
"public_default_pooled_rtfx": 192.6292762644339,
|
| 23 |
+
"test_sets": [
|
| 24 |
+
{
|
| 25 |
+
"audio_length": 28856.700437500032,
|
| 26 |
+
"cuts": 7805,
|
| 27 |
+
"dataset": "ami_cleaned",
|
| 28 |
+
"default_average": true,
|
| 29 |
+
"del": 2334,
|
| 30 |
+
"inference_time": 142.35132213500114,
|
| 31 |
+
"ins": 2053,
|
| 32 |
+
"manifest_sha256": "89462aa1a42190471e8616dd27e7d2403c126d67ed05e6a57e31a86bdee991d6",
|
| 33 |
+
"result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_ami_cleaned_test.jsonl",
|
| 34 |
+
"revision": "de79e57adf9983993bcc72586d7313460825f5eb",
|
| 35 |
+
"rtfx": 202.7147,
|
| 36 |
+
"scoring_rows": 7805,
|
| 37 |
+
"scoring_unit": "utterance",
|
| 38 |
+
"split": "test",
|
| 39 |
+
"sub": 3514,
|
| 40 |
+
"wer": 9.59
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"audio_length": 126528.78700000017,
|
| 44 |
+
"cuts": 18768,
|
| 45 |
+
"dataset": "gigaspeech_cleaned",
|
| 46 |
+
"default_average": true,
|
| 47 |
+
"del": 5098,
|
| 48 |
+
"inference_time": 652.5400200150451,
|
| 49 |
+
"ins": 8548,
|
| 50 |
+
"manifest_sha256": "cbebef3e4f0d28b348ee9aa741090f8a9a48c7925eb08872a519b6e487b910c5",
|
| 51 |
+
"result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_gigaspeech_cleaned_test.jsonl",
|
| 52 |
+
"revision": "de79e57adf9983993bcc72586d7313460825f5eb",
|
| 53 |
+
"rtfx": 193.902,
|
| 54 |
+
"scoring_rows": 18768,
|
| 55 |
+
"scoring_unit": "utterance",
|
| 56 |
+
"split": "test",
|
| 57 |
+
"sub": 18396,
|
| 58 |
+
"wer": 8.07
|
| 59 |
+
},
|
| 60 |
+
{
|
| 61 |
+
"audio_length": 7122.378374999999,
|
| 62 |
+
"cuts": 628,
|
| 63 |
+
"dataset": "voxpopuli_cleaned_aa",
|
| 64 |
+
"default_average": true,
|
| 65 |
+
"del": 226,
|
| 66 |
+
"inference_time": 37.72588397499897,
|
| 67 |
+
"ins": 130,
|
| 68 |
+
"manifest_sha256": "61e855acf35048601fdcaed1c0ca10ba30ac7fb3a887d722c31a456c545605e1",
|
| 69 |
+
"result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_voxpopuli_cleaned_aa_test.jsonl",
|
| 70 |
+
"revision": "de79e57adf9983993bcc72586d7313460825f5eb",
|
| 71 |
+
"rtfx": 188.7929,
|
| 72 |
+
"scoring_rows": 628,
|
| 73 |
+
"scoring_unit": "utterance",
|
| 74 |
+
"split": "test",
|
| 75 |
+
"sub": 225,
|
| 76 |
+
"wer": 3.27
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"audio_length": 6900.9376250000005,
|
| 80 |
+
"cuts": 341,
|
| 81 |
+
"dataset": "earnings22_cleaned_aa_chunked",
|
| 82 |
+
"default_average": true,
|
| 83 |
+
"del": 222,
|
| 84 |
+
"inference_time": 36.58612209400007,
|
| 85 |
+
"ins": 154,
|
| 86 |
+
"manifest_sha256": "d6d297c76ce3fa9208791d989bd15cbaaaff6fed545ad17ab96b7d4492beef70",
|
| 87 |
+
"result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_ArtificialAnalysis-Earnings22-Cleaned-AA-chunked_earnings22_cleaned_aa_chunked_test.jsonl",
|
| 88 |
+
"revision": "27f7d0056d593231974a752ce03c1e965da69b3f",
|
| 89 |
+
"rtfx": 188.6217,
|
| 90 |
+
"scoring_rows": 6,
|
| 91 |
+
"scoring_unit": "parent_session",
|
| 92 |
+
"split": "test",
|
| 93 |
+
"sub": 668,
|
| 94 |
+
"wer": 6.78
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"audio_length": 360007.49950000306,
|
| 98 |
+
"cuts": 39341,
|
| 99 |
+
"dataset": "spgispeech",
|
| 100 |
+
"default_average": true,
|
| 101 |
+
"del": 4664,
|
| 102 |
+
"inference_time": 1881.6296108457523,
|
| 103 |
+
"ins": 19136,
|
| 104 |
+
"manifest_sha256": "f4a401536ffbefa5db8e4c9e7336301357c8d71a3d9eac16e026556456e0f44c",
|
| 105 |
+
"result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_spgispeech_test.jsonl",
|
| 106 |
+
"revision": "de79e57adf9983993bcc72586d7313460825f5eb",
|
| 107 |
+
"rtfx": 191.3275,
|
| 108 |
+
"scoring_rows": 39341,
|
| 109 |
+
"scoring_unit": "utterance",
|
| 110 |
+
"split": "test",
|
| 111 |
+
"sub": 13919,
|
| 112 |
+
"wer": 3.89
|
| 113 |
+
},
|
| 114 |
+
{
|
| 115 |
+
"audio_length": 20247.60743749999,
|
| 116 |
+
"cuts": 2102,
|
| 117 |
+
"dataset": "monsoon_en_in",
|
| 118 |
+
"default_average": true,
|
| 119 |
+
"del": 350,
|
| 120 |
+
"inference_time": 103.29725760700066,
|
| 121 |
+
"ins": 550,
|
| 122 |
+
"manifest_sha256": "571d17a51e0df966785330f3e7e232a39bbe00f3390e47ec20bfd3fb7a74b9b1",
|
| 123 |
+
"result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_VoiceArena-Monsoon_en_IN_test_monsoon_en_in_test.jsonl",
|
| 124 |
+
"revision": "bc1da7b42ef6e2853123c97bf6d22067e4802d11",
|
| 125 |
+
"rtfx": 196.013,
|
| 126 |
+
"scoring_rows": 2102,
|
| 127 |
+
"scoring_unit": "utterance",
|
| 128 |
+
"split": "test",
|
| 129 |
+
"sub": 1270,
|
| 130 |
+
"wer": 4.44
|
| 131 |
+
},
|
| 132 |
+
{
|
| 133 |
+
"audio_length": 6928.642312499995,
|
| 134 |
+
"cuts": 1000,
|
| 135 |
+
"dataset": "urgent2024",
|
| 136 |
+
"default_average": true,
|
| 137 |
+
"del": 814,
|
| 138 |
+
"inference_time": 35.319214283000115,
|
| 139 |
+
"ins": 296,
|
| 140 |
+
"manifest_sha256": "2015c656012b35660a3469702cd54af9a55a853e35ebe017bba6d39e24651983",
|
| 141 |
+
"result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_test.jsonl",
|
| 142 |
+
"revision": "de79e57adf9983993bcc72586d7313460825f5eb",
|
| 143 |
+
"rtfx": 196.172,
|
| 144 |
+
"scoring_rows": 1000,
|
| 145 |
+
"scoring_unit": "utterance",
|
| 146 |
+
"split": "test",
|
| 147 |
+
"sub": 1170,
|
| 148 |
+
"wer": 12.29
|
| 149 |
+
},
|
| 150 |
+
{
|
| 151 |
+
"audio_length": 6928.642312499995,
|
| 152 |
+
"cuts": 1000,
|
| 153 |
+
"dataset": "urgent2024_clean",
|
| 154 |
+
"default_average": false,
|
| 155 |
+
"del": 260,
|
| 156 |
+
"inference_time": 35.618651621000595,
|
| 157 |
+
"ins": 268,
|
| 158 |
+
"manifest_sha256": "c260695c3d61864b7e220fed2993680c6e80f107519d7df0a3c6cb849d9bb535",
|
| 159 |
+
"result_file": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_clean_test.jsonl",
|
| 160 |
+
"revision": "de79e57adf9983993bcc72586d7313460825f5eb",
|
| 161 |
+
"rtfx": 194.5229,
|
| 162 |
+
"scoring_rows": 1000,
|
| 163 |
+
"scoring_unit": "utterance",
|
| 164 |
+
"split": "test",
|
| 165 |
+
"sub": 499,
|
| 166 |
+
"wer": 5.54
|
| 167 |
+
}
|
| 168 |
+
],
|
| 169 |
+
"export_note": "Local inference and final scoring statistics are unchanged. Result filenames now use the published Hub id, and audio_filepath is a stable public identifier. Original manifest SHA256 values refer to the original scored outputs; released output SHA256 values are in export_manifest.json."
|
| 170 |
+
}
|
benchmark_results/local_3090ti_20261119/export_manifest.json
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_id": "SpeedAudio261119/SpeedASR-v1-65M",
|
| 3 |
+
"transformation": "Changed filenames to the released model id; audio_filepath replaced with stable public source_id/cut_id. Raw references, predictions, durations, timing and chunk metadata remain unchanged. Original verification hashes refer to the original files; released hashes are recorded here.",
|
| 4 |
+
"files": [
|
| 5 |
+
{
|
| 6 |
+
"original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_ArtificialAnalysis-Earnings22-Cleaned-AA-chunked_earnings22_cleaned_aa_chunked_test.jsonl",
|
| 7 |
+
"original_sha256": "d6d297c76ce3fa9208791d989bd15cbaaaff6fed545ad17ab96b7d4492beef70",
|
| 8 |
+
"released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_ArtificialAnalysis-Earnings22-Cleaned-AA-chunked_earnings22_cleaned_aa_chunked_test.jsonl",
|
| 9 |
+
"released_sha256": "5010bb0dcc3024b9d3dda5956734044a0d35ccaf9bb3214271efb9f03cc6ea92",
|
| 10 |
+
"rows": 341
|
| 11 |
+
},
|
| 12 |
+
{
|
| 13 |
+
"original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_VoiceArena-Monsoon_en_IN_test_monsoon_en_in_test.jsonl",
|
| 14 |
+
"original_sha256": "571d17a51e0df966785330f3e7e232a39bbe00f3390e47ec20bfd3fb7a74b9b1",
|
| 15 |
+
"released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_VoiceArena-Monsoon_en_IN_test_monsoon_en_in_test.jsonl",
|
| 16 |
+
"released_sha256": "fd947f47d55206b7e720a96e8ce2d704b5213c3e6709c83a35a1d8138df0611c",
|
| 17 |
+
"rows": 2102
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_hf-audio-open-asr-leaderboard_ami_cleaned_test.jsonl",
|
| 21 |
+
"original_sha256": "89462aa1a42190471e8616dd27e7d2403c126d67ed05e6a57e31a86bdee991d6",
|
| 22 |
+
"released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_ami_cleaned_test.jsonl",
|
| 23 |
+
"released_sha256": "73b4f9d3c03e519f70a5897ae20b5195a9733a1f9baa8b9ff0618c590d86fb14",
|
| 24 |
+
"rows": 7805
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_hf-audio-open-asr-leaderboard_gigaspeech_cleaned_test.jsonl",
|
| 28 |
+
"original_sha256": "cbebef3e4f0d28b348ee9aa741090f8a9a48c7925eb08872a519b6e487b910c5",
|
| 29 |
+
"released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_gigaspeech_cleaned_test.jsonl",
|
| 30 |
+
"released_sha256": "e25623081b67f8c37db13966077e8f44eff25e47de61f96db2c5f2938ae1aaa0",
|
| 31 |
+
"rows": 18768
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_hf-audio-open-asr-leaderboard_spgispeech_test.jsonl",
|
| 35 |
+
"original_sha256": "f4a401536ffbefa5db8e4c9e7336301357c8d71a3d9eac16e026556456e0f44c",
|
| 36 |
+
"released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_spgispeech_test.jsonl",
|
| 37 |
+
"released_sha256": "ce7512cd10eff18d797cdc9efb167ada34b37c4beda1b5ba4bcde206ced0330b",
|
| 38 |
+
"rows": 39341
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_hf-audio-open-asr-leaderboard_urgent2024_clean_test.jsonl",
|
| 42 |
+
"original_sha256": "c260695c3d61864b7e220fed2993680c6e80f107519d7df0a3c6cb849d9bb535",
|
| 43 |
+
"released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_clean_test.jsonl",
|
| 44 |
+
"released_sha256": "bf1f804714c7ae77d742a24cb499a3ddb50934e03ca8ce8b91c0c92928a1321a",
|
| 45 |
+
"rows": 1000
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_hf-audio-open-asr-leaderboard_urgent2024_test.jsonl",
|
| 49 |
+
"original_sha256": "2015c656012b35660a3469702cd54af9a55a853e35ebe017bba6d39e24651983",
|
| 50 |
+
"released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_urgent2024_test.jsonl",
|
| 51 |
+
"released_sha256": "30edc6b2c7dcf13c664682f5ca06fb7b2dace5bb92cde4e8710688729206a887",
|
| 52 |
+
"rows": 1000
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"original_filename": "MODEL_jianyou-zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9_DATASET_hf-audio-open-asr-leaderboard_voxpopuli_cleaned_aa_test.jsonl",
|
| 56 |
+
"original_sha256": "61e855acf35048601fdcaed1c0ca10ba30ac7fb3a887d722c31a456c545605e1",
|
| 57 |
+
"released_filename": "MODEL_SpeedAudio261119-SpeedASR-v1-65M_DATASET_hf-audio-open-asr-leaderboard_voxpopuli_cleaned_aa_test.jsonl",
|
| 58 |
+
"released_sha256": "1bb76945bcb6f8b6fb89420e10615b1ab310160391894062d53f443b3ff79d51",
|
| 59 |
+
"rows": 628
|
| 60 |
+
}
|
| 61 |
+
]
|
| 62 |
+
}
|
benchmark_results/local_3090ti_20261119/independent_validation.json
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"state": "passed",
|
| 3 |
+
"complete": true,
|
| 4 |
+
"checked_at": "2026-11-19T03:54:19.864649+00:00",
|
| 5 |
+
"results_dir": "$RESULTS_DIR",
|
| 6 |
+
"provenance": {
|
| 7 |
+
"checkpoint_sha256": "b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01",
|
| 8 |
+
"tokens_sha256": "a238b7a0fa67aa6be998ca13dd129d69c50639970583e32232dd149e5448210a",
|
| 9 |
+
"runner_sha256": {
|
| 10 |
+
"benchmark_offline_pytorch.py": "e0d9b56d8334f56011ec927ea55411fe26d96317159458e9cff1bcd34601deb6",
|
| 11 |
+
"run_open_asr.py": "a8c336887d97b241cd58ea03f1ef5dc481bbb877ed4425c5dd0898d56f7b49a4",
|
| 12 |
+
"score_open_asr.py": "b3959a5a8e66e86b3f9923996009a7c3f66d62105499518206cf37bae8017d73",
|
| 13 |
+
"supervise_benchmark.py": "426862a8ad2d33d38991f9d56ee0556ad114fdc01bac23e9eaf89dca189004b5"
|
| 14 |
+
},
|
| 15 |
+
"manifest_index_sha256": "b09ba6209d2b96a092f5cc605e8b9045abc1b1d4e3ff566bd57f002b336fbedb",
|
| 16 |
+
"run_config_sha256": "4e3b9b0a0533c8d27de4118fb4a0e104729f386679f6792bb622d89020a58b97",
|
| 17 |
+
"input_provenance_sha256": "170cc1f5caab32df0f7de151fdf46673d3fa6fed7b7b8145a513c1cb7206f1b1",
|
| 18 |
+
"official_commit": "4a113dd90c9ffd011d7d3d105da76980d511d029",
|
| 19 |
+
"model_code_git_commit": "3f848bb6d0acc970c9b294a30ca0a04a7c9c78d1"
|
| 20 |
+
},
|
| 21 |
+
"test_sets": [
|
| 22 |
+
{
|
| 23 |
+
"dataset": "ami_cleaned",
|
| 24 |
+
"rows": 7805,
|
| 25 |
+
"scoring_rows": 7805,
|
| 26 |
+
"default_average": true,
|
| 27 |
+
"result_sha256": "89462aa1a42190471e8616dd27e7d2403c126d67ed05e6a57e31a86bdee991d6",
|
| 28 |
+
"cuts_manifest_sha256": "5153cc565bc9115d0ab8c48a18a14effb5a372ad781740a105ebefc68e98b590",
|
| 29 |
+
"wer": 9.59,
|
| 30 |
+
"ins": 2053,
|
| 31 |
+
"del": 2334,
|
| 32 |
+
"sub": 3514,
|
| 33 |
+
"audio_length": 28856.700437500032,
|
| 34 |
+
"inference_time": 142.35132213500114,
|
| 35 |
+
"rtfx": 202.7147
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"dataset": "gigaspeech_cleaned",
|
| 39 |
+
"rows": 18768,
|
| 40 |
+
"scoring_rows": 18768,
|
| 41 |
+
"default_average": true,
|
| 42 |
+
"result_sha256": "cbebef3e4f0d28b348ee9aa741090f8a9a48c7925eb08872a519b6e487b910c5",
|
| 43 |
+
"cuts_manifest_sha256": "79a86209fcbdbd0e2263a0b28fd58c801a29583fe0073102cec302de8fe025e0",
|
| 44 |
+
"wer": 8.07,
|
| 45 |
+
"ins": 8548,
|
| 46 |
+
"del": 5098,
|
| 47 |
+
"sub": 18396,
|
| 48 |
+
"audio_length": 126528.78700000017,
|
| 49 |
+
"inference_time": 652.5400200150451,
|
| 50 |
+
"rtfx": 193.902
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"dataset": "voxpopuli_cleaned_aa",
|
| 54 |
+
"rows": 628,
|
| 55 |
+
"scoring_rows": 628,
|
| 56 |
+
"default_average": true,
|
| 57 |
+
"result_sha256": "61e855acf35048601fdcaed1c0ca10ba30ac7fb3a887d722c31a456c545605e1",
|
| 58 |
+
"cuts_manifest_sha256": "2e59bbe6fb67984168b3708def118f7b0ca2ece581f5f52f79a291f047e8f92c",
|
| 59 |
+
"wer": 3.27,
|
| 60 |
+
"ins": 130,
|
| 61 |
+
"del": 226,
|
| 62 |
+
"sub": 225,
|
| 63 |
+
"audio_length": 7122.378374999999,
|
| 64 |
+
"inference_time": 37.72588397499897,
|
| 65 |
+
"rtfx": 188.7929
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"dataset": "earnings22_cleaned_aa_chunked",
|
| 69 |
+
"rows": 341,
|
| 70 |
+
"scoring_rows": 6,
|
| 71 |
+
"default_average": true,
|
| 72 |
+
"result_sha256": "d6d297c76ce3fa9208791d989bd15cbaaaff6fed545ad17ab96b7d4492beef70",
|
| 73 |
+
"cuts_manifest_sha256": "05828c77118824dbe506395891e930ac5222b6cadd67f19323eb03f67235e8c5",
|
| 74 |
+
"wer": 6.78,
|
| 75 |
+
"ins": 154,
|
| 76 |
+
"del": 222,
|
| 77 |
+
"sub": 668,
|
| 78 |
+
"audio_length": 6900.9376250000005,
|
| 79 |
+
"inference_time": 36.58612209400007,
|
| 80 |
+
"rtfx": 188.6217,
|
| 81 |
+
"parent_chunk_counts": {
|
| 82 |
+
"4468919_trimmed": 51,
|
| 83 |
+
"4471586": 62,
|
| 84 |
+
"4474955": 44,
|
| 85 |
+
"4475604": 61,
|
| 86 |
+
"4482968": 66,
|
| 87 |
+
"4483046": 57
|
| 88 |
+
},
|
| 89 |
+
"earnings_order_independence": "passed"
|
| 90 |
+
},
|
| 91 |
+
{
|
| 92 |
+
"dataset": "spgispeech",
|
| 93 |
+
"rows": 39341,
|
| 94 |
+
"scoring_rows": 39341,
|
| 95 |
+
"default_average": true,
|
| 96 |
+
"result_sha256": "f4a401536ffbefa5db8e4c9e7336301357c8d71a3d9eac16e026556456e0f44c",
|
| 97 |
+
"cuts_manifest_sha256": "878e6e0c4a5b6bad505d50d5f66ce22dd9e24aec327d6d06a3ec161e66f47112",
|
| 98 |
+
"wer": 3.89,
|
| 99 |
+
"ins": 19136,
|
| 100 |
+
"del": 4664,
|
| 101 |
+
"sub": 13919,
|
| 102 |
+
"audio_length": 360007.49950000306,
|
| 103 |
+
"inference_time": 1881.6296108457523,
|
| 104 |
+
"rtfx": 191.3275
|
| 105 |
+
},
|
| 106 |
+
{
|
| 107 |
+
"dataset": "monsoon_en_in",
|
| 108 |
+
"rows": 2102,
|
| 109 |
+
"scoring_rows": 2102,
|
| 110 |
+
"default_average": true,
|
| 111 |
+
"result_sha256": "571d17a51e0df966785330f3e7e232a39bbe00f3390e47ec20bfd3fb7a74b9b1",
|
| 112 |
+
"cuts_manifest_sha256": "7fac09464f571bf0777fb7d19015704946033fa64bd7ce0fd5a20658dc5fe719",
|
| 113 |
+
"wer": 4.44,
|
| 114 |
+
"ins": 550,
|
| 115 |
+
"del": 350,
|
| 116 |
+
"sub": 1270,
|
| 117 |
+
"audio_length": 20247.60743749999,
|
| 118 |
+
"inference_time": 103.29725760700066,
|
| 119 |
+
"rtfx": 196.013
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"dataset": "urgent2024",
|
| 123 |
+
"rows": 1000,
|
| 124 |
+
"scoring_rows": 1000,
|
| 125 |
+
"default_average": true,
|
| 126 |
+
"result_sha256": "2015c656012b35660a3469702cd54af9a55a853e35ebe017bba6d39e24651983",
|
| 127 |
+
"cuts_manifest_sha256": "bbf812820676c8beaf2b1f7bdfca9a0a1a33b8adadc50a43076e8970d7af6dba",
|
| 128 |
+
"wer": 12.29,
|
| 129 |
+
"ins": 296,
|
| 130 |
+
"del": 814,
|
| 131 |
+
"sub": 1170,
|
| 132 |
+
"audio_length": 6928.642312499995,
|
| 133 |
+
"inference_time": 35.319214283000115,
|
| 134 |
+
"rtfx": 196.172
|
| 135 |
+
},
|
| 136 |
+
{
|
| 137 |
+
"dataset": "urgent2024_clean",
|
| 138 |
+
"rows": 1000,
|
| 139 |
+
"scoring_rows": 1000,
|
| 140 |
+
"default_average": false,
|
| 141 |
+
"result_sha256": "c260695c3d61864b7e220fed2993680c6e80f107519d7df0a3c6cb849d9bb535",
|
| 142 |
+
"cuts_manifest_sha256": "39f91cd92c672659ba4d6f399ff6007e9279811dc4931a7162b82460e51b9f99",
|
| 143 |
+
"wer": 5.54,
|
| 144 |
+
"ins": 268,
|
| 145 |
+
"del": 260,
|
| 146 |
+
"sub": 499,
|
| 147 |
+
"audio_length": 6928.642312499995,
|
| 148 |
+
"inference_time": 35.618651621000595,
|
| 149 |
+
"rtfx": 194.5229
|
| 150 |
+
}
|
| 151 |
+
],
|
| 152 |
+
"source_alignment_report_sha256": "5c881b63e09401193d48c972668bdde4650c5c8ed000bc4aedfe9acb761e0877",
|
| 153 |
+
"scope": "Independent exported-row/provenance validation and official CPU rescoring; no inference or raw dataset download rehash.",
|
| 154 |
+
"total_rows": 70985,
|
| 155 |
+
"test_set_count": 8,
|
| 156 |
+
"public_default_test_sets": 7,
|
| 157 |
+
"public_default_average_wer": 6.904285714285714,
|
| 158 |
+
"public_default_pooled_rtfx": 192.6292762644339,
|
| 159 |
+
"official_default_ranking_average": null,
|
| 160 |
+
"current_scores_sha256": "b510e95ac08af92186551bbd402e03e6a6dd3e0f1bf3e6812c859c688682364a",
|
| 161 |
+
"official_scorer_sha256": "ddb5bc93e77b3bc2e5397bed09f32bf3aa9953546c822d25f30b957d6a06e9fd"
|
| 162 |
+
}
|
benchmark_results/local_3090ti_20261119/input_provenance.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"checkpoint": "$MODEL_DIR/pretrained.pt",
|
| 3 |
+
"checkpoint_sha256": "b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01",
|
| 4 |
+
"tokens": "$ICEFALL_ROOT/egs/librispeech/ASR/data/lang_bpe_500/tokens.txt",
|
| 5 |
+
"tokens_sha256": "a238b7a0fa67aa6be998ca13dd129d69c50639970583e32232dd149e5448210a",
|
| 6 |
+
"official_commit": "4a113dd90c9ffd011d7d3d105da76980d511d029",
|
| 7 |
+
"manifest_dir": "$BENCHMARK_ROOT/data/english_shortform_20261118",
|
| 8 |
+
"runner_script": "$BENCHMARK_ROOT/scripts/benchmark_epoch30_20261119/run_open_asr.py",
|
| 9 |
+
"runner_sha256": {
|
| 10 |
+
"benchmark_offline_pytorch.py": "e0d9b56d8334f56011ec927ea55411fe26d96317159458e9cff1bcd34601deb6",
|
| 11 |
+
"run_open_asr.py": "a8c336887d97b241cd58ea03f1ef5dc481bbb877ed4425c5dd0898d56f7b49a4",
|
| 12 |
+
"score_open_asr.py": "b3959a5a8e66e86b3f9923996009a7c3f66d62105499518206cf37bae8017d73",
|
| 13 |
+
"supervise_benchmark.py": "426862a8ad2d33d38991f9d56ee0556ad114fdc01bac23e9eaf89dca189004b5"
|
| 14 |
+
},
|
| 15 |
+
"model_code_git_commit": "3f848bb6d0acc970c9b294a30ca0a04a7c9c78d1"
|
| 16 |
+
}
|
benchmark_results/local_3090ti_20261119/run_config.json
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"audio_workers": 8,
|
| 3 |
+
"batch_audio_seconds": 5000.0,
|
| 4 |
+
"beam_size": 4,
|
| 5 |
+
"causal": false,
|
| 6 |
+
"checkpoint": "$MODEL_DIR/pretrained.pt",
|
| 7 |
+
"checkpoint_sha256": "b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01",
|
| 8 |
+
"command": [
|
| 9 |
+
"$BENCHMARK_ROOT/.venv/bin/python",
|
| 10 |
+
"-u",
|
| 11 |
+
"-B",
|
| 12 |
+
"$BENCHMARK_ROOT/scripts/benchmark_epoch30_20261119/benchmark_offline_pytorch.py",
|
| 13 |
+
"--work-dir",
|
| 14 |
+
"$BENCHMARK_ROOT",
|
| 15 |
+
"--open-asr-repo",
|
| 16 |
+
"$BENCHMARK_ROOT/open_asr_leaderboard_20261119",
|
| 17 |
+
"--manifest-dir",
|
| 18 |
+
"$BENCHMARK_ROOT/data/english_shortform_20261118",
|
| 19 |
+
"--results-dir",
|
| 20 |
+
"$RESULTS_DIR",
|
| 21 |
+
"--model-id",
|
| 22 |
+
"jianyou/zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9",
|
| 23 |
+
"--model-dir",
|
| 24 |
+
"$MODEL_DIR",
|
| 25 |
+
"--checkpoint",
|
| 26 |
+
"$MODEL_DIR/pretrained.pt",
|
| 27 |
+
"--tokens",
|
| 28 |
+
"$ICEFALL_ROOT/egs/librispeech/ASR/data/lang_bpe_500/tokens.txt",
|
| 29 |
+
"--icefall-root",
|
| 30 |
+
"$ICEFALL_ROOT",
|
| 31 |
+
"--zipformer-dir",
|
| 32 |
+
"$ICEFALL_ROOT/egs/librispeech/ASR/zipformer",
|
| 33 |
+
"--context-size",
|
| 34 |
+
"3",
|
| 35 |
+
"--decoding-method",
|
| 36 |
+
"modified_beam_search",
|
| 37 |
+
"--num-active-paths",
|
| 38 |
+
"4",
|
| 39 |
+
"--num-decode-streams",
|
| 40 |
+
"2048",
|
| 41 |
+
"--batch-audio-seconds",
|
| 42 |
+
"5000.0",
|
| 43 |
+
"--long-chunk-batch-size",
|
| 44 |
+
"512",
|
| 45 |
+
"--torch-threads",
|
| 46 |
+
"16",
|
| 47 |
+
"--audio-workers",
|
| 48 |
+
"8",
|
| 49 |
+
"--device-id",
|
| 50 |
+
"0",
|
| 51 |
+
"--voicea"
|
| 52 |
+
],
|
| 53 |
+
"context_size": 3,
|
| 54 |
+
"decoding_method": "modified_beam_search",
|
| 55 |
+
"dry_run": false,
|
| 56 |
+
"force": false,
|
| 57 |
+
"gpu": 0,
|
| 58 |
+
"icefall_root": "$ICEFALL_ROOT",
|
| 59 |
+
"long_chunk_batch_size": 512,
|
| 60 |
+
"manifest_dir": "$BENCHMARK_ROOT/data/english_shortform_20261118",
|
| 61 |
+
"manifest_index_sha256": "b09ba6209d2b96a092f5cc605e8b9045abc1b1d4e3ff566bd57f002b336fbedb",
|
| 62 |
+
"model_id": "jianyou/zipformer-6wHour-offline-no-delay-penalty-epoch30-avg9",
|
| 63 |
+
"num_decode_streams": 2048,
|
| 64 |
+
"official_commit": "4a113dd90c9ffd011d7d3d105da76980d511d029",
|
| 65 |
+
"open_asr_repo": "$BENCHMARK_ROOT/open_asr_leaderboard_20261119",
|
| 66 |
+
"precision": "fp32",
|
| 67 |
+
"preflight_only": false,
|
| 68 |
+
"results_dir": "$RESULTS_DIR",
|
| 69 |
+
"score_workers": 4,
|
| 70 |
+
"tokens": "$ICEFALL_ROOT/egs/librispeech/ASR/data/lang_bpe_500/tokens.txt",
|
| 71 |
+
"torch_threads": 16
|
| 72 |
+
}
|
benchmark_results/portable_validation.json
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"parameters": 65551059,
|
| 3 |
+
"device": "cuda:0",
|
| 4 |
+
"torch": "2.6.0+cu124",
|
| 5 |
+
"compile": "passed",
|
| 6 |
+
"strict_checkpoint": "passed",
|
| 7 |
+
"cases": [
|
| 8 |
+
{
|
| 9 |
+
"dataset": "ArtificialAnalysis__Earnings22-Cleaned-AA-chunked__earnings22_cleaned_aa_chunked__test.cuts.jsonl.gz",
|
| 10 |
+
"id": "00000124_2b6f28ab-45a2-59fc-b4ba-ff23567dd270_1c6057d858",
|
| 11 |
+
"seconds": 2.322,
|
| 12 |
+
"matches_original": true,
|
| 13 |
+
"original_prediction": "Is slower than expected .",
|
| 14 |
+
"portable_prediction": "Is slower than expected ."
|
| 15 |
+
},
|
| 16 |
+
{
|
| 17 |
+
"dataset": "ArtificialAnalysis__Earnings22-Cleaned-AA-chunked__earnings22_cleaned_aa_chunked__test.cuts.jsonl.gz",
|
| 18 |
+
"id": "00000075_9d3ce16b-47c2-5d75-8ed2-5a4989527b34_652f49e385",
|
| 19 |
+
"seconds": 24.416,
|
| 20 |
+
"matches_original": true,
|
| 21 |
+
"original_prediction": "So in November we received a score of fifty seven and S P Global CSA and sustainables ESG risk rating of twenty five point zero . And also MSCI confirmed our MSCI ESG rating at A . That's that's uh all uh as for the uh management comments , and we are ready to take your questions .",
|
| 22 |
+
"portable_prediction": "So in November we received a score of fifty seven and S P Global CSA and sustainables ESG risk rating of twenty five point zero . And also MSCI confirmed our MSCI ESG rating at A . That's that's uh all uh as for the uh management comments , and we are ready to take your questions ."
|
| 23 |
+
},
|
| 24 |
+
{
|
| 25 |
+
"dataset": "ArtificialAnalysis__Earnings22-Cleaned-AA-chunked__earnings22_cleaned_aa_chunked__test.cuts.jsonl.gz",
|
| 26 |
+
"id": "00000005_10793ea8-44a5-5699-aa20-4e74cd1976d6_ba691e780d",
|
| 27 |
+
"seconds": 29.984,
|
| 28 |
+
"matches_original": true,
|
| 29 |
+
"original_prediction": "We assume no obligation to update or revise any forward looking statements to reflect new or changed events or circumstances . This conference call will also include discussion on non gap financial measures . The full reconciliation to the corresponding financial measures is included in the ernest press release . Now I would like to turn the call over to Sargo . Thank you , Diego . Hello everyone and thank you for showing us today .",
|
| 30 |
+
"portable_prediction": "We assume no obligation to update or revise any forward looking statements to reflect new or changed events or circumstances . This conference call will also include discussion on non gap financial measures . The full reconciliation to the corresponding financial measures is included in the ernest press release . Now I would like to turn the call over to Sargo . Thank you , Diego . Hello everyone and thank you for showing us today ."
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"dataset": "VoiceArena__Monsoon_en_IN_test__monsoon_en_in__test.cuts.jsonl.gz",
|
| 34 |
+
"id": "00000384_80041517_12_72_13_95_86406836b9",
|
| 35 |
+
"seconds": 1.23,
|
| 36 |
+
"matches_original": true,
|
| 37 |
+
"original_prediction": "Can you hear me ?",
|
| 38 |
+
"portable_prediction": "Can you hear me ?"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"dataset": "VoiceArena__Monsoon_en_IN_test__monsoon_en_in__test.cuts.jsonl.gz",
|
| 42 |
+
"id": "00000881_25383873_120_3_130_71_51f9103de9",
|
| 43 |
+
"seconds": 10.41,
|
| 44 |
+
"matches_original": true,
|
| 45 |
+
"original_prediction": "Yeah . I have also felt that on that particular time everyone is just advising me to left this or to do some other things .",
|
| 46 |
+
"portable_prediction": "Yeah . I have also felt that on that particular time everyone is just advising me to left this or to do some other things ."
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"dataset": "VoiceArena__Monsoon_en_IN_test__monsoon_en_in__test.cuts.jsonl.gz",
|
| 50 |
+
"id": "00000848_92653570_353_07_368_07_602c1b0efd",
|
| 51 |
+
"seconds": 15.0,
|
| 52 |
+
"matches_original": true,
|
| 53 |
+
"original_prediction": "Fruit . My my favorite fruit is apple because I am farmer from Himachal Pradesh , I grow apples in my farms . Which is your favorite fruit ? Oh , mango , very nice choice .",
|
| 54 |
+
"portable_prediction": "Fruit . My my favorite fruit is apple because I am farmer from Himachal Pradesh , I grow apples in my farms . Which is your favorite fruit ? Oh , mango , very nice choice ."
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"dataset": "hf-audio__open-asr-leaderboard__ami_cleaned__test.cuts.jsonl.gz",
|
| 58 |
+
"id": "00007778_AMI_EN2002a_H03_MEE071_0047016_0047116_1cf8ac5dc8",
|
| 59 |
+
"seconds": 1.0,
|
| 60 |
+
"matches_original": true,
|
| 61 |
+
"original_prediction": "I didn't quite understand .",
|
| 62 |
+
"portable_prediction": "I didn't quite understand ."
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"dataset": "hf-audio__open-asr-leaderboard__ami_cleaned__test.cuts.jsonl.gz",
|
| 66 |
+
"id": "00003890_AMI_EN2002a_H01_FEO070_0212701_0212989_7cb11459f8",
|
| 67 |
+
"seconds": 2.88,
|
| 68 |
+
"matches_original": true,
|
| 69 |
+
"original_prediction": "Yeah .",
|
| 70 |
+
"portable_prediction": "Yeah ."
|
| 71 |
+
},
|
| 72 |
+
{
|
| 73 |
+
"dataset": "hf-audio__open-asr-leaderboard__ami_cleaned__test.cuts.jsonl.gz",
|
| 74 |
+
"id": "00000000_AMI_IS1009b_H03_FIO089_0023485_0026102_c32dfaec4b",
|
| 75 |
+
"seconds": 26.1699375,
|
| 76 |
+
"matches_original": true,
|
| 77 |
+
"original_prediction": "Um and then I'll be coming to you as the user interface person to try to tell me from your point of view what are the most friendly features that we could put on it and try to h prior help me with that prioritizing of uh of the features and of the the look and the color and I'll be coming back to you to help weed out those suggestions from that point of view .",
|
| 78 |
+
"portable_prediction": "Um and then I'll be coming to you as the user interface person to try to tell me from your point of view what are the most friendly features that we could put on it and try to h prior help me with that prioritizing of uh of the features and of the the look and the color and I'll be coming back to you to help weed out those suggestions from that point of view ."
|
| 79 |
+
},
|
| 80 |
+
{
|
| 81 |
+
"dataset": "hf-audio__open-asr-leaderboard__gigaspeech_cleaned__test.cuts.jsonl.gz",
|
| 82 |
+
"id": "00018750_POD1000000040_S0000787_13c0467f52",
|
| 83 |
+
"seconds": 1.0,
|
| 84 |
+
"matches_original": true,
|
| 85 |
+
"original_prediction": "Oh I can't see it .",
|
| 86 |
+
"portable_prediction": "Oh I can't see it ."
|
| 87 |
+
},
|
| 88 |
+
{
|
| 89 |
+
"dataset": "hf-audio__open-asr-leaderboard__gigaspeech_cleaned__test.cuts.jsonl.gz",
|
| 90 |
+
"id": "00009382_YOU1000000140_S0000081_47adf75a06",
|
| 91 |
+
"seconds": 6.509,
|
| 92 |
+
"matches_original": true,
|
| 93 |
+
"original_prediction": "it's only y'all two out here , Dad ain't nobody else out here with y'all . So you telling me he that fast ? They not jumping me ? Ain't nobody jumping me ?",
|
| 94 |
+
"portable_prediction": "it's only y'all two out here , Dad ain't nobody else out here with y'all . So you telling me he that fast ? They not jumping me ? Ain't nobody jumping me ?"
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"dataset": "hf-audio__open-asr-leaderboard__gigaspeech_cleaned__test.cuts.jsonl.gz",
|
| 98 |
+
"id": "00000000_POD1000000005_S0000339_7d69478636",
|
| 99 |
+
"seconds": 22.025,
|
| 100 |
+
"matches_original": true,
|
| 101 |
+
"original_prediction": "And , you know , those stores look at trend reports and they look at what's on the runway . And part of their great uh skill in both marketing and manufacturing is being able to take those ideas and turn them around at just a rapid fire pace and get them into stores .",
|
| 102 |
+
"portable_prediction": "And , you know , those stores look at trend reports and they look at what's on the runway . And part of their great uh skill in both marketing and manufacturing is being able to take those ideas and turn them around at just a rapid fire pace and get them into stores ."
|
| 103 |
+
},
|
| 104 |
+
{
|
| 105 |
+
"dataset": "hf-audio__open-asr-leaderboard__spgispeech__test.cuts.jsonl.gz",
|
| 106 |
+
"id": "00039340_403903eb119bfc1cec44d86ec907077a__123.wav_30a8eca6f1",
|
| 107 |
+
"seconds": 5.0099375,
|
| 108 |
+
"matches_original": true,
|
| 109 |
+
"original_prediction": "Has sort of put a little bit of um sand in the gears in the near short term .",
|
| 110 |
+
"portable_prediction": "Has sort of put a little bit of um sand in the gears in the near short term ."
|
| 111 |
+
},
|
| 112 |
+
{
|
| 113 |
+
"dataset": "hf-audio__open-asr-leaderboard__spgispeech__test.cuts.jsonl.gz",
|
| 114 |
+
"id": "00019757_9826162bc1bb67bf8d65e4cf50e656e9__85.wav_351da744aa",
|
| 115 |
+
"seconds": 8.85,
|
| 116 |
+
"matches_original": true,
|
| 117 |
+
"original_prediction": "The first thing that we wanted to drive during the second half was generating more proximity with our customer to drive collaboration .",
|
| 118 |
+
"portable_prediction": "The first thing that we wanted to drive during the second half was generating more proximity with our customer to drive collaboration ."
|
| 119 |
+
},
|
| 120 |
+
{
|
| 121 |
+
"dataset": "hf-audio__open-asr-leaderboard__spgispeech__test.cuts.jsonl.gz",
|
| 122 |
+
"id": "00000051_97b02b7a41ecb22cb04e7331052cefc8__39.wav_ad39d97ffb",
|
| 123 |
+
"seconds": 15.0,
|
| 124 |
+
"matches_original": true,
|
| 125 |
+
"original_prediction": "They'll ask us to look at that . So it's really it's a little bit of a two way street . a lot of our opportunities come from client driven needs that they're talking to our account management and sales team about all the time .",
|
| 126 |
+
"portable_prediction": "They'll ask us to look at that . So it's really it's a little bit of a two way street . a lot of our opportunities come from client driven needs that they're talking to our account management and sales team about all the time ."
|
| 127 |
+
},
|
| 128 |
+
{
|
| 129 |
+
"dataset": "hf-audio__open-asr-leaderboard__urgent2024__test.cuts.jsonl.gz",
|
| 130 |
+
"id": "00000999_fileid_636_94308ea578",
|
| 131 |
+
"seconds": 2.51,
|
| 132 |
+
"matches_original": true,
|
| 133 |
+
"original_prediction": "There were benches for twenty men along each side .",
|
| 134 |
+
"portable_prediction": "There were benches for twenty men along each side ."
|
| 135 |
+
},
|
| 136 |
+
{
|
| 137 |
+
"dataset": "hf-audio__open-asr-leaderboard__urgent2024__test.cuts.jsonl.gz",
|
| 138 |
+
"id": "00000499_fileid_59_7291623332",
|
| 139 |
+
"seconds": 6.9499375,
|
| 140 |
+
"matches_original": true,
|
| 141 |
+
"original_prediction": "Uh I think I should like it of all things replied Philip , some hesitation . But what more ?",
|
| 142 |
+
"portable_prediction": "Uh I think I should like it of all things replied Philip , some hesitation . But what more ?"
|
| 143 |
+
},
|
| 144 |
+
{
|
| 145 |
+
"dataset": "hf-audio__open-asr-leaderboard__urgent2024__test.cuts.jsonl.gz",
|
| 146 |
+
"id": "00000000_fileid_401_ff740dcae0",
|
| 147 |
+
"seconds": 14.73,
|
| 148 |
+
"matches_original": true,
|
| 149 |
+
"original_prediction": "We may even have used words to describe it to ourselves while it was happening . In that case , the habit of these words may fulfill the function of Bergson's true memory , while in reality it is nothing but habit moving .",
|
| 150 |
+
"portable_prediction": "We may even have used words to describe it to ourselves while it was happening . In that case , the habit of these words may fulfill the function of Bergson's true memory , while in reality it is nothing but habit moving ."
|
| 151 |
+
},
|
| 152 |
+
{
|
| 153 |
+
"dataset": "hf-audio__open-asr-leaderboard__urgent2024_clean__test.cuts.jsonl.gz",
|
| 154 |
+
"id": "00000999_fileid_636_f03b4eb095",
|
| 155 |
+
"seconds": 2.51,
|
| 156 |
+
"matches_original": true,
|
| 157 |
+
"original_prediction": "There were benches for twenty men along each side .",
|
| 158 |
+
"portable_prediction": "There were benches for twenty men along each side ."
|
| 159 |
+
},
|
| 160 |
+
{
|
| 161 |
+
"dataset": "hf-audio__open-asr-leaderboard__urgent2024_clean__test.cuts.jsonl.gz",
|
| 162 |
+
"id": "00000499_fileid_59_2c39dc0950",
|
| 163 |
+
"seconds": 6.9499375,
|
| 164 |
+
"matches_original": true,
|
| 165 |
+
"original_prediction": "I think I should like it , of all things , replied Philip , with some hesitation . But what for ?",
|
| 166 |
+
"portable_prediction": "I think I should like it , of all things , replied Philip , with some hesitation . But what for ?"
|
| 167 |
+
},
|
| 168 |
+
{
|
| 169 |
+
"dataset": "hf-audio__open-asr-leaderboard__urgent2024_clean__test.cuts.jsonl.gz",
|
| 170 |
+
"id": "00000000_fileid_401_27b267791c",
|
| 171 |
+
"seconds": 14.73,
|
| 172 |
+
"matches_original": true,
|
| 173 |
+
"original_prediction": "We may even have used words to describe it to ourselves while it was happening . In that case , the habit of these words may fulfill the function of Bergson's true memory , while in reality it is nothing but habit memory .",
|
| 174 |
+
"portable_prediction": "We may even have used words to describe it to ourselves while it was happening . In that case , the habit of these words may fulfill the function of Bergson's true memory , while in reality it is nothing but habit memory ."
|
| 175 |
+
},
|
| 176 |
+
{
|
| 177 |
+
"dataset": "hf-audio__open-asr-leaderboard__voxpopuli_cleaned_aa__test.cuts.jsonl.gz",
|
| 178 |
+
"id": "00000627_20090421-0900-PLENARY-17-en_20090421-22_22_06_2_f1fcb752d2",
|
| 179 |
+
"seconds": 5.0194375,
|
| 180 |
+
"matches_original": true,
|
| 181 |
+
"original_prediction": "Cently though , I have been able to assure people that the policy is undergoing significant change .",
|
| 182 |
+
"portable_prediction": "Cently though , I have been able to assure people that the policy is undergoing significant change ."
|
| 183 |
+
},
|
| 184 |
+
{
|
| 185 |
+
"dataset": "hf-audio__open-asr-leaderboard__voxpopuli_cleaned_aa__test.cuts.jsonl.gz",
|
| 186 |
+
"id": "00000314_20110405-0900-PLENARY-7-en_20110405-15_47_36_0_d327a19898",
|
| 187 |
+
"seconds": 9.94,
|
| 188 |
+
"matches_original": true,
|
| 189 |
+
"original_prediction": "Mr President , the March European Council confirmed the EU role as a promoter of the highest nuclear safety standards worldwide .",
|
| 190 |
+
"portable_prediction": "Mr President , the March European Council confirmed the EU role as a promoter of the highest nuclear safety standards worldwide ."
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"dataset": "hf-audio__open-asr-leaderboard__voxpopuli_cleaned_aa__test.cuts.jsonl.gz",
|
| 194 |
+
"id": "00000000_20181210-0900-PLENARY-en_20181210-22_04_34_4_5456d2323a",
|
| 195 |
+
"seconds": 38.435625,
|
| 196 |
+
"matches_original": true,
|
| 197 |
+
"original_prediction": "Cedefop's role is to contribute to the shaping and implementing of VET , but also shaping skills and qualification policies at Union level . The Agency can do this by providing crucial evidence and services for policy making and knowledge sharing amongst the Union and national actors , in particular to governments and social partners . I believe that the revision of the functioning of this tripartite agency has been a very good one and I fully appreciate the agency the agency's great work in supporting and developing inclusive and quality VAT systems . Now we have to work together that we put in place .",
|
| 198 |
+
"portable_prediction": "Cedefop's role is to contribute to the shaping and implementing of VET , but also shaping skills and qualification policies at Union level . The Agency can do this by providing crucial evidence and services for policy making and knowledge sharing amongst the Union and national actors , in particular to governments and social partners . I believe that the revision of the functioning of this tripartite agency has been a very good one and I fully appreciate the agency the agency's great work in supporting and developing inclusive and quality VAT systems . Now we have to work together that we put in place ."
|
| 199 |
+
}
|
| 200 |
+
],
|
| 201 |
+
"original_runner_sha256": "e0d9b56d8334f56011ec927ea55411fe26d96317159458e9cff1bcd34601deb6",
|
| 202 |
+
"inference_sha256": "17ea7c209dc40b0540b3ccca49cef04e265211b38b3daf00333e1180abdaa39e",
|
| 203 |
+
"core_sha256": {
|
| 204 |
+
"__init__.py": "596ac4de65fae9944440e22cf00ec4629b80da70c2e9b309e0702e4686ac988e",
|
| 205 |
+
"beam_search.py": "d2e0297d1626b14b834a9b357caa875a5b11ce8f0aabc475debb7d20dec3b8c1",
|
| 206 |
+
"decoder.py": "a9aeeeef278f4e5357d4e49253d24f5f1c60df88cfcb623ccbd01481dc1503ce",
|
| 207 |
+
"encoder_interface.py": "fc70f530c33e2489435ff2cb4bb0dad8b9d6f58a248c425cbd6100c62f628569",
|
| 208 |
+
"joiner.py": "81faad6e0d07342d1abab28f2271201256f2139f5816b04296d6a8cbe0e2c887",
|
| 209 |
+
"model.py": "46b8e181fb326e8cd82c92a4c0ffd0917eb1474a26bd5b60b39b48daac28e90f",
|
| 210 |
+
"scaling.py": "679b1fe2655902ba13ef3f5cf7c924129a16681e3edc4343e766ff7086f5e6d7",
|
| 211 |
+
"subsampling.py": "dc33ac662d2508e191f4da5562f0c8d23444819b707624736429a85517db9145",
|
| 212 |
+
"utils.py": "682d0733f0e149be0ae9b62cb8de60d7baef65be4e64816001b6bb089b4e787b",
|
| 213 |
+
"zipformer.py": "5cea6da601de9f37de2e1db6c2dadcd931220b45341472efaa426a25130a5a36"
|
| 214 |
+
},
|
| 215 |
+
"same_batch_decode": {
|
| 216 |
+
"examples": 16,
|
| 217 |
+
"exact_prediction_match": true
|
| 218 |
+
},
|
| 219 |
+
"public_batch_api": {
|
| 220 |
+
"examples": 16,
|
| 221 |
+
"output_count_ok": true,
|
| 222 |
+
"all_strings": true,
|
| 223 |
+
"empty_batch_ok": true
|
| 224 |
+
},
|
| 225 |
+
"invalid_audio_rejected": true,
|
| 226 |
+
"elapsed_seconds": 14.379630783998437,
|
| 227 |
+
"all_passed": true
|
| 228 |
+
}
|
inference.py
ADDED
|
@@ -0,0 +1,407 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Portable FP32 offline SpeedASR inference, including batched beam search.
|
| 3 |
+
|
| 4 |
+
model = SpeedASR.from_pretrained(revision="<Hub commit SHA>")
|
| 5 |
+
texts = model.transcribe_batch([mono_float32_audio], sample_rate=16000)
|
| 6 |
+
|
| 7 |
+
This frontend returns decoded text with VoiceA casing/punctuation restored.
|
| 8 |
+
WER normalization belongs to the benchmark scorer, not the recognizer.
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import argparse
|
| 13 |
+
import hashlib
|
| 14 |
+
import json
|
| 15 |
+
import logging
|
| 16 |
+
import math
|
| 17 |
+
import re
|
| 18 |
+
from itertools import pairwise
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
from typing import Iterable, Sequence
|
| 21 |
+
|
| 22 |
+
import kaldifeat
|
| 23 |
+
import numpy as np
|
| 24 |
+
import soundfile as sf
|
| 25 |
+
import torch
|
| 26 |
+
from torch.nn.utils.rnn import pad_sequence
|
| 27 |
+
|
| 28 |
+
from speedasr_core.beam_search import modified_beam_search
|
| 29 |
+
from speedasr_core.decoder import Decoder
|
| 30 |
+
from speedasr_core.joiner import Joiner
|
| 31 |
+
from speedasr_core.model import AsrModel
|
| 32 |
+
from speedasr_core.scaling import ScheduledFloat
|
| 33 |
+
from speedasr_core.subsampling import Conv2dSubsampling
|
| 34 |
+
from speedasr_core.zipformer import Zipformer2
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _sha256(path: Path) -> str:
|
| 38 |
+
digest = hashlib.sha256()
|
| 39 |
+
with path.open("rb") as stream:
|
| 40 |
+
for block in iter(lambda: stream.read(1 << 20), b""):
|
| 41 |
+
digest.update(block)
|
| 42 |
+
return digest.hexdigest()
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _read_tokens(path: Path) -> dict[int, str]:
|
| 46 |
+
table = {}
|
| 47 |
+
for line in path.read_text(encoding="utf-8").splitlines():
|
| 48 |
+
parts = line.split()
|
| 49 |
+
if len(parts) >= 2:
|
| 50 |
+
token_id = int(parts[-1])
|
| 51 |
+
if token_id in table:
|
| 52 |
+
raise ValueError(f"Duplicate token ID: {token_id}")
|
| 53 |
+
table[token_id] = " ".join(parts[:-1])
|
| 54 |
+
if not table or "<blk>" not in table.values() or "<unk>" not in table.values():
|
| 55 |
+
raise ValueError("tokens.txt must contain <blk> and <unk>")
|
| 56 |
+
return table
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def restore_voicea_text(text: str) -> str:
|
| 60 |
+
"""The exact VoiceA postprocessing used by the local benchmark."""
|
| 61 |
+
if not text:
|
| 62 |
+
return text
|
| 63 |
+
processed = (
|
| 64 |
+
text.replace("<,>", ",").replace("<.>", ".")
|
| 65 |
+
.replace("<?>", "?").replace("<!>", "!").replace("<unk>", "")
|
| 66 |
+
)
|
| 67 |
+
words = processed.split()
|
| 68 |
+
casing_set = {"<cap>", "<upp>", "<mix>"}
|
| 69 |
+
ans = []
|
| 70 |
+
for w1, w2 in pairwise(words):
|
| 71 |
+
if w1 in casing_set:
|
| 72 |
+
continue
|
| 73 |
+
if w2 in casing_set:
|
| 74 |
+
if w2 == "<cap>":
|
| 75 |
+
w1 = w1.capitalize()
|
| 76 |
+
elif w2 == "<upp>":
|
| 77 |
+
w1 = w1.upper()
|
| 78 |
+
ans.append(w1)
|
| 79 |
+
if words and words[-1] not in casing_set:
|
| 80 |
+
ans.append(words[-1])
|
| 81 |
+
processed = " ".join(ans)
|
| 82 |
+
processed = re.sub(r"<[^>]+>", "", processed)
|
| 83 |
+
return re.sub(r"\s+", " ", processed).strip()
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _ints(value) -> tuple[int, ...]:
|
| 87 |
+
if isinstance(value, str):
|
| 88 |
+
return tuple(map(int, value.split(",")))
|
| 89 |
+
if isinstance(value, int):
|
| 90 |
+
return (value,)
|
| 91 |
+
return tuple(map(int, value))
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _build_model(config: dict, vocab_size: int, blank_id: int) -> AsrModel:
|
| 95 |
+
"""Same construction as the frozen Icefall train.get_model, without training imports."""
|
| 96 |
+
if config.get("causal") is not False or int(config["context_size"]) != 3:
|
| 97 |
+
raise ValueError("This checkpoint requires offline causal=False and context_size=3")
|
| 98 |
+
if not config.get("use_transducer", True) or any(
|
| 99 |
+
config.get(name, False) for name in ("use_ctc", "use_attention_decoder", "use_cr_ctc")
|
| 100 |
+
):
|
| 101 |
+
raise ValueError("Expected the transducer-only checkpoint")
|
| 102 |
+
encoder_dims = _ints(config["encoder_dim"])
|
| 103 |
+
encoder_embed = Conv2dSubsampling(
|
| 104 |
+
in_channels=int(config.get("feature_dim", 80)),
|
| 105 |
+
out_channels=encoder_dims[0],
|
| 106 |
+
dropout=ScheduledFloat((0.0, 0.3), (20000.0, 0.1)),
|
| 107 |
+
)
|
| 108 |
+
encoder = Zipformer2(
|
| 109 |
+
output_downsampling_factor=2,
|
| 110 |
+
downsampling_factor=_ints(config["downsampling_factor"]),
|
| 111 |
+
num_encoder_layers=_ints(config["num_encoder_layers"]),
|
| 112 |
+
encoder_dim=encoder_dims,
|
| 113 |
+
encoder_unmasked_dim=_ints(config["encoder_unmasked_dim"]),
|
| 114 |
+
query_head_dim=_ints(config["query_head_dim"]),
|
| 115 |
+
pos_head_dim=_ints(config["pos_head_dim"]),
|
| 116 |
+
value_head_dim=_ints(config["value_head_dim"]),
|
| 117 |
+
pos_dim=int(config["pos_dim"]),
|
| 118 |
+
num_heads=_ints(config["num_heads"]),
|
| 119 |
+
feedforward_dim=_ints(config["feedforward_dim"]),
|
| 120 |
+
cnn_module_kernel=_ints(config["cnn_module_kernel"]),
|
| 121 |
+
dropout=ScheduledFloat((0.0, 0.3), (20000.0, 0.1)),
|
| 122 |
+
warmup_batches=4000.0,
|
| 123 |
+
causal=False,
|
| 124 |
+
chunk_size=_ints(config["chunk_size"]),
|
| 125 |
+
left_context_frames=_ints(config["left_context_frames"]),
|
| 126 |
+
)
|
| 127 |
+
decoder_dim = int(config["decoder_dim"])
|
| 128 |
+
decoder = Decoder(vocab_size, decoder_dim, blank_id, int(config["context_size"]))
|
| 129 |
+
joiner = Joiner(max(encoder_dims), decoder_dim, int(config["joiner_dim"]), vocab_size)
|
| 130 |
+
return AsrModel(
|
| 131 |
+
encoder_embed=encoder_embed, encoder=encoder, decoder=decoder, joiner=joiner,
|
| 132 |
+
attention_decoder=None, encoder_dim=max(encoder_dims), decoder_dim=decoder_dim,
|
| 133 |
+
vocab_size=vocab_size, use_transducer=True, use_ctc=False, use_attention_decoder=False,
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
class SpeedASR:
|
| 138 |
+
"""65.55M-parameter offline English recognizer.
|
| 139 |
+
|
| 140 |
+
Inputs are finite, mono, one-dimensional float waveform arrays at 16 kHz.
|
| 141 |
+
``transcribe_batch`` preserves input order and adapts GPU batches to memory.
|
| 142 |
+
No audio, transcripts or credentials are sent to a service during inference.
|
| 143 |
+
"""
|
| 144 |
+
|
| 145 |
+
def __init__(
|
| 146 |
+
self,
|
| 147 |
+
checkpoint: str | Path | None = None,
|
| 148 |
+
tokens: str | Path | None = None,
|
| 149 |
+
config: str | Path | None = None,
|
| 150 |
+
device: str | torch.device = "auto",
|
| 151 |
+
batch_size: int = 2048,
|
| 152 |
+
batch_audio_seconds: float = 5000.0,
|
| 153 |
+
torch_threads: int = 16,
|
| 154 |
+
verify_hashes: bool = True,
|
| 155 |
+
):
|
| 156 |
+
root = Path(__file__).resolve().parent
|
| 157 |
+
checkpoint_path = Path(checkpoint).expanduser().resolve() if checkpoint else root / "pretrained.pt"
|
| 158 |
+
if checkpoint_path.is_dir():
|
| 159 |
+
checkpoint_path = checkpoint_path / "pretrained.pt"
|
| 160 |
+
tokens_path = Path(tokens).expanduser().resolve() if tokens else checkpoint_path.parent / "tokens.txt"
|
| 161 |
+
config_path = Path(config).expanduser().resolve() if config else root / "model_config.json"
|
| 162 |
+
for path in (checkpoint_path, tokens_path, config_path):
|
| 163 |
+
if not path.is_file():
|
| 164 |
+
raise FileNotFoundError(path)
|
| 165 |
+
self.config = json.loads(config_path.read_text(encoding="utf-8"))
|
| 166 |
+
architecture = self.config.get("model_args", self.config)
|
| 167 |
+
inference = self.config.get("inference", {})
|
| 168 |
+
expected = self.config.get("expected", {})
|
| 169 |
+
if verify_hashes:
|
| 170 |
+
for path, key in ((checkpoint_path, "checkpoint_sha256"), (tokens_path, "tokens_sha256")):
|
| 171 |
+
if expected.get(key) and _sha256(path) != expected[key]:
|
| 172 |
+
raise ValueError(f"Artifact hash mismatch: {path.name}")
|
| 173 |
+
self.token_table = _read_tokens(tokens_path)
|
| 174 |
+
inverse_tokens = {text: token_id for token_id, text in self.token_table.items()}
|
| 175 |
+
# Equivalent to icefall.utils.num_tokens(table) + 1: exclude disambiguation symbols.
|
| 176 |
+
token_ids = [token_id for token_id, token in self.token_table.items()
|
| 177 |
+
if not re.match(r"^#\d+$", token)]
|
| 178 |
+
vocab_size = len(token_ids) - int(0 in token_ids) + 1
|
| 179 |
+
for key, value in (("vocab_size", vocab_size), ("blank_id", inverse_tokens["<blk>"]),
|
| 180 |
+
("unk_id", inverse_tokens["<unk>"])):
|
| 181 |
+
if key in architecture and int(architecture[key]) != value:
|
| 182 |
+
raise ValueError(f"Token table and model configuration disagree on {key}")
|
| 183 |
+
torch.set_num_threads(int(torch_threads))
|
| 184 |
+
if str(device) == "auto":
|
| 185 |
+
device = "cuda:0" if torch.cuda.is_available() else "cpu"
|
| 186 |
+
self.device = torch.device(device)
|
| 187 |
+
if self.device.type not in {"cpu", "cuda"}:
|
| 188 |
+
raise ValueError("Supported devices are cpu and cuda")
|
| 189 |
+
if self.device.type == "cuda":
|
| 190 |
+
if not torch.cuda.is_available():
|
| 191 |
+
raise RuntimeError("CUDA was requested but is unavailable")
|
| 192 |
+
torch.cuda.set_device(self.device)
|
| 193 |
+
self.model = _build_model(architecture, vocab_size, inverse_tokens["<blk>"])
|
| 194 |
+
checkpoint_data = torch.load(checkpoint_path, map_location="cpu", weights_only=True)
|
| 195 |
+
if not isinstance(checkpoint_data, dict) or not isinstance(checkpoint_data.get("model"), dict):
|
| 196 |
+
raise ValueError("Expected an Icefall checkpoint with a model state dictionary")
|
| 197 |
+
self.model.load_state_dict(checkpoint_data["model"], strict=True)
|
| 198 |
+
del checkpoint_data
|
| 199 |
+
self.num_parameters = sum(parameter.numel() for parameter in self.model.parameters())
|
| 200 |
+
if expected.get("parameters") and self.num_parameters != int(expected["parameters"]):
|
| 201 |
+
raise ValueError(f"Parameter count mismatch: {self.num_parameters}")
|
| 202 |
+
# Deliberately FP32. Quantized/FP16 accuracy was not measured by this benchmark.
|
| 203 |
+
self.model.float().to(self.device).eval()
|
| 204 |
+
self.max_gpu_allocated_bytes = 0
|
| 205 |
+
self.sample_rate = int(inference.get("sample_rate", 16000))
|
| 206 |
+
self.min_input_seconds = float(inference.get("min_input_seconds", 1.0))
|
| 207 |
+
self.long_chunk_seconds = float(inference.get("long_chunk_seconds", 30.0))
|
| 208 |
+
self.long_chunk_batch_size = int(inference.get("long_chunk_batch_size", 512))
|
| 209 |
+
self.beam_size = int(inference.get("beam_size", 4))
|
| 210 |
+
self.batch_size = int(batch_size)
|
| 211 |
+
self.batch_audio_seconds = float(batch_audio_seconds)
|
| 212 |
+
if self.batch_size < 1 or self.batch_audio_seconds <= 0:
|
| 213 |
+
raise ValueError("Batch limits must be positive")
|
| 214 |
+
opts = kaldifeat.FbankOptions()
|
| 215 |
+
opts.device = self.device
|
| 216 |
+
opts.frame_opts.dither = 0
|
| 217 |
+
opts.frame_opts.snip_edges = False
|
| 218 |
+
opts.frame_opts.samp_freq = self.sample_rate
|
| 219 |
+
opts.mel_opts.num_bins = int(architecture.get("feature_dim", 80))
|
| 220 |
+
opts.mel_opts.high_freq = -400
|
| 221 |
+
self.fbank = kaldifeat.Fbank(opts)
|
| 222 |
+
|
| 223 |
+
@classmethod
|
| 224 |
+
def from_pretrained(
|
| 225 |
+
cls,
|
| 226 |
+
repo_id: str = "SpeedAudio261119/SpeedASR-v1-65M",
|
| 227 |
+
revision: str = "main",
|
| 228 |
+
token: str | bool | None = None,
|
| 229 |
+
cache_dir: str | Path | None = None,
|
| 230 |
+
**kwargs,
|
| 231 |
+
) -> "SpeedASR":
|
| 232 |
+
"""Download only weights/tokens/config. Pin a commit SHA for reproducibility."""
|
| 233 |
+
from huggingface_hub import hf_hub_download
|
| 234 |
+
paths = {
|
| 235 |
+
name: hf_hub_download(repo_id=repo_id, filename=name, revision=revision,
|
| 236 |
+
token=token, cache_dir=cache_dir)
|
| 237 |
+
for name in ("pretrained.pt", "tokens.txt", "model_config.json")
|
| 238 |
+
}
|
| 239 |
+
return cls(checkpoint=paths["pretrained.pt"], tokens=paths["tokens.txt"],
|
| 240 |
+
config=paths["model_config.json"], **kwargs)
|
| 241 |
+
|
| 242 |
+
def _wave(self, audio) -> torch.Tensor:
|
| 243 |
+
if isinstance(audio, torch.Tensor):
|
| 244 |
+
if not audio.is_floating_point():
|
| 245 |
+
raise ValueError("Waveforms must contain floating-point samples, not PCM integers")
|
| 246 |
+
wave = audio.detach().to(device="cpu", dtype=torch.float32).contiguous()
|
| 247 |
+
else:
|
| 248 |
+
array = np.asarray(audio)
|
| 249 |
+
if not np.issubdtype(array.dtype, np.floating):
|
| 250 |
+
raise ValueError("Waveforms must contain floating-point samples, not PCM integers")
|
| 251 |
+
wave = torch.from_numpy(np.ascontiguousarray(array, dtype=np.float32))
|
| 252 |
+
if wave.ndim != 1 or wave.numel() == 0:
|
| 253 |
+
raise ValueError("Expected a nonempty one-dimensional mono waveform")
|
| 254 |
+
if not bool(torch.isfinite(wave).all()):
|
| 255 |
+
raise ValueError("Waveform contains non-finite samples")
|
| 256 |
+
return wave
|
| 257 |
+
|
| 258 |
+
def _pad(self, wave: torch.Tensor) -> torch.Tensor:
|
| 259 |
+
minimum = int(round(self.min_input_seconds * self.sample_rate))
|
| 260 |
+
if wave.numel() < minimum:
|
| 261 |
+
wave = torch.nn.functional.pad(wave, (0, minimum - wave.numel()))
|
| 262 |
+
return wave
|
| 263 |
+
|
| 264 |
+
@torch.no_grad()
|
| 265 |
+
def _decode(self, waves: list[torch.Tensor]) -> list[str]:
|
| 266 |
+
waves = [self._pad(wave).to(self.device) for wave in waves]
|
| 267 |
+
features = self.fbank(waves)
|
| 268 |
+
lengths = torch.tensor([f.size(0) for f in features], device=self.device, dtype=torch.int64)
|
| 269 |
+
features = pad_sequence(features, batch_first=True, padding_value=math.log(1e-10))
|
| 270 |
+
with torch.autocast(device_type=self.device.type, enabled=False):
|
| 271 |
+
encoded, encoded_lengths = self.model.forward_encoder(features, lengths)
|
| 272 |
+
hypotheses = modified_beam_search(
|
| 273 |
+
model=self.model, encoder_out=encoded, encoder_out_lens=encoded_lengths,
|
| 274 |
+
beam=self.beam_size,
|
| 275 |
+
)
|
| 276 |
+
if len(hypotheses) != len(waves):
|
| 277 |
+
raise RuntimeError("Decoder returned the wrong number of hypotheses")
|
| 278 |
+
result = []
|
| 279 |
+
for hypothesis in hypotheses:
|
| 280 |
+
text = "".join(self.token_table.get(int(token), "") for token in hypothesis)
|
| 281 |
+
result.append(restore_voicea_text(text.replace("\u2581", " ").strip()))
|
| 282 |
+
if self.device.type == "cuda":
|
| 283 |
+
self.max_gpu_allocated_bytes = max(
|
| 284 |
+
self.max_gpu_allocated_bytes, torch.cuda.max_memory_allocated(self.device)
|
| 285 |
+
)
|
| 286 |
+
return result
|
| 287 |
+
|
| 288 |
+
@staticmethod
|
| 289 |
+
def _is_oom(exc: RuntimeError) -> bool:
|
| 290 |
+
return isinstance(exc, torch.cuda.OutOfMemoryError) or "out of memory" in str(exc).lower()
|
| 291 |
+
|
| 292 |
+
def _decode_long(self, wave: torch.Tensor) -> str:
|
| 293 |
+
samples = max(1, int(round(self.long_chunk_seconds * self.sample_rate)))
|
| 294 |
+
chunks = [wave[start:start + samples] for start in range(0, wave.numel(), samples)]
|
| 295 |
+
batch_size = max(1, min(self.long_chunk_batch_size, len(chunks)))
|
| 296 |
+
output = []
|
| 297 |
+
offset = 0
|
| 298 |
+
while offset < len(chunks):
|
| 299 |
+
count = min(batch_size, len(chunks) - offset)
|
| 300 |
+
try:
|
| 301 |
+
output.extend(self._decode(chunks[offset:offset + count]))
|
| 302 |
+
except RuntimeError as exc:
|
| 303 |
+
if self.device.type == "cuda" and self._is_oom(exc) and count > 1:
|
| 304 |
+
torch.cuda.empty_cache()
|
| 305 |
+
batch_size = max(1, count // 2)
|
| 306 |
+
continue
|
| 307 |
+
raise
|
| 308 |
+
offset += count
|
| 309 |
+
if batch_size < self.long_chunk_batch_size and count == batch_size:
|
| 310 |
+
batch_size = min(self.long_chunk_batch_size, batch_size * 2)
|
| 311 |
+
if self.device.type == "cuda":
|
| 312 |
+
torch.cuda.empty_cache()
|
| 313 |
+
return re.sub(r"\s+", " ", " ".join(part for part in output if part)).strip()
|
| 314 |
+
|
| 315 |
+
@torch.no_grad()
|
| 316 |
+
def transcribe_batch(
|
| 317 |
+
self,
|
| 318 |
+
waveforms: Sequence,
|
| 319 |
+
sample_rate: int = 16000,
|
| 320 |
+
batch_size: int | None = None,
|
| 321 |
+
) -> list[str]:
|
| 322 |
+
"""Decode mono float waveforms and return one string per input, in input order."""
|
| 323 |
+
if int(sample_rate) != self.sample_rate:
|
| 324 |
+
raise ValueError(f"Expected {self.sample_rate} Hz; resample before calling inference")
|
| 325 |
+
waves = [self._wave(wave) for wave in waveforms]
|
| 326 |
+
if not waves:
|
| 327 |
+
return []
|
| 328 |
+
target_batch_size = self.batch_size if batch_size is None else int(batch_size)
|
| 329 |
+
if target_batch_size < 1:
|
| 330 |
+
raise ValueError("batch_size must be positive")
|
| 331 |
+
order = sorted(range(len(waves)), key=lambda i: waves[i].numel(), reverse=True)
|
| 332 |
+
outputs = [""] * len(waves)
|
| 333 |
+
offset = 0
|
| 334 |
+
safe_seconds = self.batch_audio_seconds
|
| 335 |
+
while offset < len(order):
|
| 336 |
+
first = waves[order[offset]]
|
| 337 |
+
seconds = first.numel() / self.sample_rate
|
| 338 |
+
if seconds > self.long_chunk_seconds:
|
| 339 |
+
outputs[order[offset]] = self._decode_long(first)
|
| 340 |
+
offset += 1
|
| 341 |
+
continue
|
| 342 |
+
count = min(target_batch_size, max(1, int(safe_seconds / max(1.0, seconds))))
|
| 343 |
+
count = min(count, len(order) - offset)
|
| 344 |
+
indices = order[offset:offset + count]
|
| 345 |
+
try:
|
| 346 |
+
if self.device.type == "cuda":
|
| 347 |
+
torch.cuda.reset_peak_memory_stats(self.device)
|
| 348 |
+
texts = self._decode([waves[i] for i in indices])
|
| 349 |
+
except RuntimeError as exc:
|
| 350 |
+
if self.device.type != "cuda" or not self._is_oom(exc):
|
| 351 |
+
raise
|
| 352 |
+
torch.cuda.empty_cache()
|
| 353 |
+
if count == 1:
|
| 354 |
+
outputs[indices[0]] = self._decode_long(first)
|
| 355 |
+
offset += 1
|
| 356 |
+
else:
|
| 357 |
+
safe_seconds = max(1, count // 2) * max(1.0, seconds)
|
| 358 |
+
logging.warning("CUDA memory limit; retrying with a smaller batch")
|
| 359 |
+
continue
|
| 360 |
+
for index, text in zip(indices, texts):
|
| 361 |
+
outputs[index] = text
|
| 362 |
+
if self.device.type == "cuda":
|
| 363 |
+
peak = torch.cuda.max_memory_allocated(self.device)
|
| 364 |
+
target_memory = torch.cuda.get_device_properties(self.device).total_memory * 0.80
|
| 365 |
+
if peak > 0 and peak < target_memory * 0.85:
|
| 366 |
+
safe_seconds *= min(1.15, target_memory / peak)
|
| 367 |
+
offset += count
|
| 368 |
+
if self.device.type == "cuda":
|
| 369 |
+
torch.cuda.synchronize(self.device)
|
| 370 |
+
return outputs
|
| 371 |
+
|
| 372 |
+
def transcribe(self, waveform, sample_rate: int = 16000) -> str:
|
| 373 |
+
"""Decode one mono float waveform at 16 kHz."""
|
| 374 |
+
return self.transcribe_batch([waveform], sample_rate=sample_rate)[0]
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
def main() -> None:
|
| 378 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 379 |
+
parser.add_argument("audio", nargs="+", type=Path, help="16 kHz WAV/FLAC files")
|
| 380 |
+
parser.add_argument("--checkpoint", type=Path)
|
| 381 |
+
parser.add_argument("--tokens", type=Path)
|
| 382 |
+
parser.add_argument("--config", type=Path)
|
| 383 |
+
parser.add_argument("--repo-id", help="Download model artifacts from this Hub repository")
|
| 384 |
+
parser.add_argument("--revision", default="main", help="Prefer an immutable Hub commit SHA")
|
| 385 |
+
parser.add_argument("--device", default="auto")
|
| 386 |
+
parser.add_argument("--batch-size", type=int, default=2048)
|
| 387 |
+
parser.add_argument("--batch-audio-seconds", type=float, default=5000.0)
|
| 388 |
+
parser.add_argument("--torch-threads", type=int, default=16)
|
| 389 |
+
args = parser.parse_args()
|
| 390 |
+
options = dict(device=args.device, batch_size=args.batch_size,
|
| 391 |
+
batch_audio_seconds=args.batch_audio_seconds, torch_threads=args.torch_threads)
|
| 392 |
+
if args.repo_id:
|
| 393 |
+
model = SpeedASR.from_pretrained(args.repo_id, revision=args.revision, **options)
|
| 394 |
+
else:
|
| 395 |
+
model = SpeedASR(args.checkpoint, args.tokens, args.config, **options)
|
| 396 |
+
waves = []
|
| 397 |
+
for path in args.audio:
|
| 398 |
+
audio, sample_rate = sf.read(path, dtype="float32", always_2d=True)
|
| 399 |
+
if sample_rate != model.sample_rate:
|
| 400 |
+
parser.error(f"{path}: expected {model.sample_rate} Hz, got {sample_rate}")
|
| 401 |
+
waves.append(np.ascontiguousarray(audio[:, 0]))
|
| 402 |
+
for path, text in zip(args.audio, model.transcribe_batch(waves)):
|
| 403 |
+
print(json.dumps({"audio_filepath": str(path), "pred_text": text}, ensure_ascii=False))
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
if __name__ == "__main__":
|
| 407 |
+
main()
|
leaderboard/Dockerfile
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Build from the model package root:
|
| 2 |
+
# docker build -f leaderboard/Dockerfile -t speedasr-open-asr .
|
| 3 |
+
FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
|
| 4 |
+
ARG OPEN_ASR_COMMIT=4a113dd90c9ffd011d7d3d105da76980d511d029
|
| 5 |
+
ENV DEBIAN_FRONTEND=noninteractive \
|
| 6 |
+
VIRTUAL_ENV=/opt/venv \
|
| 7 |
+
PATH=/opt/venv/bin:$PATH \
|
| 8 |
+
PYTHONPATH=/app:/opt/open_asr_leaderboard \
|
| 9 |
+
OPEN_ASR_REPO=/opt/open_asr_leaderboard \
|
| 10 |
+
HF_AUDIO_DECODER_BACKEND=soundfile \
|
| 11 |
+
HF_HOME=/tmp/huggingface \
|
| 12 |
+
PYTHONUNBUFFERED=1
|
| 13 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 14 |
+
python3.10 python3.10-venv libsndfile1 libgomp1 git ca-certificates \
|
| 15 |
+
&& rm -rf /var/lib/apt/lists/* \
|
| 16 |
+
&& python3.10 -m venv /opt/venv \
|
| 17 |
+
&& python -m pip install --no-cache-dir pip==25.0.1
|
| 18 |
+
WORKDIR /app
|
| 19 |
+
COPY requirements-linux-cu124.txt /app/requirements-linux-cu124.txt
|
| 20 |
+
COPY requirements-core.txt /app/requirements-core.txt
|
| 21 |
+
COPY leaderboard/requirements.txt /app/leaderboard/requirements.txt
|
| 22 |
+
RUN python -m pip install --no-cache-dir \
|
| 23 |
+
-r requirements-linux-cu124.txt -r leaderboard/requirements.txt \
|
| 24 |
+
&& python -m pip check
|
| 25 |
+
RUN git init /opt/open_asr_leaderboard \
|
| 26 |
+
&& git -C /opt/open_asr_leaderboard remote add origin https://github.com/huggingface/open_asr_leaderboard.git \
|
| 27 |
+
&& git -C /opt/open_asr_leaderboard fetch --depth 1 origin ${OPEN_ASR_COMMIT} \
|
| 28 |
+
&& git -C /opt/open_asr_leaderboard checkout --detach FETCH_HEAD
|
| 29 |
+
COPY . /app
|
| 30 |
+
EXPOSE 7860
|
| 31 |
+
# The Space stays lightweight; Jobs override this command with run_eval.py.
|
| 32 |
+
CMD ["python", "leaderboard/space_app.py"]
|
leaderboard/cloud_submission.txt
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
SpeedASR-v1-65M: reproducible Open ASR Leaderboard submission package
|
| 2 |
+
Prepared 2026-11-19
|
| 3 |
+
|
| 4 |
+
Model: https://huggingface.co/SpeedAudio261119/SpeedASR-v1-65M
|
| 5 |
+
Official evaluation/scorer revision:
|
| 6 |
+
4a113dd90c9ffd011d7d3d105da76980d511d029
|
| 7 |
+
|
| 8 |
+
STATUS AND SCOPE
|
| 9 |
+
|
| 10 |
+
Uploading this model package publishes the checkpoint, tokenizer, code and local
|
| 11 |
+
results. It does not automatically create an evaluation Space, run paid HF Jobs,
|
| 12 |
+
open a GitHub pull request, or add a row to the official leaderboard. The files
|
| 13 |
+
in this directory prepare those subsequent steps for an owner/maintainer.
|
| 14 |
+
|
| 15 |
+
The supplied local benchmark was performed on an RTX 3090 Ti in FP32. Its WER
|
| 16 |
+
and throughput must not be labeled as an H200 cloud run. Private-set evaluation
|
| 17 |
+
requires the leaderboard maintainers and is not included in the local results.
|
| 18 |
+
|
| 19 |
+
MODEL AND DECODE SETTINGS
|
| 20 |
+
|
| 21 |
+
Offline Zipformer transducer; 65,551,059 total parameters; 16 kHz mono audio;
|
| 22 |
+
context size 3; modified beam search; beam size 4; FP32; the same decoder and
|
| 23 |
+
text postprocessor for every dataset. No external language model or per-dataset
|
| 24 |
+
prompt/vocabulary. The model package's inference.py exposes:
|
| 25 |
+
|
| 26 |
+
from inference import SpeedASR
|
| 27 |
+
model = SpeedASR.from_pretrained(
|
| 28 |
+
"SpeedAudio261119/SpeedASR-v1-65M", revision="<MODEL_COMMIT_SHA>",
|
| 29 |
+
device="cuda:0", batch_size=2048, batch_audio_seconds=5000,
|
| 30 |
+
)
|
| 31 |
+
predictions = model.transcribe_batch(waveforms, sample_rate=16000)
|
| 32 |
+
|
| 33 |
+
For a local package, pass checkpoint=..., tokens=..., config=... to SpeedASR.
|
| 34 |
+
The adapter uses the same inference code and model settings. Batching can differ
|
| 35 |
+
from the earlier local runner, so throughput is independently measured.
|
| 36 |
+
|
| 37 |
+
1. BUILD A DOCKER EVALUATION IMAGE / HF SPACE
|
| 38 |
+
|
| 39 |
+
Use a separate directory or checkout for a Docker Space. Copy the model package
|
| 40 |
+
code, vendor sources, model_config.json, requirements files and leaderboard/
|
| 41 |
+
directory into it. Copy leaderboard/Dockerfile to the Space's root Dockerfile,
|
| 42 |
+
leaderboard/space_card.txt to its root README.md, and
|
| 43 |
+
leaderboard/dockerignore.txt to its root .dockerignore. Checkpoint/ONNX weights
|
| 44 |
+
and benchmark artifacts do not need to be baked into the image: the adapter
|
| 45 |
+
downloads the explicitly pinned model snapshot at job startup.
|
| 46 |
+
|
| 47 |
+
The Space README is distinct from the model README; do not replace the model
|
| 48 |
+
card in the model repository. Create a Docker Space in your own account, e.g.
|
| 49 |
+
your-user/open-asr-leaderboard-speedasr, and upload that build directory. A CPU
|
| 50 |
+
Space is sufficient to build the image; the Jobs command selects the H200.
|
| 51 |
+
Wait for the Docker build to succeed. No benchmark starts from the landing page.
|
| 52 |
+
|
| 53 |
+
The Dockerfile uses CUDA 12.4.1 / Ubuntu 22.04 / Python 3.10 and the package's
|
| 54 |
+
requirements-linux-cu124.txt with compatible Torch/k2/kaldifeat wheels. It pins
|
| 55 |
+
the official normalizer checkout to the revision above. Do not independently
|
| 56 |
+
upgrade Torch without installing matching k2 and kaldifeat wheels.
|
| 57 |
+
|
| 58 |
+
Optional local Docker build from this model package root:
|
| 59 |
+
|
| 60 |
+
docker build -f leaderboard/Dockerfile -t speedasr-open-asr .
|
| 61 |
+
|
| 62 |
+
2. SMOKE TEST THE ADAPTER
|
| 63 |
+
|
| 64 |
+
From a fresh working directory (results are written relative to that directory),
|
| 65 |
+
with the package dependencies installed and an official checkout at the pinned
|
| 66 |
+
revision:
|
| 67 |
+
|
| 68 |
+
python /path/to/SpeedASR-v1-65M/leaderboard/run_eval.py \
|
| 69 |
+
--model_id SpeedAudio261119/SpeedASR-v1-65M \
|
| 70 |
+
--revision <MODEL_COMMIT_SHA> \
|
| 71 |
+
--open-asr-repo /path/to/pinned/open_asr_leaderboard \
|
| 72 |
+
--dataset_path hf-audio/open-asr-leaderboard \
|
| 73 |
+
--dataset ami_cleaned --split test --device 0 \
|
| 74 |
+
--batch_size 16 --batch-audio-seconds 200 --max_eval_samples 16
|
| 75 |
+
|
| 76 |
+
Use --model-dir /path/to/downloaded/model/package instead of --revision to load
|
| 77 |
+
local weights. Do not truncate Earnings22: every chunk is required to reconstruct
|
| 78 |
+
its complete parent sessions. The adapter rejects chunked partial runs.
|
| 79 |
+
|
| 80 |
+
The adapter accepts only the stated official Git revision so a normalizer change
|
| 81 |
+
cannot silently alter scores. Later updates should deliberately update the pin,
|
| 82 |
+
Docker image and score provenance together.
|
| 83 |
+
|
| 84 |
+
3. LAUNCH THE SEVEN PUBLIC SETS ON HF JOBS (PAID)
|
| 85 |
+
|
| 86 |
+
HF Jobs requires credits and a write token, plus an existing writable results
|
| 87 |
+
bucket. Credentials belong in the process environment / HF Job secrets, not in
|
| 88 |
+
the model repository, model card, scripts or command logs.
|
| 89 |
+
|
| 90 |
+
Install a current huggingface_hub CLI (the working local environment used
|
| 91 |
+
huggingface_hub 1.18.0), then set the following environment variables:
|
| 92 |
+
|
| 93 |
+
export HF_TOKEN=<your_write_token>
|
| 94 |
+
export SPACE=your-user/open-asr-leaderboard-speedasr
|
| 95 |
+
export RESULTS_BUCKET=your-user/speedasr-results
|
| 96 |
+
export MODEL_REVISION=<FULL_MODEL_UPLOAD_COMMIT_SHA>
|
| 97 |
+
bash leaderboard/submit_jobs.sh
|
| 98 |
+
|
| 99 |
+
This script deliberately runs one H200 job per dataset, sequentially. It waits
|
| 100 |
+
for each job, fails on a job error and downloads this run's results. A unique
|
| 101 |
+
UTC RUN_ID keeps repeated result sets separate. ORG_NAME selects organization
|
| 102 |
+
billing if supplied. FLAVOR defaults to h200; TIMEOUT defaults to 8h per job.
|
| 103 |
+
|
| 104 |
+
Before a final official throughput run, determine the maximum practical H200
|
| 105 |
+
batch settings and record them. The shipped defaults reproduce the local
|
| 106 |
+
batch-size/duration limits; they are not claimed to saturate an H200. Override:
|
| 107 |
+
|
| 108 |
+
BATCH_SIZE=... BATCH_AUDIO_SECONDS=... TORCH_THREADS=... \
|
| 109 |
+
bash leaderboard/submit_jobs.sh
|
| 110 |
+
|
| 111 |
+
All decoding settings stay fixed when changing these throughput controls. Do
|
| 112 |
+
not tune decoder settings or prompts separately for individual datasets.
|
| 113 |
+
|
| 114 |
+
To select a subset, e.g. for a complete single-dataset cloud check:
|
| 115 |
+
|
| 116 |
+
ONLY_DATASETS="ami_cleaned" bash leaderboard/submit_jobs.sh
|
| 117 |
+
|
| 118 |
+
The default seven sets are:
|
| 119 |
+
ami_cleaned hf-audio/open-asr-leaderboard
|
| 120 |
+
gigaspeech_cleaned hf-audio/open-asr-leaderboard
|
| 121 |
+
voxpopuli_cleaned_aa hf-audio/open-asr-leaderboard
|
| 122 |
+
earnings22_cleaned_aa_chunked
|
| 123 |
+
ArtificialAnalysis/Earnings22-Cleaned-AA-chunked
|
| 124 |
+
spgispeech hf-audio/open-asr-leaderboard
|
| 125 |
+
urgent2024 hf-audio/open-asr-leaderboard
|
| 126 |
+
monsoon_en_in VoiceArena/Monsoon_en_IN_test
|
| 127 |
+
|
| 128 |
+
All use split=test. Dedicated Earnings22 and Monsoon repositories use an empty
|
| 129 |
+
config name. INCLUDE_URGENT_CLEAN=1 adds urgent2024_clean as an optional eighth
|
| 130 |
+
public set; it is outside the default seven-set average. Legacy LibriSpeech and
|
| 131 |
+
uncleaned variants are not needed for this default seven-set average.
|
| 132 |
+
|
| 133 |
+
4. OUTPUTS AND SCORING
|
| 134 |
+
|
| 135 |
+
Each run writes official-format JSONL files under ./results. Standard row keys:
|
| 136 |
+
audio_filepath, duration, time, text, pred_text
|
| 137 |
+
|
| 138 |
+
text and pred_text are raw, unnormalized strings. duration is real audio seconds,
|
| 139 |
+
and time is each batch's synchronized wall time divided equally among its rows.
|
| 140 |
+
For Earnings22 the adapter also saves parent_id and chunk_index. The official
|
| 141 |
+
scorer merges chunk predictions in index order and scores against one parent
|
| 142 |
+
reference per call. No chunk or long audio is truncated.
|
| 143 |
+
|
| 144 |
+
The metadata sidecar records model/data revisions, total model parameters,
|
| 145 |
+
environment versions, batching, raw manifest hash, timing scope and GPU details.
|
| 146 |
+
The official chunk loader resolves main internally: observed chunk/parent
|
| 147 |
+
revisions are recorded, but that loader does not offer snapshot locking. The
|
| 148 |
+
adapter explicitly rejects a dataset-revision override for that case.
|
| 149 |
+
|
| 150 |
+
Audio loading/resampling, staging/sorting, model loading, synthetic warmup and
|
| 151 |
+
scoring are excluded from throughput timing. Timing includes the complete
|
| 152 |
+
transcribe_batch call (features, transfers, neural inference, decode and text).
|
| 153 |
+
Synthetic warmup is independent of evaluation examples; any work not covered
|
| 154 |
+
by warmup remains inside the timed evaluation calls.
|
| 155 |
+
|
| 156 |
+
To print a combined public-set Markdown result table, use the pinned official
|
| 157 |
+
repo and the downloaded cloud results (or the included local raw results):
|
| 158 |
+
|
| 159 |
+
PYTHONPATH=/path/to/pinned/open_asr_leaderboard python - <<'PY'
|
| 160 |
+
from normalizer.eval_utils import score_results
|
| 161 |
+
score_results('/path/to/results', 'SpeedAudio261119/SpeedASR-v1-65M',
|
| 162 |
+
families=['public'], num_workers=1)
|
| 163 |
+
PY
|
| 164 |
+
|
| 165 |
+
Serial scoring is intentional; it avoids a multiprocessing alignment stall
|
| 166 |
+
observed on the local machine. Normalization is applied only at scoring time.
|
| 167 |
+
The default public WER is an equal-weight mean across the seven testsets, not
|
| 168 |
+
a duration/word-weighted pooled WER. RTFx uses real audio duration divided by
|
| 169 |
+
measured inference wall time. Private scores are unavailable in this package.
|
| 170 |
+
|
| 171 |
+
5. PREPARE THE OFFICIAL LEADERBOARD PR
|
| 172 |
+
|
| 173 |
+
Follow the current official PR checklist. Add a model-family folder containing
|
| 174 |
+
the adapter and submit_jobs.sh to the official GitHub repository, point its
|
| 175 |
+
launcher to the built evaluation Space and results bucket, and provide the
|
| 176 |
+
complete seven-set WER/RTFx table and raw result provenance. Provide a public
|
| 177 |
+
model link and metadata: license, total size=0.065551059B, language count=1,
|
| 178 |
+
encoder=Zipformer, decoder=stateless transducer prediction network/joiner, and
|
| 179 |
+
a link to the model card's training-data disclosure. Do not infer an undisclosed
|
| 180 |
+
training-data composition from experiment directory names.
|
| 181 |
+
|
| 182 |
+
Uploading this package has not opened that PR. Maintainers decide inclusion,
|
| 183 |
+
reproduce results and evaluate the held-out private sets separately.
|
| 184 |
+
|
| 185 |
+
PRIMARY REFERENCES
|
| 186 |
+
|
| 187 |
+
https://github.com/huggingface/open_asr_leaderboard/blob/4a113dd90c9ffd011d7d3d105da76980d511d029/.github/PULL_REQUEST_TEMPLATE.md
|
| 188 |
+
https://github.com/huggingface/open_asr_leaderboard/blob/4a113dd90c9ffd011d7d3d105da76980d511d029/README.md
|
| 189 |
+
https://github.com/huggingface/open_asr_leaderboard/blob/4a113dd90c9ffd011d7d3d105da76980d511d029/normalizer/data_utils.py
|
| 190 |
+
https://github.com/huggingface/open_asr_leaderboard/blob/4a113dd90c9ffd011d7d3d105da76980d511d029/normalizer/eval_utils.py
|
| 191 |
+
https://huggingface.co/docs/huggingface_hub/guides/jobs
|
leaderboard/dockerignore.txt
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.git
|
| 2 |
+
*.pt
|
| 3 |
+
*.onnx
|
| 4 |
+
*.safetensors
|
| 5 |
+
benchmarks
|
| 6 |
+
results
|
| 7 |
+
__pycache__
|
| 8 |
+
*.pyc
|
| 9 |
+
.env
|
| 10 |
+
*token*
|
| 11 |
+
!tokens.txt
|
| 12 |
+
|
| 13 |
+
benchmark_results
|
leaderboard/model_metadata.json
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_id": "SpeedAudio261119/SpeedASR-v1-65M",
|
| 3 |
+
"model_name": "SpeedASR-v1-65M",
|
| 4 |
+
"architecture": "Offline Zipformer2 transducer",
|
| 5 |
+
"total_parameters": 65551059,
|
| 6 |
+
"components": {
|
| 7 |
+
"audio_encoder_including_subsampling": 63994103,
|
| 8 |
+
"prediction_network": 262144,
|
| 9 |
+
"joiner": 781812,
|
| 10 |
+
"auxiliary_training_projections_retained_in_checkpoint": 513000
|
| 11 |
+
},
|
| 12 |
+
"languages": [
|
| 13 |
+
"en"
|
| 14 |
+
],
|
| 15 |
+
"language_count": 1,
|
| 16 |
+
"license": "apache-2.0",
|
| 17 |
+
"license_basis": "Preserves the pre-existing target repository declaration and LICENSE.",
|
| 18 |
+
"encoder": {
|
| 19 |
+
"type": "Non-causal Zipformer2",
|
| 20 |
+
"stacks": 6,
|
| 21 |
+
"layers": 16
|
| 22 |
+
},
|
| 23 |
+
"decoder": {
|
| 24 |
+
"type": "Transducer stateless prediction network",
|
| 25 |
+
"context_size": 3,
|
| 26 |
+
"dimension": 512
|
| 27 |
+
},
|
| 28 |
+
"inference": {
|
| 29 |
+
"precision": "fp32",
|
| 30 |
+
"sample_rate": 16000,
|
| 31 |
+
"decoding": "modified_beam_search",
|
| 32 |
+
"beam": 4,
|
| 33 |
+
"causal": false,
|
| 34 |
+
"external_language_model": false,
|
| 35 |
+
"voicea_postprocessing": true
|
| 36 |
+
},
|
| 37 |
+
"training_data": {
|
| 38 |
+
"composition": "Not disclosed",
|
| 39 |
+
"default_seven_public_testsets_excluded": true,
|
| 40 |
+
"evidence": "Author confirmation in this release task; not an independent overlap audit."
|
| 41 |
+
},
|
| 42 |
+
"evaluation": {
|
| 43 |
+
"status": "Local public evaluation; official cloud/private evaluation not yet performed",
|
| 44 |
+
"date": "2026-11-19",
|
| 45 |
+
"hardware": "NVIDIA GeForce RTX 3090 Ti",
|
| 46 |
+
"public_seven_set_macro_wer": 6.904285714285714,
|
| 47 |
+
"pooled_public_rtfx": 192.6292762644339,
|
| 48 |
+
"official_scorer_commit": "4a113dd90c9ffd011d7d3d105da76980d511d029",
|
| 49 |
+
"result_directory": "benchmark_results/local_3090ti_20261119"
|
| 50 |
+
},
|
| 51 |
+
"checkpoint_sha256": "b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01",
|
| 52 |
+
"tokens_sha256": "a238b7a0fa67aa6be998ca13dd129d69c50639970583e32232dd149e5448210a"
|
| 53 |
+
}
|
leaderboard/requirements.txt
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# English-only leaderboard loading and official scorer dependencies.
|
| 2 |
+
# Install together with ../requirements-linux-cu124.txt to retain the tested
|
| 3 |
+
# torch/k2/kaldifeat ABI. No Transformers or training packages are needed.
|
| 4 |
+
datasets==3.6.0
|
| 5 |
+
librosa==0.10.2.post1
|
| 6 |
+
pandas==2.2.3
|
| 7 |
+
pyarrow==20.0.0
|
| 8 |
+
fsspec==2025.3.0
|
| 9 |
+
kaldialign==0.12.0
|
| 10 |
+
num2words==0.5.14
|
| 11 |
+
regex==2024.11.6
|
| 12 |
+
tqdm==4.66.4
|
leaderboard/run_eval.py
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Batched SpeedASR evaluation using the official Open ASR data/manifest code.
|
| 3 |
+
|
| 4 |
+
Run from the model package root. The Dockerfile installs the official scorer at
|
| 5 |
+
the pinned commit; for a local run set --open-asr-repo to that checkout. Raw
|
| 6 |
+
references/predictions are retained, including Earnings22 session metadata.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import hashlib
|
| 12 |
+
import importlib.metadata
|
| 13 |
+
import json
|
| 14 |
+
import math
|
| 15 |
+
import os
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
import platform
|
| 18 |
+
import subprocess
|
| 19 |
+
import sys
|
| 20 |
+
import time
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
OFFICIAL_COMMIT = "4a113dd90c9ffd011d7d3d105da76980d511d029"
|
| 24 |
+
PACKAGE_ROOT = Path(__file__).resolve().parents[1]
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def parse_args():
|
| 28 |
+
p = argparse.ArgumentParser(description=__doc__)
|
| 29 |
+
p.add_argument("--model_id", "--model-id", default="SpeedAudio261119/SpeedASR-v1-65M")
|
| 30 |
+
p.add_argument("--revision", help="Pinned model commit/tag; required for a Hub download.")
|
| 31 |
+
p.add_argument("--model-dir", type=Path, help="Use an already downloaded package instead of Hub weights.")
|
| 32 |
+
p.add_argument("--dataset_path", "--dataset-path", default="hf-audio/open-asr-leaderboard")
|
| 33 |
+
p.add_argument("--dataset", default="", help="Configuration name, or empty for a dedicated dataset repo.")
|
| 34 |
+
p.add_argument("--dataset-revision", default=None)
|
| 35 |
+
p.add_argument("--split", default="test")
|
| 36 |
+
p.add_argument("--device", default="0", help="CUDA index, cuda:N, or cpu.")
|
| 37 |
+
p.add_argument("--batch_size", "--batch-size", type=int, default=2048)
|
| 38 |
+
p.add_argument("--batch-audio-seconds", type=float, default=5000.0)
|
| 39 |
+
p.add_argument("--torch-threads", type=int, default=16)
|
| 40 |
+
p.add_argument("--warmup-steps", type=int, default=5)
|
| 41 |
+
p.add_argument("--warmup-seconds", type=float, default=10.0,
|
| 42 |
+
help="Fixed synthetic audio duration, independent of the test set.")
|
| 43 |
+
p.add_argument("--max_eval_samples", "--max-eval-samples", type=int, default=-1)
|
| 44 |
+
p.add_argument("--open-asr-repo", type=Path,
|
| 45 |
+
default=Path(os.getenv("OPEN_ASR_REPO", "/opt/open_asr_leaderboard")))
|
| 46 |
+
p.add_argument("--skip-score", action="store_true", help="Write raw manifests without local scoring.")
|
| 47 |
+
a = p.parse_args()
|
| 48 |
+
a.streaming = False
|
| 49 |
+
if (a.batch_size < 1 or not math.isfinite(a.batch_audio_seconds)
|
| 50 |
+
or a.batch_audio_seconds <= 0 or a.torch_threads < 1):
|
| 51 |
+
p.error("Batch size, batch audio seconds and torch threads must be positive.")
|
| 52 |
+
if a.warmup_steps < 0 or not math.isfinite(a.warmup_seconds) or a.warmup_seconds <= 0:
|
| 53 |
+
p.error("Warmup steps must be nonnegative and warmup duration positive.")
|
| 54 |
+
if a.max_eval_samples == 0 or a.max_eval_samples < -1:
|
| 55 |
+
p.error("max_eval_samples must be -1 or positive.")
|
| 56 |
+
if a.model_dir is None and not a.revision:
|
| 57 |
+
p.error("Set --revision to the pinned Hub model commit/tag, or use --model-dir.")
|
| 58 |
+
return a
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def sha256(path):
|
| 62 |
+
h = hashlib.sha256()
|
| 63 |
+
with open(path, "rb") as f:
|
| 64 |
+
for block in iter(lambda: f.read(1024 * 1024), b""):
|
| 65 |
+
h.update(block)
|
| 66 |
+
return h.hexdigest()
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def prepared_batches(dataset, batch_size, batch_audio_seconds, np):
|
| 70 |
+
"""Decode bounded staging windows, then batch by count and padded duration.
|
| 71 |
+
|
| 72 |
+
Audio decoding/resampling and sorting occur before the timed inference call.
|
| 73 |
+
No example is truncated or dropped. Long single examples form their own
|
| 74 |
+
batch, even when they exceed the nominal batch audio budget.
|
| 75 |
+
"""
|
| 76 |
+
def stage(samples):
|
| 77 |
+
rows = []
|
| 78 |
+
for sample in samples:
|
| 79 |
+
audio = sample["audio"]
|
| 80 |
+
waveform = np.asarray(audio["array"], dtype=np.float32)
|
| 81 |
+
if waveform.ndim != 1 or waveform.size == 0 or not np.isfinite(waveform).all():
|
| 82 |
+
raise ValueError("Expected nonempty finite mono audio from official prepare_data.")
|
| 83 |
+
if int(audio["sampling_rate"]) != 16000:
|
| 84 |
+
raise ValueError("Official prepare_data must resample audio to 16000 Hz.")
|
| 85 |
+
rows.append((sample, waveform))
|
| 86 |
+
rows.sort(key=lambda r: len(r[1]), reverse=True)
|
| 87 |
+
batch = []
|
| 88 |
+
max_samples = 0
|
| 89 |
+
for row in rows:
|
| 90 |
+
proposed_max = max(max_samples, len(row[1]))
|
| 91 |
+
if batch and (len(batch) >= batch_size or
|
| 92 |
+
proposed_max * (len(batch) + 1) / 16000 > batch_audio_seconds):
|
| 93 |
+
yield batch
|
| 94 |
+
batch, max_samples = [], 0
|
| 95 |
+
batch.append(row)
|
| 96 |
+
max_samples = max(max_samples, len(row[1]))
|
| 97 |
+
if batch:
|
| 98 |
+
yield batch
|
| 99 |
+
|
| 100 |
+
window = []
|
| 101 |
+
for sample in dataset:
|
| 102 |
+
window.append(sample)
|
| 103 |
+
if len(window) >= batch_size:
|
| 104 |
+
yield from stage(window)
|
| 105 |
+
window = []
|
| 106 |
+
if window:
|
| 107 |
+
yield from stage(window)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def main(a):
|
| 111 |
+
sys.path.insert(0, str(PACKAGE_ROOT))
|
| 112 |
+
sys.path.insert(0, str(a.open_asr_repo.resolve()))
|
| 113 |
+
try:
|
| 114 |
+
commit = subprocess.check_output(
|
| 115 |
+
["git", "-C", str(a.open_asr_repo), "rev-parse", "HEAD"], text=True
|
| 116 |
+
).strip()
|
| 117 |
+
except (OSError, subprocess.CalledProcessError) as e:
|
| 118 |
+
raise RuntimeError("--open-asr-repo must point to the pinned official Git checkout.") from e
|
| 119 |
+
if commit != OFFICIAL_COMMIT:
|
| 120 |
+
raise RuntimeError(f"Expected official scorer {OFFICIAL_COMMIT}, found {commit}.")
|
| 121 |
+
|
| 122 |
+
import numpy as np
|
| 123 |
+
import torch
|
| 124 |
+
from huggingface_hub import HfApi
|
| 125 |
+
from normalizer import data_utils
|
| 126 |
+
from normalizer.eval_utils import score_manifest
|
| 127 |
+
from inference import SpeedASR
|
| 128 |
+
|
| 129 |
+
is_chunked = data_utils.is_chunked_dataset(a.dataset_path)
|
| 130 |
+
if is_chunked and a.max_eval_samples > 0:
|
| 131 |
+
raise ValueError("Do not truncate Earnings22 chunks: complete parent sessions are required.")
|
| 132 |
+
if is_chunked and a.dataset_revision:
|
| 133 |
+
raise ValueError("This pinned official chunk loader does not support --dataset-revision.")
|
| 134 |
+
if a.device in {"cpu", "-1"}:
|
| 135 |
+
device = "cpu"
|
| 136 |
+
else:
|
| 137 |
+
device = a.device if a.device.startswith("cuda") else f"cuda:{int(a.device)}"
|
| 138 |
+
if not torch.cuda.is_available():
|
| 139 |
+
raise RuntimeError("CUDA was requested but is unavailable.")
|
| 140 |
+
torch.set_num_threads(a.torch_threads)
|
| 141 |
+
|
| 142 |
+
api = HfApi()
|
| 143 |
+
model_revision = a.revision
|
| 144 |
+
if a.model_dir:
|
| 145 |
+
model_dir = a.model_dir.resolve()
|
| 146 |
+
model = SpeedASR(checkpoint=model_dir / "pretrained.pt", tokens=model_dir / "tokens.txt",
|
| 147 |
+
config=model_dir / "model_config.json", device=device,
|
| 148 |
+
batch_size=a.batch_size, batch_audio_seconds=a.batch_audio_seconds,
|
| 149 |
+
torch_threads=a.torch_threads)
|
| 150 |
+
else:
|
| 151 |
+
model_revision = api.model_info(a.model_id, revision=a.revision).sha
|
| 152 |
+
model = SpeedASR.from_pretrained(
|
| 153 |
+
a.model_id, revision=model_revision, device=device, batch_size=a.batch_size,
|
| 154 |
+
batch_audio_seconds=a.batch_audio_seconds, torch_threads=a.torch_threads,
|
| 155 |
+
)
|
| 156 |
+
count = sum(p.numel() for p in model.model.parameters())
|
| 157 |
+
if model.beam_size != 4:
|
| 158 |
+
raise ValueError("This submission uses the same fixed beam size 4 on every dataset.")
|
| 159 |
+
print(f"Model size: {count} parameters ({count / 1e9:.9f}B)", flush=True)
|
| 160 |
+
|
| 161 |
+
dataset_revision = api.dataset_info(a.dataset_path, revision=a.dataset_revision).sha
|
| 162 |
+
parent_revision = None
|
| 163 |
+
if is_chunked:
|
| 164 |
+
parent_revision = api.dataset_info(
|
| 165 |
+
data_utils.CHUNKED_DATASETS[a.dataset_path.lower()]["parent_dataset_path"]
|
| 166 |
+
).sha
|
| 167 |
+
# Dedicated dataset repos use no config. Keep the manifest config as an empty
|
| 168 |
+
# string while passing None to load_dataset through the official helper.
|
| 169 |
+
load_args = argparse.Namespace(**vars(a))
|
| 170 |
+
load_args.dataset = a.dataset or None
|
| 171 |
+
if not is_chunked:
|
| 172 |
+
load_args.dataset_revision = dataset_revision
|
| 173 |
+
dataset = data_utils.prepare_data(data_utils.load_data(load_args))
|
| 174 |
+
if a.max_eval_samples > 0:
|
| 175 |
+
dataset = dataset.select(range(min(a.max_eval_samples, len(dataset))))
|
| 176 |
+
|
| 177 |
+
def sync():
|
| 178 |
+
if device.startswith("cuda"):
|
| 179 |
+
torch.cuda.synchronize(device)
|
| 180 |
+
|
| 181 |
+
dummy = np.zeros(round(16000 * a.warmup_seconds), dtype=np.float32)
|
| 182 |
+
for _ in range(a.warmup_steps):
|
| 183 |
+
model.transcribe_batch([dummy], sample_rate=16000)
|
| 184 |
+
sync()
|
| 185 |
+
if device.startswith("cuda"):
|
| 186 |
+
torch.cuda.reset_peak_memory_stats(device)
|
| 187 |
+
model.max_gpu_allocated_bytes = 0
|
| 188 |
+
|
| 189 |
+
refs, preds, durations, times, paths = [], [], [], [], []
|
| 190 |
+
extra = {key: [] for key in data_utils.CHUNK_METADATA_KEYS} if is_chunked else {}
|
| 191 |
+
batches = 0
|
| 192 |
+
for batch in prepared_batches(dataset, a.batch_size, a.batch_audio_seconds, np):
|
| 193 |
+
waveforms = [waveform for _, waveform in batch]
|
| 194 |
+
sync()
|
| 195 |
+
started = time.perf_counter()
|
| 196 |
+
output = model.transcribe_batch(waveforms, sample_rate=16000)
|
| 197 |
+
sync()
|
| 198 |
+
runtime = time.perf_counter() - started
|
| 199 |
+
if len(output) != len(batch) or any(not isinstance(text, str) for text in output):
|
| 200 |
+
raise RuntimeError("Inference returned an invalid number/type of predictions.")
|
| 201 |
+
for (sample, waveform), prediction in zip(batch, output):
|
| 202 |
+
refs.append(sample["original_text"])
|
| 203 |
+
preds.append(prediction)
|
| 204 |
+
durations.append(len(waveform) / 16000)
|
| 205 |
+
times.append(runtime / len(batch))
|
| 206 |
+
paths.append(data_utils.extract_audio_filepath_from_sample(sample))
|
| 207 |
+
for key in extra:
|
| 208 |
+
extra[key].append(sample[key])
|
| 209 |
+
batches += 1
|
| 210 |
+
print(f"Finished batch {batches}: {len(refs)} examples", flush=True)
|
| 211 |
+
if not refs:
|
| 212 |
+
raise RuntimeError("No evaluation examples remain after official reference filtering.")
|
| 213 |
+
|
| 214 |
+
manifest = Path(data_utils.write_manifest(
|
| 215 |
+
refs, preds, a.model_id, a.dataset_path, a.dataset or "", a.split,
|
| 216 |
+
audio_length=durations, transcription_time=times,
|
| 217 |
+
audio_filepaths=paths, extra_fields=extra or None,
|
| 218 |
+
))
|
| 219 |
+
metadata = {
|
| 220 |
+
"model_id": a.model_id, "model_revision": model_revision,
|
| 221 |
+
"model_parameters": count, "dataset_path": a.dataset_path,
|
| 222 |
+
"dataset": a.dataset, "dataset_revision": dataset_revision,
|
| 223 |
+
"parent_dataset_revision_observed": parent_revision,
|
| 224 |
+
"chunk_revision_note": "Official chunk loader resolves main; observed revisions recorded, not a snapshot lock." if is_chunked else None,
|
| 225 |
+
"split": a.split, "official_commit": commit, "examples": len(refs),
|
| 226 |
+
"batches": batches, "precision": "fp32", "causal": False,
|
| 227 |
+
"decoding_method": "modified_beam_search", "beam_size": 4,
|
| 228 |
+
"context_size": 3, "batch_size": a.batch_size,
|
| 229 |
+
"long_chunk_seconds": model.long_chunk_seconds,
|
| 230 |
+
"long_chunk_batch_size": model.long_chunk_batch_size,
|
| 231 |
+
"min_input_seconds": model.min_input_seconds,
|
| 232 |
+
"batch_audio_seconds": a.batch_audio_seconds,
|
| 233 |
+
"warmup_steps": a.warmup_steps, "warmup_seconds": a.warmup_seconds,
|
| 234 |
+
"warmup_input": "synthetic zeros, independent of evaluation data",
|
| 235 |
+
"audio_seconds": sum(durations), "inference_seconds": sum(times),
|
| 236 |
+
"rtfx": sum(durations) / sum(times),
|
| 237 |
+
"timing_scope": "Synchronized wall time of complete transcribe_batch calls; excludes audio loading/resampling, staging/sorting, model loading, warmup and scoring.",
|
| 238 |
+
"device": device,
|
| 239 |
+
"gpu_name": torch.cuda.get_device_name(device) if device.startswith("cuda") else None,
|
| 240 |
+
"peak_gpu_allocated_bytes": model.max_gpu_allocated_bytes if device.startswith("cuda") else None,
|
| 241 |
+
"python": platform.python_version(), "torch": torch.__version__,
|
| 242 |
+
"package_versions": {n: importlib.metadata.version(n) for n in ("k2", "kaldifeat", "numpy", "datasets", "huggingface_hub")},
|
| 243 |
+
"manifest_sha256": sha256(manifest),
|
| 244 |
+
"run_eval_sha256": sha256(Path(__file__)),
|
| 245 |
+
"inference_code_sha256": sha256(PACKAGE_ROOT / "inference.py"),
|
| 246 |
+
"vendor_code_sha256": {p.name: sha256(p) for p in sorted((PACKAGE_ROOT / "speedasr_core").glob("*.py"))},
|
| 247 |
+
"model_configuration": model.config,
|
| 248 |
+
"partial_run": a.max_eval_samples > 0,
|
| 249 |
+
}
|
| 250 |
+
if not a.skip_score:
|
| 251 |
+
rows = data_utils.read_manifest(manifest)
|
| 252 |
+
if is_chunked:
|
| 253 |
+
rows = data_utils.merge_chunked_manifest(rows)
|
| 254 |
+
score = score_manifest(rows, language="en", multilingual=False, num_workers=1)
|
| 255 |
+
metadata["official_score"] = score
|
| 256 |
+
print("Official score:", json.dumps(score, ensure_ascii=False), flush=True)
|
| 257 |
+
manifest.with_suffix(".meta.json").write_text(
|
| 258 |
+
json.dumps(metadata, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
|
| 259 |
+
)
|
| 260 |
+
print(f"Manifest: {manifest.resolve()}\nRTFx: {metadata['rtfx']:.4f}", flush=True)
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
if __name__ == "__main__":
|
| 264 |
+
main(parse_args())
|
leaderboard/space_app.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Minimal landing page for the reproducible Docker evaluation Space."""
|
| 2 |
+
from http.server import BaseHTTPRequestHandler, HTTPServer
|
| 3 |
+
|
| 4 |
+
BODY = b"""<!doctype html><html lang=en><meta charset=utf-8>
|
| 5 |
+
<title>SpeedASR Open ASR evaluation image</title>
|
| 6 |
+
<body style="max-width:780px;margin:60px auto;padding:20px;font:17px/1.6 system-ui">
|
| 7 |
+
<h1>SpeedASR Open ASR evaluation image</h1>
|
| 8 |
+
<p>This Docker Space supplies the reproducible inference environment for
|
| 9 |
+
<a href="https://huggingface.co/SpeedAudio261119/SpeedASR-v1-65M">SpeedASR-v1-65M</a>.
|
| 10 |
+
It does not run a benchmark automatically.</p>
|
| 11 |
+
<p>Read <code>leaderboard/cloud_submission.txt</code> in the model repository for
|
| 12 |
+
the pinned evaluation command, result schema and HF Jobs launcher. Jobs use
|
| 13 |
+
separate paid hardware; the Space landing page performs no inference.</p></body></html>"""
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class Handler(BaseHTTPRequestHandler):
|
| 17 |
+
def do_GET(self):
|
| 18 |
+
self.send_response(200)
|
| 19 |
+
self.send_header("Content-Type", "text/html; charset=utf-8")
|
| 20 |
+
self.send_header("Content-Length", str(len(BODY)))
|
| 21 |
+
self.end_headers()
|
| 22 |
+
self.wfile.write(BODY)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
if __name__ == "__main__":
|
| 26 |
+
HTTPServer(("0.0.0.0", 7860), Handler).serve_forever()
|
leaderboard/space_card.txt
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: SpeedASR Open ASR evaluation
|
| 3 |
+
emoji: 🎙️
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: green
|
| 6 |
+
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
+
---
|
| 9 |
+
|
| 10 |
+
Reproducible Docker image for the SpeedASR-v1-65M offline English ASR model.
|
| 11 |
+
|
| 12 |
+
Model: https://huggingface.co/SpeedAudio261119/SpeedASR-v1-65M
|
| 13 |
+
|
| 14 |
+
See leaderboard/cloud_submission.txt for build and HF Jobs commands. The landing
|
| 15 |
+
page does not launch paid jobs or run inference automatically.
|
leaderboard/submit_jobs.sh
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# Run this script deliberately: it launches PAID HF Jobs, one H200 per dataset.
|
| 3 |
+
# It does not submit a GitHub PR or register the model on the leaderboard.
|
| 4 |
+
set -euo pipefail
|
| 5 |
+
|
| 6 |
+
: "${HF_TOKEN:?Set a write token in the process environment; do not commit it.}"
|
| 7 |
+
: "${SPACE:?Set the built Docker Space, e.g. your-user/open-asr-leaderboard-speedasr.}"
|
| 8 |
+
: "${RESULTS_BUCKET:?Set an existing writable bucket, e.g. your-user/speedasr-results.}"
|
| 9 |
+
: "${MODEL_REVISION:?Set the full model commit SHA from the Hub upload.}"
|
| 10 |
+
|
| 11 |
+
MODEL_ID=${MODEL_ID:-SpeedAudio261119/SpeedASR-v1-65M}
|
| 12 |
+
FLAVOR=${FLAVOR:-h200}
|
| 13 |
+
TIMEOUT=${TIMEOUT:-8h}
|
| 14 |
+
BATCH_SIZE=${BATCH_SIZE:-2048}
|
| 15 |
+
BATCH_AUDIO_SECONDS=${BATCH_AUDIO_SECONDS:-5000}
|
| 16 |
+
TORCH_THREADS=${TORCH_THREADS:-16}
|
| 17 |
+
RUN_ID=${RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)}
|
| 18 |
+
MODEL_FOLDER=${MODEL_ID//\//-}
|
| 19 |
+
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
|
| 20 |
+
|
| 21 |
+
DATASETS=(
|
| 22 |
+
"ami_cleaned|hf-audio/open-asr-leaderboard|ami_cleaned"
|
| 23 |
+
"gigaspeech_cleaned|hf-audio/open-asr-leaderboard|gigaspeech_cleaned"
|
| 24 |
+
"voxpopuli_cleaned_aa|hf-audio/open-asr-leaderboard|voxpopuli_cleaned_aa"
|
| 25 |
+
"earnings22_cleaned_aa_chunked|ArtificialAnalysis/Earnings22-Cleaned-AA-chunked|"
|
| 26 |
+
"spgispeech|hf-audio/open-asr-leaderboard|spgispeech"
|
| 27 |
+
"urgent2024|hf-audio/open-asr-leaderboard|urgent2024"
|
| 28 |
+
"monsoon_en_in|VoiceArena/Monsoon_en_IN_test|"
|
| 29 |
+
)
|
| 30 |
+
if [[ ${INCLUDE_URGENT_CLEAN:-0} == 1 ]]; then
|
| 31 |
+
DATASETS+=("urgent2024_clean|hf-audio/open-asr-leaderboard|urgent2024_clean")
|
| 32 |
+
fi
|
| 33 |
+
namespace_args=()
|
| 34 |
+
if [[ -n ${ORG_NAME:-} ]]; then
|
| 35 |
+
namespace_args=(--namespace "$ORG_NAME")
|
| 36 |
+
fi
|
| 37 |
+
|
| 38 |
+
# Variables enter the container as environment values; the fixed command below
|
| 39 |
+
# contains no interpolated token or user-supplied shell fragments.
|
| 40 |
+
JOB_COMMAND=$(cat <<'JOB'
|
| 41 |
+
set -euo pipefail
|
| 42 |
+
cd /app
|
| 43 |
+
OUT="/results/${SPEEDASR_MODEL_FOLDER}/${SPEEDASR_RUN_ID}/${SPEEDASR_DATASET_LABEL}"
|
| 44 |
+
mkdir -p "$OUT"
|
| 45 |
+
persist_results() {
|
| 46 |
+
cp results/*.jsonl results/*.meta.json "$OUT/" 2>/dev/null || true
|
| 47 |
+
}
|
| 48 |
+
trap persist_results EXIT
|
| 49 |
+
python leaderboard/run_eval.py \
|
| 50 |
+
--model_id "$SPEEDASR_MODEL_ID" --revision "$SPEEDASR_MODEL_REVISION" \
|
| 51 |
+
--dataset_path "$SPEEDASR_DATASET_PATH" --dataset "$SPEEDASR_DATASET_CONFIG" \
|
| 52 |
+
--split test --device 0 --batch_size "$SPEEDASR_BATCH_SIZE" \
|
| 53 |
+
--batch-audio-seconds "$SPEEDASR_BATCH_AUDIO_SECONDS" \
|
| 54 |
+
--torch-threads "$SPEEDASR_TORCH_THREADS" --warmup-steps 5 \
|
| 55 |
+
--warmup-seconds 10 --max_eval_samples -1 2>&1 | tee "$OUT/job.log"
|
| 56 |
+
JOB
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
selected=0
|
| 60 |
+
for cfg in "${DATASETS[@]}"; do
|
| 61 |
+
IFS='|' read -r label dataset_path dataset_config <<< "$cfg"
|
| 62 |
+
if [[ -n ${ONLY_DATASETS:-} && " ${ONLY_DATASETS} " != *" ${label} "* ]]; then
|
| 63 |
+
continue
|
| 64 |
+
fi
|
| 65 |
+
selected=$((selected + 1))
|
| 66 |
+
printf 'Running %s, model revision %s, flavor %s\n' "$label" "$MODEL_REVISION" "$FLAVOR"
|
| 67 |
+
# Without --detach the current hf CLI waits and returns nonzero on job failure.
|
| 68 |
+
hf jobs run --flavor "$FLAVOR" --timeout "$TIMEOUT" \
|
| 69 |
+
"${namespace_args[@]}" \
|
| 70 |
+
--secrets "HF_TOKEN=$HF_TOKEN" \
|
| 71 |
+
--env "HF_AUDIO_DECODER_BACKEND=soundfile" \
|
| 72 |
+
--env "SPEEDASR_MODEL_ID=$MODEL_ID" \
|
| 73 |
+
--env "SPEEDASR_MODEL_REVISION=$MODEL_REVISION" \
|
| 74 |
+
--env "SPEEDASR_MODEL_FOLDER=$MODEL_FOLDER" \
|
| 75 |
+
--env "SPEEDASR_RUN_ID=$RUN_ID" \
|
| 76 |
+
--env "SPEEDASR_DATASET_LABEL=$label" \
|
| 77 |
+
--env "SPEEDASR_DATASET_PATH=$dataset_path" \
|
| 78 |
+
--env "SPEEDASR_DATASET_CONFIG=$dataset_config" \
|
| 79 |
+
--env "SPEEDASR_BATCH_SIZE=$BATCH_SIZE" \
|
| 80 |
+
--env "SPEEDASR_BATCH_AUDIO_SECONDS=$BATCH_AUDIO_SECONDS" \
|
| 81 |
+
--env "SPEEDASR_TORCH_THREADS=$TORCH_THREADS" \
|
| 82 |
+
--volume "hf://buckets/${RESULTS_BUCKET}:/results" \
|
| 83 |
+
"hf.co/spaces/${SPACE}" bash -c "$JOB_COMMAND"
|
| 84 |
+
done
|
| 85 |
+
if (( selected == 0 )); then
|
| 86 |
+
printf 'No datasets selected; check ONLY_DATASETS.\n' >&2
|
| 87 |
+
exit 2
|
| 88 |
+
fi
|
| 89 |
+
|
| 90 |
+
LOCAL_RESULTS="$SCRIPT_DIR/cloud_results/$RUN_ID"
|
| 91 |
+
mkdir -p "$LOCAL_RESULTS"
|
| 92 |
+
hf buckets sync \
|
| 93 |
+
"hf://buckets/${RESULTS_BUCKET}/${MODEL_FOLDER}/${RUN_ID}" "$LOCAL_RESULTS"
|
| 94 |
+
printf 'Downloaded cloud results to %s\n' "$LOCAL_RESULTS"
|
| 95 |
+
printf 'Score all seven sets using the pinned official scorer; see cloud_submission.txt.\n'
|
model_config.json
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"model_type": "offline_zipformer_transducer",
|
| 4 |
+
"model_args": {
|
| 5 |
+
"num_encoder_layers": "2,2,3,4,3,2",
|
| 6 |
+
"downsampling_factor": "1,2,4,8,4,2",
|
| 7 |
+
"feedforward_dim": "512,768,1024,1536,1024,768",
|
| 8 |
+
"num_heads": "4,4,4,8,4,4",
|
| 9 |
+
"encoder_dim": "192,256,384,512,384,256",
|
| 10 |
+
"query_head_dim": "32",
|
| 11 |
+
"value_head_dim": "12",
|
| 12 |
+
"pos_head_dim": "4",
|
| 13 |
+
"pos_dim": 48,
|
| 14 |
+
"encoder_unmasked_dim": "192,192,256,256,256,192",
|
| 15 |
+
"cnn_module_kernel": "31,31,15,15,15,31",
|
| 16 |
+
"decoder_dim": 512,
|
| 17 |
+
"joiner_dim": 512,
|
| 18 |
+
"context_size": 3,
|
| 19 |
+
"causal": false,
|
| 20 |
+
"chunk_size": "-1",
|
| 21 |
+
"left_context_frames": "-1",
|
| 22 |
+
"use_transducer": true,
|
| 23 |
+
"use_ctc": false,
|
| 24 |
+
"use_attention_decoder": false,
|
| 25 |
+
"use_cr_ctc": false,
|
| 26 |
+
"feature_dim": 80,
|
| 27 |
+
"attention_decoder_dim": 512,
|
| 28 |
+
"attention_decoder_num_layers": 6,
|
| 29 |
+
"attention_decoder_attention_dim": 512,
|
| 30 |
+
"attention_decoder_num_heads": 8,
|
| 31 |
+
"attention_decoder_feedforward_dim": 2048,
|
| 32 |
+
"blank_id": 0,
|
| 33 |
+
"unk_id": 7,
|
| 34 |
+
"vocab_size": 500
|
| 35 |
+
},
|
| 36 |
+
"inference": {
|
| 37 |
+
"sample_rate": 16000,
|
| 38 |
+
"feature_type": "kaldifeat.Fbank",
|
| 39 |
+
"num_mel_bins": 80,
|
| 40 |
+
"dither": 0,
|
| 41 |
+
"snip_edges": false,
|
| 42 |
+
"high_freq": -400,
|
| 43 |
+
"precision": "fp32",
|
| 44 |
+
"decoding_method": "modified_beam_search",
|
| 45 |
+
"beam_size": 4,
|
| 46 |
+
"voicea_postprocessing": true,
|
| 47 |
+
"min_input_seconds": 1,
|
| 48 |
+
"long_chunk_seconds": 30,
|
| 49 |
+
"long_chunk_batch_size": 512,
|
| 50 |
+
"batch_size": 2048,
|
| 51 |
+
"batch_audio_seconds": 5000,
|
| 52 |
+
"torch_threads": 16
|
| 53 |
+
},
|
| 54 |
+
"expected": {
|
| 55 |
+
"parameters": 65551059,
|
| 56 |
+
"checkpoint_sha256": "b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01",
|
| 57 |
+
"tokens_sha256": "a238b7a0fa67aa6be998ca13dd129d69c50639970583e32232dd149e5448210a"
|
| 58 |
+
}
|
| 59 |
+
}
|
onnx/decoder-epoch-99-avg-1.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1fc497358cc2f63d4dcd3f249da1e4f29a04315c5d7e4aa358cfaf17ea8d5db1
|
| 3 |
+
size 2101273
|
onnx/encoder-epoch-99-avg-1.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:768fedad168901e872e46477b347adc95cc304ed9ee60ee57c95b60916da42f0
|
| 3 |
+
size 263111425
|
onnx/joiner-epoch-99-avg-1.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c26540ac6d9ac00191436193bc22a8dd3ceae7bd3a83058aa2d18e1470bed5f4
|
| 3 |
+
size 1026462
|
pretrained.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b0475d24e50f810fbcf997e66061243a1fe212251f6d032457b2711641906c01
|
| 3 |
+
size 262613934
|
requirements-core.txt
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Install matching torch / k2 / kaldifeat wheels first. See requirements-linux-cu124.txt.
|
| 2 |
+
numpy==1.24.4
|
| 3 |
+
soundfile==0.13.1
|
| 4 |
+
packaging>=20
|
| 5 |
+
huggingface_hub==1.18.0
|
requirements-linux-cu124.txt
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Reproduces the benchmark ABI: Linux x86_64, CPython 3.10, CUDA 12.4.
|
| 2 |
+
--extra-index-url https://download.pytorch.org/whl/cu124
|
| 3 |
+
torch==2.6.0+cu124
|
| 4 |
+
torchaudio==2.6.0+cu124
|
| 5 |
+
k2 @ https://huggingface.co/csukuangfj2/k2/resolve/main/ubuntu-cuda/1.24.4.dev20260625/k2-1.24.4.dev20260625+cuda12.4.torch2.6.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl#sha256=1057e7c004c332433b422ef7c7744b99515a71d0ec19ae18fc9ff80d46139237
|
| 6 |
+
kaldifeat @ https://huggingface.co/csukuangfj/kaldifeat/resolve/main/cuda/1.25.5.dev20260626/linux/kaldifeat-1.25.5.dev20260626+cuda12.4.torch2.6.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl#sha256=2991f7a70876e69bbf507d22fc8117e887e3098e892e7d28a1febeacee306cdb
|
| 7 |
+
-r requirements-core.txt
|
requirements.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Reference CUDA build; use CPython 3.10 on Linux x86_64.
|
| 2 |
+
# CPU/other Python builds require torch-compatible k2 and kaldifeat wheels.
|
| 3 |
+
-r requirements-linux-cu124.txt
|
runtime_source_manifest.json
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"source_repository": "https://github.com/k2-fsa/icefall",
|
| 3 |
+
"source_git_revision": "3f848bb6d0acc970c9b294a30ca0a04a7c9c78d1",
|
| 4 |
+
"notes": "Inference-only extraction from the actual 3090 model source. Original copyright and Apache2 notices retained. Imports made package-relative; no training/data modules are imported. Text line endings normalized to LF.",
|
| 5 |
+
"files": [
|
| 6 |
+
{
|
| 7 |
+
"path": "speedasr_core/encoder_interface.py",
|
| 8 |
+
"origin": "egs/librispeech/ASR/zipformer/encoder_interface.py",
|
| 9 |
+
"original_sha256": "fc70f530c33e2489435ff2cb4bb0dad8b9d6f58a248c425cbd6100c62f628569",
|
| 10 |
+
"sha256": "fc70f530c33e2489435ff2cb4bb0dad8b9d6f58a248c425cbd6100c62f628569",
|
| 11 |
+
"change": "Relative imports only; implementation retained."
|
| 12 |
+
},
|
| 13 |
+
{
|
| 14 |
+
"path": "speedasr_core/scaling.py",
|
| 15 |
+
"origin": "egs/librispeech/ASR/zipformer/scaling.py",
|
| 16 |
+
"original_sha256": "d5a5db141cd00b4beaa990fe6827599b3248b2fddd87e8dc7809b0dbd9d6633b",
|
| 17 |
+
"sha256": "679b1fe2655902ba13ef3f5cf7c924129a16681e3edc4343e766ff7086f5e6d7",
|
| 18 |
+
"change": "Relative imports only; implementation retained."
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"path": "speedasr_core/subsampling.py",
|
| 22 |
+
"origin": "egs/librispeech/ASR/zipformer/subsampling.py",
|
| 23 |
+
"original_sha256": "e04de281f136560e2d44b5f7df174d0364b5feb2c22608173b60de02630c2a66",
|
| 24 |
+
"sha256": "dc33ac662d2508e191f4da5562f0c8d23444819b707624736429a85517db9145",
|
| 25 |
+
"change": "Relative imports only; implementation retained."
|
| 26 |
+
},
|
| 27 |
+
{
|
| 28 |
+
"path": "speedasr_core/zipformer.py",
|
| 29 |
+
"origin": "egs/librispeech/ASR/zipformer/zipformer.py",
|
| 30 |
+
"original_sha256": "4d5365ed19dee07f5a51fcaaa26e18310f7fafbdf2a391e24a7cc01d1b2e8cfe",
|
| 31 |
+
"sha256": "5cea6da601de9f37de2e1db6c2dadcd931220b45341472efaa426a25130a5a36",
|
| 32 |
+
"change": "Relative imports only; implementation retained."
|
| 33 |
+
},
|
| 34 |
+
{
|
| 35 |
+
"path": "speedasr_core/decoder.py",
|
| 36 |
+
"origin": "egs/librispeech/ASR/zipformer/decoder.py",
|
| 37 |
+
"original_sha256": "92ee967e4b927a3ef2e4cc2d900756609a191f859c956a400cfe642d6ef0cf52",
|
| 38 |
+
"sha256": "a9aeeeef278f4e5357d4e49253d24f5f1c60df88cfcb623ccbd01481dc1503ce",
|
| 39 |
+
"change": "Relative imports only; implementation retained."
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"path": "speedasr_core/joiner.py",
|
| 43 |
+
"origin": "egs/librispeech/ASR/zipformer/joiner.py",
|
| 44 |
+
"original_sha256": "2d924a5a00f7a2bb2340c1a991f8a71037bcfbc7e98cbabe26527ffa7d4030d4",
|
| 45 |
+
"sha256": "81faad6e0d07342d1abab28f2271201256f2139f5816b04296d6a8cbe0e2c887",
|
| 46 |
+
"change": "Relative imports only; implementation retained."
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"path": "speedasr_core/model.py",
|
| 50 |
+
"origin": "egs/librispeech/ASR/zipformer/model.py",
|
| 51 |
+
"original_sha256": "1e516fec1ad65220470a7db69bf3d16e4736a0b9943ef8f5ad75bb962d7a3296",
|
| 52 |
+
"sha256": "46b8e181fb326e8cd82c92a4c0ffd0917eb1474a26bd5b60b39b48daac28e90f",
|
| 53 |
+
"change": "Keep AsrModel constructor and forward_encoder unchanged; remove training loss methods and training imports."
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"path": "speedasr_core/beam_search.py",
|
| 57 |
+
"origin": "egs/librispeech/ASR/zipformer/beam_search.py",
|
| 58 |
+
"original_sha256": "ae6afa2654215175ba5be630c7c34ee885a42df4cc79b1abb743c707d716212a",
|
| 59 |
+
"sha256": "d2e0297d1626b14b834a9b357caa875a5b11ce8f0aabc475debb7d20dec3b8c1",
|
| 60 |
+
"change": "Extract Hypothesis/HypothesisList/get_hyps_shape/modified_beam_search unchanged; remove unrelated decoders and LM/training imports; retain annotation-only context names."
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"path": "speedasr_core/utils.py",
|
| 64 |
+
"origin": "icefall/utils.py",
|
| 65 |
+
"original_sha256": "35beb5077b458872621094d7b176adf58fb6a7d4de81a60f033184052b6555d6",
|
| 66 |
+
"sha256": "682d0733f0e149be0ae9b62cb8de60d7baef65be4e64816001b6bb089b4e787b",
|
| 67 |
+
"change": "Extract only torch_autocast and make_pad_mask unchanged; remove unrelated training/data helpers."
|
| 68 |
+
}
|
| 69 |
+
]
|
| 70 |
+
}
|
speedasr_core/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
speedasr_core/__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Vendored Apache-2.0 Icefall inference components; see LICENSE.
|
| 2 |
+
|
| 3 |
+
No external Icefall checkout is required.
|
| 4 |
+
"""
|
speedasr_core/beam_search.py
ADDED
|
@@ -0,0 +1,423 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang
|
| 2 |
+
# Xiaoyu Yang)
|
| 3 |
+
#
|
| 4 |
+
# See ../../../../LICENSE for clarification regarding multiple authors
|
| 5 |
+
#
|
| 6 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 7 |
+
# you may not use this file except in compliance with the License.
|
| 8 |
+
# You may obtain a copy of the License at
|
| 9 |
+
#
|
| 10 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 11 |
+
#
|
| 12 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 13 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 14 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 15 |
+
# See the License for the specific language governing permissions and
|
| 16 |
+
# limitations under the License.
|
| 17 |
+
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import warnings
|
| 21 |
+
from dataclasses import dataclass, field
|
| 22 |
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
| 23 |
+
|
| 24 |
+
import k2
|
| 25 |
+
import torch
|
| 26 |
+
from torch import nn
|
| 27 |
+
|
| 28 |
+
# Optional context/LM types are annotations only; SpeedASR uses no context graph.
|
| 29 |
+
ContextGraph = ContextState = NgramLmStateCost = Any
|
| 30 |
+
|
| 31 |
+
@dataclass
|
| 32 |
+
class DecodingResults:
|
| 33 |
+
hyps: List[List[int]]
|
| 34 |
+
timestamps: List[List[int]]
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
@dataclass
|
| 38 |
+
class Hypothesis:
|
| 39 |
+
# The predicted tokens so far.
|
| 40 |
+
# Newly predicted tokens are appended to `ys`.
|
| 41 |
+
ys: List[int]
|
| 42 |
+
|
| 43 |
+
# The log prob of ys.
|
| 44 |
+
# It contains only one entry.
|
| 45 |
+
log_prob: torch.Tensor
|
| 46 |
+
|
| 47 |
+
ac_probs: Optional[List[float]] = None
|
| 48 |
+
|
| 49 |
+
# timestamp[i] is the frame index after subsampling
|
| 50 |
+
# on which ys[i] is decoded
|
| 51 |
+
timestamp: List[int] = field(default_factory=list)
|
| 52 |
+
|
| 53 |
+
# the lm score for next token given the current ys
|
| 54 |
+
lm_score: Optional[torch.Tensor] = None
|
| 55 |
+
|
| 56 |
+
# the RNNLM states (h and c in LSTM)
|
| 57 |
+
state: Optional[Tuple[torch.Tensor, torch.Tensor]] = None
|
| 58 |
+
|
| 59 |
+
# N-gram LM state
|
| 60 |
+
state_cost: Optional[NgramLmStateCost] = None
|
| 61 |
+
|
| 62 |
+
# Context graph state
|
| 63 |
+
context_state: Optional[ContextState] = None
|
| 64 |
+
|
| 65 |
+
num_tailing_blanks: int = 0
|
| 66 |
+
|
| 67 |
+
@property
|
| 68 |
+
def key(self) -> str:
|
| 69 |
+
"""Return a string representation of self.ys"""
|
| 70 |
+
return "_".join(map(str, self.ys))
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class HypothesisList(object):
|
| 74 |
+
def __init__(self, data: Optional[Dict[str, Hypothesis]] = None) -> None:
|
| 75 |
+
"""
|
| 76 |
+
Args:
|
| 77 |
+
data:
|
| 78 |
+
A dict of Hypotheses. Its key is its `value.key`.
|
| 79 |
+
"""
|
| 80 |
+
if data is None:
|
| 81 |
+
self._data = {}
|
| 82 |
+
else:
|
| 83 |
+
self._data = data
|
| 84 |
+
|
| 85 |
+
@property
|
| 86 |
+
def data(self) -> Dict[str, Hypothesis]:
|
| 87 |
+
return self._data
|
| 88 |
+
|
| 89 |
+
def add(self, hyp: Hypothesis) -> None:
|
| 90 |
+
"""Add a Hypothesis to `self`.
|
| 91 |
+
|
| 92 |
+
If `hyp` already exists in `self`, its probability is updated using
|
| 93 |
+
`log-sum-exp` with the existed one.
|
| 94 |
+
|
| 95 |
+
Args:
|
| 96 |
+
hyp:
|
| 97 |
+
The hypothesis to be added.
|
| 98 |
+
"""
|
| 99 |
+
key = hyp.key
|
| 100 |
+
if key in self:
|
| 101 |
+
old_hyp = self._data[key] # shallow copy
|
| 102 |
+
torch.logaddexp(old_hyp.log_prob, hyp.log_prob, out=old_hyp.log_prob)
|
| 103 |
+
else:
|
| 104 |
+
self._data[key] = hyp
|
| 105 |
+
|
| 106 |
+
def get_most_probable(self, length_norm: bool = False) -> Hypothesis:
|
| 107 |
+
"""Get the most probable hypothesis, i.e., the one with
|
| 108 |
+
the largest `log_prob`.
|
| 109 |
+
|
| 110 |
+
Args:
|
| 111 |
+
length_norm:
|
| 112 |
+
If True, the `log_prob` of a hypothesis is normalized by the
|
| 113 |
+
number of tokens in it.
|
| 114 |
+
Returns:
|
| 115 |
+
Return the hypothesis that has the largest `log_prob`.
|
| 116 |
+
"""
|
| 117 |
+
if length_norm:
|
| 118 |
+
return max(self._data.values(), key=lambda hyp: hyp.log_prob / len(hyp.ys))
|
| 119 |
+
else:
|
| 120 |
+
return max(self._data.values(), key=lambda hyp: hyp.log_prob)
|
| 121 |
+
|
| 122 |
+
def remove(self, hyp: Hypothesis) -> None:
|
| 123 |
+
"""Remove a given hypothesis.
|
| 124 |
+
|
| 125 |
+
Caution:
|
| 126 |
+
`self` is modified **in-place**.
|
| 127 |
+
|
| 128 |
+
Args:
|
| 129 |
+
hyp:
|
| 130 |
+
The hypothesis to be removed from `self`.
|
| 131 |
+
Note: It must be contained in `self`. Otherwise,
|
| 132 |
+
an exception is raised.
|
| 133 |
+
"""
|
| 134 |
+
key = hyp.key
|
| 135 |
+
assert key in self, f"{key} does not exist"
|
| 136 |
+
del self._data[key]
|
| 137 |
+
|
| 138 |
+
def filter(self, threshold: torch.Tensor) -> "HypothesisList":
|
| 139 |
+
"""Remove all Hypotheses whose log_prob is less than threshold.
|
| 140 |
+
|
| 141 |
+
Caution:
|
| 142 |
+
`self` is not modified. Instead, a new HypothesisList is returned.
|
| 143 |
+
|
| 144 |
+
Returns:
|
| 145 |
+
Return a new HypothesisList containing all hypotheses from `self`
|
| 146 |
+
with `log_prob` being greater than the given `threshold`.
|
| 147 |
+
"""
|
| 148 |
+
ans = HypothesisList()
|
| 149 |
+
for _, hyp in self._data.items():
|
| 150 |
+
if hyp.log_prob > threshold:
|
| 151 |
+
ans.add(hyp) # shallow copy
|
| 152 |
+
return ans
|
| 153 |
+
|
| 154 |
+
def topk(self, k: int, length_norm: bool = False) -> "HypothesisList":
|
| 155 |
+
"""Return the top-k hypothesis.
|
| 156 |
+
|
| 157 |
+
Args:
|
| 158 |
+
length_norm:
|
| 159 |
+
If True, the `log_prob` of a hypothesis is normalized by the
|
| 160 |
+
number of tokens in it.
|
| 161 |
+
"""
|
| 162 |
+
hyps = list(self._data.items())
|
| 163 |
+
|
| 164 |
+
if length_norm:
|
| 165 |
+
hyps = sorted(
|
| 166 |
+
hyps, key=lambda h: h[1].log_prob / len(h[1].ys), reverse=True
|
| 167 |
+
)[:k]
|
| 168 |
+
else:
|
| 169 |
+
hyps = sorted(hyps, key=lambda h: h[1].log_prob, reverse=True)[:k]
|
| 170 |
+
|
| 171 |
+
ans = HypothesisList(dict(hyps))
|
| 172 |
+
return ans
|
| 173 |
+
|
| 174 |
+
def __contains__(self, key: str):
|
| 175 |
+
return key in self._data
|
| 176 |
+
|
| 177 |
+
def __iter__(self):
|
| 178 |
+
return iter(self._data.values())
|
| 179 |
+
|
| 180 |
+
def __len__(self) -> int:
|
| 181 |
+
return len(self._data)
|
| 182 |
+
|
| 183 |
+
def __str__(self) -> str:
|
| 184 |
+
s = []
|
| 185 |
+
for key in self:
|
| 186 |
+
s.append(key)
|
| 187 |
+
return ", ".join(s)
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def get_hyps_shape(hyps: List[HypothesisList]) -> k2.RaggedShape:
|
| 191 |
+
"""Return a ragged shape with axes [utt][num_hyps].
|
| 192 |
+
|
| 193 |
+
Args:
|
| 194 |
+
hyps:
|
| 195 |
+
len(hyps) == batch_size. It contains the current hypothesis for
|
| 196 |
+
each utterance in the batch.
|
| 197 |
+
Returns:
|
| 198 |
+
Return a ragged shape with 2 axes [utt][num_hyps]. Note that
|
| 199 |
+
the shape is on CPU.
|
| 200 |
+
"""
|
| 201 |
+
num_hyps = [len(h) for h in hyps]
|
| 202 |
+
|
| 203 |
+
# torch.cumsum() is inclusive sum, so we put a 0 at the beginning
|
| 204 |
+
# to get exclusive sum later.
|
| 205 |
+
num_hyps.insert(0, 0)
|
| 206 |
+
|
| 207 |
+
num_hyps = torch.tensor(num_hyps)
|
| 208 |
+
row_splits = torch.cumsum(num_hyps, dim=0, dtype=torch.int32)
|
| 209 |
+
ans = k2.ragged.create_ragged_shape2(
|
| 210 |
+
row_splits=row_splits, cached_tot_size=row_splits[-1].item()
|
| 211 |
+
)
|
| 212 |
+
return ans
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def modified_beam_search(
|
| 217 |
+
model: nn.Module,
|
| 218 |
+
encoder_out: torch.Tensor,
|
| 219 |
+
encoder_out_lens: torch.Tensor,
|
| 220 |
+
context_graph: Optional[ContextGraph] = None,
|
| 221 |
+
beam: int = 4,
|
| 222 |
+
temperature: float = 1.0,
|
| 223 |
+
blank_penalty: float = 0.0,
|
| 224 |
+
return_timestamps: bool = False,
|
| 225 |
+
) -> Union[List[List[int]], DecodingResults]:
|
| 226 |
+
"""Beam search in batch mode with --max-sym-per-frame=1 being hardcoded.
|
| 227 |
+
|
| 228 |
+
Args:
|
| 229 |
+
model:
|
| 230 |
+
The transducer model.
|
| 231 |
+
encoder_out:
|
| 232 |
+
Output from the encoder. Its shape is (N, T, C).
|
| 233 |
+
encoder_out_lens:
|
| 234 |
+
A 1-D tensor of shape (N,), containing number of valid frames in
|
| 235 |
+
encoder_out before padding.
|
| 236 |
+
beam:
|
| 237 |
+
Number of active paths during the beam search.
|
| 238 |
+
temperature:
|
| 239 |
+
Softmax temperature.
|
| 240 |
+
return_timestamps:
|
| 241 |
+
Whether to return timestamps.
|
| 242 |
+
Returns:
|
| 243 |
+
If return_timestamps is False, return the decoded result.
|
| 244 |
+
Else, return a DecodingResults object containing
|
| 245 |
+
decoded result and corresponding timestamps.
|
| 246 |
+
"""
|
| 247 |
+
assert encoder_out.ndim == 3, encoder_out.shape
|
| 248 |
+
assert encoder_out.size(0) >= 1, encoder_out.size(0)
|
| 249 |
+
|
| 250 |
+
packed_encoder_out = torch.nn.utils.rnn.pack_padded_sequence(
|
| 251 |
+
input=encoder_out,
|
| 252 |
+
lengths=encoder_out_lens.cpu(),
|
| 253 |
+
batch_first=True,
|
| 254 |
+
enforce_sorted=False,
|
| 255 |
+
)
|
| 256 |
+
|
| 257 |
+
blank_id = model.decoder.blank_id
|
| 258 |
+
unk_id = getattr(model, "unk_id", blank_id)
|
| 259 |
+
context_size = model.decoder.context_size
|
| 260 |
+
device = next(model.parameters()).device
|
| 261 |
+
|
| 262 |
+
batch_size_list = packed_encoder_out.batch_sizes.tolist()
|
| 263 |
+
N = encoder_out.size(0)
|
| 264 |
+
assert torch.all(encoder_out_lens > 0), encoder_out_lens
|
| 265 |
+
assert N == batch_size_list[0], (N, batch_size_list)
|
| 266 |
+
|
| 267 |
+
B = [HypothesisList() for _ in range(N)]
|
| 268 |
+
for i in range(N):
|
| 269 |
+
B[i].add(
|
| 270 |
+
Hypothesis(
|
| 271 |
+
ys=[-1] * (context_size - 1) + [blank_id],
|
| 272 |
+
log_prob=torch.zeros(1, dtype=torch.float32, device=device),
|
| 273 |
+
context_state=None if context_graph is None else context_graph.root,
|
| 274 |
+
timestamp=[],
|
| 275 |
+
)
|
| 276 |
+
)
|
| 277 |
+
|
| 278 |
+
encoder_out = model.joiner.encoder_proj(packed_encoder_out.data)
|
| 279 |
+
|
| 280 |
+
offset = 0
|
| 281 |
+
finalized_B = []
|
| 282 |
+
for t, batch_size in enumerate(batch_size_list):
|
| 283 |
+
start = offset
|
| 284 |
+
end = offset + batch_size
|
| 285 |
+
current_encoder_out = encoder_out.data[start:end]
|
| 286 |
+
current_encoder_out = current_encoder_out.unsqueeze(1).unsqueeze(1)
|
| 287 |
+
# current_encoder_out's shape is (batch_size, 1, 1, encoder_out_dim)
|
| 288 |
+
offset = end
|
| 289 |
+
|
| 290 |
+
finalized_B = B[batch_size:] + finalized_B
|
| 291 |
+
B = B[:batch_size]
|
| 292 |
+
|
| 293 |
+
hyps_shape = get_hyps_shape(B).to(device)
|
| 294 |
+
|
| 295 |
+
A = [list(b) for b in B]
|
| 296 |
+
|
| 297 |
+
B = [HypothesisList() for _ in range(batch_size)]
|
| 298 |
+
|
| 299 |
+
ys_log_probs = torch.cat(
|
| 300 |
+
[hyp.log_prob.reshape(1, 1) for hyps in A for hyp in hyps]
|
| 301 |
+
) # (num_hyps, 1)
|
| 302 |
+
|
| 303 |
+
decoder_input = torch.tensor(
|
| 304 |
+
[hyp.ys[-context_size:] for hyps in A for hyp in hyps],
|
| 305 |
+
device=device,
|
| 306 |
+
dtype=torch.int64,
|
| 307 |
+
) # (num_hyps, context_size)
|
| 308 |
+
|
| 309 |
+
decoder_out = model.decoder(decoder_input, need_pad=False).unsqueeze(1)
|
| 310 |
+
decoder_out = model.joiner.decoder_proj(decoder_out)
|
| 311 |
+
# decoder_out is of shape (num_hyps, 1, 1, joiner_dim)
|
| 312 |
+
|
| 313 |
+
# Note: For torch 1.7.1 and below, it requires a torch.int64 tensor
|
| 314 |
+
# as index, so we use `to(torch.int64)` below.
|
| 315 |
+
current_encoder_out = torch.index_select(
|
| 316 |
+
current_encoder_out,
|
| 317 |
+
dim=0,
|
| 318 |
+
index=hyps_shape.row_ids(1).to(torch.int64),
|
| 319 |
+
) # (num_hyps, 1, 1, encoder_out_dim)
|
| 320 |
+
|
| 321 |
+
logits = model.joiner(
|
| 322 |
+
current_encoder_out,
|
| 323 |
+
decoder_out,
|
| 324 |
+
project_input=False,
|
| 325 |
+
) # (num_hyps, 1, 1, vocab_size)
|
| 326 |
+
|
| 327 |
+
logits = logits.squeeze(1).squeeze(1) # (num_hyps, vocab_size)
|
| 328 |
+
|
| 329 |
+
if blank_penalty != 0:
|
| 330 |
+
logits[:, 0] -= blank_penalty
|
| 331 |
+
|
| 332 |
+
log_probs = (logits / temperature).log_softmax(dim=-1) # (num_hyps, vocab_size)
|
| 333 |
+
|
| 334 |
+
log_probs.add_(ys_log_probs)
|
| 335 |
+
|
| 336 |
+
vocab_size = log_probs.size(-1)
|
| 337 |
+
|
| 338 |
+
log_probs = log_probs.reshape(-1)
|
| 339 |
+
|
| 340 |
+
row_splits = hyps_shape.row_splits(1) * vocab_size
|
| 341 |
+
log_probs_shape = k2.ragged.create_ragged_shape2(
|
| 342 |
+
row_splits=row_splits, cached_tot_size=log_probs.numel()
|
| 343 |
+
)
|
| 344 |
+
ragged_log_probs = k2.RaggedTensor(shape=log_probs_shape, value=log_probs)
|
| 345 |
+
|
| 346 |
+
for i in range(batch_size):
|
| 347 |
+
topk_log_probs, topk_indexes = ragged_log_probs[i].topk(beam)
|
| 348 |
+
|
| 349 |
+
with warnings.catch_warnings():
|
| 350 |
+
warnings.simplefilter("ignore")
|
| 351 |
+
topk_hyp_indexes = (topk_indexes // vocab_size).tolist()
|
| 352 |
+
topk_token_indexes = (topk_indexes % vocab_size).tolist()
|
| 353 |
+
|
| 354 |
+
for k in range(len(topk_hyp_indexes)):
|
| 355 |
+
hyp_idx = topk_hyp_indexes[k]
|
| 356 |
+
hyp = A[i][hyp_idx]
|
| 357 |
+
new_ys = hyp.ys[:]
|
| 358 |
+
new_token = topk_token_indexes[k]
|
| 359 |
+
new_timestamp = hyp.timestamp[:]
|
| 360 |
+
context_score = 0
|
| 361 |
+
new_context_state = None if context_graph is None else hyp.context_state
|
| 362 |
+
if new_token not in (blank_id, unk_id):
|
| 363 |
+
new_ys.append(new_token)
|
| 364 |
+
new_timestamp.append(t)
|
| 365 |
+
if context_graph is not None:
|
| 366 |
+
(
|
| 367 |
+
context_score,
|
| 368 |
+
new_context_state,
|
| 369 |
+
_,
|
| 370 |
+
) = context_graph.forward_one_step(
|
| 371 |
+
hyp.context_state, new_token, strict_mode=False
|
| 372 |
+
)
|
| 373 |
+
|
| 374 |
+
new_log_prob = topk_log_probs[k] + context_score
|
| 375 |
+
|
| 376 |
+
new_hyp = Hypothesis(
|
| 377 |
+
ys=new_ys,
|
| 378 |
+
log_prob=new_log_prob,
|
| 379 |
+
timestamp=new_timestamp,
|
| 380 |
+
context_state=new_context_state,
|
| 381 |
+
)
|
| 382 |
+
B[i].add(new_hyp)
|
| 383 |
+
|
| 384 |
+
B = B + finalized_B
|
| 385 |
+
|
| 386 |
+
# finalize context_state, if the matched contexts do not reach final state
|
| 387 |
+
# we need to add the score on the corresponding backoff arc
|
| 388 |
+
if context_graph is not None:
|
| 389 |
+
finalized_B = [HypothesisList() for _ in range(len(B))]
|
| 390 |
+
for i, hyps in enumerate(B):
|
| 391 |
+
for hyp in list(hyps):
|
| 392 |
+
context_score, new_context_state = context_graph.finalize(
|
| 393 |
+
hyp.context_state
|
| 394 |
+
)
|
| 395 |
+
finalized_B[i].add(
|
| 396 |
+
Hypothesis(
|
| 397 |
+
ys=hyp.ys,
|
| 398 |
+
log_prob=hyp.log_prob + context_score,
|
| 399 |
+
timestamp=hyp.timestamp,
|
| 400 |
+
context_state=new_context_state,
|
| 401 |
+
)
|
| 402 |
+
)
|
| 403 |
+
B = finalized_B
|
| 404 |
+
|
| 405 |
+
best_hyps = [b.get_most_probable(length_norm=True) for b in B]
|
| 406 |
+
|
| 407 |
+
sorted_ans = [h.ys[context_size:] for h in best_hyps]
|
| 408 |
+
sorted_timestamps = [h.timestamp for h in best_hyps]
|
| 409 |
+
ans = []
|
| 410 |
+
ans_timestamps = []
|
| 411 |
+
unsorted_indices = packed_encoder_out.unsorted_indices.tolist()
|
| 412 |
+
for i in range(N):
|
| 413 |
+
ans.append(sorted_ans[unsorted_indices[i]])
|
| 414 |
+
ans_timestamps.append(sorted_timestamps[unsorted_indices[i]])
|
| 415 |
+
|
| 416 |
+
if not return_timestamps:
|
| 417 |
+
return ans
|
| 418 |
+
else:
|
| 419 |
+
return DecodingResults(
|
| 420 |
+
hyps=ans,
|
| 421 |
+
timestamps=ans_timestamps,
|
| 422 |
+
)
|
| 423 |
+
|
speedasr_core/decoder.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang)
|
| 2 |
+
#
|
| 3 |
+
# See ../../../../LICENSE for clarification regarding multiple authors
|
| 4 |
+
#
|
| 5 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 6 |
+
# you may not use this file except in compliance with the License.
|
| 7 |
+
# You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 12 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 13 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 14 |
+
# See the License for the specific language governing permissions and
|
| 15 |
+
# limitations under the License.
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
import torch.nn as nn
|
| 19 |
+
import torch.nn.functional as F
|
| 20 |
+
from .scaling import Balancer
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class Decoder(nn.Module):
|
| 24 |
+
"""This class modifies the stateless decoder from the following paper:
|
| 25 |
+
|
| 26 |
+
RNN-transducer with stateless prediction network
|
| 27 |
+
https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=9054419
|
| 28 |
+
|
| 29 |
+
It removes the recurrent connection from the decoder, i.e., the prediction
|
| 30 |
+
network. Different from the above paper, it adds an extra Conv1d
|
| 31 |
+
right after the embedding layer.
|
| 32 |
+
|
| 33 |
+
TODO: Implement https://arxiv.org/pdf/2109.07513.pdf
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
def __init__(
|
| 37 |
+
self,
|
| 38 |
+
vocab_size: int,
|
| 39 |
+
decoder_dim: int,
|
| 40 |
+
blank_id: int,
|
| 41 |
+
context_size: int,
|
| 42 |
+
):
|
| 43 |
+
"""
|
| 44 |
+
Args:
|
| 45 |
+
vocab_size:
|
| 46 |
+
Number of tokens of the modeling unit including blank.
|
| 47 |
+
decoder_dim:
|
| 48 |
+
Dimension of the input embedding, and of the decoder output.
|
| 49 |
+
blank_id:
|
| 50 |
+
The ID of the blank symbol.
|
| 51 |
+
context_size:
|
| 52 |
+
Number of previous words to use to predict the next word.
|
| 53 |
+
1 means bigram; 2 means trigram. n means (n+1)-gram.
|
| 54 |
+
"""
|
| 55 |
+
super().__init__()
|
| 56 |
+
|
| 57 |
+
self.embedding = nn.Embedding(
|
| 58 |
+
num_embeddings=vocab_size,
|
| 59 |
+
embedding_dim=decoder_dim,
|
| 60 |
+
)
|
| 61 |
+
# the balancers are to avoid any drift in the magnitude of the
|
| 62 |
+
# embeddings, which would interact badly with parameter averaging.
|
| 63 |
+
self.balancer = Balancer(
|
| 64 |
+
decoder_dim,
|
| 65 |
+
channel_dim=-1,
|
| 66 |
+
min_positive=0.0,
|
| 67 |
+
max_positive=1.0,
|
| 68 |
+
min_abs=0.5,
|
| 69 |
+
max_abs=1.0,
|
| 70 |
+
prob=0.05,
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
self.blank_id = blank_id
|
| 74 |
+
|
| 75 |
+
assert context_size >= 1, context_size
|
| 76 |
+
self.context_size = context_size
|
| 77 |
+
self.vocab_size = vocab_size
|
| 78 |
+
|
| 79 |
+
if context_size > 1:
|
| 80 |
+
self.conv = nn.Conv1d(
|
| 81 |
+
in_channels=decoder_dim,
|
| 82 |
+
out_channels=decoder_dim,
|
| 83 |
+
kernel_size=context_size,
|
| 84 |
+
padding=0,
|
| 85 |
+
groups=decoder_dim // 4, # group size == 4
|
| 86 |
+
bias=False,
|
| 87 |
+
)
|
| 88 |
+
self.balancer2 = Balancer(
|
| 89 |
+
decoder_dim,
|
| 90 |
+
channel_dim=-1,
|
| 91 |
+
min_positive=0.0,
|
| 92 |
+
max_positive=1.0,
|
| 93 |
+
min_abs=0.5,
|
| 94 |
+
max_abs=1.0,
|
| 95 |
+
prob=0.05,
|
| 96 |
+
)
|
| 97 |
+
else:
|
| 98 |
+
# To avoid `RuntimeError: Module 'Decoder' has no attribute 'conv'`
|
| 99 |
+
# when inference with torch.jit.script and context_size == 1
|
| 100 |
+
self.conv = nn.Identity()
|
| 101 |
+
self.balancer2 = nn.Identity()
|
| 102 |
+
|
| 103 |
+
def forward(self, y: torch.Tensor, need_pad: bool = True) -> torch.Tensor:
|
| 104 |
+
"""
|
| 105 |
+
Args:
|
| 106 |
+
y:
|
| 107 |
+
A 2-D tensor of shape (N, U).
|
| 108 |
+
need_pad:
|
| 109 |
+
True to left pad the input. Should be True during training.
|
| 110 |
+
False to not pad the input. Should be False during inference.
|
| 111 |
+
Returns:
|
| 112 |
+
Return a tensor of shape (N, U, decoder_dim).
|
| 113 |
+
"""
|
| 114 |
+
y = y.to(torch.int64)
|
| 115 |
+
# this stuff about clamp() is a temporary fix for a mismatch
|
| 116 |
+
# at utterance start, we use negative ids in beam_search.py
|
| 117 |
+
embedding_out = self.embedding(y.clamp(min=0)) * (y >= 0).unsqueeze(-1)
|
| 118 |
+
|
| 119 |
+
embedding_out = self.balancer(embedding_out)
|
| 120 |
+
|
| 121 |
+
if self.context_size > 1:
|
| 122 |
+
embedding_out = embedding_out.permute(0, 2, 1)
|
| 123 |
+
if need_pad is True:
|
| 124 |
+
embedding_out = F.pad(embedding_out, pad=(self.context_size - 1, 0))
|
| 125 |
+
else:
|
| 126 |
+
# During inference time, there is no need to do extra padding
|
| 127 |
+
# as we only need one output
|
| 128 |
+
assert embedding_out.size(-1) == self.context_size
|
| 129 |
+
embedding_out = self.conv(embedding_out)
|
| 130 |
+
embedding_out = embedding_out.permute(0, 2, 1)
|
| 131 |
+
embedding_out = F.relu(embedding_out)
|
| 132 |
+
embedding_out = self.balancer2(embedding_out)
|
| 133 |
+
|
| 134 |
+
return embedding_out
|
speedasr_core/encoder_interface.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang)
|
| 2 |
+
#
|
| 3 |
+
# See ../../../../LICENSE for clarification regarding multiple authors
|
| 4 |
+
#
|
| 5 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 6 |
+
# you may not use this file except in compliance with the License.
|
| 7 |
+
# You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 12 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 13 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 14 |
+
# See the License for the specific language governing permissions and
|
| 15 |
+
# limitations under the License.
|
| 16 |
+
|
| 17 |
+
from typing import Tuple
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
import torch.nn as nn
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class EncoderInterface(nn.Module):
|
| 24 |
+
def forward(
|
| 25 |
+
self, x: torch.Tensor, x_lens: torch.Tensor
|
| 26 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 27 |
+
"""
|
| 28 |
+
Args:
|
| 29 |
+
x:
|
| 30 |
+
A tensor of shape (batch_size, input_seq_len, num_features)
|
| 31 |
+
containing the input features.
|
| 32 |
+
x_lens:
|
| 33 |
+
A tensor of shape (batch_size,) containing the number of frames
|
| 34 |
+
in `x` before padding.
|
| 35 |
+
Returns:
|
| 36 |
+
Return a tuple containing two tensors:
|
| 37 |
+
- encoder_out, a tensor of (batch_size, out_seq_len, output_dim)
|
| 38 |
+
containing unnormalized probabilities, i.e., the output of a
|
| 39 |
+
linear layer.
|
| 40 |
+
- encoder_out_lens, a tensor of shape (batch_size,) containing
|
| 41 |
+
the number of frames in `encoder_out` before padding.
|
| 42 |
+
"""
|
| 43 |
+
raise NotImplementedError("Please implement it in a subclass")
|
speedasr_core/joiner.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang)
|
| 2 |
+
#
|
| 3 |
+
# See ../../../../LICENSE for clarification regarding multiple authors
|
| 4 |
+
#
|
| 5 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 6 |
+
# you may not use this file except in compliance with the License.
|
| 7 |
+
# You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 12 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 13 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 14 |
+
# See the License for the specific language governing permissions and
|
| 15 |
+
# limitations under the License.
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
import torch.nn as nn
|
| 19 |
+
from .scaling import ScaledLinear
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class Joiner(nn.Module):
|
| 23 |
+
def __init__(
|
| 24 |
+
self,
|
| 25 |
+
encoder_dim: int,
|
| 26 |
+
decoder_dim: int,
|
| 27 |
+
joiner_dim: int,
|
| 28 |
+
vocab_size: int,
|
| 29 |
+
):
|
| 30 |
+
super().__init__()
|
| 31 |
+
|
| 32 |
+
self.encoder_proj = ScaledLinear(encoder_dim, joiner_dim, initial_scale=0.25)
|
| 33 |
+
self.decoder_proj = ScaledLinear(decoder_dim, joiner_dim, initial_scale=0.25)
|
| 34 |
+
self.output_linear = nn.Linear(joiner_dim, vocab_size)
|
| 35 |
+
|
| 36 |
+
def forward(
|
| 37 |
+
self,
|
| 38 |
+
encoder_out: torch.Tensor,
|
| 39 |
+
decoder_out: torch.Tensor,
|
| 40 |
+
project_input: bool = True,
|
| 41 |
+
) -> torch.Tensor:
|
| 42 |
+
"""
|
| 43 |
+
Args:
|
| 44 |
+
encoder_out:
|
| 45 |
+
Output from the encoder. Its shape is (N, T, s_range, C).
|
| 46 |
+
decoder_out:
|
| 47 |
+
Output from the decoder. Its shape is (N, T, s_range, C).
|
| 48 |
+
project_input:
|
| 49 |
+
If true, apply input projections encoder_proj and decoder_proj.
|
| 50 |
+
If this is false, it is the user's responsibility to do this
|
| 51 |
+
manually.
|
| 52 |
+
Returns:
|
| 53 |
+
Return a tensor of shape (N, T, s_range, C).
|
| 54 |
+
"""
|
| 55 |
+
assert encoder_out.ndim == decoder_out.ndim, (
|
| 56 |
+
encoder_out.shape,
|
| 57 |
+
decoder_out.shape,
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
if project_input:
|
| 61 |
+
logit = self.encoder_proj(encoder_out) + self.decoder_proj(decoder_out)
|
| 62 |
+
else:
|
| 63 |
+
logit = encoder_out + decoder_out
|
| 64 |
+
|
| 65 |
+
logit = self.output_linear(torch.tanh(logit))
|
| 66 |
+
|
| 67 |
+
return logit
|
speedasr_core/model.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2021-2023 Xiaomi Corp. (authors: Fangjun Kuang,
|
| 2 |
+
# Wei Kang,
|
| 3 |
+
# Zengwei Yao)
|
| 4 |
+
#
|
| 5 |
+
# See ../../../../LICENSE for clarification regarding multiple authors
|
| 6 |
+
#
|
| 7 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 8 |
+
# you may not use this file except in compliance with the License.
|
| 9 |
+
# You may obtain a copy of the License at
|
| 10 |
+
#
|
| 11 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 12 |
+
#
|
| 13 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 14 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 15 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 16 |
+
# See the License for the specific language governing permissions and
|
| 17 |
+
# limitations under the License.
|
| 18 |
+
|
| 19 |
+
from typing import Optional, Tuple
|
| 20 |
+
|
| 21 |
+
import torch
|
| 22 |
+
from torch import nn
|
| 23 |
+
from .encoder_interface import EncoderInterface
|
| 24 |
+
from .scaling import ScaledLinear
|
| 25 |
+
from .utils import make_pad_mask
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class AsrModel(nn.Module):
|
| 29 |
+
def __init__(
|
| 30 |
+
self,
|
| 31 |
+
encoder_embed: nn.Module,
|
| 32 |
+
encoder: EncoderInterface,
|
| 33 |
+
decoder: Optional[nn.Module] = None,
|
| 34 |
+
joiner: Optional[nn.Module] = None,
|
| 35 |
+
attention_decoder: Optional[nn.Module] = None,
|
| 36 |
+
encoder_dim: int = 384,
|
| 37 |
+
decoder_dim: int = 512,
|
| 38 |
+
vocab_size: int = 500,
|
| 39 |
+
use_transducer: bool = True,
|
| 40 |
+
use_ctc: bool = False,
|
| 41 |
+
use_attention_decoder: bool = False,
|
| 42 |
+
):
|
| 43 |
+
"""A joint CTC & Transducer ASR model.
|
| 44 |
+
|
| 45 |
+
- Connectionist temporal classification: labelling unsegmented sequence data with recurrent neural networks (http://imagine.enpc.fr/~obozinsg/teaching/mva_gm/papers/ctc.pdf)
|
| 46 |
+
- Sequence Transduction with Recurrent Neural Networks (https://arxiv.org/pdf/1211.3711.pdf)
|
| 47 |
+
- Pruned RNN-T for fast, memory-efficient ASR training (https://arxiv.org/pdf/2206.13236.pdf)
|
| 48 |
+
|
| 49 |
+
Args:
|
| 50 |
+
encoder_embed:
|
| 51 |
+
It is a Convolutional 2D subsampling module. It converts
|
| 52 |
+
an input of shape (N, T, idim) to an output of of shape
|
| 53 |
+
(N, T', odim), where T' = (T-3)//2-2 = (T-7)//2.
|
| 54 |
+
encoder:
|
| 55 |
+
It is the transcription network in the paper. Its accepts
|
| 56 |
+
two inputs: `x` of (N, T, encoder_dim) and `x_lens` of shape (N,).
|
| 57 |
+
It returns two tensors: `logits` of shape (N, T, encoder_dim) and
|
| 58 |
+
`logit_lens` of shape (N,).
|
| 59 |
+
decoder:
|
| 60 |
+
It is the prediction network in the paper. Its input shape
|
| 61 |
+
is (N, U) and its output shape is (N, U, decoder_dim).
|
| 62 |
+
It should contain one attribute: `blank_id`.
|
| 63 |
+
It is used when use_transducer is True.
|
| 64 |
+
joiner:
|
| 65 |
+
It has two inputs with shapes: (N, T, encoder_dim) and (N, U, decoder_dim).
|
| 66 |
+
Its output shape is (N, T, U, vocab_size). Note that its output contains
|
| 67 |
+
unnormalized probs, i.e., not processed by log-softmax.
|
| 68 |
+
It is used when use_transducer is True.
|
| 69 |
+
use_transducer:
|
| 70 |
+
Whether use transducer head. Default: True.
|
| 71 |
+
use_ctc:
|
| 72 |
+
Whether use CTC head. Default: False.
|
| 73 |
+
use_attention_decoder:
|
| 74 |
+
Whether use attention-decoder head. Default: False.
|
| 75 |
+
"""
|
| 76 |
+
super().__init__()
|
| 77 |
+
|
| 78 |
+
assert (
|
| 79 |
+
use_transducer or use_ctc
|
| 80 |
+
), f"At least one of them should be True, but got use_transducer={use_transducer}, use_ctc={use_ctc}"
|
| 81 |
+
|
| 82 |
+
assert isinstance(encoder, EncoderInterface), type(encoder)
|
| 83 |
+
|
| 84 |
+
self.encoder_embed = encoder_embed
|
| 85 |
+
self.encoder = encoder
|
| 86 |
+
|
| 87 |
+
self.use_transducer = use_transducer
|
| 88 |
+
if use_transducer:
|
| 89 |
+
# Modules for Transducer head
|
| 90 |
+
assert decoder is not None
|
| 91 |
+
assert hasattr(decoder, "blank_id")
|
| 92 |
+
assert joiner is not None
|
| 93 |
+
|
| 94 |
+
self.decoder = decoder
|
| 95 |
+
self.joiner = joiner
|
| 96 |
+
|
| 97 |
+
self.simple_am_proj = ScaledLinear(
|
| 98 |
+
encoder_dim, vocab_size, initial_scale=0.25
|
| 99 |
+
)
|
| 100 |
+
self.simple_lm_proj = ScaledLinear(
|
| 101 |
+
decoder_dim, vocab_size, initial_scale=0.25
|
| 102 |
+
)
|
| 103 |
+
else:
|
| 104 |
+
assert decoder is None
|
| 105 |
+
assert joiner is None
|
| 106 |
+
|
| 107 |
+
self.use_ctc = use_ctc
|
| 108 |
+
if use_ctc:
|
| 109 |
+
# Modules for CTC head
|
| 110 |
+
self.ctc_output = nn.Sequential(
|
| 111 |
+
nn.Dropout(p=0.1),
|
| 112 |
+
nn.Linear(encoder_dim, vocab_size),
|
| 113 |
+
nn.LogSoftmax(dim=-1),
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
self.use_attention_decoder = use_attention_decoder
|
| 117 |
+
if use_attention_decoder:
|
| 118 |
+
self.attention_decoder = attention_decoder
|
| 119 |
+
else:
|
| 120 |
+
assert attention_decoder is None
|
| 121 |
+
|
| 122 |
+
def forward_encoder(
|
| 123 |
+
self, x: torch.Tensor, x_lens: torch.Tensor
|
| 124 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 125 |
+
"""Compute encoder outputs.
|
| 126 |
+
Args:
|
| 127 |
+
x:
|
| 128 |
+
A 3-D tensor of shape (N, T, C).
|
| 129 |
+
x_lens:
|
| 130 |
+
A 1-D tensor of shape (N,). It contains the number of frames in `x`
|
| 131 |
+
before padding.
|
| 132 |
+
|
| 133 |
+
Returns:
|
| 134 |
+
encoder_out:
|
| 135 |
+
Encoder output, of shape (N, T, C).
|
| 136 |
+
encoder_out_lens:
|
| 137 |
+
Encoder output lengths, of shape (N,).
|
| 138 |
+
"""
|
| 139 |
+
# logging.info(f"Memory allocated at entry: {torch.cuda.memory_allocated() // 1000000}M")
|
| 140 |
+
x, x_lens = self.encoder_embed(x, x_lens)
|
| 141 |
+
# logging.info(f"Memory allocated after encoder_embed: {torch.cuda.memory_allocated() // 1000000}M")
|
| 142 |
+
|
| 143 |
+
src_key_padding_mask = make_pad_mask(x_lens)
|
| 144 |
+
x = x.permute(1, 0, 2) # (N, T, C) -> (T, N, C)
|
| 145 |
+
|
| 146 |
+
encoder_out, encoder_out_lens = self.encoder(x, x_lens, src_key_padding_mask)
|
| 147 |
+
|
| 148 |
+
encoder_out = encoder_out.permute(1, 0, 2) # (T, N, C) ->(N, T, C)
|
| 149 |
+
assert torch.all(encoder_out_lens > 0), (x_lens, encoder_out_lens)
|
| 150 |
+
|
| 151 |
+
return encoder_out, encoder_out_lens
|
| 152 |
+
|
speedasr_core/scaling.py
ADDED
|
@@ -0,0 +1,1913 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2022-2023 Xiaomi Corp. (authors: Daniel Povey)
|
| 2 |
+
#
|
| 3 |
+
# See ../../../../LICENSE for clarification regarding multiple authors
|
| 4 |
+
#
|
| 5 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 6 |
+
# you may not use this file except in compliance with the License.
|
| 7 |
+
# You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 12 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 13 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 14 |
+
# See the License for the specific language governing permissions and
|
| 15 |
+
# limitations under the License.
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
import logging
|
| 19 |
+
import math
|
| 20 |
+
import random
|
| 21 |
+
from typing import Optional, Tuple, Union
|
| 22 |
+
|
| 23 |
+
import k2
|
| 24 |
+
import torch
|
| 25 |
+
import torch.nn as nn
|
| 26 |
+
from torch import Tensor
|
| 27 |
+
from torch.cuda.amp import custom_bwd, custom_fwd
|
| 28 |
+
|
| 29 |
+
from .utils import torch_autocast
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def logaddexp_onnx(x: Tensor, y: Tensor) -> Tensor:
|
| 33 |
+
max_value = torch.max(x, y)
|
| 34 |
+
diff = torch.abs(x - y)
|
| 35 |
+
return max_value + torch.log1p(torch.exp(-diff))
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
# RuntimeError: Exporting the operator logaddexp to ONNX opset version
|
| 39 |
+
# 14 is not supported. Please feel free to request support or submit
|
| 40 |
+
# a pull request on PyTorch GitHub.
|
| 41 |
+
#
|
| 42 |
+
# The following function is to solve the above error when exporting
|
| 43 |
+
# models to ONNX via torch.jit.trace()
|
| 44 |
+
def logaddexp(x: Tensor, y: Tensor) -> Tensor:
|
| 45 |
+
# Caution(fangjun): Put torch.jit.is_scripting() before
|
| 46 |
+
# torch.onnx.is_in_onnx_export();
|
| 47 |
+
# otherwise, it will cause errors for torch.jit.script().
|
| 48 |
+
#
|
| 49 |
+
# torch.logaddexp() works for both torch.jit.script() and
|
| 50 |
+
# torch.jit.trace() but it causes errors for ONNX export.
|
| 51 |
+
#
|
| 52 |
+
if torch.jit.is_scripting():
|
| 53 |
+
# Note: We cannot use torch.jit.is_tracing() here as it also
|
| 54 |
+
# matches torch.onnx.export().
|
| 55 |
+
return torch.logaddexp(x, y)
|
| 56 |
+
elif torch.onnx.is_in_onnx_export():
|
| 57 |
+
return logaddexp_onnx(x, y)
|
| 58 |
+
else:
|
| 59 |
+
# for torch.jit.trace()
|
| 60 |
+
return torch.logaddexp(x, y)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class PiecewiseLinear(object):
|
| 64 |
+
"""
|
| 65 |
+
Piecewise linear function, from float to float, specified as nonempty list of (x,y) pairs with
|
| 66 |
+
the x values in order. x values <[initial x] or >[final x] are map to [initial y], [final y]
|
| 67 |
+
respectively.
|
| 68 |
+
"""
|
| 69 |
+
|
| 70 |
+
def __init__(self, *args):
|
| 71 |
+
assert len(args) >= 1, len(args)
|
| 72 |
+
if len(args) == 1 and isinstance(args[0], PiecewiseLinear):
|
| 73 |
+
self.pairs = list(args[0].pairs)
|
| 74 |
+
else:
|
| 75 |
+
self.pairs = [(float(x), float(y)) for x, y in args]
|
| 76 |
+
for x, y in self.pairs:
|
| 77 |
+
assert isinstance(x, (float, int)), type(x)
|
| 78 |
+
assert isinstance(y, (float, int)), type(y)
|
| 79 |
+
|
| 80 |
+
for i in range(len(self.pairs) - 1):
|
| 81 |
+
assert self.pairs[i + 1][0] > self.pairs[i][0], (
|
| 82 |
+
i,
|
| 83 |
+
self.pairs[i],
|
| 84 |
+
self.pairs[i + 1],
|
| 85 |
+
)
|
| 86 |
+
|
| 87 |
+
def __str__(self):
|
| 88 |
+
# e.g. 'PiecewiseLinear((0., 10.), (100., 0.))'
|
| 89 |
+
return f"PiecewiseLinear({str(self.pairs)[1:-1]})"
|
| 90 |
+
|
| 91 |
+
def __call__(self, x):
|
| 92 |
+
if x <= self.pairs[0][0]:
|
| 93 |
+
return self.pairs[0][1]
|
| 94 |
+
elif x >= self.pairs[-1][0]:
|
| 95 |
+
return self.pairs[-1][1]
|
| 96 |
+
else:
|
| 97 |
+
cur_x, cur_y = self.pairs[0]
|
| 98 |
+
for i in range(1, len(self.pairs)):
|
| 99 |
+
next_x, next_y = self.pairs[i]
|
| 100 |
+
if x >= cur_x and x <= next_x:
|
| 101 |
+
return cur_y + (next_y - cur_y) * (x - cur_x) / (next_x - cur_x)
|
| 102 |
+
cur_x, cur_y = next_x, next_y
|
| 103 |
+
assert False
|
| 104 |
+
|
| 105 |
+
def __mul__(self, alpha):
|
| 106 |
+
return PiecewiseLinear(*[(x, y * alpha) for x, y in self.pairs])
|
| 107 |
+
|
| 108 |
+
def __add__(self, x):
|
| 109 |
+
if isinstance(x, (float, int)):
|
| 110 |
+
return PiecewiseLinear(*[(p[0], p[1] + x) for p in self.pairs])
|
| 111 |
+
s, x = self.get_common_basis(x)
|
| 112 |
+
return PiecewiseLinear(
|
| 113 |
+
*[(sp[0], sp[1] + xp[1]) for sp, xp in zip(s.pairs, x.pairs)]
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
def max(self, x):
|
| 117 |
+
if isinstance(x, (float, int)):
|
| 118 |
+
x = PiecewiseLinear((0, x))
|
| 119 |
+
s, x = self.get_common_basis(x, include_crossings=True)
|
| 120 |
+
return PiecewiseLinear(
|
| 121 |
+
*[(sp[0], max(sp[1], xp[1])) for sp, xp in zip(s.pairs, x.pairs)]
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
def min(self, x):
|
| 125 |
+
if isinstance(x, float) or isinstance(x, int):
|
| 126 |
+
x = PiecewiseLinear((0, x))
|
| 127 |
+
s, x = self.get_common_basis(x, include_crossings=True)
|
| 128 |
+
return PiecewiseLinear(
|
| 129 |
+
*[(sp[0], min(sp[1], xp[1])) for sp, xp in zip(s.pairs, x.pairs)]
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
def __eq__(self, other):
|
| 133 |
+
return self.pairs == other.pairs
|
| 134 |
+
|
| 135 |
+
def get_common_basis(self, p: "PiecewiseLinear", include_crossings: bool = False):
|
| 136 |
+
"""
|
| 137 |
+
Returns (self_mod, p_mod) which are equivalent piecewise linear
|
| 138 |
+
functions to self and p, but with the same x values.
|
| 139 |
+
|
| 140 |
+
p: the other piecewise linear function
|
| 141 |
+
include_crossings: if true, include in the x values positions
|
| 142 |
+
where the functions indicate by this and p cross.
|
| 143 |
+
"""
|
| 144 |
+
assert isinstance(p, PiecewiseLinear), type(p)
|
| 145 |
+
|
| 146 |
+
# get sorted x-values without repetition.
|
| 147 |
+
x_vals = sorted(set([x for x, _ in self.pairs] + [x for x, _ in p.pairs]))
|
| 148 |
+
y_vals1 = [self(x) for x in x_vals]
|
| 149 |
+
y_vals2 = [p(x) for x in x_vals]
|
| 150 |
+
|
| 151 |
+
if include_crossings:
|
| 152 |
+
extra_x_vals = []
|
| 153 |
+
for i in range(len(x_vals) - 1):
|
| 154 |
+
if (y_vals1[i] > y_vals2[i]) != (y_vals1[i + 1] > y_vals2[i + 1]):
|
| 155 |
+
# if the two lines in this subsegment potentially cross each other..
|
| 156 |
+
diff_cur = abs(y_vals1[i] - y_vals2[i])
|
| 157 |
+
diff_next = abs(y_vals1[i + 1] - y_vals2[i + 1])
|
| 158 |
+
# `pos`, between 0 and 1, gives the relative x position,
|
| 159 |
+
# with 0 being x_vals[i] and 1 being x_vals[i+1].
|
| 160 |
+
pos = diff_cur / (diff_cur + diff_next)
|
| 161 |
+
extra_x_val = x_vals[i] + pos * (x_vals[i + 1] - x_vals[i])
|
| 162 |
+
extra_x_vals.append(extra_x_val)
|
| 163 |
+
if len(extra_x_vals) > 0:
|
| 164 |
+
x_vals = sorted(set(x_vals + extra_x_vals))
|
| 165 |
+
|
| 166 |
+
y_vals1 = [self(x) for x in x_vals]
|
| 167 |
+
y_vals2 = [p(x) for x in x_vals]
|
| 168 |
+
|
| 169 |
+
return (
|
| 170 |
+
PiecewiseLinear(*zip(x_vals, y_vals1)),
|
| 171 |
+
PiecewiseLinear(*zip(x_vals, y_vals2)),
|
| 172 |
+
)
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
class ScheduledFloat(torch.nn.Module):
|
| 176 |
+
"""
|
| 177 |
+
This object is a torch.nn.Module only because we want it to show up in [top_level module].modules();
|
| 178 |
+
it does not have a working forward() function. You are supposed to cast it to float, as
|
| 179 |
+
in, float(parent_module.whatever), and use it as something like a dropout prob.
|
| 180 |
+
|
| 181 |
+
It is a floating point value whose value changes depending on the batch count of the
|
| 182 |
+
training loop. It is a piecewise linear function where you specify the (x,y) pairs
|
| 183 |
+
in sorted order on x; x corresponds to the batch index. For batch-index values before the
|
| 184 |
+
first x or after the last x, we just use the first or last y value.
|
| 185 |
+
|
| 186 |
+
Example:
|
| 187 |
+
self.dropout = ScheduledFloat((0.0, 0.2), (4000.0, 0.0), default=0.0)
|
| 188 |
+
|
| 189 |
+
`default` is used when self.batch_count is not set or not in training mode or in
|
| 190 |
+
torch.jit scripting mode.
|
| 191 |
+
"""
|
| 192 |
+
|
| 193 |
+
def __init__(self, *args, default: float = 0.0):
|
| 194 |
+
super().__init__()
|
| 195 |
+
# self.batch_count and self.name will be written to in the training loop.
|
| 196 |
+
self.batch_count = None
|
| 197 |
+
self.name = None
|
| 198 |
+
self.default = default
|
| 199 |
+
self.schedule = PiecewiseLinear(*args)
|
| 200 |
+
|
| 201 |
+
def extra_repr(self) -> str:
|
| 202 |
+
return (
|
| 203 |
+
f"batch_count={self.batch_count}, schedule={str(self.schedule.pairs[1:-1])}"
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
def __float__(self):
|
| 207 |
+
batch_count = self.batch_count
|
| 208 |
+
if (
|
| 209 |
+
batch_count is None
|
| 210 |
+
or not self.training
|
| 211 |
+
or torch.jit.is_scripting()
|
| 212 |
+
or torch.jit.is_tracing()
|
| 213 |
+
):
|
| 214 |
+
return float(self.default)
|
| 215 |
+
else:
|
| 216 |
+
ans = self.schedule(self.batch_count)
|
| 217 |
+
if random.random() < 0.0002:
|
| 218 |
+
logging.info(
|
| 219 |
+
f"ScheduledFloat: name={self.name}, batch_count={self.batch_count}, ans={ans}"
|
| 220 |
+
)
|
| 221 |
+
return ans
|
| 222 |
+
|
| 223 |
+
def __add__(self, x):
|
| 224 |
+
if isinstance(x, float) or isinstance(x, int):
|
| 225 |
+
return ScheduledFloat(self.schedule + x, default=self.default)
|
| 226 |
+
else:
|
| 227 |
+
return ScheduledFloat(
|
| 228 |
+
self.schedule + x.schedule, default=self.default + x.default
|
| 229 |
+
)
|
| 230 |
+
|
| 231 |
+
def max(self, x):
|
| 232 |
+
if isinstance(x, float) or isinstance(x, int):
|
| 233 |
+
return ScheduledFloat(self.schedule.max(x), default=self.default)
|
| 234 |
+
else:
|
| 235 |
+
return ScheduledFloat(
|
| 236 |
+
self.schedule.max(x.schedule), default=max(self.default, x.default)
|
| 237 |
+
)
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
FloatLike = Union[float, ScheduledFloat]
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def random_cast_to_half(x: Tensor, min_abs: float = 5.0e-06) -> Tensor:
|
| 244 |
+
"""
|
| 245 |
+
A randomized way of casting a floating point value to half precision.
|
| 246 |
+
"""
|
| 247 |
+
if x.dtype == torch.float16:
|
| 248 |
+
return x
|
| 249 |
+
x_abs = x.abs()
|
| 250 |
+
is_too_small = x_abs < min_abs
|
| 251 |
+
# for elements where is_too_small is true, random_val will contain +-min_abs with
|
| 252 |
+
# probability (x.abs() / min_abs), and 0.0 otherwise. [so this preserves expectations,
|
| 253 |
+
# for those elements].
|
| 254 |
+
random_val = min_abs * x.sign() * (torch.rand_like(x) * min_abs < x_abs)
|
| 255 |
+
return torch.where(is_too_small, random_val, x).to(torch.float16)
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
class CutoffEstimator:
|
| 259 |
+
"""
|
| 260 |
+
Estimates cutoffs of an arbitrary numerical quantity such that a specified
|
| 261 |
+
proportion of items will be above the cutoff on average.
|
| 262 |
+
|
| 263 |
+
p is the proportion of items that should be above the cutoff.
|
| 264 |
+
"""
|
| 265 |
+
|
| 266 |
+
def __init__(self, p: float):
|
| 267 |
+
self.p = p
|
| 268 |
+
# total count of items
|
| 269 |
+
self.count = 0
|
| 270 |
+
# total count of items that were above the cutoff
|
| 271 |
+
self.count_above = 0
|
| 272 |
+
# initial cutoff value
|
| 273 |
+
self.cutoff = 0
|
| 274 |
+
|
| 275 |
+
def __call__(self, x: float) -> bool:
|
| 276 |
+
"""
|
| 277 |
+
Returns true if x is above the cutoff.
|
| 278 |
+
"""
|
| 279 |
+
ans = x > self.cutoff
|
| 280 |
+
self.count += 1
|
| 281 |
+
if ans:
|
| 282 |
+
self.count_above += 1
|
| 283 |
+
cur_p = self.count_above / self.count
|
| 284 |
+
delta_p = cur_p - self.p
|
| 285 |
+
if (delta_p > 0) == ans:
|
| 286 |
+
q = abs(delta_p)
|
| 287 |
+
self.cutoff = x * q + self.cutoff * (1 - q)
|
| 288 |
+
return ans
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
class SoftmaxFunction(torch.autograd.Function):
|
| 292 |
+
"""
|
| 293 |
+
Tries to handle half-precision derivatives in a randomized way that should
|
| 294 |
+
be more accurate for training than the default behavior.
|
| 295 |
+
"""
|
| 296 |
+
|
| 297 |
+
@staticmethod
|
| 298 |
+
def forward(ctx, x: Tensor, dim: int):
|
| 299 |
+
ans = x.softmax(dim=dim)
|
| 300 |
+
# if x dtype is float16, x.softmax() returns a float32 because
|
| 301 |
+
# (presumably) that op does not support float16, and autocast
|
| 302 |
+
# is enabled.
|
| 303 |
+
if torch.is_autocast_enabled():
|
| 304 |
+
ans = ans.to(torch.get_autocast_gpu_dtype())
|
| 305 |
+
ctx.save_for_backward(ans)
|
| 306 |
+
ctx.x_dtype = x.dtype
|
| 307 |
+
ctx.dim = dim
|
| 308 |
+
return ans
|
| 309 |
+
|
| 310 |
+
@staticmethod
|
| 311 |
+
def backward(ctx, ans_grad: Tensor):
|
| 312 |
+
(ans,) = ctx.saved_tensors
|
| 313 |
+
with torch_autocast(enabled=False):
|
| 314 |
+
ans_grad = ans_grad.to(torch.float32)
|
| 315 |
+
ans = ans.to(torch.float32)
|
| 316 |
+
x_grad = ans_grad * ans
|
| 317 |
+
x_grad = x_grad - ans * x_grad.sum(dim=ctx.dim, keepdim=True)
|
| 318 |
+
return x_grad, None
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def softmax(x: Tensor, dim: int):
|
| 322 |
+
if not x.requires_grad or torch.jit.is_scripting() or torch.jit.is_tracing():
|
| 323 |
+
return x.softmax(dim=dim)
|
| 324 |
+
|
| 325 |
+
return SoftmaxFunction.apply(x, dim)
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
class MaxEigLimiterFunction(torch.autograd.Function):
|
| 329 |
+
@staticmethod
|
| 330 |
+
def forward(
|
| 331 |
+
ctx,
|
| 332 |
+
x: Tensor,
|
| 333 |
+
coeffs: Tensor,
|
| 334 |
+
direction: Tensor,
|
| 335 |
+
channel_dim: int,
|
| 336 |
+
grad_scale: float,
|
| 337 |
+
) -> Tensor:
|
| 338 |
+
ctx.channel_dim = channel_dim
|
| 339 |
+
ctx.grad_scale = grad_scale
|
| 340 |
+
ctx.save_for_backward(x.detach(), coeffs.detach(), direction.detach())
|
| 341 |
+
return x
|
| 342 |
+
|
| 343 |
+
@staticmethod
|
| 344 |
+
def backward(ctx, x_grad, *args):
|
| 345 |
+
with torch.enable_grad():
|
| 346 |
+
(x_orig, coeffs, new_direction) = ctx.saved_tensors
|
| 347 |
+
x_orig.requires_grad = True
|
| 348 |
+
num_channels = x_orig.shape[ctx.channel_dim]
|
| 349 |
+
x = x_orig.transpose(ctx.channel_dim, -1).reshape(-1, num_channels)
|
| 350 |
+
new_direction.requires_grad = False
|
| 351 |
+
x = x - x.mean(dim=0)
|
| 352 |
+
x_var = (x**2).mean()
|
| 353 |
+
x_residual = x - coeffs * new_direction
|
| 354 |
+
x_residual_var = (x_residual**2).mean()
|
| 355 |
+
# `variance_proportion` is the proportion of the variance accounted for
|
| 356 |
+
# by the top eigen-direction. This is to be minimized.
|
| 357 |
+
variance_proportion = (x_var - x_residual_var) / (x_var + 1.0e-20)
|
| 358 |
+
variance_proportion.backward()
|
| 359 |
+
x_orig_grad = x_orig.grad
|
| 360 |
+
x_extra_grad = (
|
| 361 |
+
x_orig.grad
|
| 362 |
+
* ctx.grad_scale
|
| 363 |
+
* x_grad.norm()
|
| 364 |
+
/ (x_orig_grad.norm() + 1.0e-20)
|
| 365 |
+
)
|
| 366 |
+
return x_grad + x_extra_grad.detach(), None, None, None, None
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
class BiasNormFunction(torch.autograd.Function):
|
| 370 |
+
# This computes:
|
| 371 |
+
# scales = (torch.mean((x - bias) ** 2, keepdim=True)) ** -0.5 * log_scale.exp()
|
| 372 |
+
# return x * scales
|
| 373 |
+
# (after unsqueezing the bias), but it does it in a memory-efficient way so that
|
| 374 |
+
# it can just store the returned value (chances are, this will also be needed for
|
| 375 |
+
# some other reason, related to the next operation, so we can save memory).
|
| 376 |
+
@staticmethod
|
| 377 |
+
def forward(
|
| 378 |
+
ctx,
|
| 379 |
+
x: Tensor,
|
| 380 |
+
bias: Tensor,
|
| 381 |
+
log_scale: Tensor,
|
| 382 |
+
channel_dim: int,
|
| 383 |
+
store_output_for_backprop: bool,
|
| 384 |
+
) -> Tensor:
|
| 385 |
+
assert bias.ndim == 1
|
| 386 |
+
if channel_dim < 0:
|
| 387 |
+
channel_dim = channel_dim + x.ndim
|
| 388 |
+
ctx.store_output_for_backprop = store_output_for_backprop
|
| 389 |
+
ctx.channel_dim = channel_dim
|
| 390 |
+
for _ in range(channel_dim + 1, x.ndim):
|
| 391 |
+
bias = bias.unsqueeze(-1)
|
| 392 |
+
scales = (
|
| 393 |
+
torch.mean((x - bias) ** 2, dim=channel_dim, keepdim=True) ** -0.5
|
| 394 |
+
) * log_scale.exp()
|
| 395 |
+
ans = x * scales
|
| 396 |
+
ctx.save_for_backward(
|
| 397 |
+
ans.detach() if store_output_for_backprop else x,
|
| 398 |
+
scales.detach(),
|
| 399 |
+
bias.detach(),
|
| 400 |
+
log_scale.detach(),
|
| 401 |
+
)
|
| 402 |
+
return ans
|
| 403 |
+
|
| 404 |
+
@staticmethod
|
| 405 |
+
def backward(ctx, ans_grad: Tensor) -> Tensor:
|
| 406 |
+
ans_or_x, scales, bias, log_scale = ctx.saved_tensors
|
| 407 |
+
if ctx.store_output_for_backprop:
|
| 408 |
+
x = ans_or_x / scales
|
| 409 |
+
else:
|
| 410 |
+
x = ans_or_x
|
| 411 |
+
x = x.detach()
|
| 412 |
+
x.requires_grad = True
|
| 413 |
+
bias.requires_grad = True
|
| 414 |
+
log_scale.requires_grad = True
|
| 415 |
+
with torch.enable_grad():
|
| 416 |
+
# recompute scales from x, bias and log_scale.
|
| 417 |
+
scales = (
|
| 418 |
+
torch.mean((x - bias) ** 2, dim=ctx.channel_dim, keepdim=True) ** -0.5
|
| 419 |
+
) * log_scale.exp()
|
| 420 |
+
ans = x * scales
|
| 421 |
+
ans.backward(gradient=ans_grad)
|
| 422 |
+
return x.grad, bias.grad.flatten(), log_scale.grad, None, None
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
class BiasNorm(torch.nn.Module):
|
| 426 |
+
"""
|
| 427 |
+
This is intended to be a simpler, and hopefully cheaper, replacement for
|
| 428 |
+
LayerNorm. The observation this is based on, is that Transformer-type
|
| 429 |
+
networks, especially with pre-norm, sometimes seem to set one of the
|
| 430 |
+
feature dimensions to a large constant value (e.g. 50), which "defeats"
|
| 431 |
+
the LayerNorm because the output magnitude is then not strongly dependent
|
| 432 |
+
on the other (useful) features. Presumably the weight and bias of the
|
| 433 |
+
LayerNorm are required to allow it to do this.
|
| 434 |
+
|
| 435 |
+
Instead, we give the BiasNorm a trainable bias that it can use when
|
| 436 |
+
computing the scale for normalization. We also give it a (scalar)
|
| 437 |
+
trainable scale on the output.
|
| 438 |
+
|
| 439 |
+
|
| 440 |
+
Args:
|
| 441 |
+
num_channels: the number of channels, e.g. 512.
|
| 442 |
+
channel_dim: the axis/dimension corresponding to the channel,
|
| 443 |
+
interpreted as an offset from the input's ndim if negative.
|
| 444 |
+
This is NOT the num_channels; it should typically be one of
|
| 445 |
+
{-2, -1, 0, 1, 2, 3}.
|
| 446 |
+
log_scale: the initial log-scale that we multiply the output by; this
|
| 447 |
+
is learnable.
|
| 448 |
+
log_scale_min: FloatLike, minimum allowed value of log_scale
|
| 449 |
+
log_scale_max: FloatLike, maximum allowed value of log_scale
|
| 450 |
+
store_output_for_backprop: only possibly affects memory use; recommend
|
| 451 |
+
to set to True if you think the output of this module is more likely
|
| 452 |
+
than the input of this module to be required to be stored for the
|
| 453 |
+
backprop.
|
| 454 |
+
"""
|
| 455 |
+
|
| 456 |
+
def __init__(
|
| 457 |
+
self,
|
| 458 |
+
num_channels: int,
|
| 459 |
+
channel_dim: int = -1, # CAUTION: see documentation.
|
| 460 |
+
log_scale: float = 1.0,
|
| 461 |
+
log_scale_min: float = -1.5,
|
| 462 |
+
log_scale_max: float = 1.5,
|
| 463 |
+
store_output_for_backprop: bool = False,
|
| 464 |
+
) -> None:
|
| 465 |
+
super(BiasNorm, self).__init__()
|
| 466 |
+
self.num_channels = num_channels
|
| 467 |
+
self.channel_dim = channel_dim
|
| 468 |
+
self.log_scale = nn.Parameter(torch.tensor(log_scale))
|
| 469 |
+
self.bias = nn.Parameter(torch.empty(num_channels).normal_(mean=0, std=1e-4))
|
| 470 |
+
|
| 471 |
+
self.log_scale_min = log_scale_min
|
| 472 |
+
self.log_scale_max = log_scale_max
|
| 473 |
+
|
| 474 |
+
self.store_output_for_backprop = store_output_for_backprop
|
| 475 |
+
|
| 476 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 477 |
+
assert x.shape[self.channel_dim] == self.num_channels
|
| 478 |
+
|
| 479 |
+
if torch.jit.is_scripting() or torch.jit.is_tracing():
|
| 480 |
+
channel_dim = self.channel_dim
|
| 481 |
+
if channel_dim < 0:
|
| 482 |
+
channel_dim += x.ndim
|
| 483 |
+
bias = self.bias
|
| 484 |
+
for _ in range(channel_dim + 1, x.ndim):
|
| 485 |
+
bias = bias.unsqueeze(-1)
|
| 486 |
+
scales = (
|
| 487 |
+
torch.mean((x - bias) ** 2, dim=channel_dim, keepdim=True) ** -0.5
|
| 488 |
+
) * self.log_scale.exp()
|
| 489 |
+
return x * scales
|
| 490 |
+
|
| 491 |
+
log_scale = limit_param_value(
|
| 492 |
+
self.log_scale,
|
| 493 |
+
min=float(self.log_scale_min),
|
| 494 |
+
max=float(self.log_scale_max),
|
| 495 |
+
training=self.training,
|
| 496 |
+
)
|
| 497 |
+
|
| 498 |
+
return BiasNormFunction.apply(
|
| 499 |
+
x, self.bias, log_scale, self.channel_dim, self.store_output_for_backprop
|
| 500 |
+
)
|
| 501 |
+
|
| 502 |
+
|
| 503 |
+
def ScaledLinear(*args, initial_scale: float = 1.0, **kwargs) -> nn.Linear:
|
| 504 |
+
"""
|
| 505 |
+
Behaves like a constructor of a modified version of nn.Linear
|
| 506 |
+
that gives an easy way to set the default initial parameter scale.
|
| 507 |
+
|
| 508 |
+
Args:
|
| 509 |
+
Accepts the standard args and kwargs that nn.Linear accepts
|
| 510 |
+
e.g. in_features, out_features, bias=False.
|
| 511 |
+
|
| 512 |
+
initial_scale: you can override this if you want to increase
|
| 513 |
+
or decrease the initial magnitude of the module's output
|
| 514 |
+
(affects the initialization of weight_scale and bias_scale).
|
| 515 |
+
Another option, if you want to do something like this, is
|
| 516 |
+
to re-initialize the parameters.
|
| 517 |
+
"""
|
| 518 |
+
ans = nn.Linear(*args, **kwargs)
|
| 519 |
+
with torch.no_grad():
|
| 520 |
+
ans.weight[:] *= initial_scale
|
| 521 |
+
if ans.bias is not None:
|
| 522 |
+
torch.nn.init.uniform_(ans.bias, -0.1 * initial_scale, 0.1 * initial_scale)
|
| 523 |
+
return ans
|
| 524 |
+
|
| 525 |
+
|
| 526 |
+
def ScaledConv1d(*args, initial_scale: float = 1.0, **kwargs) -> nn.Conv1d:
|
| 527 |
+
"""
|
| 528 |
+
Behaves like a constructor of a modified version of nn.Conv1d
|
| 529 |
+
that gives an easy way to set the default initial parameter scale.
|
| 530 |
+
|
| 531 |
+
Args:
|
| 532 |
+
Accepts the standard args and kwargs that nn.Linear accepts
|
| 533 |
+
e.g. in_features, out_features, bias=False.
|
| 534 |
+
|
| 535 |
+
initial_scale: you can override this if you want to increase
|
| 536 |
+
or decrease the initial magnitude of the module's output
|
| 537 |
+
(affects the initialization of weight_scale and bias_scale).
|
| 538 |
+
Another option, if you want to do something like this, is
|
| 539 |
+
to re-initialize the parameters.
|
| 540 |
+
"""
|
| 541 |
+
ans = nn.Conv1d(*args, **kwargs)
|
| 542 |
+
with torch.no_grad():
|
| 543 |
+
ans.weight[:] *= initial_scale
|
| 544 |
+
if ans.bias is not None:
|
| 545 |
+
torch.nn.init.uniform_(ans.bias, -0.1 * initial_scale, 0.1 * initial_scale)
|
| 546 |
+
return ans
|
| 547 |
+
|
| 548 |
+
|
| 549 |
+
def ScaledConv2d(*args, initial_scale: float = 1.0, **kwargs) -> nn.Conv2d:
|
| 550 |
+
"""
|
| 551 |
+
Behaves like a constructor of a modified version of nn.Conv2d
|
| 552 |
+
that gives an easy way to set the default initial parameter scale.
|
| 553 |
+
|
| 554 |
+
Args:
|
| 555 |
+
Accepts the standard args and kwargs that nn.Linear accepts
|
| 556 |
+
e.g. in_features, out_features, bias=False, but:
|
| 557 |
+
NO PADDING-RELATED ARGS.
|
| 558 |
+
|
| 559 |
+
initial_scale: you can override this if you want to increase
|
| 560 |
+
or decrease the initial magnitude of the module's output
|
| 561 |
+
(affects the initialization of weight_scale and bias_scale).
|
| 562 |
+
Another option, if you want to do something like this, is
|
| 563 |
+
to re-initialize the parameters.
|
| 564 |
+
"""
|
| 565 |
+
ans = nn.Conv2d(*args, **kwargs)
|
| 566 |
+
with torch.no_grad():
|
| 567 |
+
ans.weight[:] *= initial_scale
|
| 568 |
+
if ans.bias is not None:
|
| 569 |
+
torch.nn.init.uniform_(ans.bias, -0.1 * initial_scale, 0.1 * initial_scale)
|
| 570 |
+
return ans
|
| 571 |
+
|
| 572 |
+
|
| 573 |
+
class ChunkCausalDepthwiseConv1d(torch.nn.Module):
|
| 574 |
+
"""
|
| 575 |
+
Behaves like a depthwise 1d convolution, except that it is causal in
|
| 576 |
+
a chunkwise way, as if we had a block-triangular attention mask.
|
| 577 |
+
The chunk size is provided at test time (it should probably be
|
| 578 |
+
kept in sync with the attention mask).
|
| 579 |
+
|
| 580 |
+
This has a little more than twice the parameters of a conventional
|
| 581 |
+
depthwise conv1d module: we implement it by having one
|
| 582 |
+
depthwise convolution, of half the width, that is causal (via
|
| 583 |
+
right-padding); and one depthwise convolution that is applied only
|
| 584 |
+
within chunks, that we multiply by a scaling factor which depends
|
| 585 |
+
on the position within the chunk.
|
| 586 |
+
|
| 587 |
+
Args:
|
| 588 |
+
Accepts the standard args and kwargs that nn.Linear accepts
|
| 589 |
+
e.g. in_features, out_features, bias=False.
|
| 590 |
+
|
| 591 |
+
initial_scale: you can override this if you want to increase
|
| 592 |
+
or decrease the initial magnitude of the module's output
|
| 593 |
+
(affects the initialization of weight_scale and bias_scale).
|
| 594 |
+
Another option, if you want to do something like this, is
|
| 595 |
+
to re-initialize the parameters.
|
| 596 |
+
"""
|
| 597 |
+
|
| 598 |
+
def __init__(
|
| 599 |
+
self,
|
| 600 |
+
channels: int,
|
| 601 |
+
kernel_size: int,
|
| 602 |
+
initial_scale: float = 1.0,
|
| 603 |
+
bias: bool = True,
|
| 604 |
+
):
|
| 605 |
+
super().__init__()
|
| 606 |
+
assert kernel_size % 2 == 1
|
| 607 |
+
|
| 608 |
+
half_kernel_size = (kernel_size + 1) // 2
|
| 609 |
+
# will pad manually, on one side.
|
| 610 |
+
self.causal_conv = nn.Conv1d(
|
| 611 |
+
in_channels=channels,
|
| 612 |
+
out_channels=channels,
|
| 613 |
+
groups=channels,
|
| 614 |
+
kernel_size=half_kernel_size,
|
| 615 |
+
padding=0,
|
| 616 |
+
bias=True,
|
| 617 |
+
)
|
| 618 |
+
|
| 619 |
+
self.chunkwise_conv = nn.Conv1d(
|
| 620 |
+
in_channels=channels,
|
| 621 |
+
out_channels=channels,
|
| 622 |
+
groups=channels,
|
| 623 |
+
kernel_size=kernel_size,
|
| 624 |
+
padding=kernel_size // 2,
|
| 625 |
+
bias=bias,
|
| 626 |
+
)
|
| 627 |
+
|
| 628 |
+
# first row is correction factors added to the scale near the left edge of the chunk,
|
| 629 |
+
# second row is correction factors added to the scale near the right edge of the chunk,
|
| 630 |
+
# both of these are added to a default scale of 1.0.
|
| 631 |
+
self.chunkwise_conv_scale = nn.Parameter(torch.zeros(2, channels, kernel_size))
|
| 632 |
+
self.kernel_size = kernel_size
|
| 633 |
+
|
| 634 |
+
with torch.no_grad():
|
| 635 |
+
self.causal_conv.weight[:] *= initial_scale
|
| 636 |
+
self.chunkwise_conv.weight[:] *= initial_scale
|
| 637 |
+
if bias:
|
| 638 |
+
torch.nn.init.uniform_(
|
| 639 |
+
self.causal_conv.bias, -0.1 * initial_scale, 0.1 * initial_scale
|
| 640 |
+
)
|
| 641 |
+
|
| 642 |
+
def forward(self, x: Tensor, chunk_size: int = -1) -> Tensor:
|
| 643 |
+
"""Forward function.
|
| 644 |
+
|
| 645 |
+
Args:
|
| 646 |
+
x: a Tensor of shape (batch_size, channels, seq_len)
|
| 647 |
+
chunk_size: the chunk size, in frames; does not have to divide seq_len exactly.
|
| 648 |
+
"""
|
| 649 |
+
(batch_size, num_channels, seq_len) = x.shape
|
| 650 |
+
|
| 651 |
+
# half_kernel_size = self.kernel_size + 1 // 2
|
| 652 |
+
# left_pad is half_kernel_size - 1 where half_kernel_size is the size used
|
| 653 |
+
# in the causal conv. It's the amount by which we must pad on the left,
|
| 654 |
+
# to make the convolution causal.
|
| 655 |
+
left_pad = self.kernel_size // 2
|
| 656 |
+
|
| 657 |
+
if chunk_size < 0 or chunk_size > seq_len:
|
| 658 |
+
chunk_size = seq_len
|
| 659 |
+
right_pad = -seq_len % chunk_size
|
| 660 |
+
|
| 661 |
+
x = torch.nn.functional.pad(x, (left_pad, right_pad))
|
| 662 |
+
|
| 663 |
+
x_causal = self.causal_conv(x[..., : left_pad + seq_len])
|
| 664 |
+
assert x_causal.shape == (batch_size, num_channels, seq_len)
|
| 665 |
+
|
| 666 |
+
x_chunk = x[..., left_pad:]
|
| 667 |
+
num_chunks = x_chunk.shape[2] // chunk_size
|
| 668 |
+
x_chunk = x_chunk.reshape(batch_size, num_channels, num_chunks, chunk_size)
|
| 669 |
+
x_chunk = x_chunk.permute(0, 2, 1, 3).reshape(
|
| 670 |
+
batch_size * num_chunks, num_channels, chunk_size
|
| 671 |
+
)
|
| 672 |
+
x_chunk = self.chunkwise_conv(x_chunk) # does not change shape
|
| 673 |
+
|
| 674 |
+
chunk_scale = self._get_chunk_scale(chunk_size)
|
| 675 |
+
|
| 676 |
+
x_chunk = x_chunk * chunk_scale
|
| 677 |
+
x_chunk = x_chunk.reshape(
|
| 678 |
+
batch_size, num_chunks, num_channels, chunk_size
|
| 679 |
+
).permute(0, 2, 1, 3)
|
| 680 |
+
x_chunk = x_chunk.reshape(batch_size, num_channels, num_chunks * chunk_size)[
|
| 681 |
+
..., :seq_len
|
| 682 |
+
]
|
| 683 |
+
|
| 684 |
+
return x_chunk + x_causal
|
| 685 |
+
|
| 686 |
+
def _get_chunk_scale(self, chunk_size: int):
|
| 687 |
+
"""Returns tensor of shape (num_channels, chunk_size) that will be used to
|
| 688 |
+
scale the output of self.chunkwise_conv."""
|
| 689 |
+
left_edge = self.chunkwise_conv_scale[0]
|
| 690 |
+
right_edge = self.chunkwise_conv_scale[1]
|
| 691 |
+
if chunk_size < self.kernel_size:
|
| 692 |
+
left_edge = left_edge[:, :chunk_size]
|
| 693 |
+
right_edge = right_edge[:, -chunk_size:]
|
| 694 |
+
else:
|
| 695 |
+
t = chunk_size - self.kernel_size
|
| 696 |
+
channels = left_edge.shape[0]
|
| 697 |
+
pad = torch.zeros(
|
| 698 |
+
channels, t, device=left_edge.device, dtype=left_edge.dtype
|
| 699 |
+
)
|
| 700 |
+
left_edge = torch.cat((left_edge, pad), dim=-1)
|
| 701 |
+
right_edge = torch.cat((pad, right_edge), dim=-1)
|
| 702 |
+
return 1.0 + (left_edge + right_edge)
|
| 703 |
+
|
| 704 |
+
def streaming_forward(
|
| 705 |
+
self,
|
| 706 |
+
x: Tensor,
|
| 707 |
+
cache: Tensor,
|
| 708 |
+
) -> Tuple[Tensor, Tensor]:
|
| 709 |
+
"""Streaming Forward function.
|
| 710 |
+
|
| 711 |
+
Args:
|
| 712 |
+
x: a Tensor of shape (batch_size, channels, seq_len)
|
| 713 |
+
cache: cached left context of shape (batch_size, channels, left_pad)
|
| 714 |
+
"""
|
| 715 |
+
(batch_size, num_channels, seq_len) = x.shape
|
| 716 |
+
|
| 717 |
+
# left_pad is half_kernel_size - 1 where half_kernel_size is the size used
|
| 718 |
+
# in the causal conv. It's the amount by which we must pad on the left,
|
| 719 |
+
# to make the convolution causal.
|
| 720 |
+
left_pad = self.kernel_size // 2
|
| 721 |
+
|
| 722 |
+
# Pad cache
|
| 723 |
+
assert cache.shape[-1] == left_pad, (cache.shape[-1], left_pad)
|
| 724 |
+
x = torch.cat([cache, x], dim=2)
|
| 725 |
+
# Update cache
|
| 726 |
+
cache = x[..., -left_pad:]
|
| 727 |
+
|
| 728 |
+
x_causal = self.causal_conv(x)
|
| 729 |
+
assert x_causal.shape == (batch_size, num_channels, seq_len)
|
| 730 |
+
|
| 731 |
+
x_chunk = x[..., left_pad:]
|
| 732 |
+
x_chunk = self.chunkwise_conv(x_chunk) # does not change shape
|
| 733 |
+
|
| 734 |
+
chunk_scale = self._get_chunk_scale(chunk_size=seq_len)
|
| 735 |
+
x_chunk = x_chunk * chunk_scale
|
| 736 |
+
|
| 737 |
+
return x_chunk + x_causal, cache
|
| 738 |
+
|
| 739 |
+
|
| 740 |
+
class BalancerFunction(torch.autograd.Function):
|
| 741 |
+
@staticmethod
|
| 742 |
+
def forward(
|
| 743 |
+
ctx,
|
| 744 |
+
x: Tensor,
|
| 745 |
+
min_mean: float,
|
| 746 |
+
max_mean: float,
|
| 747 |
+
min_rms: float,
|
| 748 |
+
max_rms: float,
|
| 749 |
+
grad_scale: float,
|
| 750 |
+
channel_dim: int,
|
| 751 |
+
) -> Tensor:
|
| 752 |
+
if channel_dim < 0:
|
| 753 |
+
channel_dim += x.ndim
|
| 754 |
+
ctx.channel_dim = channel_dim
|
| 755 |
+
ctx.save_for_backward(x)
|
| 756 |
+
ctx.config = (min_mean, max_mean, min_rms, max_rms, grad_scale, channel_dim)
|
| 757 |
+
return x
|
| 758 |
+
|
| 759 |
+
@staticmethod
|
| 760 |
+
def backward(ctx, x_grad: Tensor) -> Tuple[Tensor, None, None, None, None, None]:
|
| 761 |
+
(x,) = ctx.saved_tensors
|
| 762 |
+
(min_mean, max_mean, min_rms, max_rms, grad_scale, channel_dim) = ctx.config
|
| 763 |
+
|
| 764 |
+
try:
|
| 765 |
+
with torch.enable_grad():
|
| 766 |
+
with torch_autocast(enabled=False):
|
| 767 |
+
x = x.to(torch.float32)
|
| 768 |
+
x = x.detach()
|
| 769 |
+
x.requires_grad = True
|
| 770 |
+
mean_dims = [i for i in range(x.ndim) if i != channel_dim]
|
| 771 |
+
uncentered_var = (x**2).mean(dim=mean_dims, keepdim=True)
|
| 772 |
+
mean = x.mean(dim=mean_dims, keepdim=True)
|
| 773 |
+
stddev = (uncentered_var - (mean * mean)).clamp(min=1.0e-20).sqrt()
|
| 774 |
+
rms = uncentered_var.clamp(min=1.0e-20).sqrt()
|
| 775 |
+
|
| 776 |
+
m = mean / stddev
|
| 777 |
+
# part of loss that relates to mean / stddev
|
| 778 |
+
m_loss = (m - m.clamp(min=min_mean, max=max_mean)).abs()
|
| 779 |
+
|
| 780 |
+
# put a much larger scale on the RMS-max-limit loss, so that if both it and the
|
| 781 |
+
# m_loss are violated we fix the RMS loss first.
|
| 782 |
+
rms_clamped = rms.clamp(min=min_rms, max=max_rms)
|
| 783 |
+
r_loss = (rms_clamped / rms).log().abs()
|
| 784 |
+
|
| 785 |
+
loss = m_loss + r_loss
|
| 786 |
+
|
| 787 |
+
loss.backward(gradient=torch.ones_like(loss))
|
| 788 |
+
loss_grad = x.grad
|
| 789 |
+
loss_grad_rms = (
|
| 790 |
+
(loss_grad**2)
|
| 791 |
+
.mean(dim=mean_dims, keepdim=True)
|
| 792 |
+
.sqrt()
|
| 793 |
+
.clamp(min=1.0e-20)
|
| 794 |
+
)
|
| 795 |
+
|
| 796 |
+
loss_grad = loss_grad * (grad_scale / loss_grad_rms)
|
| 797 |
+
|
| 798 |
+
x_grad_float = x_grad.to(torch.float32)
|
| 799 |
+
# scale each element of loss_grad by the absolute value of the corresponding
|
| 800 |
+
# element of x_grad, which we view as a noisy estimate of its magnitude for that
|
| 801 |
+
# (frame and dimension). later we can consider factored versions.
|
| 802 |
+
x_grad_mod = x_grad_float + (x_grad_float.abs() * loss_grad)
|
| 803 |
+
x_grad = x_grad_mod.to(x_grad.dtype)
|
| 804 |
+
except Exception as e:
|
| 805 |
+
logging.info(
|
| 806 |
+
f"Caught exception in Balancer backward: {e}, size={list(x_grad.shape)}, will continue."
|
| 807 |
+
)
|
| 808 |
+
|
| 809 |
+
return x_grad, None, None, None, None, None, None
|
| 810 |
+
|
| 811 |
+
|
| 812 |
+
class Balancer(torch.nn.Module):
|
| 813 |
+
"""
|
| 814 |
+
Modifies the backpropped derivatives of a function to try to encourage, for
|
| 815 |
+
each channel, that it is positive at least a proportion `threshold` of the
|
| 816 |
+
time. It does this by multiplying negative derivative values by up to
|
| 817 |
+
(1+max_factor), and positive derivative values by up to (1-max_factor),
|
| 818 |
+
interpolated from 1 at the threshold to those extremal values when none
|
| 819 |
+
of the inputs are positive.
|
| 820 |
+
|
| 821 |
+
Args:
|
| 822 |
+
num_channels: the number of channels
|
| 823 |
+
channel_dim: the dimension/axis corresponding to the channel, e.g.
|
| 824 |
+
-1, 0, 1, 2; will be interpreted as an offset from x.ndim if negative.
|
| 825 |
+
min_positive: the minimum, per channel, of the proportion of the time
|
| 826 |
+
that (x > 0), below which we start to modify the derivatives.
|
| 827 |
+
max_positive: the maximum, per channel, of the proportion of the time
|
| 828 |
+
that (x > 0), above which we start to modify the derivatives.
|
| 829 |
+
scale_gain_factor: determines the 'gain' with which we increase the
|
| 830 |
+
change in gradient once the constraints on min_abs and max_abs
|
| 831 |
+
are violated.
|
| 832 |
+
min_abs: the minimum average-absolute-value difference from the mean
|
| 833 |
+
value per channel, which we allow, before we start to modify
|
| 834 |
+
the derivatives to prevent this.
|
| 835 |
+
max_abs: the maximum average-absolute-value difference from the mean
|
| 836 |
+
value per channel, which we allow, before we start to modify
|
| 837 |
+
the derivatives to prevent this.
|
| 838 |
+
prob: determines the minimum probability with which we modify the
|
| 839 |
+
gradients for the {min,max}_positive and {min,max}_abs constraints,
|
| 840 |
+
on each forward(). This is done randomly to prevent all layers
|
| 841 |
+
from doing it at the same time.
|
| 842 |
+
"""
|
| 843 |
+
|
| 844 |
+
def __init__(
|
| 845 |
+
self,
|
| 846 |
+
num_channels: int,
|
| 847 |
+
channel_dim: int,
|
| 848 |
+
min_positive: FloatLike = 0.05,
|
| 849 |
+
max_positive: FloatLike = 0.95,
|
| 850 |
+
min_abs: FloatLike = 0.2,
|
| 851 |
+
max_abs: FloatLike = 100.0,
|
| 852 |
+
grad_scale: FloatLike = 0.04,
|
| 853 |
+
prob: Optional[FloatLike] = None,
|
| 854 |
+
):
|
| 855 |
+
super().__init__()
|
| 856 |
+
|
| 857 |
+
if prob is None:
|
| 858 |
+
prob = ScheduledFloat((0.0, 0.5), (8000.0, 0.125), default=0.4)
|
| 859 |
+
self.prob = prob
|
| 860 |
+
# 5% of the time we will return and do nothing because memory usage is
|
| 861 |
+
# too high.
|
| 862 |
+
self.mem_cutoff = CutoffEstimator(0.05)
|
| 863 |
+
|
| 864 |
+
# actually self.num_channels is no longer needed except for an assertion.
|
| 865 |
+
self.num_channels = num_channels
|
| 866 |
+
self.channel_dim = channel_dim
|
| 867 |
+
self.min_positive = min_positive
|
| 868 |
+
self.max_positive = max_positive
|
| 869 |
+
self.min_abs = min_abs
|
| 870 |
+
self.max_abs = max_abs
|
| 871 |
+
self.grad_scale = grad_scale
|
| 872 |
+
|
| 873 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 874 |
+
if (
|
| 875 |
+
torch.jit.is_scripting()
|
| 876 |
+
or not x.requires_grad
|
| 877 |
+
or (x.is_cuda and self.mem_cutoff(torch.cuda.memory_allocated()))
|
| 878 |
+
):
|
| 879 |
+
return _no_op(x)
|
| 880 |
+
|
| 881 |
+
prob = float(self.prob)
|
| 882 |
+
if random.random() < prob:
|
| 883 |
+
# The following inner-functions convert from the way we historically specified
|
| 884 |
+
# these limitations, as limits on the absolute value and the proportion of positive
|
| 885 |
+
# values, to limits on the RMS value and the (mean / stddev).
|
| 886 |
+
def _abs_to_rms(x):
|
| 887 |
+
# for normally distributed data, if the expected absolute value is x, the
|
| 888 |
+
# expected rms value will be sqrt(pi/2) * x.
|
| 889 |
+
return 1.25331413732 * x
|
| 890 |
+
|
| 891 |
+
def _proportion_positive_to_mean(x):
|
| 892 |
+
def _atanh(x):
|
| 893 |
+
eps = 1.0e-10
|
| 894 |
+
# eps is to prevent crashes if x is exactly 0 or 1.
|
| 895 |
+
# we'll just end up returning a fairly large value.
|
| 896 |
+
return (math.log(1 + x + eps) - math.log(1 - x + eps)) / 2.0
|
| 897 |
+
|
| 898 |
+
def _approx_inverse_erf(x):
|
| 899 |
+
# 1 / (sqrt(pi) * ln(2)),
|
| 900 |
+
# see https://math.stackexchange.com/questions/321569/approximating-the-error-function-erf-by-analytical-functions
|
| 901 |
+
# this approximation is extremely crude and gets progressively worse for
|
| 902 |
+
# x very close to -1 or +1, but we mostly care about the "middle" region
|
| 903 |
+
# e.g. _approx_inverse_erf(0.05) = 0.0407316414078772,
|
| 904 |
+
# and math.erf(0.0407316414078772) = 0.045935330944660666,
|
| 905 |
+
# which is pretty close to 0.05.
|
| 906 |
+
return 0.8139535143 * _atanh(x)
|
| 907 |
+
|
| 908 |
+
# first convert x from the range 0..1 to the range -1..1 which the error
|
| 909 |
+
# function returns
|
| 910 |
+
x = -1 + (2 * x)
|
| 911 |
+
return _approx_inverse_erf(x)
|
| 912 |
+
|
| 913 |
+
min_mean = _proportion_positive_to_mean(float(self.min_positive))
|
| 914 |
+
max_mean = _proportion_positive_to_mean(float(self.max_positive))
|
| 915 |
+
min_rms = _abs_to_rms(float(self.min_abs))
|
| 916 |
+
max_rms = _abs_to_rms(float(self.max_abs))
|
| 917 |
+
grad_scale = float(self.grad_scale)
|
| 918 |
+
|
| 919 |
+
assert x.shape[self.channel_dim] == self.num_channels
|
| 920 |
+
|
| 921 |
+
return BalancerFunction.apply(
|
| 922 |
+
x, min_mean, max_mean, min_rms, max_rms, grad_scale, self.channel_dim
|
| 923 |
+
)
|
| 924 |
+
else:
|
| 925 |
+
return _no_op(x)
|
| 926 |
+
|
| 927 |
+
|
| 928 |
+
def penalize_abs_values_gt(
|
| 929 |
+
x: Tensor, limit: float, penalty: float, name: str = None
|
| 930 |
+
) -> Tensor:
|
| 931 |
+
"""
|
| 932 |
+
Returns x unmodified, but in backprop will put a penalty for the excess of
|
| 933 |
+
the absolute values of elements of x over the limit "limit". E.g. if
|
| 934 |
+
limit == 10.0, then if x has any values over 10 it will get a penalty.
|
| 935 |
+
|
| 936 |
+
Caution: the value of this penalty will be affected by grad scaling used
|
| 937 |
+
in automatic mixed precision training. For this reasons we use this,
|
| 938 |
+
it shouldn't really matter, or may even be helpful; we just use this
|
| 939 |
+
to disallow really implausible values of scores to be given to softmax.
|
| 940 |
+
|
| 941 |
+
The name is for randomly printed debug info.
|
| 942 |
+
"""
|
| 943 |
+
x_sign = x.sign()
|
| 944 |
+
over_limit = (x.abs() - limit) > 0
|
| 945 |
+
# The following is a memory efficient way to penalize the absolute values of
|
| 946 |
+
# x that's over the limit. (The memory efficiency comes when you think
|
| 947 |
+
# about which items torch needs to cache for the autograd, and which ones it
|
| 948 |
+
# can throw away). The numerical value of aux_loss as computed here will
|
| 949 |
+
# actually be larger than it should be, by limit * over_limit.sum(), but it
|
| 950 |
+
# has the same derivative as the real aux_loss which is penalty * (x.abs() -
|
| 951 |
+
# limit).relu().
|
| 952 |
+
aux_loss = penalty * ((x_sign * over_limit).to(torch.int8) * x)
|
| 953 |
+
# note: we don't do sum() here on aux)_loss, but it's as if we had done
|
| 954 |
+
# sum() due to how with_loss() works.
|
| 955 |
+
x = with_loss(x, aux_loss, name)
|
| 956 |
+
# you must use x for something, or this will be ineffective.
|
| 957 |
+
return x
|
| 958 |
+
|
| 959 |
+
|
| 960 |
+
def _diag(x: Tensor): # like .diag(), but works for tensors with 3 dims.
|
| 961 |
+
if x.ndim == 2:
|
| 962 |
+
return x.diag()
|
| 963 |
+
else:
|
| 964 |
+
(batch, dim, dim) = x.shape
|
| 965 |
+
x = x.reshape(batch, dim * dim)
|
| 966 |
+
x = x[:, :: dim + 1]
|
| 967 |
+
assert x.shape == (batch, dim)
|
| 968 |
+
return x
|
| 969 |
+
|
| 970 |
+
|
| 971 |
+
def _whitening_metric(x: Tensor, num_groups: int):
|
| 972 |
+
"""
|
| 973 |
+
Computes the "whitening metric", a value which will be 1.0 if all the eigenvalues of
|
| 974 |
+
of the centered feature covariance are the same within each group's covariance matrix
|
| 975 |
+
and also between groups.
|
| 976 |
+
Args:
|
| 977 |
+
x: a Tensor of shape (*, num_channels)
|
| 978 |
+
num_groups: the number of groups of channels, a number >=1 that divides num_channels
|
| 979 |
+
Returns:
|
| 980 |
+
Returns a scalar Tensor that will be 1.0 if the data is "perfectly white" and
|
| 981 |
+
greater than 1.0 otherwise.
|
| 982 |
+
"""
|
| 983 |
+
assert x.dtype != torch.float16
|
| 984 |
+
x = x.reshape(-1, x.shape[-1])
|
| 985 |
+
(num_frames, num_channels) = x.shape
|
| 986 |
+
assert num_channels % num_groups == 0
|
| 987 |
+
channels_per_group = num_channels // num_groups
|
| 988 |
+
x = x.reshape(num_frames, num_groups, channels_per_group).transpose(0, 1)
|
| 989 |
+
# x now has shape (num_groups, num_frames, channels_per_group)
|
| 990 |
+
# subtract the mean so we use the centered, not uncentered, covariance.
|
| 991 |
+
# My experience has been that when we "mess with the gradients" like this,
|
| 992 |
+
# it's better not do anything that tries to move the mean around, because
|
| 993 |
+
# that can easily cause instability.
|
| 994 |
+
x = x - x.mean(dim=1, keepdim=True)
|
| 995 |
+
# x_covar: (num_groups, channels_per_group, channels_per_group)
|
| 996 |
+
x_covar = torch.matmul(x.transpose(1, 2), x)
|
| 997 |
+
x_covar_mean_diag = _diag(x_covar).mean()
|
| 998 |
+
# the following expression is what we'd get if we took the matrix product
|
| 999 |
+
# of each covariance and measured the mean of its trace, i.e.
|
| 1000 |
+
# the same as _diag(torch.matmul(x_covar, x_covar)).mean().
|
| 1001 |
+
x_covarsq_mean_diag = (x_covar**2).sum() / (num_groups * channels_per_group)
|
| 1002 |
+
# this metric will be >= 1.0; the larger it is, the less 'white' the data was.
|
| 1003 |
+
metric = x_covarsq_mean_diag / (x_covar_mean_diag**2 + 1.0e-20)
|
| 1004 |
+
return metric
|
| 1005 |
+
|
| 1006 |
+
|
| 1007 |
+
class WhiteningPenaltyFunction(torch.autograd.Function):
|
| 1008 |
+
@staticmethod
|
| 1009 |
+
def forward(ctx, x: Tensor, module: nn.Module) -> Tensor:
|
| 1010 |
+
ctx.save_for_backward(x)
|
| 1011 |
+
ctx.module = module
|
| 1012 |
+
return x
|
| 1013 |
+
|
| 1014 |
+
@staticmethod
|
| 1015 |
+
def backward(ctx, x_grad: Tensor):
|
| 1016 |
+
(x_orig,) = ctx.saved_tensors
|
| 1017 |
+
w = ctx.module
|
| 1018 |
+
|
| 1019 |
+
try:
|
| 1020 |
+
with torch.enable_grad():
|
| 1021 |
+
with torch_autocast(enabled=False):
|
| 1022 |
+
x_detached = x_orig.to(torch.float32).detach()
|
| 1023 |
+
x_detached.requires_grad = True
|
| 1024 |
+
|
| 1025 |
+
metric = _whitening_metric(x_detached, w.num_groups)
|
| 1026 |
+
|
| 1027 |
+
if random.random() < 0.005 or __name__ == "__main__":
|
| 1028 |
+
logging.info(
|
| 1029 |
+
f"Whitening: name={w.name}, num_groups={w.num_groups}, num_channels={x_orig.shape[-1]}, "
|
| 1030 |
+
f"metric={metric.item():.2f} vs. limit={float(w.whitening_limit)}"
|
| 1031 |
+
)
|
| 1032 |
+
|
| 1033 |
+
if metric < float(w.whitening_limit):
|
| 1034 |
+
w.prob = w.min_prob
|
| 1035 |
+
return x_grad, None
|
| 1036 |
+
else:
|
| 1037 |
+
w.prob = w.max_prob
|
| 1038 |
+
metric.backward()
|
| 1039 |
+
penalty_grad = x_detached.grad
|
| 1040 |
+
scale = float(w.grad_scale) * (
|
| 1041 |
+
x_grad.to(torch.float32).norm()
|
| 1042 |
+
/ (penalty_grad.norm() + 1.0e-20)
|
| 1043 |
+
)
|
| 1044 |
+
penalty_grad = penalty_grad * scale
|
| 1045 |
+
return x_grad + penalty_grad.to(x_grad.dtype), None
|
| 1046 |
+
except Exception as e:
|
| 1047 |
+
logging.info(
|
| 1048 |
+
f"Caught exception in Whiten backward: {e}, size={list(x_grad.shape)}, will continue."
|
| 1049 |
+
)
|
| 1050 |
+
return x_grad, None
|
| 1051 |
+
|
| 1052 |
+
|
| 1053 |
+
class Whiten(nn.Module):
|
| 1054 |
+
def __init__(
|
| 1055 |
+
self,
|
| 1056 |
+
num_groups: int,
|
| 1057 |
+
whitening_limit: FloatLike,
|
| 1058 |
+
prob: Union[float, Tuple[float, float]],
|
| 1059 |
+
grad_scale: FloatLike,
|
| 1060 |
+
):
|
| 1061 |
+
"""
|
| 1062 |
+
Args:
|
| 1063 |
+
num_groups: the number of groups to divide the channel dim into before
|
| 1064 |
+
whitening. We will attempt to make the feature covariance
|
| 1065 |
+
within each group, after mean subtraction, as "white" as possible,
|
| 1066 |
+
while having the same trace across all groups.
|
| 1067 |
+
whitening_limit: a value greater than 1.0, that dictates how much
|
| 1068 |
+
freedom we have to violate the constraints. 1.0 would mean perfectly
|
| 1069 |
+
white, with exactly the same trace across groups; larger values
|
| 1070 |
+
give more freedom. E.g. 2.0.
|
| 1071 |
+
prob: the probability with which we apply the gradient modification
|
| 1072 |
+
(also affects the grad scale). May be supplied as a float,
|
| 1073 |
+
or as a pair (min_prob, max_prob)
|
| 1074 |
+
|
| 1075 |
+
grad_scale: determines the scale on the gradient term from this object,
|
| 1076 |
+
relative to the rest of the gradient on the attention weights.
|
| 1077 |
+
E.g. 0.02 (you may want to use smaller values than this if prob is large)
|
| 1078 |
+
"""
|
| 1079 |
+
super(Whiten, self).__init__()
|
| 1080 |
+
assert num_groups >= 1
|
| 1081 |
+
assert float(whitening_limit) >= 1
|
| 1082 |
+
assert float(grad_scale) >= 0
|
| 1083 |
+
self.num_groups = num_groups
|
| 1084 |
+
self.whitening_limit = whitening_limit
|
| 1085 |
+
self.grad_scale = grad_scale
|
| 1086 |
+
|
| 1087 |
+
if isinstance(prob, float):
|
| 1088 |
+
prob = (prob, prob)
|
| 1089 |
+
(self.min_prob, self.max_prob) = prob
|
| 1090 |
+
assert 0 < self.min_prob <= self.max_prob <= 1
|
| 1091 |
+
self.prob = self.max_prob
|
| 1092 |
+
self.name = None # will be set in training loop
|
| 1093 |
+
|
| 1094 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 1095 |
+
"""
|
| 1096 |
+
In the forward pass, this function just returns the input unmodified.
|
| 1097 |
+
In the backward pass, it will modify the gradients to ensure that the
|
| 1098 |
+
distribution in each group has close to (lambda times I) as the covariance
|
| 1099 |
+
after mean subtraction, with the same lambda across groups.
|
| 1100 |
+
For whitening_limit > 1, there will be more freedom to violate this
|
| 1101 |
+
constraint.
|
| 1102 |
+
|
| 1103 |
+
Args:
|
| 1104 |
+
x: the input of shape (*, num_channels)
|
| 1105 |
+
|
| 1106 |
+
Returns:
|
| 1107 |
+
x, unmodified. You should make sure
|
| 1108 |
+
you use the returned value, or the graph will be freed
|
| 1109 |
+
and nothing will happen in backprop.
|
| 1110 |
+
"""
|
| 1111 |
+
grad_scale = float(self.grad_scale)
|
| 1112 |
+
if not x.requires_grad or random.random() > self.prob or grad_scale == 0:
|
| 1113 |
+
return _no_op(x)
|
| 1114 |
+
else:
|
| 1115 |
+
return WhiteningPenaltyFunction.apply(x, self)
|
| 1116 |
+
|
| 1117 |
+
|
| 1118 |
+
class WithLoss(torch.autograd.Function):
|
| 1119 |
+
@staticmethod
|
| 1120 |
+
def forward(ctx, x: Tensor, y: Tensor, name: str):
|
| 1121 |
+
ctx.y_shape = y.shape
|
| 1122 |
+
if random.random() < 0.002 and name is not None:
|
| 1123 |
+
loss_sum = y.sum().item()
|
| 1124 |
+
logging.info(f"WithLoss: name={name}, loss-sum={loss_sum:.3e}")
|
| 1125 |
+
return x
|
| 1126 |
+
|
| 1127 |
+
@staticmethod
|
| 1128 |
+
def backward(ctx, ans_grad: Tensor):
|
| 1129 |
+
return (
|
| 1130 |
+
ans_grad,
|
| 1131 |
+
torch.ones(ctx.y_shape, dtype=ans_grad.dtype, device=ans_grad.device),
|
| 1132 |
+
None,
|
| 1133 |
+
)
|
| 1134 |
+
|
| 1135 |
+
|
| 1136 |
+
def with_loss(x, y, name):
|
| 1137 |
+
# returns x but adds y.sum() to the loss function.
|
| 1138 |
+
return WithLoss.apply(x, y, name)
|
| 1139 |
+
|
| 1140 |
+
|
| 1141 |
+
class ScaleGradFunction(torch.autograd.Function):
|
| 1142 |
+
@staticmethod
|
| 1143 |
+
def forward(ctx, x: Tensor, alpha: float) -> Tensor:
|
| 1144 |
+
ctx.alpha = alpha
|
| 1145 |
+
return x
|
| 1146 |
+
|
| 1147 |
+
@staticmethod
|
| 1148 |
+
def backward(ctx, grad: Tensor):
|
| 1149 |
+
return grad * ctx.alpha, None
|
| 1150 |
+
|
| 1151 |
+
|
| 1152 |
+
def scale_grad(x: Tensor, alpha: float):
|
| 1153 |
+
return ScaleGradFunction.apply(x, alpha)
|
| 1154 |
+
|
| 1155 |
+
|
| 1156 |
+
class ScaleGrad(nn.Module):
|
| 1157 |
+
def __init__(self, alpha: float):
|
| 1158 |
+
super().__init__()
|
| 1159 |
+
self.alpha = alpha
|
| 1160 |
+
|
| 1161 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 1162 |
+
if torch.jit.is_scripting() or torch.jit.is_tracing() or not self.training:
|
| 1163 |
+
return x
|
| 1164 |
+
return scale_grad(x, self.alpha)
|
| 1165 |
+
|
| 1166 |
+
|
| 1167 |
+
class LimitParamValue(torch.autograd.Function):
|
| 1168 |
+
@staticmethod
|
| 1169 |
+
def forward(ctx, x: Tensor, min: float, max: float):
|
| 1170 |
+
ctx.save_for_backward(x)
|
| 1171 |
+
assert max >= min
|
| 1172 |
+
ctx.min = min
|
| 1173 |
+
ctx.max = max
|
| 1174 |
+
return x
|
| 1175 |
+
|
| 1176 |
+
@staticmethod
|
| 1177 |
+
def backward(ctx, x_grad: Tensor):
|
| 1178 |
+
(x,) = ctx.saved_tensors
|
| 1179 |
+
# where x < ctx.min, ensure all grads are negative (this will tend to make
|
| 1180 |
+
# x more positive).
|
| 1181 |
+
x_grad = x_grad * torch.where(
|
| 1182 |
+
torch.logical_and(x_grad > 0, x < ctx.min), -1.0, 1.0
|
| 1183 |
+
)
|
| 1184 |
+
# where x > ctx.max, ensure all grads are positive (this will tend to make
|
| 1185 |
+
# x more negative).
|
| 1186 |
+
x_grad *= torch.where(torch.logical_and(x_grad < 0, x > ctx.max), -1.0, 1.0)
|
| 1187 |
+
return x_grad, None, None
|
| 1188 |
+
|
| 1189 |
+
|
| 1190 |
+
def limit_param_value(
|
| 1191 |
+
x: Tensor, min: float, max: float, prob: float = 0.6, training: bool = True
|
| 1192 |
+
):
|
| 1193 |
+
# You apply this to (typically) an nn.Parameter during training to ensure that its
|
| 1194 |
+
# (elements mostly) stays within a supplied range. This is done by modifying the
|
| 1195 |
+
# gradients in backprop.
|
| 1196 |
+
# It's not necessary to do this on every batch: do it only some of the time,
|
| 1197 |
+
# to save a little time.
|
| 1198 |
+
if training and random.random() < prob:
|
| 1199 |
+
return LimitParamValue.apply(x, min, max)
|
| 1200 |
+
else:
|
| 1201 |
+
return x
|
| 1202 |
+
|
| 1203 |
+
|
| 1204 |
+
def _no_op(x: Tensor) -> Tensor:
|
| 1205 |
+
if torch.jit.is_scripting() or torch.jit.is_tracing():
|
| 1206 |
+
return x
|
| 1207 |
+
else:
|
| 1208 |
+
# a no-op function that will have a node in the autograd graph,
|
| 1209 |
+
# to avoid certain bugs relating to backward hooks
|
| 1210 |
+
return x.chunk(1, dim=-1)[0]
|
| 1211 |
+
|
| 1212 |
+
|
| 1213 |
+
class Identity(torch.nn.Module):
|
| 1214 |
+
def __init__(self):
|
| 1215 |
+
super(Identity, self).__init__()
|
| 1216 |
+
|
| 1217 |
+
def forward(self, x):
|
| 1218 |
+
return _no_op(x)
|
| 1219 |
+
|
| 1220 |
+
|
| 1221 |
+
class DoubleSwishFunction(torch.autograd.Function):
|
| 1222 |
+
"""
|
| 1223 |
+
double_swish(x) = x * torch.sigmoid(x-1)
|
| 1224 |
+
|
| 1225 |
+
This is a definition, originally motivated by its close numerical
|
| 1226 |
+
similarity to swish(swish(x)), where swish(x) = x * sigmoid(x).
|
| 1227 |
+
|
| 1228 |
+
Memory-efficient derivative computation:
|
| 1229 |
+
double_swish(x) = x * s, where s(x) = torch.sigmoid(x-1)
|
| 1230 |
+
double_swish'(x) = d/dx double_swish(x) = x * s'(x) + x' * s(x) = x * s'(x) + s(x).
|
| 1231 |
+
Now, s'(x) = s(x) * (1-s(x)).
|
| 1232 |
+
double_swish'(x) = x * s'(x) + s(x).
|
| 1233 |
+
= x * s(x) * (1-s(x)) + s(x).
|
| 1234 |
+
= double_swish(x) * (1-s(x)) + s(x)
|
| 1235 |
+
... so we just need to remember s(x) but not x itself.
|
| 1236 |
+
"""
|
| 1237 |
+
|
| 1238 |
+
@staticmethod
|
| 1239 |
+
def forward(ctx, x: Tensor) -> Tensor:
|
| 1240 |
+
requires_grad = x.requires_grad
|
| 1241 |
+
if x.dtype == torch.float16 or x.dtype == torch.bfloat16:
|
| 1242 |
+
x = x.to(torch.float32)
|
| 1243 |
+
|
| 1244 |
+
s = torch.sigmoid(x - 1.0)
|
| 1245 |
+
y = x * s
|
| 1246 |
+
|
| 1247 |
+
if requires_grad:
|
| 1248 |
+
deriv = y * (1 - s) + s
|
| 1249 |
+
|
| 1250 |
+
# notes on derivative of x * sigmoid(x - 1):
|
| 1251 |
+
# https://www.wolframalpha.com/input?i=d%2Fdx+%28x+*+sigmoid%28x-1%29%29
|
| 1252 |
+
# min \simeq -0.043638. Take floor as -0.044 so it's a lower bund
|
| 1253 |
+
# max \simeq 1.1990. Take ceil to be 1.2 so it's an upper bound.
|
| 1254 |
+
# the combination of "+ torch.rand_like(deriv)" and casting to torch.uint8 (which
|
| 1255 |
+
# floors), should be expectation-preserving.
|
| 1256 |
+
floor = -0.044
|
| 1257 |
+
ceil = 1.2
|
| 1258 |
+
d_scaled = (deriv - floor) * (255.0 / (ceil - floor)) + torch.rand_like(
|
| 1259 |
+
deriv
|
| 1260 |
+
)
|
| 1261 |
+
if __name__ == "__main__":
|
| 1262 |
+
# for self-testing only.
|
| 1263 |
+
assert d_scaled.min() >= 0.0
|
| 1264 |
+
assert d_scaled.max() < 256.0
|
| 1265 |
+
d_int = d_scaled.to(torch.uint8)
|
| 1266 |
+
ctx.save_for_backward(d_int)
|
| 1267 |
+
if x.dtype == torch.float16 or torch.is_autocast_enabled():
|
| 1268 |
+
y = y.to(torch.float16)
|
| 1269 |
+
return y
|
| 1270 |
+
|
| 1271 |
+
@staticmethod
|
| 1272 |
+
def backward(ctx, y_grad: Tensor) -> Tensor:
|
| 1273 |
+
(d,) = ctx.saved_tensors
|
| 1274 |
+
# the same constants as used in forward pass.
|
| 1275 |
+
floor = -0.043637
|
| 1276 |
+
ceil = 1.2
|
| 1277 |
+
|
| 1278 |
+
d = d * ((ceil - floor) / 255.0) + floor
|
| 1279 |
+
return y_grad * d
|
| 1280 |
+
|
| 1281 |
+
|
| 1282 |
+
class DoubleSwish(torch.nn.Module):
|
| 1283 |
+
def __init__(self):
|
| 1284 |
+
super().__init__()
|
| 1285 |
+
|
| 1286 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 1287 |
+
"""Return double-swish activation function which is an approximation to Swish(Swish(x)),
|
| 1288 |
+
that we approximate closely with x * sigmoid(x-1).
|
| 1289 |
+
"""
|
| 1290 |
+
if torch.jit.is_scripting() or torch.jit.is_tracing():
|
| 1291 |
+
return x * torch.sigmoid(x - 1.0)
|
| 1292 |
+
return DoubleSwishFunction.apply(x)
|
| 1293 |
+
|
| 1294 |
+
|
| 1295 |
+
# Dropout2 is just like normal dropout, except it supports schedules on the dropout rates.
|
| 1296 |
+
class Dropout2(nn.Module):
|
| 1297 |
+
def __init__(self, p: FloatLike):
|
| 1298 |
+
super().__init__()
|
| 1299 |
+
self.p = p
|
| 1300 |
+
|
| 1301 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 1302 |
+
return torch.nn.functional.dropout(x, p=float(self.p), training=self.training)
|
| 1303 |
+
|
| 1304 |
+
|
| 1305 |
+
class MulForDropout3(torch.autograd.Function):
|
| 1306 |
+
# returns (x * y * alpha) where alpha is a float and y doesn't require
|
| 1307 |
+
# grad and is zero-or-one.
|
| 1308 |
+
@staticmethod
|
| 1309 |
+
@custom_fwd
|
| 1310 |
+
def forward(ctx, x, y, alpha):
|
| 1311 |
+
assert not y.requires_grad
|
| 1312 |
+
ans = x * y * alpha
|
| 1313 |
+
ctx.save_for_backward(ans)
|
| 1314 |
+
ctx.alpha = alpha
|
| 1315 |
+
return ans
|
| 1316 |
+
|
| 1317 |
+
@staticmethod
|
| 1318 |
+
@custom_bwd
|
| 1319 |
+
def backward(ctx, ans_grad):
|
| 1320 |
+
(ans,) = ctx.saved_tensors
|
| 1321 |
+
x_grad = ctx.alpha * ans_grad * (ans != 0)
|
| 1322 |
+
return x_grad, None, None
|
| 1323 |
+
|
| 1324 |
+
|
| 1325 |
+
# Dropout3 is just like normal dropout, except it supports schedules on the dropout rates,
|
| 1326 |
+
# and it lets you choose one dimension to share the dropout mask over
|
| 1327 |
+
class Dropout3(nn.Module):
|
| 1328 |
+
def __init__(self, p: FloatLike, shared_dim: int):
|
| 1329 |
+
super().__init__()
|
| 1330 |
+
self.p = p
|
| 1331 |
+
self.shared_dim = shared_dim
|
| 1332 |
+
|
| 1333 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 1334 |
+
p = float(self.p)
|
| 1335 |
+
if not self.training or p == 0:
|
| 1336 |
+
return _no_op(x)
|
| 1337 |
+
scale = 1.0 / (1 - p)
|
| 1338 |
+
rand_shape = list(x.shape)
|
| 1339 |
+
rand_shape[self.shared_dim] = 1
|
| 1340 |
+
mask = torch.rand(*rand_shape, device=x.device) > p
|
| 1341 |
+
ans = MulForDropout3.apply(x, mask, scale)
|
| 1342 |
+
return ans
|
| 1343 |
+
|
| 1344 |
+
|
| 1345 |
+
class SwooshLFunction(torch.autograd.Function):
|
| 1346 |
+
"""
|
| 1347 |
+
swoosh_l(x) = log(1 + exp(x-4)) - 0.08*x - 0.035
|
| 1348 |
+
"""
|
| 1349 |
+
|
| 1350 |
+
@staticmethod
|
| 1351 |
+
def forward(ctx, x: Tensor) -> Tensor:
|
| 1352 |
+
requires_grad = x.requires_grad
|
| 1353 |
+
if x.dtype == torch.float16 or x.dtype == torch.bfloat16:
|
| 1354 |
+
x = x.to(torch.float32)
|
| 1355 |
+
|
| 1356 |
+
zero = torch.tensor(0.0, dtype=x.dtype, device=x.device)
|
| 1357 |
+
|
| 1358 |
+
coeff = -0.08
|
| 1359 |
+
|
| 1360 |
+
with torch_autocast(enabled=False):
|
| 1361 |
+
with torch.enable_grad():
|
| 1362 |
+
x = x.detach()
|
| 1363 |
+
x.requires_grad = True
|
| 1364 |
+
y = torch.logaddexp(zero, x - 4.0) + coeff * x - 0.035
|
| 1365 |
+
|
| 1366 |
+
if not requires_grad:
|
| 1367 |
+
return y
|
| 1368 |
+
|
| 1369 |
+
y.backward(gradient=torch.ones_like(y))
|
| 1370 |
+
|
| 1371 |
+
grad = x.grad
|
| 1372 |
+
floor = coeff
|
| 1373 |
+
ceil = 1.0 + coeff + 0.005
|
| 1374 |
+
|
| 1375 |
+
d_scaled = (grad - floor) * (255.0 / (ceil - floor)) + torch.rand_like(
|
| 1376 |
+
grad
|
| 1377 |
+
)
|
| 1378 |
+
if __name__ == "__main__":
|
| 1379 |
+
# for self-testing only.
|
| 1380 |
+
assert d_scaled.min() >= 0.0
|
| 1381 |
+
assert d_scaled.max() < 256.0
|
| 1382 |
+
|
| 1383 |
+
d_int = d_scaled.to(torch.uint8)
|
| 1384 |
+
ctx.save_for_backward(d_int)
|
| 1385 |
+
if x.dtype == torch.float16 or torch.is_autocast_enabled():
|
| 1386 |
+
y = y.to(torch.get_autocast_gpu_dtype())
|
| 1387 |
+
return y
|
| 1388 |
+
|
| 1389 |
+
@staticmethod
|
| 1390 |
+
def backward(ctx, y_grad: Tensor) -> Tensor:
|
| 1391 |
+
(d,) = ctx.saved_tensors
|
| 1392 |
+
# the same constants as used in forward pass.
|
| 1393 |
+
|
| 1394 |
+
coeff = -0.08
|
| 1395 |
+
floor = coeff
|
| 1396 |
+
ceil = 1.0 + coeff + 0.005
|
| 1397 |
+
d = d * ((ceil - floor) / 255.0) + floor
|
| 1398 |
+
return y_grad * d
|
| 1399 |
+
|
| 1400 |
+
|
| 1401 |
+
class SwooshL(torch.nn.Module):
|
| 1402 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 1403 |
+
"""Return Swoosh-L activation."""
|
| 1404 |
+
if torch.jit.is_scripting() or torch.jit.is_tracing():
|
| 1405 |
+
zero = torch.tensor(0.0, dtype=x.dtype, device=x.device)
|
| 1406 |
+
return logaddexp(zero, x - 4.0) - 0.08 * x - 0.035
|
| 1407 |
+
if not x.requires_grad:
|
| 1408 |
+
return k2.swoosh_l_forward(x).to(x.dtype)
|
| 1409 |
+
else:
|
| 1410 |
+
return k2.swoosh_l(x).to(x.dtype)
|
| 1411 |
+
# return SwooshLFunction.apply(x)
|
| 1412 |
+
|
| 1413 |
+
|
| 1414 |
+
class SwooshLOnnx(torch.nn.Module):
|
| 1415 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 1416 |
+
"""Return Swoosh-L activation."""
|
| 1417 |
+
zero = torch.tensor(0.0, dtype=x.dtype, device=x.device)
|
| 1418 |
+
return logaddexp_onnx(zero, x - 4.0) - 0.08 * x - 0.035
|
| 1419 |
+
|
| 1420 |
+
|
| 1421 |
+
class SwooshRFunction(torch.autograd.Function):
|
| 1422 |
+
"""
|
| 1423 |
+
swoosh_r(x) = log(1 + exp(x-1)) - 0.08*x - 0.313261687
|
| 1424 |
+
|
| 1425 |
+
derivatives are between -0.08 and 0.92.
|
| 1426 |
+
"""
|
| 1427 |
+
|
| 1428 |
+
@staticmethod
|
| 1429 |
+
def forward(ctx, x: Tensor) -> Tensor:
|
| 1430 |
+
requires_grad = x.requires_grad
|
| 1431 |
+
|
| 1432 |
+
if x.dtype == torch.float16 or x.dtype == torch.bfloat16:
|
| 1433 |
+
x = x.to(torch.float32)
|
| 1434 |
+
|
| 1435 |
+
zero = torch.tensor(0.0, dtype=x.dtype, device=x.device)
|
| 1436 |
+
|
| 1437 |
+
with torch_autocast(enabled=False):
|
| 1438 |
+
with torch.enable_grad():
|
| 1439 |
+
x = x.detach()
|
| 1440 |
+
x.requires_grad = True
|
| 1441 |
+
y = torch.logaddexp(zero, x - 1.0) - 0.08 * x - 0.313261687
|
| 1442 |
+
|
| 1443 |
+
if not requires_grad:
|
| 1444 |
+
return y
|
| 1445 |
+
y.backward(gradient=torch.ones_like(y))
|
| 1446 |
+
|
| 1447 |
+
grad = x.grad
|
| 1448 |
+
floor = -0.08
|
| 1449 |
+
ceil = 0.925
|
| 1450 |
+
|
| 1451 |
+
d_scaled = (grad - floor) * (255.0 / (ceil - floor)) + torch.rand_like(
|
| 1452 |
+
grad
|
| 1453 |
+
)
|
| 1454 |
+
if __name__ == "__main__":
|
| 1455 |
+
# for self-testing only.
|
| 1456 |
+
assert d_scaled.min() >= 0.0
|
| 1457 |
+
assert d_scaled.max() < 256.0
|
| 1458 |
+
|
| 1459 |
+
d_int = d_scaled.to(torch.uint8)
|
| 1460 |
+
ctx.save_for_backward(d_int)
|
| 1461 |
+
if x.dtype == torch.float16 or torch.is_autocast_enabled():
|
| 1462 |
+
y = y.to(torch.get_autocast_gpu_dtype())
|
| 1463 |
+
return y
|
| 1464 |
+
|
| 1465 |
+
@staticmethod
|
| 1466 |
+
def backward(ctx, y_grad: Tensor) -> Tensor:
|
| 1467 |
+
(d,) = ctx.saved_tensors
|
| 1468 |
+
# the same constants as used in forward pass.
|
| 1469 |
+
floor = -0.08
|
| 1470 |
+
ceil = 0.925
|
| 1471 |
+
d = d * ((ceil - floor) / 255.0) + floor
|
| 1472 |
+
return y_grad * d
|
| 1473 |
+
|
| 1474 |
+
|
| 1475 |
+
class SwooshR(torch.nn.Module):
|
| 1476 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 1477 |
+
"""Return Swoosh-R activation."""
|
| 1478 |
+
if torch.jit.is_scripting() or torch.jit.is_tracing():
|
| 1479 |
+
zero = torch.tensor(0.0, dtype=x.dtype, device=x.device)
|
| 1480 |
+
return logaddexp(zero, x - 1.0) - 0.08 * x - 0.313261687
|
| 1481 |
+
if not x.requires_grad:
|
| 1482 |
+
return k2.swoosh_r_forward(x).to(x.dtype)
|
| 1483 |
+
else:
|
| 1484 |
+
return k2.swoosh_r(x).to(x.dtype)
|
| 1485 |
+
# return SwooshRFunction.apply(x)
|
| 1486 |
+
|
| 1487 |
+
|
| 1488 |
+
class SwooshROnnx(torch.nn.Module):
|
| 1489 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 1490 |
+
"""Return Swoosh-R activation."""
|
| 1491 |
+
zero = torch.tensor(0.0, dtype=x.dtype, device=x.device)
|
| 1492 |
+
return logaddexp_onnx(zero, x - 1.0) - 0.08 * x - 0.313261687
|
| 1493 |
+
|
| 1494 |
+
|
| 1495 |
+
# simple version of SwooshL that does not redefine the backprop, used in
|
| 1496 |
+
# ActivationDropoutAndLinearFunction.
|
| 1497 |
+
def SwooshLForward(x: Tensor):
|
| 1498 |
+
x_offset = x - 4.0
|
| 1499 |
+
log_sum = (1.0 + x_offset.exp()).log().to(x.dtype)
|
| 1500 |
+
log_sum = torch.where(log_sum == float("inf"), x_offset, log_sum)
|
| 1501 |
+
return log_sum - 0.08 * x - 0.035
|
| 1502 |
+
|
| 1503 |
+
|
| 1504 |
+
# simple version of SwooshR that does not redefine the backprop, used in
|
| 1505 |
+
# ActivationDropoutAndLinearFunction.
|
| 1506 |
+
def SwooshRForward(x: Tensor):
|
| 1507 |
+
x_offset = x - 1.0
|
| 1508 |
+
log_sum = (1.0 + x_offset.exp()).log().to(x.dtype)
|
| 1509 |
+
log_sum = torch.where(log_sum == float("inf"), x_offset, log_sum)
|
| 1510 |
+
return log_sum - 0.08 * x - 0.313261687
|
| 1511 |
+
|
| 1512 |
+
|
| 1513 |
+
class ActivationDropoutAndLinearFunction(torch.autograd.Function):
|
| 1514 |
+
@staticmethod
|
| 1515 |
+
@custom_fwd
|
| 1516 |
+
def forward(
|
| 1517 |
+
ctx,
|
| 1518 |
+
x: Tensor,
|
| 1519 |
+
weight: Tensor,
|
| 1520 |
+
bias: Optional[Tensor],
|
| 1521 |
+
activation: str,
|
| 1522 |
+
dropout_p: float,
|
| 1523 |
+
dropout_shared_dim: Optional[int],
|
| 1524 |
+
):
|
| 1525 |
+
if dropout_p != 0.0:
|
| 1526 |
+
dropout_shape = list(x.shape)
|
| 1527 |
+
if dropout_shared_dim is not None:
|
| 1528 |
+
dropout_shape[dropout_shared_dim] = 1
|
| 1529 |
+
# else it won't be very memory efficient.
|
| 1530 |
+
dropout_mask = (1.0 / (1.0 - dropout_p)) * (
|
| 1531 |
+
torch.rand(*dropout_shape, device=x.device, dtype=x.dtype) > dropout_p
|
| 1532 |
+
)
|
| 1533 |
+
else:
|
| 1534 |
+
dropout_mask = None
|
| 1535 |
+
|
| 1536 |
+
ctx.save_for_backward(x, weight, bias, dropout_mask)
|
| 1537 |
+
|
| 1538 |
+
ctx.activation = activation
|
| 1539 |
+
|
| 1540 |
+
forward_activation_dict = {
|
| 1541 |
+
"SwooshL": k2.swoosh_l_forward,
|
| 1542 |
+
"SwooshR": k2.swoosh_r_forward,
|
| 1543 |
+
}
|
| 1544 |
+
# it will raise a KeyError if this fails. This will be an error. We let it
|
| 1545 |
+
# propagate to the user.
|
| 1546 |
+
activation_func = forward_activation_dict[activation]
|
| 1547 |
+
x = activation_func(x)
|
| 1548 |
+
if dropout_mask is not None:
|
| 1549 |
+
x = x * dropout_mask
|
| 1550 |
+
x = torch.nn.functional.linear(x, weight, bias)
|
| 1551 |
+
return x
|
| 1552 |
+
|
| 1553 |
+
@staticmethod
|
| 1554 |
+
@custom_bwd
|
| 1555 |
+
def backward(ctx, ans_grad: Tensor):
|
| 1556 |
+
saved = ctx.saved_tensors
|
| 1557 |
+
(x, weight, bias, dropout_mask) = saved
|
| 1558 |
+
|
| 1559 |
+
forward_and_deriv_activation_dict = {
|
| 1560 |
+
"SwooshL": k2.swoosh_l_forward_and_deriv,
|
| 1561 |
+
"SwooshR": k2.swoosh_r_forward_and_deriv,
|
| 1562 |
+
}
|
| 1563 |
+
# the following lines a KeyError if the activation is unrecognized.
|
| 1564 |
+
# This will be an error. We let it propagate to the user.
|
| 1565 |
+
func = forward_and_deriv_activation_dict[ctx.activation]
|
| 1566 |
+
|
| 1567 |
+
y, func_deriv = func(x)
|
| 1568 |
+
if dropout_mask is not None:
|
| 1569 |
+
y = y * dropout_mask
|
| 1570 |
+
# now compute derivative of y w.r.t. weight and bias..
|
| 1571 |
+
# y: (..., in_channels), ans_grad: (..., out_channels),
|
| 1572 |
+
(out_channels, in_channels) = weight.shape
|
| 1573 |
+
|
| 1574 |
+
in_channels = y.shape[-1]
|
| 1575 |
+
g = ans_grad.reshape(-1, out_channels)
|
| 1576 |
+
weight_deriv = torch.matmul(g.t(), y.reshape(-1, in_channels))
|
| 1577 |
+
y_deriv = torch.matmul(ans_grad, weight)
|
| 1578 |
+
bias_deriv = None if bias is None else g.sum(dim=0)
|
| 1579 |
+
x_deriv = y_deriv * func_deriv
|
| 1580 |
+
if dropout_mask is not None:
|
| 1581 |
+
# order versus func_deriv does not matter
|
| 1582 |
+
x_deriv = x_deriv * dropout_mask
|
| 1583 |
+
|
| 1584 |
+
return x_deriv, weight_deriv, bias_deriv, None, None, None
|
| 1585 |
+
|
| 1586 |
+
|
| 1587 |
+
class ActivationDropoutAndLinear(torch.nn.Module):
|
| 1588 |
+
"""
|
| 1589 |
+
This merges an activation function followed by dropout and then a nn.Linear module;
|
| 1590 |
+
it does so in a memory efficient way so that it only stores the input to the whole
|
| 1591 |
+
module. If activation == SwooshL and dropout_shared_dim != None, this will be
|
| 1592 |
+
equivalent to:
|
| 1593 |
+
nn.Sequential(SwooshL(),
|
| 1594 |
+
Dropout3(dropout_p, shared_dim=dropout_shared_dim),
|
| 1595 |
+
ScaledLinear(in_channels, out_channels, bias=bias,
|
| 1596 |
+
initial_scale=initial_scale))
|
| 1597 |
+
If dropout_shared_dim is None, the dropout would be equivalent to
|
| 1598 |
+
Dropout2(dropout_p). Note: Dropout3 will be more memory efficient as the dropout
|
| 1599 |
+
mask is smaller.
|
| 1600 |
+
|
| 1601 |
+
Args:
|
| 1602 |
+
in_channels: number of input channels, e.g. 256
|
| 1603 |
+
out_channels: number of output channels, e.g. 256
|
| 1604 |
+
bias: if true, have a bias
|
| 1605 |
+
activation: the activation function, for now just support SwooshL.
|
| 1606 |
+
dropout_p: the dropout probability or schedule (happens after nonlinearity).
|
| 1607 |
+
dropout_shared_dim: the dimension, if any, across which the dropout mask is
|
| 1608 |
+
shared (e.g. the time dimension). If None, this may be less memory
|
| 1609 |
+
efficient if there are modules before this one that cache the input
|
| 1610 |
+
for their backprop (e.g. Balancer or Whiten).
|
| 1611 |
+
"""
|
| 1612 |
+
|
| 1613 |
+
def __init__(
|
| 1614 |
+
self,
|
| 1615 |
+
in_channels: int,
|
| 1616 |
+
out_channels: int,
|
| 1617 |
+
bias: bool = True,
|
| 1618 |
+
activation: str = "SwooshL",
|
| 1619 |
+
dropout_p: FloatLike = 0.0,
|
| 1620 |
+
dropout_shared_dim: Optional[int] = -1,
|
| 1621 |
+
initial_scale: float = 1.0,
|
| 1622 |
+
):
|
| 1623 |
+
super().__init__()
|
| 1624 |
+
# create a temporary module of nn.Linear that we'll steal the
|
| 1625 |
+
# weights and bias from
|
| 1626 |
+
l = ScaledLinear(
|
| 1627 |
+
in_channels, out_channels, bias=bias, initial_scale=initial_scale
|
| 1628 |
+
)
|
| 1629 |
+
|
| 1630 |
+
self.weight = l.weight
|
| 1631 |
+
# register_parameter properly handles making it a parameter when l.bias
|
| 1632 |
+
# is None. I think there is some reason for doing it this way rather
|
| 1633 |
+
# than just setting it to None but I don't know what it is, maybe
|
| 1634 |
+
# something to do with exporting the module..
|
| 1635 |
+
self.register_parameter("bias", l.bias)
|
| 1636 |
+
|
| 1637 |
+
self.activation = activation
|
| 1638 |
+
self.dropout_p = dropout_p
|
| 1639 |
+
self.dropout_shared_dim = dropout_shared_dim
|
| 1640 |
+
|
| 1641 |
+
def forward(self, x: Tensor):
|
| 1642 |
+
if not self.training or torch.jit.is_scripting() or torch.jit.is_tracing():
|
| 1643 |
+
if self.activation == "SwooshL":
|
| 1644 |
+
x = SwooshLForward(x)
|
| 1645 |
+
elif self.activation == "SwooshR":
|
| 1646 |
+
x = SwooshRForward(x)
|
| 1647 |
+
else:
|
| 1648 |
+
assert False, self.activation
|
| 1649 |
+
return torch.nn.functional.linear(x, self.weight, self.bias)
|
| 1650 |
+
|
| 1651 |
+
return ActivationDropoutAndLinearFunction.apply(
|
| 1652 |
+
x,
|
| 1653 |
+
self.weight,
|
| 1654 |
+
self.bias,
|
| 1655 |
+
self.activation,
|
| 1656 |
+
float(self.dropout_p),
|
| 1657 |
+
self.dropout_shared_dim,
|
| 1658 |
+
)
|
| 1659 |
+
|
| 1660 |
+
|
| 1661 |
+
def convert_num_channels(x: Tensor, num_channels: int) -> Tensor:
|
| 1662 |
+
if num_channels <= x.shape[-1]:
|
| 1663 |
+
return x[..., :num_channels]
|
| 1664 |
+
else:
|
| 1665 |
+
shape = list(x.shape)
|
| 1666 |
+
shape[-1] = num_channels - shape[-1]
|
| 1667 |
+
zeros = torch.zeros(shape, dtype=x.dtype, device=x.device)
|
| 1668 |
+
return torch.cat((x, zeros), dim=-1)
|
| 1669 |
+
|
| 1670 |
+
|
| 1671 |
+
def _test_whiten():
|
| 1672 |
+
for proportion in [0.1, 0.5, 10.0]:
|
| 1673 |
+
logging.info(f"_test_whiten(): proportion = {proportion}")
|
| 1674 |
+
x = torch.randn(100, 128)
|
| 1675 |
+
direction = torch.randn(128)
|
| 1676 |
+
coeffs = torch.randn(100, 1)
|
| 1677 |
+
x += proportion * direction * coeffs
|
| 1678 |
+
|
| 1679 |
+
x.requires_grad = True
|
| 1680 |
+
|
| 1681 |
+
m = Whiten(
|
| 1682 |
+
1, 5.0, prob=1.0, grad_scale=0.1 # num_groups # whitening_limit,
|
| 1683 |
+
) # grad_scale
|
| 1684 |
+
|
| 1685 |
+
for _ in range(4):
|
| 1686 |
+
y = m(x)
|
| 1687 |
+
|
| 1688 |
+
y_grad = torch.randn_like(x)
|
| 1689 |
+
y.backward(gradient=y_grad)
|
| 1690 |
+
|
| 1691 |
+
if proportion < 0.2:
|
| 1692 |
+
assert torch.allclose(x.grad, y_grad)
|
| 1693 |
+
elif proportion > 1.0:
|
| 1694 |
+
assert not torch.allclose(x.grad, y_grad)
|
| 1695 |
+
|
| 1696 |
+
|
| 1697 |
+
def _test_balancer_sign():
|
| 1698 |
+
probs = torch.arange(0, 1, 0.01)
|
| 1699 |
+
N = 1000
|
| 1700 |
+
x = 1.0 * ((2.0 * (torch.rand(probs.numel(), N) < probs.unsqueeze(-1))) - 1.0)
|
| 1701 |
+
x = x.detach()
|
| 1702 |
+
x.requires_grad = True
|
| 1703 |
+
m = Balancer(
|
| 1704 |
+
probs.numel(),
|
| 1705 |
+
channel_dim=0,
|
| 1706 |
+
min_positive=0.05,
|
| 1707 |
+
max_positive=0.95,
|
| 1708 |
+
min_abs=0.0,
|
| 1709 |
+
prob=1.0,
|
| 1710 |
+
)
|
| 1711 |
+
|
| 1712 |
+
y_grad = torch.sign(torch.randn(probs.numel(), N))
|
| 1713 |
+
|
| 1714 |
+
y = m(x)
|
| 1715 |
+
y.backward(gradient=y_grad)
|
| 1716 |
+
print("_test_balancer_sign: x = ", x)
|
| 1717 |
+
print("_test_balancer_sign: y grad = ", y_grad)
|
| 1718 |
+
print("_test_balancer_sign: x grad = ", x.grad)
|
| 1719 |
+
|
| 1720 |
+
|
| 1721 |
+
def _test_balancer_magnitude():
|
| 1722 |
+
magnitudes = torch.arange(0, 1, 0.01)
|
| 1723 |
+
N = 1000
|
| 1724 |
+
x = torch.sign(torch.randn(magnitudes.numel(), N)) * magnitudes.unsqueeze(-1)
|
| 1725 |
+
x = x.detach()
|
| 1726 |
+
x.requires_grad = True
|
| 1727 |
+
m = Balancer(
|
| 1728 |
+
magnitudes.numel(),
|
| 1729 |
+
channel_dim=0,
|
| 1730 |
+
min_positive=0.0,
|
| 1731 |
+
max_positive=1.0,
|
| 1732 |
+
min_abs=0.2,
|
| 1733 |
+
max_abs=0.7,
|
| 1734 |
+
prob=1.0,
|
| 1735 |
+
)
|
| 1736 |
+
|
| 1737 |
+
y_grad = torch.sign(torch.randn(magnitudes.numel(), N))
|
| 1738 |
+
|
| 1739 |
+
y = m(x)
|
| 1740 |
+
y.backward(gradient=y_grad)
|
| 1741 |
+
print("_test_balancer_magnitude: x = ", x)
|
| 1742 |
+
print("_test_balancer_magnitude: y grad = ", y_grad)
|
| 1743 |
+
print("_test_balancer_magnitude: x grad = ", x.grad)
|
| 1744 |
+
|
| 1745 |
+
|
| 1746 |
+
def _test_double_swish_deriv():
|
| 1747 |
+
x = torch.randn(10, 12, dtype=torch.double) * 3.0
|
| 1748 |
+
x.requires_grad = True
|
| 1749 |
+
m = DoubleSwish()
|
| 1750 |
+
|
| 1751 |
+
tol = (1.2 - (-0.043637)) / 255.0
|
| 1752 |
+
torch.autograd.gradcheck(m, x, atol=tol)
|
| 1753 |
+
|
| 1754 |
+
# for self-test.
|
| 1755 |
+
x = torch.randn(1000, 1000, dtype=torch.double) * 3.0
|
| 1756 |
+
x.requires_grad = True
|
| 1757 |
+
y = m(x)
|
| 1758 |
+
|
| 1759 |
+
|
| 1760 |
+
def _test_swooshl_deriv():
|
| 1761 |
+
x = torch.randn(10, 12, dtype=torch.double) * 3.0
|
| 1762 |
+
x.requires_grad = True
|
| 1763 |
+
m = SwooshL()
|
| 1764 |
+
|
| 1765 |
+
tol = 1.0 / 255.0
|
| 1766 |
+
torch.autograd.gradcheck(m, x, atol=tol, eps=0.01)
|
| 1767 |
+
|
| 1768 |
+
# for self-test.
|
| 1769 |
+
x = torch.randn(1000, 1000, dtype=torch.double) * 3.0
|
| 1770 |
+
x.requires_grad = True
|
| 1771 |
+
y = m(x)
|
| 1772 |
+
|
| 1773 |
+
|
| 1774 |
+
def _test_swooshr_deriv():
|
| 1775 |
+
x = torch.randn(10, 12, dtype=torch.double) * 3.0
|
| 1776 |
+
x.requires_grad = True
|
| 1777 |
+
m = SwooshR()
|
| 1778 |
+
|
| 1779 |
+
tol = 1.0 / 255.0
|
| 1780 |
+
torch.autograd.gradcheck(m, x, atol=tol, eps=0.01)
|
| 1781 |
+
|
| 1782 |
+
# for self-test.
|
| 1783 |
+
x = torch.randn(1000, 1000, dtype=torch.double) * 3.0
|
| 1784 |
+
x.requires_grad = True
|
| 1785 |
+
y = m(x)
|
| 1786 |
+
|
| 1787 |
+
|
| 1788 |
+
def _test_softmax():
|
| 1789 |
+
a = torch.randn(2, 10, dtype=torch.float64)
|
| 1790 |
+
b = a.clone()
|
| 1791 |
+
a.requires_grad = True
|
| 1792 |
+
b.requires_grad = True
|
| 1793 |
+
a.softmax(dim=1)[:, 0].sum().backward()
|
| 1794 |
+
print("a grad = ", a.grad)
|
| 1795 |
+
softmax(b, dim=1)[:, 0].sum().backward()
|
| 1796 |
+
print("b grad = ", b.grad)
|
| 1797 |
+
assert torch.allclose(a.grad, b.grad)
|
| 1798 |
+
|
| 1799 |
+
|
| 1800 |
+
def _test_piecewise_linear():
|
| 1801 |
+
p = PiecewiseLinear((0, 10.0))
|
| 1802 |
+
for x in [-100, 0, 100]:
|
| 1803 |
+
assert p(x) == 10.0
|
| 1804 |
+
p = PiecewiseLinear((0, 10.0), (1, 0.0))
|
| 1805 |
+
for x, y in [(-100, 10.0), (0, 10.0), (0.5, 5.0), (1, 0.0), (2, 0.0)]:
|
| 1806 |
+
print("x, y = ", x, y)
|
| 1807 |
+
assert p(x) == y, (x, p(x), y)
|
| 1808 |
+
|
| 1809 |
+
q = PiecewiseLinear((0.5, 15.0), (0.6, 1.0))
|
| 1810 |
+
x_vals = [-1.0, 0.0, 0.1, 0.2, 0.5, 0.6, 0.7, 0.9, 1.0, 2.0]
|
| 1811 |
+
pq = p.max(q)
|
| 1812 |
+
for x in x_vals:
|
| 1813 |
+
y1 = max(p(x), q(x))
|
| 1814 |
+
y2 = pq(x)
|
| 1815 |
+
assert abs(y1 - y2) < 0.001
|
| 1816 |
+
pq = p.min(q)
|
| 1817 |
+
for x in x_vals:
|
| 1818 |
+
y1 = min(p(x), q(x))
|
| 1819 |
+
y2 = pq(x)
|
| 1820 |
+
assert abs(y1 - y2) < 0.001
|
| 1821 |
+
pq = p + q
|
| 1822 |
+
for x in x_vals:
|
| 1823 |
+
y1 = p(x) + q(x)
|
| 1824 |
+
y2 = pq(x)
|
| 1825 |
+
assert abs(y1 - y2) < 0.001
|
| 1826 |
+
|
| 1827 |
+
|
| 1828 |
+
def _test_activation_dropout_and_linear():
|
| 1829 |
+
in_channels = 20
|
| 1830 |
+
out_channels = 30
|
| 1831 |
+
|
| 1832 |
+
for bias in [True, False]:
|
| 1833 |
+
# actually we don't test for dropout_p != 0.0 because forward functions will give
|
| 1834 |
+
# different answers. This is because we are using the k2 implementation of
|
| 1835 |
+
# swoosh_l an swoosh_r inside SwooshL() and SwooshR(), and they call randn()
|
| 1836 |
+
# internally, messing up the random state.
|
| 1837 |
+
for dropout_p in [0.0]:
|
| 1838 |
+
for activation in ["SwooshL", "SwooshR"]:
|
| 1839 |
+
m1 = nn.Sequential(
|
| 1840 |
+
SwooshL() if activation == "SwooshL" else SwooshR(),
|
| 1841 |
+
Dropout3(p=dropout_p, shared_dim=-1),
|
| 1842 |
+
ScaledLinear(
|
| 1843 |
+
in_channels, out_channels, bias=bias, initial_scale=0.5
|
| 1844 |
+
),
|
| 1845 |
+
)
|
| 1846 |
+
m2 = ActivationDropoutAndLinear(
|
| 1847 |
+
in_channels,
|
| 1848 |
+
out_channels,
|
| 1849 |
+
bias=bias,
|
| 1850 |
+
initial_scale=0.5,
|
| 1851 |
+
activation=activation,
|
| 1852 |
+
dropout_p=dropout_p,
|
| 1853 |
+
)
|
| 1854 |
+
with torch.no_grad():
|
| 1855 |
+
m2.weight[:] = m1[2].weight
|
| 1856 |
+
if bias:
|
| 1857 |
+
m2.bias[:] = m1[2].bias
|
| 1858 |
+
# make sure forward gives same result.
|
| 1859 |
+
x1 = torch.randn(10, in_channels)
|
| 1860 |
+
x1.requires_grad = True
|
| 1861 |
+
|
| 1862 |
+
# TEMP.
|
| 1863 |
+
assert torch.allclose(
|
| 1864 |
+
SwooshRFunction.apply(x1), SwooshRForward(x1), atol=1.0e-03
|
| 1865 |
+
)
|
| 1866 |
+
|
| 1867 |
+
x2 = x1.clone().detach()
|
| 1868 |
+
x2.requires_grad = True
|
| 1869 |
+
seed = 10
|
| 1870 |
+
torch.manual_seed(seed)
|
| 1871 |
+
y1 = m1(x1)
|
| 1872 |
+
y_grad = torch.randn_like(y1)
|
| 1873 |
+
y1.backward(gradient=y_grad)
|
| 1874 |
+
torch.manual_seed(seed)
|
| 1875 |
+
y2 = m2(x2)
|
| 1876 |
+
y2.backward(gradient=y_grad)
|
| 1877 |
+
|
| 1878 |
+
print(
|
| 1879 |
+
f"bias = {bias}, dropout_p = {dropout_p}, activation = {activation}"
|
| 1880 |
+
)
|
| 1881 |
+
print("y1 = ", y1)
|
| 1882 |
+
print("y2 = ", y2)
|
| 1883 |
+
assert torch.allclose(y1, y2, atol=0.02)
|
| 1884 |
+
assert torch.allclose(m1[2].weight.grad, m2.weight.grad, atol=1.0e-05)
|
| 1885 |
+
if bias:
|
| 1886 |
+
assert torch.allclose(m1[2].bias.grad, m2.bias.grad, atol=1.0e-05)
|
| 1887 |
+
print("x1.grad = ", x1.grad)
|
| 1888 |
+
print("x2.grad = ", x2.grad)
|
| 1889 |
+
|
| 1890 |
+
def isclose(a, b):
|
| 1891 |
+
# return true if cosine similarity is > 0.9.
|
| 1892 |
+
return (a * b).sum() > 0.9 * (
|
| 1893 |
+
(a**2).sum() * (b**2).sum()
|
| 1894 |
+
).sqrt()
|
| 1895 |
+
|
| 1896 |
+
# the SwooshL() implementation has a noisy gradient due to 1-byte
|
| 1897 |
+
# storage of it.
|
| 1898 |
+
assert isclose(x1.grad, x2.grad)
|
| 1899 |
+
|
| 1900 |
+
|
| 1901 |
+
if __name__ == "__main__":
|
| 1902 |
+
logging.getLogger().setLevel(logging.INFO)
|
| 1903 |
+
torch.set_num_threads(1)
|
| 1904 |
+
torch.set_num_interop_threads(1)
|
| 1905 |
+
_test_piecewise_linear()
|
| 1906 |
+
_test_softmax()
|
| 1907 |
+
_test_whiten()
|
| 1908 |
+
_test_balancer_sign()
|
| 1909 |
+
_test_balancer_magnitude()
|
| 1910 |
+
_test_double_swish_deriv()
|
| 1911 |
+
_test_swooshr_deriv()
|
| 1912 |
+
_test_swooshl_deriv()
|
| 1913 |
+
_test_activation_dropout_and_linear()
|
speedasr_core/subsampling.py
ADDED
|
@@ -0,0 +1,406 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
# Copyright 2023 Xiaomi Corp. (authors: Daniel Povey,
|
| 3 |
+
# Zengwei Yao)
|
| 4 |
+
#
|
| 5 |
+
# See ../../../../LICENSE for clarification regarding multiple authors
|
| 6 |
+
#
|
| 7 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 8 |
+
# you may not use this file except in compliance with the License.
|
| 9 |
+
# You may obtain a copy of the License at
|
| 10 |
+
#
|
| 11 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 12 |
+
#
|
| 13 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 14 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 15 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 16 |
+
# See the License for the specific language governing permissions and
|
| 17 |
+
# limitations under the License.
|
| 18 |
+
|
| 19 |
+
import warnings
|
| 20 |
+
from typing import Tuple
|
| 21 |
+
|
| 22 |
+
import torch
|
| 23 |
+
from .scaling import (
|
| 24 |
+
Balancer,
|
| 25 |
+
BiasNorm,
|
| 26 |
+
Dropout3,
|
| 27 |
+
FloatLike,
|
| 28 |
+
Optional,
|
| 29 |
+
ScaledConv2d,
|
| 30 |
+
ScaleGrad,
|
| 31 |
+
ScheduledFloat,
|
| 32 |
+
SwooshL,
|
| 33 |
+
SwooshR,
|
| 34 |
+
Whiten,
|
| 35 |
+
)
|
| 36 |
+
from torch import Tensor, nn
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class ConvNeXt(nn.Module):
|
| 40 |
+
"""
|
| 41 |
+
Our interpretation of the ConvNeXt module as used in https://arxiv.org/pdf/2206.14747.pdf
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
def __init__(
|
| 45 |
+
self,
|
| 46 |
+
channels: int,
|
| 47 |
+
hidden_ratio: int = 3,
|
| 48 |
+
kernel_size: Tuple[int, int] = (7, 7),
|
| 49 |
+
layerdrop_rate: FloatLike = None,
|
| 50 |
+
):
|
| 51 |
+
super().__init__()
|
| 52 |
+
self.padding = ((kernel_size[0] - 1) // 2, (kernel_size[1] - 1) // 2)
|
| 53 |
+
hidden_channels = channels * hidden_ratio
|
| 54 |
+
if layerdrop_rate is None:
|
| 55 |
+
layerdrop_rate = ScheduledFloat((0.0, 0.2), (20000.0, 0.015))
|
| 56 |
+
self.layerdrop_rate = layerdrop_rate
|
| 57 |
+
|
| 58 |
+
self.depthwise_conv = nn.Conv2d(
|
| 59 |
+
in_channels=channels,
|
| 60 |
+
out_channels=channels,
|
| 61 |
+
groups=channels,
|
| 62 |
+
kernel_size=kernel_size,
|
| 63 |
+
padding=self.padding,
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
self.pointwise_conv1 = nn.Conv2d(
|
| 67 |
+
in_channels=channels, out_channels=hidden_channels, kernel_size=1
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
self.hidden_balancer = Balancer(
|
| 71 |
+
hidden_channels,
|
| 72 |
+
channel_dim=1,
|
| 73 |
+
min_positive=0.3,
|
| 74 |
+
max_positive=1.0,
|
| 75 |
+
min_abs=0.75,
|
| 76 |
+
max_abs=5.0,
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
self.activation = SwooshL()
|
| 80 |
+
self.pointwise_conv2 = ScaledConv2d(
|
| 81 |
+
in_channels=hidden_channels,
|
| 82 |
+
out_channels=channels,
|
| 83 |
+
kernel_size=1,
|
| 84 |
+
initial_scale=0.01,
|
| 85 |
+
)
|
| 86 |
+
|
| 87 |
+
self.out_balancer = Balancer(
|
| 88 |
+
channels,
|
| 89 |
+
channel_dim=1,
|
| 90 |
+
min_positive=0.4,
|
| 91 |
+
max_positive=0.6,
|
| 92 |
+
min_abs=1.0,
|
| 93 |
+
max_abs=6.0,
|
| 94 |
+
)
|
| 95 |
+
self.out_whiten = Whiten(
|
| 96 |
+
num_groups=1,
|
| 97 |
+
whitening_limit=5.0,
|
| 98 |
+
prob=(0.025, 0.25),
|
| 99 |
+
grad_scale=0.01,
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 103 |
+
if torch.jit.is_scripting() or torch.jit.is_tracing() or not self.training:
|
| 104 |
+
return self.forward_internal(x)
|
| 105 |
+
layerdrop_rate = float(self.layerdrop_rate)
|
| 106 |
+
|
| 107 |
+
if layerdrop_rate != 0.0:
|
| 108 |
+
batch_size = x.shape[0]
|
| 109 |
+
mask = (
|
| 110 |
+
torch.rand((batch_size, 1, 1, 1), dtype=x.dtype, device=x.device)
|
| 111 |
+
> layerdrop_rate
|
| 112 |
+
)
|
| 113 |
+
else:
|
| 114 |
+
mask = None
|
| 115 |
+
# turns out this caching idea does not work with --world-size > 1
|
| 116 |
+
# return caching_eval(self.forward_internal, x, mask)
|
| 117 |
+
return self.forward_internal(x, mask)
|
| 118 |
+
|
| 119 |
+
def forward_internal(
|
| 120 |
+
self, x: Tensor, layer_skip_mask: Optional[Tensor] = None
|
| 121 |
+
) -> Tensor:
|
| 122 |
+
"""
|
| 123 |
+
x layout: (N, C, H, W), i.e. (batch_size, num_channels, num_frames, num_freqs)
|
| 124 |
+
|
| 125 |
+
The returned value has the same shape as x.
|
| 126 |
+
"""
|
| 127 |
+
bypass = x
|
| 128 |
+
x = self.depthwise_conv(x)
|
| 129 |
+
x = self.pointwise_conv1(x)
|
| 130 |
+
x = self.hidden_balancer(x)
|
| 131 |
+
x = self.activation(x)
|
| 132 |
+
x = self.pointwise_conv2(x)
|
| 133 |
+
|
| 134 |
+
if layer_skip_mask is not None:
|
| 135 |
+
x = x * layer_skip_mask
|
| 136 |
+
|
| 137 |
+
x = bypass + x
|
| 138 |
+
x = self.out_balancer(x)
|
| 139 |
+
|
| 140 |
+
if x.requires_grad:
|
| 141 |
+
x = x.transpose(1, 3) # (N, W, H, C); need channel dim to be last
|
| 142 |
+
x = self.out_whiten(x)
|
| 143 |
+
x = x.transpose(1, 3) # (N, C, H, W)
|
| 144 |
+
|
| 145 |
+
return x
|
| 146 |
+
|
| 147 |
+
def streaming_forward(
|
| 148 |
+
self,
|
| 149 |
+
x: Tensor,
|
| 150 |
+
cached_left_pad: Tensor,
|
| 151 |
+
) -> Tuple[Tensor, Tensor]:
|
| 152 |
+
"""
|
| 153 |
+
Args:
|
| 154 |
+
x layout: (N, C, H, W), i.e. (batch_size, num_channels, num_frames, num_freqs)
|
| 155 |
+
cached_left_pad: (batch_size, num_channels, left_pad, num_freqs)
|
| 156 |
+
|
| 157 |
+
Returns:
|
| 158 |
+
- The returned value has the same shape as x.
|
| 159 |
+
- Updated cached_left_pad.
|
| 160 |
+
"""
|
| 161 |
+
padding = self.padding
|
| 162 |
+
|
| 163 |
+
# The length without right padding for depth-wise conv
|
| 164 |
+
T = x.size(2) - padding[0]
|
| 165 |
+
|
| 166 |
+
bypass = x[:, :, :T, :]
|
| 167 |
+
|
| 168 |
+
# Pad left side
|
| 169 |
+
assert cached_left_pad.size(2) == padding[0], (
|
| 170 |
+
cached_left_pad.size(2),
|
| 171 |
+
padding[0],
|
| 172 |
+
)
|
| 173 |
+
x = torch.cat([cached_left_pad, x], dim=2)
|
| 174 |
+
# Update cached left padding
|
| 175 |
+
cached_left_pad = x[:, :, T : padding[0] + T, :]
|
| 176 |
+
|
| 177 |
+
# depthwise_conv
|
| 178 |
+
x = torch.nn.functional.conv2d(
|
| 179 |
+
x,
|
| 180 |
+
weight=self.depthwise_conv.weight,
|
| 181 |
+
bias=self.depthwise_conv.bias,
|
| 182 |
+
padding=(0, padding[1]),
|
| 183 |
+
groups=self.depthwise_conv.groups,
|
| 184 |
+
)
|
| 185 |
+
x = self.pointwise_conv1(x)
|
| 186 |
+
x = self.hidden_balancer(x)
|
| 187 |
+
x = self.activation(x)
|
| 188 |
+
x = self.pointwise_conv2(x)
|
| 189 |
+
|
| 190 |
+
x = bypass + x
|
| 191 |
+
return x, cached_left_pad
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
class Conv2dSubsampling(nn.Module):
|
| 195 |
+
"""Convolutional 2D subsampling (to 1/2 length).
|
| 196 |
+
|
| 197 |
+
Convert an input of shape (N, T, idim) to an output
|
| 198 |
+
with shape (N, T', odim), where
|
| 199 |
+
T' = (T-3)//2 - 2 == (T-7)//2
|
| 200 |
+
|
| 201 |
+
It is based on
|
| 202 |
+
https://github.com/espnet/espnet/blob/master/espnet/nets/pytorch_backend/transformer/subsampling.py # noqa
|
| 203 |
+
"""
|
| 204 |
+
|
| 205 |
+
def __init__(
|
| 206 |
+
self,
|
| 207 |
+
in_channels: int,
|
| 208 |
+
out_channels: int,
|
| 209 |
+
layer1_channels: int = 8,
|
| 210 |
+
layer2_channels: int = 32,
|
| 211 |
+
layer3_channels: int = 128,
|
| 212 |
+
dropout: FloatLike = 0.1,
|
| 213 |
+
) -> None:
|
| 214 |
+
"""
|
| 215 |
+
Args:
|
| 216 |
+
in_channels:
|
| 217 |
+
Number of channels in. The input shape is (N, T, in_channels).
|
| 218 |
+
Caution: It requires: T >=7, in_channels >=7
|
| 219 |
+
out_channels
|
| 220 |
+
Output dim. The output shape is (N, (T-3)//2, out_channels)
|
| 221 |
+
layer1_channels:
|
| 222 |
+
Number of channels in layer1
|
| 223 |
+
layer1_channels:
|
| 224 |
+
Number of channels in layer2
|
| 225 |
+
bottleneck:
|
| 226 |
+
bottleneck dimension for 1d squeeze-excite
|
| 227 |
+
"""
|
| 228 |
+
assert in_channels >= 7
|
| 229 |
+
super().__init__()
|
| 230 |
+
|
| 231 |
+
# The ScaleGrad module is there to prevent the gradients
|
| 232 |
+
# w.r.t. the weight or bias of the first Conv2d module in self.conv from
|
| 233 |
+
# exceeding the range of fp16 when using automatic mixed precision (amp)
|
| 234 |
+
# training. (The second one is necessary to stop its bias from getting
|
| 235 |
+
# a too-large gradient).
|
| 236 |
+
|
| 237 |
+
self.conv = nn.Sequential(
|
| 238 |
+
nn.Conv2d(
|
| 239 |
+
in_channels=1,
|
| 240 |
+
out_channels=layer1_channels,
|
| 241 |
+
kernel_size=3,
|
| 242 |
+
padding=(0, 1), # (time, freq)
|
| 243 |
+
),
|
| 244 |
+
ScaleGrad(0.2),
|
| 245 |
+
Balancer(layer1_channels, channel_dim=1, max_abs=1.0),
|
| 246 |
+
SwooshR(),
|
| 247 |
+
nn.Conv2d(
|
| 248 |
+
in_channels=layer1_channels,
|
| 249 |
+
out_channels=layer2_channels,
|
| 250 |
+
kernel_size=3,
|
| 251 |
+
stride=2,
|
| 252 |
+
padding=0,
|
| 253 |
+
),
|
| 254 |
+
Balancer(layer2_channels, channel_dim=1, max_abs=4.0),
|
| 255 |
+
SwooshR(),
|
| 256 |
+
nn.Conv2d(
|
| 257 |
+
in_channels=layer2_channels,
|
| 258 |
+
out_channels=layer3_channels,
|
| 259 |
+
kernel_size=3,
|
| 260 |
+
stride=(1, 2), # (time, freq)
|
| 261 |
+
),
|
| 262 |
+
Balancer(layer3_channels, channel_dim=1, max_abs=4.0),
|
| 263 |
+
SwooshR(),
|
| 264 |
+
)
|
| 265 |
+
|
| 266 |
+
# just one convnext layer
|
| 267 |
+
self.convnext = ConvNeXt(layer3_channels, kernel_size=(7, 7))
|
| 268 |
+
|
| 269 |
+
# (in_channels-3)//4
|
| 270 |
+
self.out_width = (((in_channels - 1) // 2) - 1) // 2
|
| 271 |
+
self.layer3_channels = layer3_channels
|
| 272 |
+
|
| 273 |
+
self.out = nn.Linear(self.out_width * layer3_channels, out_channels)
|
| 274 |
+
# use a larger than normal grad_scale on this whitening module; there is
|
| 275 |
+
# only one such module, so there is not a concern about adding together
|
| 276 |
+
# many copies of this extra gradient term.
|
| 277 |
+
self.out_whiten = Whiten(
|
| 278 |
+
num_groups=1,
|
| 279 |
+
whitening_limit=ScheduledFloat((0.0, 4.0), (20000.0, 8.0), default=4.0),
|
| 280 |
+
prob=(0.025, 0.25),
|
| 281 |
+
grad_scale=0.02,
|
| 282 |
+
)
|
| 283 |
+
|
| 284 |
+
# max_log_eps=0.0 is to prevent both eps and the output of self.out from
|
| 285 |
+
# getting large, there is an unnecessary degree of freedom.
|
| 286 |
+
self.out_norm = BiasNorm(out_channels)
|
| 287 |
+
self.dropout = Dropout3(dropout, shared_dim=1)
|
| 288 |
+
|
| 289 |
+
def forward(
|
| 290 |
+
self, x: torch.Tensor, x_lens: torch.Tensor
|
| 291 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 292 |
+
"""Subsample x.
|
| 293 |
+
|
| 294 |
+
Args:
|
| 295 |
+
x:
|
| 296 |
+
Its shape is (N, T, idim).
|
| 297 |
+
x_lens:
|
| 298 |
+
A tensor of shape (batch_size,) containing the number of frames in
|
| 299 |
+
|
| 300 |
+
Returns:
|
| 301 |
+
- a tensor of shape (N, (T-7)//2, odim)
|
| 302 |
+
- output lengths, of shape (batch_size,)
|
| 303 |
+
"""
|
| 304 |
+
# On entry, x is (N, T, idim)
|
| 305 |
+
x = x.unsqueeze(1) # (N, T, idim) -> (N, 1, T, idim) i.e., (N, C, H, W)
|
| 306 |
+
# scaling x by 0.1 allows us to use a larger grad-scale in fp16 "amp" (automatic mixed precision)
|
| 307 |
+
# training, since the weights in the first convolution are otherwise the limiting factor for getting infinite
|
| 308 |
+
# gradients.
|
| 309 |
+
x = self.conv(x)
|
| 310 |
+
x = self.convnext(x)
|
| 311 |
+
|
| 312 |
+
# Now x is of shape (N, odim, (T-7)//2, (idim-3)//4)
|
| 313 |
+
b, c, t, f = x.size()
|
| 314 |
+
|
| 315 |
+
x = x.transpose(1, 2).reshape(b, t, c * f)
|
| 316 |
+
# now x: (N, (T-7)//2, out_width * layer3_channels))
|
| 317 |
+
|
| 318 |
+
x = self.out(x)
|
| 319 |
+
# Now x is of shape (N, (T-7)//2, odim)
|
| 320 |
+
x = self.out_whiten(x)
|
| 321 |
+
x = self.out_norm(x)
|
| 322 |
+
x = self.dropout(x)
|
| 323 |
+
|
| 324 |
+
if torch.jit.is_scripting() or torch.jit.is_tracing():
|
| 325 |
+
x_lens = (x_lens - 7) // 2
|
| 326 |
+
else:
|
| 327 |
+
with warnings.catch_warnings():
|
| 328 |
+
warnings.simplefilter("ignore")
|
| 329 |
+
x_lens = (x_lens - 7) // 2
|
| 330 |
+
assert x.size(1) == x_lens.max().item(), (x.size(1), x_lens.max())
|
| 331 |
+
|
| 332 |
+
return x, x_lens
|
| 333 |
+
|
| 334 |
+
def streaming_forward(
|
| 335 |
+
self,
|
| 336 |
+
x: torch.Tensor,
|
| 337 |
+
x_lens: torch.Tensor,
|
| 338 |
+
cached_left_pad: Tensor,
|
| 339 |
+
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 340 |
+
"""Subsample x.
|
| 341 |
+
|
| 342 |
+
Args:
|
| 343 |
+
x:
|
| 344 |
+
Its shape is (N, T, idim).
|
| 345 |
+
x_lens:
|
| 346 |
+
A tensor of shape (batch_size,) containing the number of frames in
|
| 347 |
+
|
| 348 |
+
Returns:
|
| 349 |
+
- a tensor of shape (N, (T-7)//2, odim)
|
| 350 |
+
- output lengths, of shape (batch_size,)
|
| 351 |
+
- updated cache
|
| 352 |
+
"""
|
| 353 |
+
# On entry, x is (N, T, idim)
|
| 354 |
+
x = x.unsqueeze(1) # (N, T, idim) -> (N, 1, T, idim) i.e., (N, C, H, W)
|
| 355 |
+
|
| 356 |
+
# T' = (T-7)//2
|
| 357 |
+
x = self.conv(x)
|
| 358 |
+
|
| 359 |
+
# T' = (T-7)//2-3
|
| 360 |
+
x, cached_left_pad = self.convnext.streaming_forward(
|
| 361 |
+
x, cached_left_pad=cached_left_pad
|
| 362 |
+
)
|
| 363 |
+
|
| 364 |
+
# Now x is of shape (N, odim, T', ((idim-1)//2 - 1)//2)
|
| 365 |
+
b, c, t, f = x.size()
|
| 366 |
+
|
| 367 |
+
x = x.transpose(1, 2).reshape(b, t, c * f)
|
| 368 |
+
# now x: (N, T', out_width * layer3_channels))
|
| 369 |
+
|
| 370 |
+
x = self.out(x)
|
| 371 |
+
# Now x is of shape (N, T', odim)
|
| 372 |
+
x = self.out_norm(x)
|
| 373 |
+
|
| 374 |
+
if torch.jit.is_scripting() or torch.jit.is_tracing():
|
| 375 |
+
assert self.convnext.padding[0] == 3
|
| 376 |
+
# The ConvNeXt module needs 3 frames of right padding after subsampling
|
| 377 |
+
x_lens = (x_lens - 7) // 2 - 3
|
| 378 |
+
else:
|
| 379 |
+
with warnings.catch_warnings():
|
| 380 |
+
warnings.simplefilter("ignore")
|
| 381 |
+
# The ConvNeXt module needs 3 frames of right padding after subsampling
|
| 382 |
+
assert self.convnext.padding[0] == 3
|
| 383 |
+
x_lens = (x_lens - 7) // 2 - 3
|
| 384 |
+
|
| 385 |
+
assert x.size(1) == x_lens.max().item(), (x.shape, x_lens.max())
|
| 386 |
+
|
| 387 |
+
return x, x_lens, cached_left_pad
|
| 388 |
+
|
| 389 |
+
@torch.jit.export
|
| 390 |
+
def get_init_states(
|
| 391 |
+
self,
|
| 392 |
+
batch_size: int = 1,
|
| 393 |
+
device: torch.device = torch.device("cpu"),
|
| 394 |
+
) -> Tensor:
|
| 395 |
+
"""Get initial states for Conv2dSubsampling module.
|
| 396 |
+
It is the cached left padding for ConvNeXt module,
|
| 397 |
+
of shape (batch_size, num_channels, left_pad, num_freqs)
|
| 398 |
+
"""
|
| 399 |
+
left_pad = self.convnext.padding[0]
|
| 400 |
+
freq = self.out_width
|
| 401 |
+
channels = self.layer3_channels
|
| 402 |
+
cached_embed_left_pad = torch.zeros(batch_size, channels, left_pad, freq).to(
|
| 403 |
+
device
|
| 404 |
+
)
|
| 405 |
+
|
| 406 |
+
return cached_embed_left_pad
|
speedasr_core/utils.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2021 Xiaomi Corp. (authors: Fangjun Kuang,
|
| 2 |
+
# Mingshuang Luo,
|
| 3 |
+
# Zengwei Yao)
|
| 4 |
+
# 2023 Johns Hopkins University (authors: Amir Hussein)
|
| 5 |
+
#
|
| 6 |
+
# See ../../LICENSE for clarification regarding multiple authors
|
| 7 |
+
#
|
| 8 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 9 |
+
# you may not use this file except in compliance with the License.
|
| 10 |
+
# You may obtain a copy of the License at
|
| 11 |
+
#
|
| 12 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 13 |
+
#
|
| 14 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 15 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 16 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 17 |
+
# See the License for the specific language governing permissions and
|
| 18 |
+
# limitations under the License.
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
from contextlib import contextmanager
|
| 22 |
+
import warnings
|
| 23 |
+
|
| 24 |
+
import torch
|
| 25 |
+
from packaging import version
|
| 26 |
+
|
| 27 |
+
TORCH_VERSION = version.parse(torch.__version__.split("+")[0])
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@contextmanager
|
| 31 |
+
def torch_autocast(device_type="cuda", **kwargs):
|
| 32 |
+
"""
|
| 33 |
+
To fix the following warnings:
|
| 34 |
+
/icefall/egs/librispeech/ASR/zipformer/model.py:323:
|
| 35 |
+
FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated.
|
| 36 |
+
Please use `torch.amp.autocast('cuda', args...)` instead.
|
| 37 |
+
with torch.cuda.amp.autocast(enabled=False):
|
| 38 |
+
"""
|
| 39 |
+
if TORCH_VERSION >= version.parse("2.3.0"):
|
| 40 |
+
# Use new unified API
|
| 41 |
+
with torch.amp.autocast(device_type=device_type, **kwargs):
|
| 42 |
+
yield
|
| 43 |
+
else:
|
| 44 |
+
# Suppress deprecation warning and use old CUDA-specific autocast
|
| 45 |
+
with warnings.catch_warnings():
|
| 46 |
+
warnings.simplefilter("ignore", category=FutureWarning)
|
| 47 |
+
with torch.cuda.amp.autocast(**kwargs):
|
| 48 |
+
yield
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def make_pad_mask(
|
| 54 |
+
lengths: torch.Tensor,
|
| 55 |
+
max_len: int = 0,
|
| 56 |
+
pad_left: bool = False,
|
| 57 |
+
) -> torch.Tensor:
|
| 58 |
+
"""
|
| 59 |
+
Args:
|
| 60 |
+
lengths:
|
| 61 |
+
A 1-D tensor containing sentence lengths.
|
| 62 |
+
max_len:
|
| 63 |
+
The length of masks.
|
| 64 |
+
pad_left:
|
| 65 |
+
If ``False`` (default), padding is on the right.
|
| 66 |
+
If ``True``, padding is on the left.
|
| 67 |
+
Returns:
|
| 68 |
+
Return a 2-D bool tensor, where masked positions
|
| 69 |
+
are filled with `True` and non-masked positions are
|
| 70 |
+
filled with `False`.
|
| 71 |
+
|
| 72 |
+
>>> lengths = torch.tensor([1, 3, 2, 5])
|
| 73 |
+
>>> make_pad_mask(lengths)
|
| 74 |
+
tensor([[False, True, True, True, True],
|
| 75 |
+
[False, False, False, True, True],
|
| 76 |
+
[False, False, True, True, True],
|
| 77 |
+
[False, False, False, False, False]])
|
| 78 |
+
"""
|
| 79 |
+
assert lengths.ndim == 1, lengths.ndim
|
| 80 |
+
max_len = max(max_len, lengths.max())
|
| 81 |
+
n = lengths.size(0)
|
| 82 |
+
seq_range = torch.arange(0, max_len, device=lengths.device)
|
| 83 |
+
expanded_lengths = seq_range.unsqueeze(0).expand(n, max_len)
|
| 84 |
+
|
| 85 |
+
if pad_left:
|
| 86 |
+
mask = expanded_lengths < (max_len - lengths).unsqueeze(1)
|
| 87 |
+
else:
|
| 88 |
+
mask = expanded_lengths >= lengths.unsqueeze(-1)
|
| 89 |
+
|
| 90 |
+
return mask
|
| 91 |
+
|
| 92 |
+
|