Add StreamAlign R=32 tokenizer stack (mirror of js1044k/streamalign-r32-tokenizer)

README.md

@@ -0,0 +1,53 @@
+---
+license: apache-2.0
+tags:
+  - speech
+  - speech-tokenizer
+  - streaming
+  - rvq
+  - streamalign
+library_name: pytorch
+---
+
+# StreamAlign R=32 RVQ Speech Tokenizer
+
+Streaming speech tokenizer reproducing the StreamAlign paper recipe:
+a char-level RNN-T aligner (frozen encoder) with a word-aligned acoustic
+head quantized by a plain ResidualVQ (**R=32 layers, codebook 512,
+dim 256**), decoding to CosyVoice3 speech tokens for streaming
+reconstruction (chunk_size=4, left_context=32).
+
+## Checkpoint
+
+`final.pt` = phase-C (cosine) epoch 13. Dict keys:
+`{epoch, hubert_state_dict, optimizer_state_dict, train_loss}`.
+Load `hubert_state_dict` into
+`models/model_tokenizer.py::Data2VecSemanticAcousticModel`
+(streamASR, branch `refactor/tokenizer-r32`) with env
+`RVQ_R=32 RVQ_CODEBOOK_SIZE=512`.
+
+## Training (3-phase, global batch 16)
+
+| phase | trainer | LR | epochs |
+|---|---|---|---|
+| A continuous (RVQ_BYPASS=1) | train_tokenizer.py | 1e-4 | 15 |
+| B RVQ on (subalign-init from A) | train_tokenizer.py | 1e-4 | 14 |
+| C cosine finetune (from B) | train_tokenizer_cosine.py | 1e-5 -> 0 | 13 |
+
+Data: LibriSpeech 960h + Emilia-EN 400h subset (precomputed CosyVoice3
+features). Pipeline: `scripts/train_tokenizer_r32_pipeline.sh`.
+
+## Results (LibriSpeech test-clean 2620, streaming reconstruction)
+
+| metric | value | note |
+|---|---|---|
+| WER | 4.43% | whisper-large-v3 (paper: 4.41%) |
+| CER | 1.92% | |
+| UTMOS | 4.23 | versa pseudo_mos |
+| SECS | 0.585 | versa speaker, RawNet3 |
+
+## Dependencies at inference
+
+Frozen char RNN-T encoder ckpt (stage 1), word/BPE streaming ASR +
+tokenizer, boundary classifier, CosyVoice3-0.5B vocoder. See
+`train_tokenizer_r32_pipeline.sh eval` for the exact wiring.

alignment.yaml

@@ -0,0 +1,368 @@
+# Generated 2026-02-02 from:
+# /gallery_rijks/jinyoung.park/streamalign/streamASR/hparams/train_chunk_streaming.yaml
+# yamllint disable
+# ############################################################################
+# Model: E2E ASR with Conformer Transducer (Character-level)
+# Encoder: Conformer with Dynamic Chunk Training
+# Decoder: LSTM + greedy/beam search
+# Tokens: Character-level (no BPE)
+# Losses: Transducer + CTC (optional)
+# Training: Librispeech
+# 
+# Modified for character-level prediction and CER evaluation
+# ############################################################################
+
+seed: 3407
+__set_seed: !apply:speechbrain.utils.seed_everything [3407]
+output_folder: results/conformer_transducer_char/3407
+output_wer_folder: results/conformer_transducer_char/3407/
+save_folder: results/conformer_transducer_char/3407/save
+train_log: results/conformer_transducer_char/3407/train_log.txt
+
+# Data files
+data_folder: /gallery_rijks/jinyoung.park/datasets/LibriTTS
+
+# Files to hold the manifest data
+train_json: !ref <save_folder>/train.json
+valid_json: !ref <save_folder>/valid.json
+test_json: !ref <save_folder>/test.json
+
+# Data splits
+train_split: ["train-clean-100", "train-clean-360", "train-other-500"]
+valid_split: ["dev-clean", "dev-other"]
+test_split: ["test-clean", "test-other"]
+
+skip_prep: false
+ckpt_interval_minutes: 5
+####################### Training Parameters ####################################
+
+number_of_epochs: 100
+warmup_steps: 25000
+num_workers: 4
+batch_size_valid: 4
+lr: 0.0008
+weight_decay: 0.01
+number_of_ctc_epochs: 60
+ctc_weight: 0.3
+ce_weight: 0.0
+max_grad_norm: 5.0
+loss_reduction: batchmean
+precision: fp16
+
+batch_size: 8
+grad_accumulation_factor: 4
+sorting: random
+avg_checkpoints: 10
+
+# Feature parameters
+sample_rate: 16000
+n_fft: 512
+n_mels: 80
+win_length: 32
+
+# ============================================================================
+# Streaming & Dynamic Chunk Training
+# ============================================================================
+streaming: true
+
+dynchunktrain_config_sampler: !new:speechbrain.utils.dynamic_chunk_training.DynChunkTrainConfigRandomSampler
+  chunkwise_prob: 0.6
+  chunk_size_min: 8
+  chunk_size_max: 32
+  limited_left_context_prob: 0.75
+  left_context_chunks_min: 2
+  left_context_chunks_max: 32
+   # Validation uses same streaming config as test
+  valid_config: !new:speechbrain.utils.dynamic_chunk_training.DynChunkTrainConfig
+    chunk_size: 16
+    left_context_size: 8
+   # Test config with specific streaming parameters
+  test_config: !new:speechbrain.utils.dynamic_chunk_training.DynChunkTrainConfig
+    chunk_size: 16
+    left_context_size: 8
+
+# Dataloader options
+train_dataloader_opts:
+  batch_size: 8
+  num_workers: 4
+
+valid_dataloader_opts:
+  batch_size: 4
+
+test_dataloader_opts:
+  batch_size: 4
+
+# Dynamic batching
+dynamic_batching: true
+max_batch_len: 150
+max_batch_len_val: 50
+num_bucket: 200
+
+dynamic_batch_sampler:
+  max_batch_len: 150
+  max_batch_len_val: 50
+  num_buckets: 200
+  shuffle_ex: true
+  batch_ordering: random
+  max_batch_ex: 256
+
+####################### Model Parameters #######################################
+
+# Transformer/Conformer
+d_model: 512
+joint_dim: 640
+nhead: 8
+num_encoder_layers: 12
+num_decoder_layers: 0
+d_ffn: 2048
+transformer_dropout: 0.1
+activation: &id005 !name:torch.nn.GELU
+
+
+# ============================================================================
+# Character-level output
+# 72 characters + blank = 73 tokens (CTC_VOCAB_SIZE)
+# space/punct/a-z/accented (0-71) + blank (72)
+# ============================================================================
+output_neurons: 73
+
+# Decoder
+dec_dim: 512
+dec_emb_dropout: 0.2
+dec_dropout: 0.1
+attention_type: RelPosMHAXL
+
+# Decoding parameters
+blank_index: 72
+bos_index: 72
+eos_index: 72
+pad_index: 72
+beam_size: 10
+nbest: 1
+state_beam: 2.3
+expand_beam: 2.3
+
+# If True uses torchaudio loss
+use_torchaudio: false
+
+epoch_counter: &id019 !new:speechbrain.utils.epoch_loop.EpochCounter
+
+  limit: 100
+
+normalize: &id013 !new:speechbrain.processing.features.InputNormalization
+  norm_type: global
+  update_until_epoch: 4
+
+compute_features: &id021 !new:speechbrain.lobes.features.Fbank
+  sample_rate: 16000
+  n_fft: 512
+  n_mels: 80
+  win_length: 32
+
+############################## Augmentations ###################################
+
+speed_perturb: &id001 !new:speechbrain.augment.time_domain.SpeedPerturb
+  orig_freq: 16000
+  speeds: [95, 100, 105]
+
+wav_augment: !new:speechbrain.augment.augmenter.Augmenter
+  min_augmentations: 1
+  max_augmentations: 1
+  augment_prob: 1.0
+  augmentations: [*id001]
+
+time_drop: &id002 !new:speechbrain.augment.freq_domain.SpectrogramDrop
+  drop_length_low: 12
+  drop_length_high: 20
+  drop_count_low: 5
+  drop_count_high: 5
+  replace: zeros
+
+freq_drop: &id003 !new:speechbrain.augment.freq_domain.SpectrogramDrop
+  drop_length_low: 20
+  drop_length_high: 25
+  drop_count_low: 2
+  drop_count_high: 2
+  replace: zeros
+  dim: 2
+
+time_warp: &id004 !new:speechbrain.augment.freq_domain.Warping
+
+fea_augment: !new:speechbrain.augment.augmenter.Augmenter
+  parallel_augment: false
+  concat_original: false
+  repeat_augment: 1
+  shuffle_augmentations: false
+  min_augmentations: 3
+  max_augmentations: 3
+  augment_prob: 1.0
+  augmentations: [*id002, *id003, *id004]
+
+############################## Pretrained ###################################
+pretrained_source: speechbrain/asr-streaming-conformer-librispeech
+pretrained_save: results/conformer_transducer_char/3407/pretrained
+
+
+############################## Models ##########################################
+
+CNN: &id007 !new:speechbrain.lobes.models.convolution.ConvolutionFrontEnd
+  input_shape: (8, 10, 80)
+  num_blocks: 2
+  num_layers_per_block: 1
+  out_channels: (64, 32)
+  kernel_sizes: (3, 3)
+  strides: (2, 2)
+  residuals: (False, False)
+
+Transformer: &id006 !new:models.TransformerASR.TransformerASR
+
+  input_size: 640
+  tgt_vocab: 73
+  d_model: 512
+  nhead: 8
+  num_encoder_layers: 12
+  num_decoder_layers: 0
+  d_ffn: 2048
+  dropout: 0.1
+  activation: *id005
+  encoder_module: conformer
+  attention_type: RelPosMHAXL
+  normalize_before: true
+  output_hidden_states: true
+  causal: false
+
+enc: &id008 !new:models.TransformerASR.EncoderWrapper
+  transformer: *id006
+proj_ctc: &id014 !new:speechbrain.nnet.linear.Linear
+  input_size: 640
+  n_neurons: 73
+
+proj_enc: &id016 !new:speechbrain.nnet.linear.Linear
+
+  input_size: 512
+  n_neurons: 640
+  bias: false
+
+proj_dec: &id015 !new:speechbrain.nnet.linear.Linear
+  input_size: 512
+  n_neurons: 640
+  bias: false
+
+ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
+  blank_index: 72
+  reduction: batchmean
+
+# Character embedding (output_neurons - 1 because blank is handled separately)
+emb: &id009 !new:speechbrain.nnet.embedding.Embedding
+  num_embeddings: 73
+  consider_as_one_hot: true
+  blank_id: 72
+
+dec: &id010 !new:speechbrain.nnet.RNN.LSTM
+  input_shape: [null, null, 72]
+  hidden_size: 512
+  num_layers: 1
+  re_init: true
+
+ce_cost: !name:speechbrain.nnet.losses.nll_loss
+  label_smoothing: 0.1
+
+Tjoint: &id011 !new:speechbrain.nnet.transducer.transducer_joint.Transducer_joint
+  joint: sum
+  nonlinearity: *id005
+transducer_lin: &id012 !new:speechbrain.nnet.linear.Linear
+  input_size: 640
+  n_neurons: 73
+  bias: false
+
+log_softmax: !new:speechbrain.nnet.activations.Softmax
+  apply_log: true
+
+transducer_cost: !name:speechbrain.nnet.losses.transducer_loss
+  blank_index: 72
+  use_torchaudio: false
+
+modules:
+  CNN: *id007
+  enc: *id008
+  emb: *id009
+  dec: *id010
+  Tjoint: *id011
+  transducer_lin: *id012
+  normalize: *id013
+  proj_ctc: *id014
+  proj_dec: *id015
+  proj_enc: *id016
+model: &id017 !new:torch.nn.ModuleList
+- [*id007, *id008, *id009, *id010, *id016, *id015, *id014, *id012]
+Greedysearcher: &id020 !new:speechbrain.decoders.transducer.TransducerBeamSearcher
+  decode_network_lst: [*id009, *id010, *id015]
+  tjoint: *id011
+  classifier_network: [*id012]
+  blank_id: 72
+  beam_size: 1
+  nbest: 1
+
+Beamsearcher: !new:speechbrain.decoders.transducer.TransducerBeamSearcher
+  decode_network_lst: [*id009, *id010, *id015]
+  tjoint: *id011
+  classifier_network: [*id012]
+  blank_id: 72
+  beam_size: 10
+  nbest: 1
+  state_beam: 2.3
+  expand_beam: 2.3
+
+opt_class: !name:torch.optim.AdamW
+  lr: 0.0008
+  betas: (0.9, 0.98)
+  eps: 1.e-8
+  weight_decay: 0.01
+
+noam_annealing: &id018 !new:speechbrain.nnet.schedulers.NoamScheduler
+  lr_initial: 0.0008
+  n_warmup_steps: 25000
+
+
+############################## Logging and Checkpointing #######################
+
+checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
+  checkpoints_dir: results/conformer_transducer_char/3407/save
+  recoverables:
+    model: *id017
+    scheduler: *id018
+    normalizer: *id013
+    counter: *id019
+train_logger: !new:speechbrain.utils.train_logger.WandBLogger
+  initializer: !name:wandb.init
+    project: streamingAlign
+    name: subalign
+    dir: results/conformer_transducer_char/3407/wandb
+    reinit: true
+    resume: false
+
+# CER computer (character-level)
+cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+  split_tokens: true
+
+############################## Streaming Inference #############################
+
+# Streaming configuration for inference
+# chunk_size: frames per chunk (8 frames ≈ 320ms after 4x subsampling)
+# left_context_size: number of left context chunks
+streaming_chunk_size: 16
+streaming_left_context: 8
+
+# Create streaming context for the decoder
+make_decoder_streaming_context: !name:speechbrain.decoders.transducer.TransducerGreedySearcherStreamingContext
+
+# Streaming decoding function
+decoding_function: !name:speechbrain.decoders.transducer.TransducerBeamSearcher.transducer_greedy_decode_streaming
+- *id020
+fea_streaming_extractor: !new:speechbrain.lobes.features.StreamingFeatureWrapper
+  module: !new:speechbrain.nnet.containers.LengthsCapableSequential
+  - *id021
+  - *id013
+  - *id007
+  properties: !apply:speechbrain.utils.filter_analysis.stack_filter_properties
+  - [*id021, *id007]

alignment_model/CKPT.yaml

@@ -0,0 +1,5 @@
+# yamllint disable
+CER: 1.2645221513421072
+end-of-epoch: true
+epoch: 10
+unixtime: 1786244238.4986734

alignment_model/model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed0fef7ab7b7e9e0c4f0606e1b778cccc38e0eb3a70f3e7688f84e13e8bd7eca
+size 317974879

alignment_model/normalizer.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ed0e5ca389f2b8567fae0d954d3e16cbedcd95fb4f00fc65b45a1892903d276
+size 2557

boundary_classifier/best_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5f03e0d79f33c678434197b64048614acb25c83d5f3202927f363ded0a76fdc
+size 5793461

boundary_classifier/best_precision_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0dbf0d1829c16e9e3efaeaad8d7d01857bd451859407efd102d17adbbcc3f3ba
+size 5793725

rvq_tokenizer/final.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3789ae92cc1afedb96145db25f29fd5cd01aac577a27de7a13a1f6eafe03d669
+size 946819549

streaming_asr/chunk_streaming_word_fastemit.yaml

@@ -0,0 +1,428 @@
+# ############################################################################
+# Model: E2E ASR with transformer and transducer
+# Encoder: Conformer
+# Decoder: LSTM + beamsearch + RNNLM
+# Tokens: BPE with unigram
+# losses: Transducer + CTC (optional) + CE (optional)
+# Training: Librispeech 960h
+# Authors:  Titouan Parcollet 2023, Abdel HEBA, Mirco Ravanelli, Sung-Lin Yeh 2020
+# ############################################################################
+
+# Seed needs to be set at top of yaml, before objects with parameters are made
+
+seed: 3415
+__set_seed: !apply:speechbrain.utils.seed_everything [3415]
+output_folder: results/conformer_transducer_char/3415_fastemit
+output_wer_folder: results/conformer_transducer_char/3415_fastemit/
+save_folder: results/conformer_transducer_char/3407/save
+checkpoint_folder: results/conformer_transducer_char/3415_fastemit/save
+train_log: results/conformer_transducer_char/3415_fastemit/train_log.txt
+pretrain_source: speechbrain/asr-streaming-conformer-librispeech
+pretrain_folder: results/conformer_transducer_char/3415_fastemit/pretrained
+
+# Data files
+data_folder: /gallery_rijks/jinyoung.park/datasets/LibriSpeech
+emilia_data_folder: /gallery_goya/kangwook.kim/datasets/Emilia-Dataset/Emilia/EN
+emilia_train_csv: /gallery_rijks/jinyoung.park/datasets/Emilia/emilia_en_400h.csv
+
+# CSV files (LibriSpeech, pre-existing from 3409 save)
+train_csv:
+   - results/conformer_transducer_char/3409/train-clean-100.csv
+   - results/conformer_transducer_char/3409/train-clean-360.csv
+   - results/conformer_transducer_char/3409/train-other-500.csv
+
+valid_csv: results/conformer_transducer_char/3409/dev-clean.csv
+
+test_csv:
+   - results/conformer_transducer_char/3409/test-clean.csv
+   - results/conformer_transducer_char/3409/test-other.csv
+
+ckpt_interval_minutes: 5
+# Language model (LM) pretraining
+# NB: To avoid mismatch, the speech recognizer must be trained with the same
+# tokenizer used for LM training. Here, we download everything from the
+# speechbrain HuggingFace repository. However, a local path pointing to a
+# directory containing the lm.ckpt and tokenizer.ckpt may also be specified
+# instead. E.g if you want to use your own LM / tokenizer.
+pretrained_lm_tokenizer_path: speechbrain/asr-crdnn-rnnlm-librispeech
+
+
+####################### Training Parameters ####################################
+
+# To make Transformers converge, the global bath size should be large enough.
+# The global batch size is computed as batch_size * n_gpus * grad_accumulation_factor.
+# Empirically, we found that this value should be >= 128.
+# Please, set your parameters accordingly.
+number_of_epochs: 100
+num_workers: 4
+batch_size_valid: 4
+lr: 0.0004
+weight_decay: 0.01
+number_of_ctc_epochs: 60
+ctc_weight: 0.3 # Multitask with CTC for the encoder (0.0 = disabled)
+ce_weight: 0.0 # Multitask with CE for the decoder (0.0 = disabled)
+max_grad_norm: 5.0
+loss_reduction: 'batchmean'
+precision: fp16 # bf16, fp16 or fp32
+
+# The batch size is used if and only if dynamic batching is set to False
+# Validation and testing are done with fixed batches and not dynamic batching.
+batch_size: 6
+grad_accumulation_factor: 4
+sorting: random
+avg_checkpoints: 10 # Number of checkpoints to average for evaluation
+
+# Feature parameters
+sample_rate: 16000
+n_fft: 512
+n_mels: 80
+win_length: 32
+
+# Streaming & dynamic chunk training options
+# At least for the current architecture on LibriSpeech, we found out that
+# non-streaming accuracy is very similar between `streaming: True` and
+# `streaming: False`.
+streaming: True  # controls all Dynamic Chunk Training & chunk size & left context mechanisms
+
+# Configuration for Dynamic Chunk Training.
+# In this model, a chunk is roughly equivalent to 40ms of audio.
+dynchunktrain_config_sampler: !new:speechbrain.utils.dynamic_chunk_training.DynChunkTrainConfigRandomSampler # yamllint disable-line rule:line-length
+   chunkwise_prob: 1.0 # Probability during a batch to limit attention and sample a random chunk size in the following range
+   chunk_size_min: 4 # Minimum chunk size (if in a DynChunkTrain batch)
+   chunk_size_max: 4 # Maximum chunk size (if in a DynChunkTrain batch)
+   limited_left_context_prob: 1.0 # If in a DynChunkTrain batch, the probability during a batch to restrict left context to a random number of chunks
+   left_context_chunks_min: 32 # Minimum left context size (in # of chunks)
+   left_context_chunks_max: 32 # Maximum left context size (in # of chunks)
+   # If you specify a valid/test config, you can optionally have evaluation be
+   # done with a specific DynChunkTrain configuration.
+   valid_config: !new:speechbrain.utils.dynamic_chunk_training.DynChunkTrainConfig
+      chunk_size: 4
+      left_context_size: 32
+
+# Dataloader options
+train_dataloader_opts:
+   batch_size: !ref <batch_size>
+   num_workers: !ref <num_workers>
+
+valid_dataloader_opts:
+   batch_size: !ref <batch_size_valid>
+
+test_dataloader_opts:
+   batch_size: !ref <batch_size_valid>
+
+# This setup works well for 3090 24GB GPU, adapt it to your needs.
+# Adjust grad_accumulation_factor depending on the DDP node count (here 3)
+# Or turn it off (but training speed will decrease)
+dynamic_batching: True
+max_batch_len: 80
+max_batch_len_val: 50 # we reduce it as the beam is much wider (VRAM)
+num_bucket: 200
+
+dynamic_batch_sampler:
+   max_batch_len: !ref <max_batch_len>
+   max_batch_len_val: !ref <max_batch_len_val>
+   num_buckets: !ref <num_bucket>
+   shuffle_ex: True # if true re-creates batches at each epoch shuffling examples.
+   batch_ordering: random
+   max_batch_ex: 256
+
+####################### Model Parameters #######################################
+
+# Transformer
+d_model: 512
+joint_dim: 640
+nhead: 8
+num_encoder_layers: 12
+num_decoder_layers: 0
+d_ffn: 2048
+transformer_dropout: 0.1
+activation: !name:torch.nn.GELU
+output_neurons: 1000
+dec_dim: 512
+dec_emb_dropout: 0.2
+dec_dropout: 0.1
+attention_type: RelPosMHAXL
+
+# Decoding parameters
+blank_index: 0
+bos_index: 0
+eos_index: 0
+pad_index: 0
+beam_size: 10
+nbest: 1
+# by default {state,expand}_beam = 2.3 as mention in paper
+# https://arxiv.org/abs/1904.02619
+state_beam: 2.3
+expand_beam: 2.3
+lm_weight: 0.50
+
+# If True uses torchaudio loss. Otherwise, the numba one
+use_torchaudio: False
+
+epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter
+   limit: !ref <number_of_epochs>
+
+normalize: !new:speechbrain.processing.features.InputNormalization
+   norm_type: global
+   update_until_epoch: 4
+
+compute_features: !new:speechbrain.lobes.features.Fbank
+   sample_rate: !ref <sample_rate>
+   n_fft: !ref <n_fft>
+   n_mels: !ref <n_mels>
+   win_length: !ref <win_length>
+
+############################## Augmentations ###################################
+
+# Speed perturbation
+speed_perturb: !new:speechbrain.augment.time_domain.SpeedPerturb
+   orig_freq: !ref <sample_rate>
+   speeds: [95, 100, 105]
+
+# Augmenter: Combines previously defined augmentations to perform data augmentation
+wav_augment: !new:speechbrain.augment.augmenter.Augmenter
+   min_augmentations: 1
+   max_augmentations: 1
+   augment_prob: 1.0
+   augmentations: [!ref <speed_perturb>]
+
+
+# Time Drop
+time_drop: !new:speechbrain.augment.freq_domain.SpectrogramDrop
+   drop_length_low: 12
+   drop_length_high: 20
+   drop_count_low: 5
+   drop_count_high: 5
+   replace: "zeros"
+
+# Frequency Drop
+freq_drop: !new:speechbrain.augment.freq_domain.SpectrogramDrop
+   drop_length_low: 20
+   drop_length_high: 25
+   drop_count_low: 2
+   drop_count_high: 2
+   replace: "zeros"
+   dim: 2
+
+# Time warp
+time_warp: !new:speechbrain.augment.freq_domain.Warping
+
+fea_augment: !new:speechbrain.augment.augmenter.Augmenter
+   parallel_augment: False
+   concat_original: False
+   repeat_augment: 1
+   shuffle_augmentations: False
+   min_augmentations: 3
+   max_augmentations: 3
+   augment_prob: 1.0
+   augmentations: [
+      !ref <time_drop>,
+      !ref <freq_drop>,
+      !ref <time_warp>]
+
+############################## Models ##########################################
+
+CNN: !new:speechbrain.lobes.models.convolution.ConvolutionFrontEnd
+   input_shape: (8, 10, 80)
+   num_blocks: 2
+   num_layers_per_block: 1
+   out_channels: (64, 32)
+   kernel_sizes: (3, 3)
+   strides: (2, 2)
+   residuals: (False, False)
+
+Transformer: !new:speechbrain.lobes.models.transformer.TransformerASR.TransformerASR # yamllint disable-line rule:line-length
+   input_size: 640
+   tgt_vocab: !ref <output_neurons>
+   d_model: !ref <d_model>
+   nhead: !ref <nhead>
+   num_encoder_layers: !ref <num_encoder_layers>
+   num_decoder_layers: !ref <num_decoder_layers>
+   d_ffn: !ref <d_ffn>
+   dropout: !ref <transformer_dropout>
+   activation: !ref <activation>
+   encoder_module: conformer
+   attention_type: !ref <attention_type>
+   normalize_before: True
+   causal: False
+
+# We must call an encoder wrapper so the decoder isn't run (we don't have any)
+enc: !new:speechbrain.lobes.models.transformer.TransformerASR.EncoderWrapper
+   transformer: !ref <Transformer>
+
+# For MTL CTC over the encoder
+proj_ctc: !new:speechbrain.nnet.linear.Linear
+   input_size: !ref <joint_dim>
+   n_neurons: !ref <output_neurons>
+
+# Define some projection layers to make sure that enc and dec
+# output dim are the same before joining
+proj_enc: !new:speechbrain.nnet.linear.Linear
+   input_size: !ref <d_model>
+   n_neurons: !ref <joint_dim>
+   bias: False
+
+proj_dec: !new:speechbrain.nnet.linear.Linear
+   input_size: !ref <dec_dim>
+   n_neurons: !ref <joint_dim>
+   bias: False
+
+# Uncomment for MTL with CTC
+ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
+   blank_index: !ref <blank_index>
+   reduction: !ref <loss_reduction>
+
+emb: !new:speechbrain.nnet.embedding.Embedding
+   num_embeddings: !ref <output_neurons>
+   consider_as_one_hot: True
+   blank_id: !ref <blank_index>
+
+dec: !new:speechbrain.nnet.RNN.LSTM
+   input_shape: [null, null, !ref <output_neurons> - 1]
+   hidden_size: !ref <dec_dim>
+   num_layers: 1
+   re_init: True
+
+# For MTL with LM over the decoder (need to uncomment to activate)
+# dec_lin: !new:speechbrain.nnet.linear.Linear
+#   input_size: !ref <joint_dim>
+#   n_neurons: !ref <output_neurons>
+#   bias: False
+
+# For MTL
+ce_cost: !name:speechbrain.nnet.losses.nll_loss
+   label_smoothing: 0.1
+
+Tjoint: !new:speechbrain.nnet.transducer.transducer_joint.Transducer_joint
+   joint: sum # joint [sum | concat]
+   nonlinearity: !ref <activation>
+
+transducer_lin: !new:speechbrain.nnet.linear.Linear
+   input_size: !ref <joint_dim>
+   n_neurons: !ref <output_neurons>
+   bias: False
+
+log_softmax: !new:speechbrain.nnet.activations.Softmax
+   apply_log: True
+
+transducer_cost: !name:speechbrain.nnet.losses.transducer_loss
+   blank_index: !ref <blank_index>
+   use_torchaudio: !ref <use_torchaudio>
+
+# This is the RNNLM that is used according to the Huggingface repository
+# NB: It has to match the pre-trained RNNLM!!
+lm_model: !new:speechbrain.lobes.models.RNNLM.RNNLM
+   output_neurons: !ref <output_neurons>
+   embedding_dim: 128
+   activation: !name:torch.nn.LeakyReLU
+   dropout: 0.0
+   rnn_layers: 2
+   rnn_neurons: 2048
+   dnn_blocks: 1
+   dnn_neurons: 512
+   return_hidden: True  # For inference
+
+# for MTL
+# update model if any HEAD module is added
+modules:
+   CNN: !ref <CNN>
+   enc: !ref <enc>
+   emb: !ref <emb>
+   dec: !ref <dec>
+   Tjoint: !ref <Tjoint>
+   transducer_lin: !ref <transducer_lin>
+   normalize: !ref <normalize>
+   lm_model: !ref <lm_model>
+   proj_ctc: !ref <proj_ctc>
+   proj_dec: !ref <proj_dec>
+   proj_enc: !ref <proj_enc>
+#   dec_lin: !ref <dec_lin>
+
+# for MTL
+# update model if any HEAD module is added
+model: !new:torch.nn.ModuleList
+   - [!ref <CNN>, !ref <enc>, !ref <emb>, !ref <dec>, !ref <proj_enc>, !ref <proj_dec>, !ref <proj_ctc>, !ref <transducer_lin>]
+
+############################## Decoding & optimiser ############################
+
+# Tokenizer initialization
+tokenizer: !new:sentencepiece.SentencePieceProcessor
+
+Greedysearcher: !new:speechbrain.decoders.transducer.TransducerBeamSearcher
+   decode_network_lst: [!ref <emb>, !ref <dec>, !ref <proj_dec>]
+   tjoint: !ref <Tjoint>
+   classifier_network: [!ref <transducer_lin>]
+   blank_id: !ref <blank_index>
+   beam_size: 1
+   nbest: 1
+
+Beamsearcher: !new:speechbrain.decoders.transducer.TransducerBeamSearcher
+   decode_network_lst: [!ref <emb>, !ref <dec>, !ref <proj_dec>]
+   tjoint: !ref <Tjoint>
+   classifier_network: [!ref <transducer_lin>]
+   blank_id: !ref <blank_index>
+   beam_size: !ref <beam_size>
+   nbest: !ref <nbest>
+   lm_module: !ref <lm_model>
+   lm_weight: !ref <lm_weight>
+   state_beam: !ref <state_beam>
+   expand_beam: !ref <expand_beam>
+
+opt_class: !name:torch.optim.AdamW
+   lr: !ref <lr>
+   betas: (0.9, 0.98)
+   eps: 1.e-8
+   weight_decay: !ref <weight_decay>
+
+############################## Logging and Pretrainer ##########################
+
+checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
+   checkpoints_dir: !ref <checkpoint_folder>
+   recoverables:
+      model: !ref <model>
+      normalizer: !ref <normalize>
+      counter: !ref <epoch_counter>
+
+pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
+   collect_in: !ref <pretrain_folder>
+   loadables:
+      model: !ref <model>
+      tokenizer: !ref <tokenizer>
+      normalizer: !ref <normalize>
+   paths:
+      model: !ref <pretrain_source>/model.ckpt
+      tokenizer: !ref <pretrain_source>/tokenizer.ckpt
+      normalizer: !ref <pretrain_source>/normalizer.ckpt
+
+train_logger: !new:speechbrain.utils.train_logger.WandBLogger
+  initializer: !name:wandb.init
+    project: streaming-asr
+    name: conformer_transducer_char
+    dir: results/conformer_transducer_char/3415_casual/wandb
+    reinit: true
+    resume: false
+
+
+error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+
+cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+   split_tokens: True
+
+
+make_tokenizer_streaming_context: !name:speechbrain.tokenizers.SentencePiece.SentencePieceDecoderStreamingContext
+tokenizer_decode_streaming: !name:speechbrain.tokenizers.SentencePiece.spm_decode_preserve_leading_space
+
+make_decoder_streaming_context: !name:speechbrain.decoders.transducer.TransducerGreedySearcherStreamingContext # default constructor
+decoding_function: !name:speechbrain.decoders.transducer.TransducerBeamSearcher.transducer_greedy_decode_streaming
+   - !ref <Greedysearcher>  # self
+
+fea_streaming_extractor: !new:speechbrain.lobes.features.StreamingFeatureWrapper
+   module: !new:speechbrain.nnet.containers.LengthsCapableSequential
+      - !ref <compute_features>
+      - !ref <normalize>
+      - !ref <CNN>
+   # don't consider normalization as part of the input filter chain.
+   # normalization will operate at chunk level, which mismatches training
+   # somewhat, but does not appear to result in noticeable degradation.
+   properties: !apply:speechbrain.utils.filter_analysis.stack_filter_properties
+      - [!ref <compute_features>, !ref <CNN>]

streaming_asr/model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4aa2afbc47ffb8af9c9273256bac77b6dd13c9767bd5c81d32208c596c88b0c9
+size 334293791

streaming_asr/normalizer.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:752b6fe11562ef768efd418897bba1e2dbe12e72a5451aff2b5b8b214ca1d62e
+size 2621

streaming_asr/tokenizer.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37a6cba34cd520b33fd83612d5efc8ba7e351166541eb2726642bb3032234d31
+size 253217