Add duplex model checkpoint

dummy_nemo_duplex/qwen_1b/cmd-args.log

@@ -0,0 +1 @@
+/export/fs06/ahussei6/nvidia/github/NeMo/examples/speechlm2/s2s_duplex_speech_decoder_train.py --config-path=conf/train --config-name=qwen_1b ++exp_manager.checkpoint_callback_params.save_top_k=3 exp_manager.name=qwen_1b ++model.pretrained_audio_codec=/export/fs06/ahussei6/nvidia/pretrained_models/nano_codec/Low_Frame-rate_Speech_Codec++.nemo ++model.pretrained_tts_from_s2s=/export/fs06/ahussei6/nvidia/pretrained_models/magpie_tts/tts-pretraining_qwnen_2.5_81007_steps.ckpt ++model.pretrained_asr=/export/fs06/ahussei6/nvidia/pretrained_models/asr/stt_en_fastconformer_hybrid_large_streaming_multi_v1.20.0/stt_en_fastconformer_hybrid_large_streaming_multi.nemo ++model.mask_sequence_loss=True trainer.num_nodes=1 exp_manager.explicit_log_dir=/export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b data.train_ds.seed=42 ++model.audio_loss_weight=20 ++model.speech_decoder.cond_on_prev_audio_tokens=True ++model.speech_decoder.use_speaker_encoder=True ++model.speech_decoder.cond_on_char_embedding=True ++model.speech_decoder.cond_on_asr_emb=False ++model.speech_decoder.cond_on_llm_latent=False ++model.speech_decoder.cond_on_modality_adapter_emb=False ++model.speech_decoder.cond_on_text_tokens=False ++model.speech_decoder.cfg_scale=2.5 ++model.speech_decoder.kernel_size=3 ++model.speech_decoder.cfg_unconditional_prob=0.2 ++model.custom_codebook_size=2045 ++model.custom_speech_bos_id=2019 ++model.custom_speech_eos_id=2020 ++model.custom_speech_delay_id=2018 model.perception.encoder.att_context_size=[70,0] model.perception.modality_adapter.att_context_size=[70,0] ++model.pretrained_llm=/export/fs06/ahussei6/nvidia/cache/HFCACHE/hub/models--Qwen--Qwen2.5-1.5B-Instruct/snapshots/989aa7980e4cf806f80c7fef2b1adb7bc71aa306 ++trainer.limit_val_batches=1 ++trainer.val_check_interval=1000 ++model.scale_loss_by=non_sil_t ++model.scale_loss_mask=10 ++model.val_acc_tolerance=480 data.validation_ds.seed=42

dummy_nemo_duplex/qwen_1b/exp_config.yaml

@@ -0,0 +1,226 @@
+model:
+  pretrained_llm: /export/fs06/ahussei6/nvidia/cache/HFCACHE/hub/models--Qwen--Qwen2.5-1.5B-Instruct/snapshots/989aa7980e4cf806f80c7fef2b1adb7bc71aa306
+  pretrained_audio_codec: /export/fs06/ahussei6/nvidia/pretrained_models/nano_codec/Low_Frame-rate_Speech_Codec++.nemo
+  pretrained_asr: /export/fs06/ahussei6/nvidia/pretrained_models/asr/stt_en_fastconformer_hybrid_large_streaming_multi_v1.20.0/stt_en_fastconformer_hybrid_large_streaming_multi.nemo
+  scoring_asr: stt_en_fastconformer_transducer_large
+  pretrained_weights: true
+  audio_loss_weight: 20
+  text_loss_weight: 3
+  freeze_params:
+  - ^audio_codec\..+$
+  - ^speech_generation\.speaker_encoder\..+$
+  prevent_freeze_params: []
+  duplex_user_channel_weight: 1.0
+  hidden_pooling_factor: 2
+  audio_save_path: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/pred_audios
+  perception:
+    target: nemo.collections.multimodal.speech_llm.modules.perception_modules.AudioPerceptionModule
+    use_multi_layer_feat: false
+    add_sep: true
+    is_canary: true
+    preprocessor:
+      normalize: NA
+    encoder:
+      self_attention_model: rel_pos
+      att_context_size:
+      - 70
+      - 0
+      conv_context_size: causal
+      conv_norm_type: layer_norm
+    modality_adapter:
+      _target_: nemo.collections.asr.modules.ConformerEncoder
+      feat_in: 512
+      feat_out: -1
+      n_layers: 2
+      d_model: 512
+      subsampling: dw_striding
+      subsampling_factor: 1
+      subsampling_conv_channels: 256
+      causal_downsampling: true
+      ff_expansion_factor: 4
+      self_attention_model: rel_pos
+      n_heads: 8
+      att_context_size:
+      - 70
+      - 0
+      att_context_style: chunked_limited
+      xscaling: true
+      untie_biases: true
+      pos_emb_max_len: 5000
+      conv_kernel_size: 9
+      conv_norm_type: layer_norm
+      conv_context_size: causal
+      dropout: 0
+      dropout_pre_encoder: 0
+      dropout_emb: 0.0
+      dropout_att: 0
+    spec_augment:
+      _target_: nemo.collections.asr.modules.SpectrogramAugmentation
+      freq_masks: 2
+      time_masks: 10
+      freq_width: 27
+      time_width: 0.05
+  speech_decoder:
+    n_layers: 12
+    d_model: 768
+    d_ffn: 3072
+    sa_n_heads: 12
+    kernel_size: 3
+    p_dropout: 0.1
+    p_dropout_out: 0.0
+    has_xattn: false
+    xa_d_memory: 768
+    xa_n_heads: 12
+    is_causal: true
+    apply_norm_to_cond: true
+    apply_norm_out: true
+    max_length_causal_mask: 5000
+    cond_on_prev_audio_tokens: true
+    detach_input: false
+    use_learnable_pos_emb: true
+    cond_on_modality_adapter_emb: false
+    cond_on_char_embedding: true
+    cond_on_llm_latent: false
+    use_speaker_encoder: true
+    cond_on_asr_emb: false
+    cond_on_text_tokens: false
+    cfg_scale: 2.5
+    cfg_unconditional_prob: 0.2
+  optimizer:
+    _target_: torch.optim.AdamW
+    lr: 0.0001
+    betas:
+    - 0.9
+    - 0.98
+    weight_decay: 0
+    foreach: true
+  lr_scheduler:
+    _target_: nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing
+    warmup_steps: 4000
+    min_lr: 1.0e-06
+    max_steps: 100000
+  pretrained_tts_from_s2s: /export/fs06/ahussei6/nvidia/pretrained_models/magpie_tts/tts-pretraining_qwnen_2.5_81007_steps.ckpt
+  mask_sequence_loss: true
+  custom_codebook_size: 2045
+  custom_speech_bos_id: 2019
+  custom_speech_eos_id: 2020
+  custom_speech_delay_id: 2018
+  scale_loss_by: non_sil_t
+  scale_loss_mask: 10
+  val_acc_tolerance: 480
+trainer:
+  devices: -1
+  accelerator: gpu
+  num_nodes: 1
+  precision: bf16-true
+  logger: false
+  enable_checkpointing: false
+  use_distributed_sampler: false
+  max_steps: 100000
+  val_check_interval: 1000
+  limit_train_batches: 1000
+  limit_val_batches: 1
+  log_every_n_steps: 20
+  num_sanity_val_steps: 0
+  gradient_clip_val: 1.0
+  accumulate_grad_batches: 1
+  strategy:
+    _target_: lightning.pytorch.strategies.DDPStrategy
+    gradient_as_bucket_view: true
+    find_unused_parameters: true
+data:
+  frame_length: 0.08
+  source_sample_rate: 16000
+  target_sample_rate: 22050
+  input_roles:
+  - user
+  - User
+  output_roles:
+  - agent
+  - Assistant
+  - assistant
+  - Agent
+  add_delay_token: false
+  train_ds:
+    sample_rate: 22050
+    multi_config: true
+    shuffle: true
+    sampler_fusion: randomized_round_robin
+    seed: 42
+    shard_seed: trng
+    sampler_weights:
+      cv_en: 1.0
+    cv_en:
+      input_cfg: /export/fs06/ahussei6/nvidia/github/NeMo/examples/speechlm2/conf/data/en.yaml
+      bucket_buffer_size: 10000
+      shuffle_buffer_size: 10000
+      num_workers: 8
+      pin_memory: true
+      max_duration: 60
+      min_duration: 1
+      num_buckets: 30
+      bucket_duration_bins:
+      - 5.33
+      - 6.27
+      - 6.96
+      - 7.54
+      - 8.07
+      - 8.55
+      - 8.99
+      - 9.39
+      - 9.76
+      - 10.14
+      - 10.54
+      - 10.99
+      - 11.49
+      - 12.04
+      - 12.64
+      - 13.28
+      - 13.96
+      - 14.69
+      - 15.48
+      - 16.32
+      - 17.25
+      - 18.28
+      - 19.39
+      - 20.65
+      - 22.14
+      - 24.01
+      - 26.34
+      - 29.21
+      - 32.79
+      batch_duration: 400
+      use_bucketing: true
+      seed: trng
+  validation_ds:
+    datasets:
+      en_dev:
+        shar_path: /export/fs06/ahussei6/nvidia/github/debug
+    sample_rate: 22050
+    batch_size: 64
+    seed: 42
+    shard_seed: randomized
+exp_manager:
+  exp_dir: null
+  explicit_log_dir: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b
+  name: qwen_1b
+  create_tensorboard_logger: false
+  create_checkpoint_callback: true
+  use_datetime_version: true
+  max_time_per_run: 00:03:50:00
+  resume_from_checkpoint: null
+  resume_if_exists: true
+  resume_ignore_no_checkpoint: true
+  create_wandb_logger: false
+  wandb_logger_kwargs:
+    name: development-run
+    project: salm_s2s_speech_decoder_v2_char_tokens
+    resume: true
+  checkpoint_callback_params:
+    filename: '{step}'
+    monitor: val_asr_bleu
+    mode: max
+    every_n_train_steps: null
+    every_n_epochs: 1
+    save_top_k: 3
+    always_save_nemo: false

dummy_nemo_duplex/qwen_1b/lightning_logs.txt

@@ -0,0 +1,19 @@
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
+
+  | Name               | Type                       | Params | Mode 
+--------------------------------------------------------------------------
+0 | audio_codec        | AudioCodecModel            | 62.2 M | eval 
+1 | llm                | Qwen2Model                 | 1.3 B  | train
+2 | lm_head            | Linear                     | 233 M  | train
+3 | embed_tokens       | Embedding                  | 233 M  | train
+4 | perception         | AudioPerceptionModule      | 122 M  | train
+5 | speech_generation  | TransformerARSpeechDecoder | 288 M  | train
+6 | embed_audio_tokens | ModuleList                 | 40.9 M | train
+7 | audio_head         | Linear                     | 40.9 M | train
+--------------------------------------------------------------------------
+2.0 B     Trainable params
+87.5 M    Non-trainable params
+2.1 B     Total params
+8,397.045 Total estimated model params size (MB)
+1141      Modules in train mode
+2018      Modules in eval mode

dummy_nemo_duplex/qwen_1b/nemo_error_log.txt

@@ -0,0 +1,469 @@
+[NeMo W 2026-04-26 23:19:11 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo W 2026-04-26 23:19:15 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/lightning/fabric/plugins/environments/slurm.py:204: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /export/fs06/ahussei6/nvidia/github/NeMo/examples/sp ...
+    
+[NeMo W 2026-04-26 23:19:15 exp_manager:1079] Exp_manager is logging to /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b, but it already exists.
+[NeMo W 2026-04-26 23:19:15 exp_manager:997] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :/export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/checkpoints. Training from scratch.
+[NeMo W 2026-04-26 23:19:15 exp_manager:1388] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 100000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo W 2026-04-26 23:19:16 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    dataset:
+      dataset_type: tarred_vocoder
+      dataset_args:
+        dataset_meta:
+          lindy:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/tarred_audio/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/tarred_audio/audio_{0..3}.tar
+          emma:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/audio_{0..3}.tar
+          libritts_r:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/tarred_audio/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/tarred_audio/audio_{0..48}.tar
+        sample_rate: 22050
+        n_samples: 24696
+        min_duration: 0.4
+        max_duration: null
+        audio_augmentator_config:
+          impulse:
+            prob: 0.5
+            rng: 42
+            manifest_path: /lustre/fsw/portfolios/convai/projects/convai_convaird_nemo-speech/data/ROOM_Response_and_Noise_Database/processed/rir.json
+            normalize_impulse: false
+            shift_impulse: true
+          codec:
+            prob: 0.5
+            rng: 42
+            format_params:
+            - format: mp3
+              prob: 0.5
+              bit_rates:
+              - 16000
+              - 32000
+              - 64000
+              - 128000
+            - format: ogg
+              prob: 0.075
+              encoder: vorbis
+              bit_rates:
+              - 32000
+              - 48000
+              - 64000
+            - format: ogg
+              prob: 0.375
+              encoder: opus
+              bit_rates:
+              - 8000
+              - 16000
+              - 32000
+              - 64000
+              - 128000
+            - format: wav
+              prob: 0.025
+              encoder: pcm_alaw
+              bit_rates:
+              - 64000
+            - format: wav
+              prob: 0.025
+              encoder: pcm_mulaw
+              bit_rates:
+              - 64000
+        shard_strategy: replicate
+        sample_type: weighted_random
+        sample_args:
+          batch_size: 32
+          steps_per_epoch: 233
+          dataset_weights:
+          - 0.2
+          - 0.2
+          - 0.6
+        shuffle_n: 10000
+    dataloader_params:
+      batch_size: 32
+      drop_last: true
+      num_workers: 4
+    
+[NeMo W 2026-04-26 23:19:16 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    dataset:
+      dataset_type: vocoder
+      dataset_args:
+        sample_rate: 22050
+        n_samples: null
+        min_duration: null
+        max_duration: null
+        trunc_duration: 10.0
+        dataset_meta:
+          lindy:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/audio_22khz
+          emma:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/audio_22khz
+          libritts_r:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/audio_22khz
+    dataloader_params:
+      batch_size: 4
+      num_workers: 2
+    
+[NeMo W 2026-04-26 23:19:19 duplex_s2s_speech_decoder_model:148] Tokenizer does not have a `bos_token`. Setting it to '<|im_start|>'.
+[NeMo W 2026-04-26 23:19:22 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath:
+    - - /raid/local//bucket1/tarred_audio_manifest.json
+    - - /raid/local//bucket2/tarred_audio_manifest.json
+    - - /raid/local//bucket3/tarred_audio_manifest.json
+    - - /raid/local//bucket4/tarred_audio_manifest.json
+    - - /raid/local//bucket5/tarred_audio_manifest.json
+    - - /raid/local//bucket6/tarred_audio_manifest.json
+    - - /raid/local//bucket7/tarred_audio_manifest.json
+    - - /raid/local//bucket8/tarred_audio_manifest.json
+    sample_rate: 16000
+    batch_size: 1
+    shuffle: true
+    num_workers: 4
+    pin_memory: true
+    use_start_end_token: false
+    trim_silence: false
+    max_duration: 25
+    min_duration: 0.1
+    is_tarred: true
+    tarred_audio_filepaths:
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket1/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket2/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket3/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket4/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket5/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket6/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket7/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket8/audio__OP_0..8191_CL_.tar
+    shuffle_n: 2048
+    bucketing_strategy: fully_randomized
+    bucketing_batch_size:
+    - 72
+    - 64
+    - 56
+    - 48
+    - 40
+    - 32
+    - 24
+    - 16
+    
+[NeMo W 2026-04-26 23:19:22 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath:
+    - /manifests/librispeech/librivox-dev-other.json
+    - /manifests/librispeech/librivox-dev-clean.json
+    - /manifests/librispeech/librivox-test-other.json
+    - /manifests/librispeech/librivox-test-clean.json
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    num_workers: 8
+    pin_memory: true
+    use_start_end_token: false
+    
+[NeMo W 2026-04-26 23:19:22 modelPT:202] Please call the ModelPT.setup_test_data() or ModelPT.setup_multiple_test_data() method and provide a valid configuration file to setup the test data loader(s).
+    Test config : 
+    manifest_filepath:
+    - /manifests/librispeech/librivox-dev-other.json
+    - /manifests/librispeech/librivox-dev-clean.json
+    - /manifests/librispeech/librivox-test-other.json
+    - /manifests/librispeech/librivox-test-clean.json
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    num_workers: 8
+    pin_memory: true
+    use_start_end_token: false
+    
+[NeMo W 2026-04-26 23:19:24 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/train.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 64
+    shuffle: true
+    is_tarred: false
+    tarred_audio_filepaths: null
+    tarred_shard_strategy: scatter
+    augmentor:
+      noise:
+        manifest_path: /manifests/noise/rir_noise_manifest.json
+        prob: 0.5
+        min_snr_db: 0
+        max_snr_db: 15
+      speed:
+        prob: 0.5
+        sr: 16000
+        resample_type: kaiser_fast
+        min_speed_rate: 0.95
+        max_speed_rate: 1.05
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo W 2026-04-26 23:19:24 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/dev.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 128
+    shuffle: false
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo W 2026-04-26 23:19:34 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/lightning/fabric/plugins/environments/slurm.py:204: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /export/fs06/ahussei6/nvidia/github/NeMo/examples/sp ...
+    
+[NeMo W 2026-04-26 23:19:35 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=19` in the `DataLoader` to improve performance.
+    
+[NeMo W 2026-04-26 23:19:35 dataloader:480] You are using a non-tarred dataset and requested tokenization during data sampling (pretokenize=True). This will cause the tokenization to happen in the main (GPU) process,possibly impacting the training speed if your tokenizer is very large.If the impact is noticable, set pretokenize=False in dataloader config.(note: that will disable token-per-second filtering and 2D bucketing features)
+[NeMo W 2026-04-26 23:19:35 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=19` in the `DataLoader` to improve performance.
+    
+[NeMo W 2026-04-26 23:19:35 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    dataset:
+      dataset_type: tarred_vocoder
+      dataset_args:
+        dataset_meta:
+          lindy:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/tarred_audio/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/tarred_audio/audio_{0..3}.tar
+          emma:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/audio_{0..3}.tar
+          libritts_r:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/tarred_audio/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/tarred_audio/audio_{0..48}.tar
+        sample_rate: 22050
+        n_samples: 24696
+        min_duration: 0.4
+        max_duration: null
+        audio_augmentator_config:
+          impulse:
+            prob: 0.5
+            rng: 42
+            manifest_path: /lustre/fsw/portfolios/convai/projects/convai_convaird_nemo-speech/data/ROOM_Response_and_Noise_Database/processed/rir.json
+            normalize_impulse: false
+            shift_impulse: true
+          codec:
+            prob: 0.5
+            rng: 42
+            format_params:
+            - format: mp3
+              prob: 0.5
+              bit_rates:
+              - 16000
+              - 32000
+              - 64000
+              - 128000
+            - format: ogg
+              prob: 0.075
+              encoder: vorbis
+              bit_rates:
+              - 32000
+              - 48000
+              - 64000
+            - format: ogg
+              prob: 0.375
+              encoder: opus
+              bit_rates:
+              - 8000
+              - 16000
+              - 32000
+              - 64000
+              - 128000
+            - format: wav
+              prob: 0.025
+              encoder: pcm_alaw
+              bit_rates:
+              - 64000
+            - format: wav
+              prob: 0.025
+              encoder: pcm_mulaw
+              bit_rates:
+              - 64000
+        shard_strategy: replicate
+        sample_type: weighted_random
+        sample_args:
+          batch_size: 32
+          steps_per_epoch: 233
+          dataset_weights:
+          - 0.2
+          - 0.2
+          - 0.6
+        shuffle_n: 10000
+    dataloader_params:
+      batch_size: 32
+      drop_last: true
+      num_workers: 4
+    
+[NeMo W 2026-04-26 23:19:35 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    dataset:
+      dataset_type: vocoder
+      dataset_args:
+        sample_rate: 22050
+        n_samples: null
+        min_duration: null
+        max_duration: null
+        trunc_duration: 10.0
+        dataset_meta:
+          lindy:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/audio_22khz
+          emma:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/audio_22khz
+          libritts_r:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/audio_22khz
+    dataloader_params:
+      batch_size: 4
+      num_workers: 2
+    
+[NeMo W 2026-04-26 23:19:37 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/train.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 64
+    shuffle: true
+    is_tarred: false
+    tarred_audio_filepaths: null
+    tarred_shard_strategy: scatter
+    augmentor:
+      noise:
+        manifest_path: /manifests/noise/rir_noise_manifest.json
+        prob: 0.5
+        min_snr_db: 0
+        max_snr_db: 15
+      speed:
+        prob: 0.5
+        sr: 16000
+        resample_type: kaiser_fast
+        min_speed_rate: 0.95
+        max_speed_rate: 1.05
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo W 2026-04-26 23:19:37 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/dev.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 128
+    shuffle: false
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo W 2026-04-26 23:47:29 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/train.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 64
+    shuffle: true
+    is_tarred: false
+    tarred_audio_filepaths: null
+    tarred_shard_strategy: scatter
+    augmentor:
+      noise:
+        manifest_path: /manifests/noise/rir_noise_manifest.json
+        prob: 0.5
+        min_snr_db: 0
+        max_snr_db: 15
+      speed:
+        prob: 0.5
+        sr: 16000
+        resample_type: kaiser_fast
+        min_speed_rate: 0.95
+        max_speed_rate: 1.05
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo W 2026-04-26 23:47:29 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/dev.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 128
+    shuffle: false
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo W 2026-04-26 23:47:32 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 1
+    shuffle: true
+    num_workers: 8
+    pin_memory: true
+    use_start_end_token: false
+    trim_silence: false
+    max_duration: 20
+    min_duration: 0.1
+    is_tarred: false
+    tarred_audio_filepaths: null
+    shuffle_n: 2048
+    bucketing_strategy: fully_randomized
+    bucketing_batch_size: null
+    
+[NeMo W 2026-04-26 23:47:32 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 32
+    shuffle: false
+    num_workers: 8
+    pin_memory: true
+    use_start_end_token: false
+    max_duration: 20
+    
+[NeMo W 2026-04-26 23:47:32 modelPT:202] Please call the ModelPT.setup_test_data() or ModelPT.setup_multiple_test_data() method and provide a valid configuration file to setup the test data loader(s).
+    Test config : 
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    num_workers: 8
+    pin_memory: true
+    use_start_end_token: false
+    
+[NeMo W 2026-04-26 23:51:44 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/train.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 64
+    shuffle: true
+    is_tarred: false
+    tarred_audio_filepaths: null
+    tarred_shard_strategy: scatter
+    augmentor:
+      noise:
+        manifest_path: /manifests/noise/rir_noise_manifest.json
+        prob: 0.5
+        min_snr_db: 0
+        max_snr_db: 15
+      speed:
+        prob: 0.5
+        sr: 16000
+        resample_type: kaiser_fast
+        min_speed_rate: 0.95
+        max_speed_rate: 1.05
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo W 2026-04-26 23:51:44 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/dev.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 128
+    shuffle: false
+    num_workers: 15
+    pin_memory: true
+    

dummy_nemo_duplex/qwen_1b/nemo_log_globalrank-0_localrank-0.txt

@@ -0,0 +1,861 @@
+[NeMo W 2026-04-26 23:19:11 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo W 2026-04-26 23:19:15 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/lightning/fabric/plugins/environments/slurm.py:204: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /export/fs06/ahussei6/nvidia/github/NeMo/examples/sp ...
+    
+[NeMo I 2026-04-26 23:19:15 exp_manager:574] ExpManager schema
+[NeMo I 2026-04-26 23:19:15 exp_manager:575] {'explicit_log_dir': None, 'exp_dir': None, 'name': None, 'version': None, 'use_datetime_version': True, 'resume_if_exists': False, 'resume_past_end': False, 'resume_ignore_no_checkpoint': False, 'resume_from_checkpoint': None, 'create_tensorboard_logger': True, 'summary_writer_kwargs': None, 'create_wandb_logger': False, 'wandb_logger_kwargs': None, 'create_mlflow_logger': False, 'mlflow_logger_kwargs': {'experiment_name': None, 'tracking_uri': None, 'tags': None, 'save_dir': './mlruns', 'prefix': '', 'artifact_location': None, 'run_id': None, 'log_model': False}, 'create_dllogger_logger': False, 'dllogger_logger_kwargs': {'verbose': False, 'stdout': False, 'json_file': './dllogger.json'}, 'create_clearml_logger': False, 'clearml_logger_kwargs': {'project': None, 'task': None, 'connect_pytorch': False, 'model_name': None, 'tags': None, 'log_model': False, 'log_cfg': False, 'log_metrics': False}, 'create_neptune_logger': False, 'neptune_logger_kwargs': None, 'create_checkpoint_callback': True, 'checkpoint_callback_params': {'filepath': None, 'dirpath': None, 'filename': None, 'monitor': 'val_loss', 'verbose': True, 'save_last': True, 'save_top_k': 3, 'save_weights_only': False, 'mode': 'min', 'auto_insert_metric_name': True, 'every_n_epochs': 1, 'every_n_train_steps': None, 'train_time_interval': None, 'prefix': None, 'postfix': '.nemo', 'save_best_model': False, 'always_save_nemo': False, 'save_nemo_on_train_end': True, 'model_parallel_size': None, 'save_on_train_epoch_end': False, 'async_save': False, 'save_last_n_optim_states': -1, 'multistorageclient_enabled': False}, 'create_early_stopping_callback': False, 'early_stopping_callback_params': {'monitor': 'val_loss', 'mode': 'min', 'min_delta': 0.001, 'patience': 10, 'verbose': True, 'strict': True, 'check_finite': True, 'stopping_threshold': None, 'divergence_threshold': None, 'check_on_train_epoch_end': None, 'log_rank_zero_only': False}, 'create_preemption_callback': True, 'files_to_copy': None, 'log_step_timing': True, 'log_delta_step_timing': False, 'step_timing_kwargs': {'reduction': 'mean', 'sync_cuda': False, 'buffer_size': 1}, 'log_local_rank_0_only': False, 'log_global_rank_0_only': False, 'disable_validation_on_resume': True, 'ema': {'enable': False, 'decay': 0.999, 'cpu_offload': False, 'validate_original_weights': False, 'every_n_steps': 1}, 'max_time_per_run': None, 'seconds_to_sleep': 5.0, 'create_straggler_detection_callback': False, 'straggler_detection_params': {'report_time_interval': 300.0, 'calc_relative_gpu_perf': True, 'calc_individual_gpu_perf': True, 'num_gpu_perf_scores_to_log': 5, 'gpu_relative_perf_threshold': 0.7, 'gpu_individual_perf_threshold': 0.7, 'stop_if_detected': False}, 'create_fault_tolerance_callback': False, 'fault_tolerance': {'workload_check_interval': 5.0, 'initial_rank_heartbeat_timeout': 3600.0, 'rank_heartbeat_timeout': 2700.0, 'calculate_timeouts': True, 'safety_factor': 5.0, 'rank_termination_signal': <Signals.SIGKILL: 9>, 'log_level': 'INFO', 'max_rank_restarts': 0, 'max_subsequent_job_failures': 0, 'additional_ft_launcher_args': '', 'simulated_fault': None}, 'log_tflops_per_sec_per_gpu': True}
+[NeMo W 2026-04-26 23:19:15 exp_manager:1079] Exp_manager is logging to /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b, but it already exists.
+[NeMo W 2026-04-26 23:19:15 exp_manager:997] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :/export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/checkpoints. Training from scratch.
+[NeMo I 2026-04-26 23:19:15 exp_manager:635] Experiments will be logged at /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b
+[NeMo W 2026-04-26 23:19:15 exp_manager:1388] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 100000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-26 23:19:15 exp_manager:780] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo W 2026-04-26 23:19:16 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    dataset:
+      dataset_type: tarred_vocoder
+      dataset_args:
+        dataset_meta:
+          lindy:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/tarred_audio/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/tarred_audio/audio_{0..3}.tar
+          emma:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/audio_{0..3}.tar
+          libritts_r:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/tarred_audio/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/tarred_audio/audio_{0..48}.tar
+        sample_rate: 22050
+        n_samples: 24696
+        min_duration: 0.4
+        max_duration: null
+        audio_augmentator_config:
+          impulse:
+            prob: 0.5
+            rng: 42
+            manifest_path: /lustre/fsw/portfolios/convai/projects/convai_convaird_nemo-speech/data/ROOM_Response_and_Noise_Database/processed/rir.json
+            normalize_impulse: false
+            shift_impulse: true
+          codec:
+            prob: 0.5
+            rng: 42
+            format_params:
+            - format: mp3
+              prob: 0.5
+              bit_rates:
+              - 16000
+              - 32000
+              - 64000
+              - 128000
+            - format: ogg
+              prob: 0.075
+              encoder: vorbis
+              bit_rates:
+              - 32000
+              - 48000
+              - 64000
+            - format: ogg
+              prob: 0.375
+              encoder: opus
+              bit_rates:
+              - 8000
+              - 16000
+              - 32000
+              - 64000
+              - 128000
+            - format: wav
+              prob: 0.025
+              encoder: pcm_alaw
+              bit_rates:
+              - 64000
+            - format: wav
+              prob: 0.025
+              encoder: pcm_mulaw
+              bit_rates:
+              - 64000
+        shard_strategy: replicate
+        sample_type: weighted_random
+        sample_args:
+          batch_size: 32
+          steps_per_epoch: 233
+          dataset_weights:
+          - 0.2
+          - 0.2
+          - 0.6
+        shuffle_n: 10000
+    dataloader_params:
+      batch_size: 32
+      drop_last: true
+      num_workers: 4
+    
+[NeMo W 2026-04-26 23:19:16 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    dataset:
+      dataset_type: vocoder
+      dataset_args:
+        sample_rate: 22050
+        n_samples: null
+        min_duration: null
+        max_duration: null
+        trunc_duration: 10.0
+        dataset_meta:
+          lindy:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/audio_22khz
+          emma:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/audio_22khz
+          libritts_r:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/audio_22khz
+    dataloader_params:
+      batch_size: 4
+      num_workers: 2
+    
+[NeMo I 2026-04-26 23:19:16 audio_codec:101] Vector quantizer does not support commit loss.
+[NeMo I 2026-04-26 23:19:18 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:19:19 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:19:19 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:19:19 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:19:19 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:19:19 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:19:19 save_restore_connector:283] Model AudioCodecModel was successfully restored from /export/fs06/ahussei6/nvidia/pretrained_models/nano_codec/Low_Frame-rate_Speech_Codec++.nemo.
+[NeMo W 2026-04-26 23:19:19 duplex_s2s_speech_decoder_model:148] Tokenizer does not have a `bos_token`. Setting it to '<|im_start|>'.
+[NeMo I 2026-04-26 23:19:21 mixins:181] Tokenizer SentencePieceTokenizer initialized with 1024 tokens
+[NeMo W 2026-04-26 23:19:22 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath:
+    - - /raid/local//bucket1/tarred_audio_manifest.json
+    - - /raid/local//bucket2/tarred_audio_manifest.json
+    - - /raid/local//bucket3/tarred_audio_manifest.json
+    - - /raid/local//bucket4/tarred_audio_manifest.json
+    - - /raid/local//bucket5/tarred_audio_manifest.json
+    - - /raid/local//bucket6/tarred_audio_manifest.json
+    - - /raid/local//bucket7/tarred_audio_manifest.json
+    - - /raid/local//bucket8/tarred_audio_manifest.json
+    sample_rate: 16000
+    batch_size: 1
+    shuffle: true
+    num_workers: 4
+    pin_memory: true
+    use_start_end_token: false
+    trim_silence: false
+    max_duration: 25
+    min_duration: 0.1
+    is_tarred: true
+    tarred_audio_filepaths:
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket1/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket2/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket3/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket4/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket5/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket6/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket7/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket8/audio__OP_0..8191_CL_.tar
+    shuffle_n: 2048
+    bucketing_strategy: fully_randomized
+    bucketing_batch_size:
+    - 72
+    - 64
+    - 56
+    - 48
+    - 40
+    - 32
+    - 24
+    - 16
+    
+[NeMo W 2026-04-26 23:19:22 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath:
+    - /manifests/librispeech/librivox-dev-other.json
+    - /manifests/librispeech/librivox-dev-clean.json
+    - /manifests/librispeech/librivox-test-other.json
+    - /manifests/librispeech/librivox-test-clean.json
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    num_workers: 8
+    pin_memory: true
+    use_start_end_token: false
+    
+[NeMo W 2026-04-26 23:19:22 modelPT:202] Please call the ModelPT.setup_test_data() or ModelPT.setup_multiple_test_data() method and provide a valid configuration file to setup the test data loader(s).
+    Test config : 
+    manifest_filepath:
+    - /manifests/librispeech/librivox-dev-other.json
+    - /manifests/librispeech/librivox-dev-clean.json
+    - /manifests/librispeech/librivox-test-other.json
+    - /manifests/librispeech/librivox-test-clean.json
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    num_workers: 8
+    pin_memory: true
+    use_start_end_token: false
+    
+[NeMo I 2026-04-26 23:19:22 features:305] PADDING: 0
+[NeMo I 2026-04-26 23:19:23 rnnt_models:226] Using RNNT Loss : warprnnt_numba
+    Loss warprnnt_numba_kwargs: {'fastemit_lambda': 0.005, 'clamp': -1.0}
+[NeMo I 2026-04-26 23:19:23 rnnt_models:226] Using RNNT Loss : warprnnt_numba
+    Loss warprnnt_numba_kwargs: {'fastemit_lambda': 0.005, 'clamp': -1.0}
+[NeMo I 2026-04-26 23:19:23 rnnt_models:226] Using RNNT Loss : warprnnt_numba
+    Loss warprnnt_numba_kwargs: {'fastemit_lambda': 0.005, 'clamp': -1.0}
+[NeMo I 2026-04-26 23:19:24 save_restore_connector:283] Model EncDecHybridRNNTCTCBPEModel was successfully restored from /export/fs06/ahussei6/nvidia/pretrained_models/asr/stt_en_fastconformer_hybrid_large_streaming_multi_v1.20.0/stt_en_fastconformer_hybrid_large_streaming_multi.nemo.
+[NeMo I 2026-04-26 23:19:24 features:305] PADDING: 0
+[NeMo I 2026-04-26 23:19:24 cloud:58] Found existing object /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo.
+[NeMo I 2026-04-26 23:19:24 cloud:64] Re-using file from: /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo
+[NeMo I 2026-04-26 23:19:24 common:827] Instantiating model from pre-trained checkpoint
+[NeMo W 2026-04-26 23:19:24 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/train.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 64
+    shuffle: true
+    is_tarred: false
+    tarred_audio_filepaths: null
+    tarred_shard_strategy: scatter
+    augmentor:
+      noise:
+        manifest_path: /manifests/noise/rir_noise_manifest.json
+        prob: 0.5
+        min_snr_db: 0
+        max_snr_db: 15
+      speed:
+        prob: 0.5
+        sr: 16000
+        resample_type: kaiser_fast
+        min_speed_rate: 0.95
+        max_speed_rate: 1.05
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo W 2026-04-26 23:19:24 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/dev.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 128
+    shuffle: false
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo I 2026-04-26 23:19:24 features:305] PADDING: 16
+[NeMo I 2026-04-26 23:19:24 save_restore_connector:283] Model EncDecSpeakerLabelModel was successfully restored from /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo.
+[NeMo I 2026-04-26 23:19:34 pretrained:110]  | > 237 / 237 layers are restored.
+[NeMo W 2026-04-26 23:19:34 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/lightning/fabric/plugins/environments/slurm.py:204: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /export/fs06/ahussei6/nvidia/github/NeMo/examples/sp ...
+    
+[NeMo I 2026-04-26 23:19:35 optim_setup:136] Parameters | trainable=2011762688 (95.83%) | total=2099261314
+[NeMo I 2026-04-26 23:19:35 dataloader:559] Creating a Lhotse DynamicBucketingSampler (max_batch_duration=400.0 max_batch_size=None)
+[NeMo W 2026-04-26 23:19:35 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=19` in the `DataLoader` to improve performance.
+    
+[NeMo I 2026-04-26 23:19:35 dataloader:290] We will be using a Lhotse DataLoader.
+[NeMo W 2026-04-26 23:19:35 dataloader:480] You are using a non-tarred dataset and requested tokenization during data sampling (pretokenize=True). This will cause the tokenization to happen in the main (GPU) process,possibly impacting the training speed if your tokenizer is very large.If the impact is noticable, set pretokenize=False in dataloader config.(note: that will disable token-per-second filtering and 2D bucketing features)
+[NeMo I 2026-04-26 23:19:35 dataloader:583] Creating a Lhotse DynamicCutSampler (bucketing is disabled, (max_batch_duration=None max_batch_size=64)
+[NeMo W 2026-04-26 23:19:35 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:424: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=19` in the `DataLoader` to improve performance.
+    
+[NeMo W 2026-04-26 23:19:35 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    dataset:
+      dataset_type: tarred_vocoder
+      dataset_args:
+        dataset_meta:
+          lindy:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/tarred_audio/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/tarred_audio/audio_{0..3}.tar
+          emma:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/audio_{0..3}.tar
+          libritts_r:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/tarred_audio/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/tarred_audio/audio_{0..48}.tar
+        sample_rate: 22050
+        n_samples: 24696
+        min_duration: 0.4
+        max_duration: null
+        audio_augmentator_config:
+          impulse:
+            prob: 0.5
+            rng: 42
+            manifest_path: /lustre/fsw/portfolios/convai/projects/convai_convaird_nemo-speech/data/ROOM_Response_and_Noise_Database/processed/rir.json
+            normalize_impulse: false
+            shift_impulse: true
+          codec:
+            prob: 0.5
+            rng: 42
+            format_params:
+            - format: mp3
+              prob: 0.5
+              bit_rates:
+              - 16000
+              - 32000
+              - 64000
+              - 128000
+            - format: ogg
+              prob: 0.075
+              encoder: vorbis
+              bit_rates:
+              - 32000
+              - 48000
+              - 64000
+            - format: ogg
+              prob: 0.375
+              encoder: opus
+              bit_rates:
+              - 8000
+              - 16000
+              - 32000
+              - 64000
+              - 128000
+            - format: wav
+              prob: 0.025
+              encoder: pcm_alaw
+              bit_rates:
+              - 64000
+            - format: wav
+              prob: 0.025
+              encoder: pcm_mulaw
+              bit_rates:
+              - 64000
+        shard_strategy: replicate
+        sample_type: weighted_random
+        sample_args:
+          batch_size: 32
+          steps_per_epoch: 233
+          dataset_weights:
+          - 0.2
+          - 0.2
+          - 0.6
+        shuffle_n: 10000
+    dataloader_params:
+      batch_size: 32
+      drop_last: true
+      num_workers: 4
+    
+[NeMo W 2026-04-26 23:19:35 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    dataset:
+      dataset_type: vocoder
+      dataset_args:
+        sample_rate: 22050
+        n_samples: null
+        min_duration: null
+        max_duration: null
+        trunc_duration: 10.0
+        dataset_meta:
+          lindy:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/audio_22khz
+          emma:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/audio_22khz
+          libritts_r:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/audio_22khz
+    dataloader_params:
+      batch_size: 4
+      num_workers: 2
+    
+[NeMo I 2026-04-26 23:19:36 audio_codec:101] Vector quantizer does not support commit loss.
+[NeMo I 2026-04-26 23:19:36 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:19:36 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:19:36 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:19:36 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:19:36 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:19:36 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:19:37 save_restore_connector:283] Model AudioCodecModel was successfully restored from /export/fs06/ahussei6/nvidia/pretrained_models/nano_codec/Low_Frame-rate_Speech_Codec++.nemo.
+[NeMo I 2026-04-26 23:19:37 cloud:58] Found existing object /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo.
+[NeMo I 2026-04-26 23:19:37 cloud:64] Re-using file from: /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo
+[NeMo I 2026-04-26 23:19:37 common:827] Instantiating model from pre-trained checkpoint
+[NeMo W 2026-04-26 23:19:37 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/train.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 64
+    shuffle: true
+    is_tarred: false
+    tarred_audio_filepaths: null
+    tarred_shard_strategy: scatter
+    augmentor:
+      noise:
+        manifest_path: /manifests/noise/rir_noise_manifest.json
+        prob: 0.5
+        min_snr_db: 0
+        max_snr_db: 15
+      speed:
+        prob: 0.5
+        sr: 16000
+        resample_type: kaiser_fast
+        min_speed_rate: 0.95
+        max_speed_rate: 1.05
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo W 2026-04-26 23:19:37 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/dev.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 128
+    shuffle: false
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo I 2026-04-26 23:19:37 features:305] PADDING: 16
+[NeMo I 2026-04-26 23:19:37 save_restore_connector:283] Model EncDecSpeakerLabelModel was successfully restored from /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo.
+[NeMo I 2026-04-26 23:47:28 cloud:58] Found existing object /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo.
+[NeMo I 2026-04-26 23:47:28 cloud:64] Re-using file from: /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo
+[NeMo I 2026-04-26 23:47:28 common:827] Instantiating model from pre-trained checkpoint
+[NeMo W 2026-04-26 23:47:29 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/train.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 64
+    shuffle: true
+    is_tarred: false
+    tarred_audio_filepaths: null
+    tarred_shard_strategy: scatter
+    augmentor:
+      noise:
+        manifest_path: /manifests/noise/rir_noise_manifest.json
+        prob: 0.5
+        min_snr_db: 0
+        max_snr_db: 15
+      speed:
+        prob: 0.5
+        sr: 16000
+        resample_type: kaiser_fast
+        min_speed_rate: 0.95
+        max_speed_rate: 1.05
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo W 2026-04-26 23:47:29 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/dev.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 128
+    shuffle: false
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo I 2026-04-26 23:47:29 features:305] PADDING: 16
+[NeMo I 2026-04-26 23:47:29 save_restore_connector:283] Model EncDecSpeakerLabelModel was successfully restored from /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo.
+[NeMo I 2026-04-26 23:47:29 cloud:58] Found existing object /export/fs06/ahussei6/nvidia/cache/HFCACHE/stt_en_fastconformer_transducer_large/74b4e49a2b22465da737dd5345767e61/stt_en_fastconformer_transducer_large.nemo.
+[NeMo I 2026-04-26 23:47:29 cloud:64] Re-using file from: /export/fs06/ahussei6/nvidia/cache/HFCACHE/stt_en_fastconformer_transducer_large/74b4e49a2b22465da737dd5345767e61/stt_en_fastconformer_transducer_large.nemo
+[NeMo I 2026-04-26 23:47:29 common:827] Instantiating model from pre-trained checkpoint
+[NeMo I 2026-04-26 23:47:32 mixins:181] Tokenizer SentencePieceTokenizer initialized with 1024 tokens
+[NeMo W 2026-04-26 23:47:32 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 1
+    shuffle: true
+    num_workers: 8
+    pin_memory: true
+    use_start_end_token: false
+    trim_silence: false
+    max_duration: 20
+    min_duration: 0.1
+    is_tarred: false
+    tarred_audio_filepaths: null
+    shuffle_n: 2048
+    bucketing_strategy: fully_randomized
+    bucketing_batch_size: null
+    
+[NeMo W 2026-04-26 23:47:32 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 32
+    shuffle: false
+    num_workers: 8
+    pin_memory: true
+    use_start_end_token: false
+    max_duration: 20
+    
+[NeMo W 2026-04-26 23:47:32 modelPT:202] Please call the ModelPT.setup_test_data() or ModelPT.setup_multiple_test_data() method and provide a valid configuration file to setup the test data loader(s).
+    Test config : 
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    num_workers: 8
+    pin_memory: true
+    use_start_end_token: false
+    
+[NeMo I 2026-04-26 23:47:32 features:305] PADDING: 0
+[NeMo I 2026-04-26 23:47:33 rnnt_models:226] Using RNNT Loss : warprnnt_numba
+    Loss warprnnt_numba_kwargs: {'fastemit_lambda': 0.0, 'clamp': -1.0}
+[NeMo I 2026-04-26 23:47:33 rnnt_models:226] Using RNNT Loss : warprnnt_numba
+    Loss warprnnt_numba_kwargs: {'fastemit_lambda': 0.0, 'clamp': -1.0}
+[NeMo I 2026-04-26 23:47:33 rnnt_models:226] Using RNNT Loss : warprnnt_numba
+    Loss warprnnt_numba_kwargs: {'fastemit_lambda': 0.0, 'clamp': -1.0}
+[NeMo I 2026-04-26 23:47:34 save_restore_connector:283] Model EncDecRNNTBPEModel was successfully restored from /export/fs06/ahussei6/nvidia/cache/HFCACHE/stt_en_fastconformer_transducer_large/74b4e49a2b22465da737dd5345767e61/stt_en_fastconformer_transducer_large.nemo.
+[NeMo I 2026-04-26 23:47:34 optional_cuda_graphs:53] Disabled CUDA graphs for module <class 'nemo.collections.asr.models.rnnt_bpe_models.EncDecRNNTBPEModel'>.decoding.decoding
+[NeMo I 2026-04-26 23:47:34 optional_cuda_graphs:53] Disabled CUDA graphs for module <class 'nemo.collections.asr.metrics.wer.WER'>joint._wer.decoding.decoding
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	i responded in a bad way straightening myself in the chair
+    [ASR]	the [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	previously she was friends of the cruella from school
+    [ASR]	one [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	at present democrat jim mcdermott is the district representative
+    [ASR]	you d rather the dividend of the leader [4.87]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	it is found in the afro tropical and southeast asia eco zone
+    [ASR]	the love of a [2.16]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	poor things they love you so much yes they do love me wow don't tell them
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	this is developed at different times of the year in the processions and stations of the cross
+    [ASR]	the tongue but the figure didn't [1.54]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	they have high abrasion resistance
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	the award included three categories science art and literature
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	his children are paloma yerma and natalio
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	my friend sara just got her backpack stolen
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	it included salafist groups inspired by the muslim brothers
+    [ASR]	yes [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	it's related to the cows
+    [ASR]	he's the pee [14.13]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	this vowel system is the same as in vulgar latin
+    [ASR]	was the [0.92]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	the fruit comprises a couple of drupes
+    [ASR]	they didn't need to [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	its meaning is about the path
+    [ASR]	d is the debate that to be d [6.57]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	it does not grow naturally in the city of jericho
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	the capital of the region is mena
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	this is a mess
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	again do we have the right to know the private lives of celebrities
+    [ASR]	you learned that the [1.68]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	for this reason in august of the same year she temporarily withdrew from television
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	it s native to the hawaiian islands
+    [ASR]	did they [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	she led us to question everything we had learned about design
+    [ASR]	blue the beer [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	there's no advice for the past
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	the ships of the second one were destined for hong kong
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	it lives in afghanistan iran pakistan and turkey
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	sams' creativity and the king's promotional campaigns worked
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	in the united states it was released as the volkswagen quantum
+    [ASR]	di [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	that mutation is more prominent in male voices than in female ones
+    [ASR]	di [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	the grand master elected the defender to join his band
+    [ASR]	detected a war [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	declared protected areas
+    [ASR]	in the end and [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	degree in social communication
+    [ASR]	the [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	at eighteen he could produce and sell many pieces
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	they had several guest bands as in previous tours
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	like all the rapsids the location of the head was tilted back
+    [ASR]	the above the above [2.57]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	it had as a goal scorer the colombian victor renteria
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	finally he defeats the attackers and kills miller in mortal combat
+    [ASR]	let me [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	listen whoever he is he doesn't want us to find him
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	he lives in paris
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	his patron saint's name was saint justus
+    [ASR]	is the [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	its habitat extends between the guianas and venezuela
+    [ASR]	did you [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	montenegro and serbia divided the sandzak and albania was created
+    [ASR]	d [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	come on friend come on
+    [ASR]	and [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	there are older references
+    [ASR]	r [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	he witnesses the doctor and astrid teleporting back to the intergalactic cruise ship titanic
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	the oblivion relegated the emotions and even the color
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	look i cut the cookies in little pieces the cookies
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	their playing usually lasts six minutes
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	two of these pterosaurs have not been named yet
+    [ASR]	the deep [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	with her he premiered at the teatro infanta isabel plays such as oh doctor
+    [ASR]	no [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	it is found in the northern region of the amazon basin
+    [ASR]	the [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	thus the number of active participants will be reduced to only twenty five
+    [ASR]	nay they did [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	the organization was authorized to build gates and dikes
+    [ASR]	worry year [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	the beak is almost black with the base of the jaw light grey
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	he has a younger brother benjamin ben draiman who makes folk rock and ambient music
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	overcoming this problem he has returned to the team for the current season
+    [ASR]	okay [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	he created a radio network in new york
+    [ASR]	did the thing he did the d [5.69]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	it landed vertically backward
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	the inside is mainly built in wood
+    [ASR]	oh [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	he scored three goals and assisted on a fourth
+    [ASR]	two [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	each of these ayllus has its own neighborhood and square
+    [ASR]	the d n n [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	it lasted less than candy in a child's mouth
+    [ASR]	is the [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	this symbolic act represents the end of the crisis
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	they think margo has left clues so that quentin can find her
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 asr_bleu:80] [REF]	the painter is being interviewed on the radio
+    [ASR]	 [0.00]
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/864c60f6-3cfe-46d4-8137-dda948fedadf.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/47414fd7-2788-4671-a1f7-7deb1e7a8043.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/a51c5f30-cb3b-4c52-bf5a-fea3f3b308ae.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/93ce123f-4d93-48fd-ba3a-fa6cb1912502.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/a7623f82-9030-41f7-a2a8-f6070fdc00ae.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/182f4c63-9602-4a71-98b8-d8d50845e25c.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/6baffb94-af59-42b6-bbe1-cb0776e68d0f.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/e587f7d8-0a8e-4286-9d77-f5d8edae3ace.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/09aecae5-b953-4f02-a22b-296fdc73cc06.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/8ec1d43f-7b47-4cda-b453-d8cb77a2e1e1.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/2eb2c5b8-1e52-4b23-b180-ff766626b1a8.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/9f19424b-c820-487f-99b5-6e7516e9b934.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/5c6a8b18-8918-43f6-b04d-605e108ba994.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/9d8dbddc-7316-407c-a65d-7e688a99e045.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/e9310fa9-0dcd-4a92-ae97-d7e886b3066b.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/1b540a01-969f-4fe5-a449-f7a8e18f7ad1.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/b70e4973-ae52-46f6-8b57-094b77d72f97.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/f96c7b27-afda-460f-9899-db11a143a389.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/b7298c70-dc12-447c-8ff4-b90a886cd38a.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/f7a2d6d2-6d69-439b-a8a2-76ed71fbabdc.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/e352f1c7-8e9f-4984-8d49-70819616617c.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/bd12b66a-8b65-475c-9fa6-0f7b2b44bba1.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/a4608886-ec28-45a2-b5e7-db98328d95f9.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/905bea97-4b68-456c-bb1a-fd38c278c8de.wav
+[NeMo I 2026-04-26 23:47:53 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/3a42d3b8-5c35-4a13-bce1-5e559273aed7.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/7e96c80f-b460-40b5-bbbd-53f938c84523.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/414f9639-0caa-4db3-981d-a0e7f93f1030.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/233b572b-a3e7-472e-922d-c2ce9e1fe6db.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/c5d41939-29b9-4333-ab95-4d9bb5cba837.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/35209995-5fc9-498f-a1a1-9fe5ae7e0501.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/5364a9dc-4b88-4656-91a4-e08b928df2f9.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/8988b56b-9a8e-40c9-9191-ad4273846c09.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/4ff65a04-79d5-47bb-bac4-1573dcf8fe93.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/628e28c0-aa88-45aa-9cd4-0371f5cfe9a5.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/603c944c-f666-4fe0-9ec1-752698168608.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/5f3624b4-12f6-4bdd-b510-daebc84350b8.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/bede7c1d-e6c9-4345-8794-dbaf69766190.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/08d0c818-3b13-41f4-9082-2394c0f332de.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/416572c3-7838-49ac-aaf0-244dffed08c0.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/d7fc60c2-77d6-4aaf-8b69-a2a998d504e2.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/78b555eb-3470-4279-848c-af3c72208dda.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/64429b85-ab36-4b9f-a3ea-66ae17ffc47f.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/da47d7af-6da4-4d2e-8dde-2b469d95c79d.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/82835111-4af5-4dba-889c-e8c5d6c2cb6e.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/a0f90b27-57ac-4de1-8b8f-2943a2539f6e.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/64d99fe4-c70a-4084-b21e-7591cfdfc688.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/100db0bc-b3fe-4514-bb70-4a7bff839029.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/3c868384-211c-4845-9fdd-c0c8d8275cec.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/e70a297d-0607-4c3a-973c-fc6926eb25bb.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/08b61e60-fdbe-4386-8ba7-2e10ec3bf948.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/3a89894f-19ea-4613-9e7b-639a07a23d24.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/2cab0072-a929-40bc-aaaf-d94eddb02380.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/315156bf-45dc-4bb3-b6b6-8571b53097a1.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/4da5c489-586e-4758-9757-144a298871c2.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/002a68c1-69c3-4439-98f2-89cd2b650d16.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/bf07db1a-4b32-42b5-ab18-fe50b8431c03.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/fc3b1cd0-86d6-42fd-babf-af5a86d3507f.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/4eb6a1f3-c866-4aae-9cea-af7e410a5298.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/27a5f6ac-975d-4185-9880-7cd902973579.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/4c211e5a-5299-4025-81e9-e41c9de20712.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/81a195f5-70d1-49b3-bd55-51a3156ac0d7.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/5125c20c-23c0-4291-86de-1157e5ece331.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/4c96af03-b294-400c-becc-cf5028945b1a.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:72] Audio saved at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/pred_wavs/en_dev/9d4edfb3-7021-4083-b7f9-dc7346452bb5.wav
+[NeMo I 2026-04-26 23:47:54 results_logger:129] Metadata file for en_dev dataset updated at: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/validation_logs/metadatas/en_dev.json
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	i responded in a bad way straightening myself in the chair
+    [HYP]	the is the the the the the the the the the the [3.39]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	previously she was friends of the cruella from school
+    [HYP]	the is the the the the the the the the the [3.75]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	at present democrat jim mcdermott is the district representative
+    [HYP]	 the the the the the the the the the the the the the the the [2.63]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	it is found in the afro tropical and southeast asia eco zone
+    [HYP]	the is the the the the the the the the the the the [3.67]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	poor things they love you so much yes they do love me wow don't tell them
+    [HYP]	the the the the thethe the the the the the the thethe the the the the the the the the the the and and [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	this is developed at different times of the year in the processions and stations of the cross
+    [HYP]	the the the the the the the the the the the the the the [3.02]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	they have high abrasion resistance
+    [HYP]	the the the the the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	the award included three categories science art and literature
+    [HYP]	the the the the the the the the the the the the the the [2.84]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	his children are paloma yerma and natalio
+    [HYP]	 is is the the the the the the the the the the the the the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	my friend sara just got her backpack stolen
+    [HYP]	the the the the the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	it included salafist groups inspired by the muslim brothers
+    [HYP]	 the the the the the the the the the the the the [3.39]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	it's related to the cows
+    [HYP]	the is the the the the the the [5.52]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	this vowel system is the same as in vulgar latin
+    [HYP]	the the the the the the the the the the the the the [3.09]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	the fruit comprises a couple of drupes
+    [HYP]	the the the the the the the the the the [4.20]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	its meaning is about the path
+    [HYP]	the is the the the the the the the the [4.99]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	it does not grow naturally in the city of jericho
+    [HYP]	 the the the the the the the the the the the the [3.39]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	the capital of the region is mena
+    [HYP]	the the the the the the the [7.81]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	this is a mess
+    [HYP]	the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	again do we have the right to know the private lives of celebrities
+    [HYP]	the the the the the the the the the the the the the the the [3.13]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	for this reason in august of the same year she temporarily withdrew from television
+    [HYP]	the the the the the the the the the the the the the the the the the the the the [1.91]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	it s native to the hawaiian islands
+    [HYP]	the the the the the the the the the the the [3.75]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	she led us to question everything we had learned about design
+    [HYP]	the the the the the the the the the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	there's no advice for the past
+    [HYP]	the the the the the the the the the the [4.20]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	the ships of the second one were destined for hong kong
+    [HYP]	 thethe the the the the the the the the the the the the the the [3.13]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	it lives in afghanistan iran pakistan and turkey
+    [HYP]	 are and and and and and and and and and and and and and and and and [2.29]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	sams' creativity and the king's promotional campaigns worked
+    [HYP]	the the the the the the the the the the the the the the the the the [2.29]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	in the united states it was released as the volkswagen quantum
+    [HYP]	the the the the the the the the the the the the the the the the [2.91]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	that mutation is more prominent in male voices than in female ones
+    [HYP]	the are the the the the the the the the the the the the the the the the and and and [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	the grand master elected the defender to join his band
+    [HYP]	the the the the the the the the the the the the the the the [3.13]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	declared protected areas
+    [HYP]	the the the the the the the the the the the and and and and [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	degree in social communication
+    [HYP]	the the the the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	at eighteen he could produce and sell many pieces
+    [HYP]	the the the the the the the the the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	they had several guest bands as in previous tours
+    [HYP]	 the the the the the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	like all the rapsids the location of the head was tilted back
+    [HYP]	the the the the the the the the of of the of of [7.35]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	it had as a goal scorer the colombian victor renteria
+    [HYP]	i a a a a the the the the the the the the the the the [2.91]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	finally he defeats the attackers and kills miller in mortal combat
+    [HYP]	thethe the the the the the the the the the the the the the the the the [2.29]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	listen whoever he is he doesn't want us to find him
+    [HYP]	the is is the the the the the the the the the the the the the the the [2.15]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	he lives in paris
+    [HYP]	 is the the the the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	his patron saint's name was saint justus
+    [HYP]	the is is the the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	its habitat extends between the guianas and venezuela
+    [HYP]	the the the the the the the the the [4.77]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	montenegro and serbia divided the sandzak and albania was created
+    [HYP]	the the the the the the the the the the the and and and and and [3.22]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	come on friend come on
+    [HYP]	the are are are the and and and and [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	there are older references
+    [HYP]	 are are are the the the the the the the the [3.75]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	he witnesses the doctor and astrid teleporting back to the intergalactic cruise ship titanic
+    [HYP]	the the the the the the the the the the the the the the the the the the the [2.41]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	the oblivion relegated the emotions and even the color
+    [HYP]	the the the the the the the the the the the the the the the the the the the the [2.52]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	look i cut the cookies in little pieces the cookies
+    [HYP]	the the the the the the the the the the the the the the and and [2.91]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	their playing usually lasts six minutes
+    [HYP]	the the the the the the the the the the the the the the the the the and and [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	two of these pterosaurs have not been named yet
+    [HYP]	the the the the the the the the the the the the the and and and and and [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	with her he premiered at the teatro infanta isabel plays such as oh doctor
+    [HYP]	the the the the the the the the the the the the the the the [2.63]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	it is found in the northern region of the amazon basin
+    [HYP]	 the the the the the the the the the the the the [4.03]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	thus the number of active participants will be reduced to only twenty five
+    [HYP]	the the the the the the the the the the the the the the the [2.63]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	the organization was authorized to build gates and dikes
+    [HYP]	 the the the the the the the the the the the the the the the [2.63]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	the beak is almost black with the base of the jaw light grey
+    [HYP]	 the the the the the the the the the the the the the the [3.74]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	he has a younger brother benjamin ben draiman who makes folk rock and ambient music
+    [HYP]	the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	overcoming this problem he has returned to the team for the current season
+    [HYP]	the the the the the the the the the the the the the the the the the the the [2.41]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	he created a radio network in new york
+    [HYP]	the the the the the the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	it landed vertically backward
+    [HYP]	the are are the the the the the the the the and [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	the inside is mainly built in wood
+    [HYP]	the is the the the the the the the the the the [4.03]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	he scored three goals and assisted on a fourth
+    [HYP]	the the the the the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	each of these ayllus has its own neighborhood and square
+    [HYP]	the the the the the the and and and and and and and and [2.84]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	it lasted less than candy in a child's mouth
+    [HYP]	 is the the the the the the the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	this symbolic act represents the end of the crisis
+    [HYP]	the is the the the the the the the the the the [4.03]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	they think margo has left clues so that quentin can find her
+    [HYP]	the the the the the the the the the the the the the the [0.00]
+[NeMo I 2026-04-26 23:47:54 bleu:51] [REF]	the painter is being interviewed on the radio
+    [HYP]	the't the the the the the the the the the [4.99]
+[NeMo I 2026-04-26 23:47:55 nemo_model_checkpoint:569] Checkpoint save for step 1000 started at 1777261675.269628.
+[NeMo I 2026-04-26 23:49:52 nemo_model_checkpoint:569] Checkpoint save for step 1000 started at 1777261792.7167933.
+[NeMo I 2026-04-26 23:51:44 cloud:58] Found existing object /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo.
+[NeMo I 2026-04-26 23:51:44 cloud:64] Re-using file from: /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo
+[NeMo I 2026-04-26 23:51:44 common:827] Instantiating model from pre-trained checkpoint
+[NeMo W 2026-04-26 23:51:44 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/train.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 64
+    shuffle: true
+    is_tarred: false
+    tarred_audio_filepaths: null
+    tarred_shard_strategy: scatter
+    augmentor:
+      noise:
+        manifest_path: /manifests/noise/rir_noise_manifest.json
+        prob: 0.5
+        min_snr_db: 0
+        max_snr_db: 15
+      speed:
+        prob: 0.5
+        sr: 16000
+        resample_type: kaiser_fast
+        min_speed_rate: 0.95
+        max_speed_rate: 1.05
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo W 2026-04-26 23:51:44 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/dev.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 128
+    shuffle: false
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo I 2026-04-26 23:51:44 features:305] PADDING: 16
+[NeMo I 2026-04-26 23:51:45 save_restore_connector:283] Model EncDecSpeakerLabelModel was successfully restored from /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo.

dummy_nemo_duplex/qwen_1b/run_0/cmd-args.log

@@ -0,0 +1 @@
+/export/fs06/ahussei6/nvidia/github/NeMo/examples/speechlm2/s2s_duplex_speech_decoder_train.py --config-path=conf/train --config-name=qwen_1b ++exp_manager.checkpoint_callback_params.save_top_k=3 exp_manager.name=qwen_1b ++model.pretrained_audio_codec=/export/fs06/ahussei6/nvidia/pretrained_models/nano_codec/Low_Frame-rate_Speech_Codec++.nemo ++model.pretrained_tts_from_s2s=/export/fs06/ahussei6/nvidia/pretrained_models/magpie_tts/tts-pretraining_qwnen_2.5_81007_steps.ckpt ++model.pretrained_asr=/export/fs06/ahussei6/nvidia/pretrained_models/asr/stt_en_fastconformer_hybrid_large_streaming_multi_v1.20.0/stt_en_fastconformer_hybrid_large_streaming_multi.nemo ++model.mask_sequence_loss=True trainer.num_nodes=1 exp_manager.explicit_log_dir=/export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b data.train_ds.seed= ++model.audio_loss_weight=20 ++model.speech_decoder.cond_on_prev_audio_tokens=True ++model.speech_decoder.use_speaker_encoder=True ++model.speech_decoder.cond_on_char_embedding=True ++model.speech_decoder.cond_on_asr_emb=False ++model.speech_decoder.cond_on_llm_latent=False ++model.speech_decoder.cond_on_modality_adapter_emb=False ++model.speech_decoder.cond_on_text_tokens=False ++model.speech_decoder.cfg_scale=2.5 ++model.speech_decoder.kernel_size=3 ++model.speech_decoder.cfg_unconditional_prob=0.2 ++model.custom_codebook_size=2045 ++model.custom_speech_bos_id=2019 ++model.custom_speech_eos_id=2020 ++model.custom_speech_delay_id=2018 model.perception.encoder.att_context_size=[70,0] model.perception.modality_adapter.att_context_size=[70,0] ++model.pretrained_llm=/export/fs06/ahussei6/nvidia/cache/HFCACHE/hub/models--Qwen--Qwen2.5-1.5B-Instruct/snapshots/989aa7980e4cf806f80c7fef2b1adb7bc71aa306 ++trainer.limit_val_batches=1 ++trainer.val_check_interval=1000 ++model.scale_loss_by=non_sil_t ++model.scale_loss_mask=10 ++model.val_acc_tolerance=480 data.validation_ds.seed=

dummy_nemo_duplex/qwen_1b/run_0/exp_config.yaml

@@ -0,0 +1,226 @@
+model:
+  pretrained_llm: /export/fs06/ahussei6/nvidia/cache/HFCACHE/hub/models--Qwen--Qwen2.5-1.5B-Instruct/snapshots/989aa7980e4cf806f80c7fef2b1adb7bc71aa306
+  pretrained_audio_codec: /export/fs06/ahussei6/nvidia/pretrained_models/nano_codec/Low_Frame-rate_Speech_Codec++.nemo
+  pretrained_asr: /export/fs06/ahussei6/nvidia/pretrained_models/asr/stt_en_fastconformer_hybrid_large_streaming_multi_v1.20.0/stt_en_fastconformer_hybrid_large_streaming_multi.nemo
+  scoring_asr: stt_en_fastconformer_transducer_large
+  pretrained_weights: true
+  audio_loss_weight: 20
+  text_loss_weight: 3
+  freeze_params:
+  - ^audio_codec\..+$
+  - ^speech_generation\.speaker_encoder\..+$
+  prevent_freeze_params: []
+  duplex_user_channel_weight: 1.0
+  hidden_pooling_factor: 2
+  audio_save_path: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/pred_audios
+  perception:
+    target: nemo.collections.multimodal.speech_llm.modules.perception_modules.AudioPerceptionModule
+    use_multi_layer_feat: false
+    add_sep: true
+    is_canary: true
+    preprocessor:
+      normalize: NA
+    encoder:
+      self_attention_model: rel_pos
+      att_context_size:
+      - 70
+      - 0
+      conv_context_size: causal
+      conv_norm_type: layer_norm
+    modality_adapter:
+      _target_: nemo.collections.asr.modules.ConformerEncoder
+      feat_in: 512
+      feat_out: -1
+      n_layers: 2
+      d_model: 512
+      subsampling: dw_striding
+      subsampling_factor: 1
+      subsampling_conv_channels: 256
+      causal_downsampling: true
+      ff_expansion_factor: 4
+      self_attention_model: rel_pos
+      n_heads: 8
+      att_context_size:
+      - 70
+      - 0
+      att_context_style: chunked_limited
+      xscaling: true
+      untie_biases: true
+      pos_emb_max_len: 5000
+      conv_kernel_size: 9
+      conv_norm_type: layer_norm
+      conv_context_size: causal
+      dropout: 0
+      dropout_pre_encoder: 0
+      dropout_emb: 0.0
+      dropout_att: 0
+    spec_augment:
+      _target_: nemo.collections.asr.modules.SpectrogramAugmentation
+      freq_masks: 2
+      time_masks: 10
+      freq_width: 27
+      time_width: 0.05
+  speech_decoder:
+    n_layers: 12
+    d_model: 768
+    d_ffn: 3072
+    sa_n_heads: 12
+    kernel_size: 3
+    p_dropout: 0.1
+    p_dropout_out: 0.0
+    has_xattn: false
+    xa_d_memory: 768
+    xa_n_heads: 12
+    is_causal: true
+    apply_norm_to_cond: true
+    apply_norm_out: true
+    max_length_causal_mask: 5000
+    cond_on_prev_audio_tokens: true
+    detach_input: false
+    use_learnable_pos_emb: true
+    cond_on_modality_adapter_emb: false
+    cond_on_char_embedding: true
+    cond_on_llm_latent: false
+    use_speaker_encoder: true
+    cond_on_asr_emb: false
+    cond_on_text_tokens: false
+    cfg_scale: 2.5
+    cfg_unconditional_prob: 0.2
+  optimizer:
+    _target_: torch.optim.AdamW
+    lr: 0.0001
+    betas:
+    - 0.9
+    - 0.98
+    weight_decay: 0
+    foreach: true
+  lr_scheduler:
+    _target_: nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing
+    warmup_steps: 4000
+    min_lr: 1.0e-06
+    max_steps: 100000
+  pretrained_tts_from_s2s: /export/fs06/ahussei6/nvidia/pretrained_models/magpie_tts/tts-pretraining_qwnen_2.5_81007_steps.ckpt
+  mask_sequence_loss: true
+  custom_codebook_size: 2045
+  custom_speech_bos_id: 2019
+  custom_speech_eos_id: 2020
+  custom_speech_delay_id: 2018
+  scale_loss_by: non_sil_t
+  scale_loss_mask: 10
+  val_acc_tolerance: 480
+trainer:
+  devices: -1
+  accelerator: gpu
+  num_nodes: 1
+  precision: bf16-true
+  logger: false
+  enable_checkpointing: false
+  use_distributed_sampler: false
+  max_steps: 100000
+  val_check_interval: 1000
+  limit_train_batches: 1000
+  limit_val_batches: 1
+  log_every_n_steps: 20
+  num_sanity_val_steps: 0
+  gradient_clip_val: 1.0
+  accumulate_grad_batches: 1
+  strategy:
+    _target_: lightning.pytorch.strategies.DDPStrategy
+    gradient_as_bucket_view: true
+    find_unused_parameters: true
+data:
+  frame_length: 0.08
+  source_sample_rate: 16000
+  target_sample_rate: 22050
+  input_roles:
+  - user
+  - User
+  output_roles:
+  - agent
+  - Assistant
+  - assistant
+  - Agent
+  add_delay_token: false
+  train_ds:
+    sample_rate: 22050
+    multi_config: true
+    shuffle: true
+    sampler_fusion: randomized_round_robin
+    seed: ''
+    shard_seed: trng
+    sampler_weights:
+      cv_en: 1.0
+    cv_en:
+      input_cfg: /export/fs06/ahussei6/nvidia/github/NeMo/examples/speechlm2/conf/data/en.yaml
+      bucket_buffer_size: 10000
+      shuffle_buffer_size: 10000
+      num_workers: 8
+      pin_memory: true
+      max_duration: 60
+      min_duration: 1
+      num_buckets: 30
+      bucket_duration_bins:
+      - 5.33
+      - 6.27
+      - 6.96
+      - 7.54
+      - 8.07
+      - 8.55
+      - 8.99
+      - 9.39
+      - 9.76
+      - 10.14
+      - 10.54
+      - 10.99
+      - 11.49
+      - 12.04
+      - 12.64
+      - 13.28
+      - 13.96
+      - 14.69
+      - 15.48
+      - 16.32
+      - 17.25
+      - 18.28
+      - 19.39
+      - 20.65
+      - 22.14
+      - 24.01
+      - 26.34
+      - 29.21
+      - 32.79
+      batch_duration: 400
+      use_bucketing: true
+      seed: trng
+  validation_ds:
+    datasets:
+      en_dev:
+        shar_path: /export/fs06/ahussei6/nvidia/github/debug
+    sample_rate: 22050
+    batch_size: 64
+    seed: ''
+    shard_seed: randomized
+exp_manager:
+  exp_dir: null
+  explicit_log_dir: /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b
+  name: qwen_1b
+  create_tensorboard_logger: false
+  create_checkpoint_callback: true
+  use_datetime_version: true
+  max_time_per_run: 00:03:50:00
+  resume_from_checkpoint: null
+  resume_if_exists: true
+  resume_ignore_no_checkpoint: true
+  create_wandb_logger: false
+  wandb_logger_kwargs:
+    name: development-run
+    project: salm_s2s_speech_decoder_v2_char_tokens
+    resume: true
+  checkpoint_callback_params:
+    filename: '{step}'
+    monitor: val_asr_bleu
+    mode: max
+    every_n_train_steps: null
+    every_n_epochs: 1
+    save_top_k: 3
+    always_save_nemo: false

dummy_nemo_duplex/qwen_1b/run_0/lightning_logs.txt

@@ -0,0 +1,19 @@
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
+
+  | Name               | Type                       | Params | Mode 
+--------------------------------------------------------------------------
+0 | audio_codec        | AudioCodecModel            | 62.2 M | eval 
+1 | llm                | Qwen2Model                 | 1.3 B  | train
+2 | lm_head            | Linear                     | 233 M  | train
+3 | embed_tokens       | Embedding                  | 233 M  | train
+4 | perception         | AudioPerceptionModule      | 122 M  | train
+5 | speech_generation  | TransformerARSpeechDecoder | 288 M  | train
+6 | embed_audio_tokens | ModuleList                 | 40.9 M | train
+7 | audio_head         | Linear                     | 40.9 M | train
+--------------------------------------------------------------------------
+2.0 B     Trainable params
+87.5 M    Non-trainable params
+2.1 B     Total params
+8,397.045 Total estimated model params size (MB)
+1141      Modules in train mode
+2018      Modules in eval mode

dummy_nemo_duplex/qwen_1b/run_0/nemo_error_log.txt

@@ -0,0 +1,218 @@
+[NeMo W 2026-04-26 23:14:55 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo W 2026-04-26 23:14:59 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/lightning/fabric/plugins/environments/slurm.py:204: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /export/fs06/ahussei6/nvidia/github/NeMo/examples/sp ...
+    
+[NeMo W 2026-04-26 23:14:59 exp_manager:1079] Exp_manager is logging to /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b, but it already exists.
+[NeMo W 2026-04-26 23:14:59 exp_manager:997] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :/export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/checkpoints. Training from scratch.
+[NeMo W 2026-04-26 23:14:59 exp_manager:1388] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 100000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo W 2026-04-26 23:15:02 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    dataset:
+      dataset_type: tarred_vocoder
+      dataset_args:
+        dataset_meta:
+          lindy:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/tarred_audio/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/tarred_audio/audio_{0..3}.tar
+          emma:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/audio_{0..3}.tar
+          libritts_r:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/tarred_audio/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/tarred_audio/audio_{0..48}.tar
+        sample_rate: 22050
+        n_samples: 24696
+        min_duration: 0.4
+        max_duration: null
+        audio_augmentator_config:
+          impulse:
+            prob: 0.5
+            rng: 42
+            manifest_path: /lustre/fsw/portfolios/convai/projects/convai_convaird_nemo-speech/data/ROOM_Response_and_Noise_Database/processed/rir.json
+            normalize_impulse: false
+            shift_impulse: true
+          codec:
+            prob: 0.5
+            rng: 42
+            format_params:
+            - format: mp3
+              prob: 0.5
+              bit_rates:
+              - 16000
+              - 32000
+              - 64000
+              - 128000
+            - format: ogg
+              prob: 0.075
+              encoder: vorbis
+              bit_rates:
+              - 32000
+              - 48000
+              - 64000
+            - format: ogg
+              prob: 0.375
+              encoder: opus
+              bit_rates:
+              - 8000
+              - 16000
+              - 32000
+              - 64000
+              - 128000
+            - format: wav
+              prob: 0.025
+              encoder: pcm_alaw
+              bit_rates:
+              - 64000
+            - format: wav
+              prob: 0.025
+              encoder: pcm_mulaw
+              bit_rates:
+              - 64000
+        shard_strategy: replicate
+        sample_type: weighted_random
+        sample_args:
+          batch_size: 32
+          steps_per_epoch: 233
+          dataset_weights:
+          - 0.2
+          - 0.2
+          - 0.6
+        shuffle_n: 10000
+    dataloader_params:
+      batch_size: 32
+      drop_last: true
+      num_workers: 4
+    
+[NeMo W 2026-04-26 23:15:02 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    dataset:
+      dataset_type: vocoder
+      dataset_args:
+        sample_rate: 22050
+        n_samples: null
+        min_duration: null
+        max_duration: null
+        trunc_duration: 10.0
+        dataset_meta:
+          lindy:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/audio_22khz
+          emma:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/audio_22khz
+          libritts_r:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/audio_22khz
+    dataloader_params:
+      batch_size: 4
+      num_workers: 2
+    
+[NeMo W 2026-04-26 23:15:19 duplex_s2s_speech_decoder_model:148] Tokenizer does not have a `bos_token`. Setting it to '<|im_start|>'.
+[NeMo W 2026-04-26 23:15:36 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath:
+    - - /raid/local//bucket1/tarred_audio_manifest.json
+    - - /raid/local//bucket2/tarred_audio_manifest.json
+    - - /raid/local//bucket3/tarred_audio_manifest.json
+    - - /raid/local//bucket4/tarred_audio_manifest.json
+    - - /raid/local//bucket5/tarred_audio_manifest.json
+    - - /raid/local//bucket6/tarred_audio_manifest.json
+    - - /raid/local//bucket7/tarred_audio_manifest.json
+    - - /raid/local//bucket8/tarred_audio_manifest.json
+    sample_rate: 16000
+    batch_size: 1
+    shuffle: true
+    num_workers: 4
+    pin_memory: true
+    use_start_end_token: false
+    trim_silence: false
+    max_duration: 25
+    min_duration: 0.1
+    is_tarred: true
+    tarred_audio_filepaths:
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket1/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket2/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket3/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket4/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket5/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket6/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket7/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket8/audio__OP_0..8191_CL_.tar
+    shuffle_n: 2048
+    bucketing_strategy: fully_randomized
+    bucketing_batch_size:
+    - 72
+    - 64
+    - 56
+    - 48
+    - 40
+    - 32
+    - 24
+    - 16
+    
+[NeMo W 2026-04-26 23:15:36 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath:
+    - /manifests/librispeech/librivox-dev-other.json
+    - /manifests/librispeech/librivox-dev-clean.json
+    - /manifests/librispeech/librivox-test-other.json
+    - /manifests/librispeech/librivox-test-clean.json
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    num_workers: 8
+    pin_memory: true
+    use_start_end_token: false
+    
+[NeMo W 2026-04-26 23:15:36 modelPT:202] Please call the ModelPT.setup_test_data() or ModelPT.setup_multiple_test_data() method and provide a valid configuration file to setup the test data loader(s).
+    Test config : 
+    manifest_filepath:
+    - /manifests/librispeech/librivox-dev-other.json
+    - /manifests/librispeech/librivox-dev-clean.json
+    - /manifests/librispeech/librivox-test-other.json
+    - /manifests/librispeech/librivox-test-clean.json
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    num_workers: 8
+    pin_memory: true
+    use_start_end_token: false
+    
+[NeMo W 2026-04-26 23:15:39 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/train.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 64
+    shuffle: true
+    is_tarred: false
+    tarred_audio_filepaths: null
+    tarred_shard_strategy: scatter
+    augmentor:
+      noise:
+        manifest_path: /manifests/noise/rir_noise_manifest.json
+        prob: 0.5
+        min_snr_db: 0
+        max_snr_db: 15
+      speed:
+        prob: 0.5
+        sr: 16000
+        resample_type: kaiser_fast
+        min_speed_rate: 0.95
+        max_speed_rate: 1.05
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo W 2026-04-26 23:15:39 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/dev.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 128
+    shuffle: false
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo W 2026-04-26 23:17:10 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/lightning/fabric/plugins/environments/slurm.py:204: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /export/fs06/ahussei6/nvidia/github/NeMo/examples/sp ...
+    

dummy_nemo_duplex/qwen_1b/run_0/nemo_log_globalrank-0_localrank-0.txt

@@ -0,0 +1,247 @@
+[NeMo W 2026-04-26 23:14:55 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo W 2026-04-26 23:14:59 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/lightning/fabric/plugins/environments/slurm.py:204: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /export/fs06/ahussei6/nvidia/github/NeMo/examples/sp ...
+    
+[NeMo I 2026-04-26 23:14:59 exp_manager:574] ExpManager schema
+[NeMo I 2026-04-26 23:14:59 exp_manager:575] {'explicit_log_dir': None, 'exp_dir': None, 'name': None, 'version': None, 'use_datetime_version': True, 'resume_if_exists': False, 'resume_past_end': False, 'resume_ignore_no_checkpoint': False, 'resume_from_checkpoint': None, 'create_tensorboard_logger': True, 'summary_writer_kwargs': None, 'create_wandb_logger': False, 'wandb_logger_kwargs': None, 'create_mlflow_logger': False, 'mlflow_logger_kwargs': {'experiment_name': None, 'tracking_uri': None, 'tags': None, 'save_dir': './mlruns', 'prefix': '', 'artifact_location': None, 'run_id': None, 'log_model': False}, 'create_dllogger_logger': False, 'dllogger_logger_kwargs': {'verbose': False, 'stdout': False, 'json_file': './dllogger.json'}, 'create_clearml_logger': False, 'clearml_logger_kwargs': {'project': None, 'task': None, 'connect_pytorch': False, 'model_name': None, 'tags': None, 'log_model': False, 'log_cfg': False, 'log_metrics': False}, 'create_neptune_logger': False, 'neptune_logger_kwargs': None, 'create_checkpoint_callback': True, 'checkpoint_callback_params': {'filepath': None, 'dirpath': None, 'filename': None, 'monitor': 'val_loss', 'verbose': True, 'save_last': True, 'save_top_k': 3, 'save_weights_only': False, 'mode': 'min', 'auto_insert_metric_name': True, 'every_n_epochs': 1, 'every_n_train_steps': None, 'train_time_interval': None, 'prefix': None, 'postfix': '.nemo', 'save_best_model': False, 'always_save_nemo': False, 'save_nemo_on_train_end': True, 'model_parallel_size': None, 'save_on_train_epoch_end': False, 'async_save': False, 'save_last_n_optim_states': -1, 'multistorageclient_enabled': False}, 'create_early_stopping_callback': False, 'early_stopping_callback_params': {'monitor': 'val_loss', 'mode': 'min', 'min_delta': 0.001, 'patience': 10, 'verbose': True, 'strict': True, 'check_finite': True, 'stopping_threshold': None, 'divergence_threshold': None, 'check_on_train_epoch_end': None, 'log_rank_zero_only': False}, 'create_preemption_callback': True, 'files_to_copy': None, 'log_step_timing': True, 'log_delta_step_timing': False, 'step_timing_kwargs': {'reduction': 'mean', 'sync_cuda': False, 'buffer_size': 1}, 'log_local_rank_0_only': False, 'log_global_rank_0_only': False, 'disable_validation_on_resume': True, 'ema': {'enable': False, 'decay': 0.999, 'cpu_offload': False, 'validate_original_weights': False, 'every_n_steps': 1}, 'max_time_per_run': None, 'seconds_to_sleep': 5.0, 'create_straggler_detection_callback': False, 'straggler_detection_params': {'report_time_interval': 300.0, 'calc_relative_gpu_perf': True, 'calc_individual_gpu_perf': True, 'num_gpu_perf_scores_to_log': 5, 'gpu_relative_perf_threshold': 0.7, 'gpu_individual_perf_threshold': 0.7, 'stop_if_detected': False}, 'create_fault_tolerance_callback': False, 'fault_tolerance': {'workload_check_interval': 5.0, 'initial_rank_heartbeat_timeout': 3600.0, 'rank_heartbeat_timeout': 2700.0, 'calculate_timeouts': True, 'safety_factor': 5.0, 'rank_termination_signal': <Signals.SIGKILL: 9>, 'log_level': 'INFO', 'max_rank_restarts': 0, 'max_subsequent_job_failures': 0, 'additional_ft_launcher_args': '', 'simulated_fault': None}, 'log_tflops_per_sec_per_gpu': True}
+[NeMo W 2026-04-26 23:14:59 exp_manager:1079] Exp_manager is logging to /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b, but it already exists.
+[NeMo W 2026-04-26 23:14:59 exp_manager:997] There were no checkpoints found in checkpoint_dir or no checkpoint folder at checkpoint_dir :/export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b/checkpoints. Training from scratch.
+[NeMo I 2026-04-26 23:14:59 exp_manager:635] Experiments will be logged at /export/fs06/ahussei6/nvidia/ara_duplex_test/qwen_1b
+[NeMo W 2026-04-26 23:14:59 exp_manager:1388] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 100000. Please ensure that max_steps will run for at least 1 epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-26 23:14:59 exp_manager:780] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo W 2026-04-26 23:15:02 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    dataset:
+      dataset_type: tarred_vocoder
+      dataset_args:
+        dataset_meta:
+          lindy:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/tarred_audio/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/tarred_audio/audio_{0..3}.tar
+          emma:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/audio_{0..3}.tar
+          libritts_r:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/tarred_audio/train_manifest.json
+            tar_filepath: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/tarred_audio/audio_{0..48}.tar
+        sample_rate: 22050
+        n_samples: 24696
+        min_duration: 0.4
+        max_duration: null
+        audio_augmentator_config:
+          impulse:
+            prob: 0.5
+            rng: 42
+            manifest_path: /lustre/fsw/portfolios/convai/projects/convai_convaird_nemo-speech/data/ROOM_Response_and_Noise_Database/processed/rir.json
+            normalize_impulse: false
+            shift_impulse: true
+          codec:
+            prob: 0.5
+            rng: 42
+            format_params:
+            - format: mp3
+              prob: 0.5
+              bit_rates:
+              - 16000
+              - 32000
+              - 64000
+              - 128000
+            - format: ogg
+              prob: 0.075
+              encoder: vorbis
+              bit_rates:
+              - 32000
+              - 48000
+              - 64000
+            - format: ogg
+              prob: 0.375
+              encoder: opus
+              bit_rates:
+              - 8000
+              - 16000
+              - 32000
+              - 64000
+              - 128000
+            - format: wav
+              prob: 0.025
+              encoder: pcm_alaw
+              bit_rates:
+              - 64000
+            - format: wav
+              prob: 0.025
+              encoder: pcm_mulaw
+              bit_rates:
+              - 64000
+        shard_strategy: replicate
+        sample_type: weighted_random
+        sample_args:
+          batch_size: 32
+          steps_per_epoch: 233
+          dataset_weights:
+          - 0.2
+          - 0.2
+          - 0.6
+        shuffle_n: 10000
+    dataloader_params:
+      batch_size: 32
+      drop_last: true
+      num_workers: 4
+    
+[NeMo W 2026-04-26 23:15:02 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    dataset:
+      dataset_type: vocoder
+      dataset_args:
+        sample_rate: 22050
+        n_samples: null
+        min_duration: null
+        max_duration: null
+        trunc_duration: 10.0
+        dataset_meta:
+          lindy:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/lindy/22khz/audio_22khz
+          emma:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/Blackwell-Demo/emma_audio_tar/tarred_audio_22khz/audio_22khz
+          libritts_r:
+            manifest_path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/val_manifest.json
+            audio_dir: /lustre/fsw/portfolios/llmservice/projects/llmservice_nemo_speechlm/data/TTS/libritts_r/22khz/audio_22khz
+    dataloader_params:
+      batch_size: 4
+      num_workers: 2
+    
+[NeMo I 2026-04-26 23:15:02 audio_codec:101] Vector quantizer does not support commit loss.
+[NeMo I 2026-04-26 23:15:14 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:15:18 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:15:18 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:15:18 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:15:18 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:15:18 features:305] PADDING: 1
+[NeMo I 2026-04-26 23:15:18 save_restore_connector:283] Model AudioCodecModel was successfully restored from /export/fs06/ahussei6/nvidia/pretrained_models/nano_codec/Low_Frame-rate_Speech_Codec++.nemo.
+[NeMo W 2026-04-26 23:15:19 duplex_s2s_speech_decoder_model:148] Tokenizer does not have a `bos_token`. Setting it to '<|im_start|>'.
+[NeMo I 2026-04-26 23:15:35 mixins:181] Tokenizer SentencePieceTokenizer initialized with 1024 tokens
+[NeMo W 2026-04-26 23:15:36 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath:
+    - - /raid/local//bucket1/tarred_audio_manifest.json
+    - - /raid/local//bucket2/tarred_audio_manifest.json
+    - - /raid/local//bucket3/tarred_audio_manifest.json
+    - - /raid/local//bucket4/tarred_audio_manifest.json
+    - - /raid/local//bucket5/tarred_audio_manifest.json
+    - - /raid/local//bucket6/tarred_audio_manifest.json
+    - - /raid/local//bucket7/tarred_audio_manifest.json
+    - - /raid/local//bucket8/tarred_audio_manifest.json
+    sample_rate: 16000
+    batch_size: 1
+    shuffle: true
+    num_workers: 4
+    pin_memory: true
+    use_start_end_token: false
+    trim_silence: false
+    max_duration: 25
+    min_duration: 0.1
+    is_tarred: true
+    tarred_audio_filepaths:
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket1/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket2/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket3/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket4/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket5/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket6/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket7/audio__OP_0..8191_CL_.tar
+    - - /data2/nemo_asr/nemo_asr_set_3.0//bucket8/audio__OP_0..8191_CL_.tar
+    shuffle_n: 2048
+    bucketing_strategy: fully_randomized
+    bucketing_batch_size:
+    - 72
+    - 64
+    - 56
+    - 48
+    - 40
+    - 32
+    - 24
+    - 16
+    
+[NeMo W 2026-04-26 23:15:36 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath:
+    - /manifests/librispeech/librivox-dev-other.json
+    - /manifests/librispeech/librivox-dev-clean.json
+    - /manifests/librispeech/librivox-test-other.json
+    - /manifests/librispeech/librivox-test-clean.json
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    num_workers: 8
+    pin_memory: true
+    use_start_end_token: false
+    
+[NeMo W 2026-04-26 23:15:36 modelPT:202] Please call the ModelPT.setup_test_data() or ModelPT.setup_multiple_test_data() method and provide a valid configuration file to setup the test data loader(s).
+    Test config : 
+    manifest_filepath:
+    - /manifests/librispeech/librivox-dev-other.json
+    - /manifests/librispeech/librivox-dev-clean.json
+    - /manifests/librispeech/librivox-test-other.json
+    - /manifests/librispeech/librivox-test-clean.json
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    num_workers: 8
+    pin_memory: true
+    use_start_end_token: false
+    
+[NeMo I 2026-04-26 23:15:36 features:305] PADDING: 0
+[NeMo I 2026-04-26 23:15:36 rnnt_models:226] Using RNNT Loss : warprnnt_numba
+    Loss warprnnt_numba_kwargs: {'fastemit_lambda': 0.005, 'clamp': -1.0}
+[NeMo I 2026-04-26 23:15:36 rnnt_models:226] Using RNNT Loss : warprnnt_numba
+    Loss warprnnt_numba_kwargs: {'fastemit_lambda': 0.005, 'clamp': -1.0}
+[NeMo I 2026-04-26 23:15:37 rnnt_models:226] Using RNNT Loss : warprnnt_numba
+    Loss warprnnt_numba_kwargs: {'fastemit_lambda': 0.005, 'clamp': -1.0}
+[NeMo I 2026-04-26 23:15:38 save_restore_connector:283] Model EncDecHybridRNNTCTCBPEModel was successfully restored from /export/fs06/ahussei6/nvidia/pretrained_models/asr/stt_en_fastconformer_hybrid_large_streaming_multi_v1.20.0/stt_en_fastconformer_hybrid_large_streaming_multi.nemo.
+[NeMo I 2026-04-26 23:15:38 features:305] PADDING: 0
+[NeMo I 2026-04-26 23:15:38 cloud:58] Found existing object /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo.
+[NeMo I 2026-04-26 23:15:38 cloud:64] Re-using file from: /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo
+[NeMo I 2026-04-26 23:15:38 common:827] Instantiating model from pre-trained checkpoint
+[NeMo W 2026-04-26 23:15:39 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/train.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 64
+    shuffle: true
+    is_tarred: false
+    tarred_audio_filepaths: null
+    tarred_shard_strategy: scatter
+    augmentor:
+      noise:
+        manifest_path: /manifests/noise/rir_noise_manifest.json
+        prob: 0.5
+        min_snr_db: 0
+        max_snr_db: 15
+      speed:
+        prob: 0.5
+        sr: 16000
+        resample_type: kaiser_fast
+        min_speed_rate: 0.95
+        max_speed_rate: 1.05
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo W 2026-04-26 23:15:39 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/dev.json
+    sample_rate: 16000
+    labels: null
+    batch_size: 128
+    shuffle: false
+    num_workers: 15
+    pin_memory: true
+    
+[NeMo I 2026-04-26 23:15:39 features:305] PADDING: 16
+[NeMo I 2026-04-26 23:15:39 save_restore_connector:283] Model EncDecSpeakerLabelModel was successfully restored from /export/fs06/ahussei6/nvidia/cache/HFCACHE/titanet-l/11ba0924fdf87c049e339adbf6899d48/titanet-l.nemo.
+[NeMo I 2026-04-26 23:17:09 pretrained:110]  | > 237 / 237 layers are restored.
+[NeMo W 2026-04-26 23:17:10 nemo_logging:361] /home/ahussein/miniconda3/envs/nemo/lib/python3.10/site-packages/lightning/fabric/plugins/environments/slurm.py:204: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /export/fs06/ahussei6/nvidia/github/NeMo/examples/sp ...
+    
+[NeMo I 2026-04-26 23:17:10 optim_setup:136] Parameters | trainable=2011762688 (95.83%) | total=2099261314