Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,57 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/2026-04-09_09-06-28/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/checkpoints/Speech_To_Text_Finetuning.nemo filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/run_0/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/run_1/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/run_10/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/run_2/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/run_3/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/run_4/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/run_5/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/run_6/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/run_7/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/run_8/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/run_9/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Finetuning/run_9/nemo_log_globalrank-0_localrank-0.txt filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Phase2/2026-04-09_09-13-43/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Phase2/2026-04-09_09-21-07/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Phase2/2026-04-09_09-32-14/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Phase2/2026-04-09_09-33-32/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Phase2/2026-04-09_09-34-13/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Phase2/2026-04-09_09-52-34/checkpoints/Speech_To_Text_Phase2.nemo filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Phase2/2026-04-09_09-52-34/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Phase2/2026-04-09_13-04-20/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Phase2/2026-04-09_13-28-11/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Phase2/2026-04-09_13-29-07/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Phase2/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Phase2/run_0/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Phase2/run_1/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/Speech_To_Text_Phase2/run_2/git-info.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260404_122123-2026-04-04_12-20-06/run-2026-04-04_12-20-06.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260404_122542-2026-04-04_12-24-26/run-2026-04-04_12-24-26.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260404_123043-2026-04-04_12-29-27/run-2026-04-04_12-29-27.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260404_124317-2026-04-04_12-42-01/run-2026-04-04_12-42-01.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260404_125341-2026-04-04_12-49-00/run-2026-04-04_12-49-00.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260408_114122-0jtbmf55/run-0jtbmf55.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260408_120315-fl4jp2jy/run-fl4jp2jy.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260408_142513-1q7swtxr/run-1q7swtxr.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260408_152645-22lidhir/run-22lidhir.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260408_171021-jszyom5l/run-jszyom5l.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260408_171313-xy9wfyvz/run-xy9wfyvz.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260408_181741-0xkhtecb/run-0xkhtecb.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260408_185350-vp5mr58h/run-vp5mr58h.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260408_202234-pyv7fhv7/files/output.log filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260408_202234-pyv7fhv7/run-pyv7fhv7.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260409_083758-ick94apk/run-ick94apk.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260409_090414-6q8ded8v/run-6q8ded8v.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260409_090749-2026-04-09_09-06-28/run-2026-04-09_09-06-28.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260409_092141-2026-04-09_09-21-07/run-2026-04-09_09-21-07.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260409_093449-2026-04-09_09-34-13/run-2026-04-09_09-34-13.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260409_095308-2026-04-09_09-52-34/run-2026-04-09_09-52-34.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260409_130455-2026-04-09_13-04-20/run-2026-04-09_13-04-20.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260409_130627-dyzduaki/run-dyzduaki.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260409_131150-50lfjpsn/run-50lfjpsn.wandb filter=lfs diff=lfs merge=lfs -text
+nemo_experiments/wandb/run-20260409_133028-2026-04-09_13-29-07/run-2026-04-09_13-29-07.wandb filter=lfs diff=lfs merge=lfs -text

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_11-18-03/cmd-args.log

@@ -0,0 +1 @@
+./speech_to_text_finetune.py --config-path=. --config-name=speech_to_text_finetune trainer.accumulate_grad_batches=16 trainer.devices=1 trainer.accelerator=gpu +trainer.limit_train_batches=null exp_manager.create_wandb_logger=True exp_manager.wandb_logger_kwargs.name=parakeet_v3_finetune_fixed exp_manager.wandb_logger_kwargs.project=parakeet-hebrew-asr

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_11-18-03/nemo_error_log.txt

@@ -0,0 +1,45 @@
+[NeMo W 2026-04-04 11:18:02 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo W 2026-04-04 11:18:03 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000000. Please ensure that max_steps will run for at least None epochs to ensure that checkpointing will not error out.
+[NeMo W 2026-04-04 11:23:01 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    use_lhotse: true
+    skip_missing_manifest_entries: true
+    input_cfg: null
+    tarred_audio_filepaths: null
+    manifest_filepath: null
+    sample_rate: 16000
+    shuffle: true
+    num_workers: 2
+    pin_memory: true
+    max_duration: 10.0
+    min_duration: 1.0
+    text_field: answer
+    batch_duration: null
+    max_tps: null
+    use_bucketing: true
+    bucket_duration_bins: null
+    bucket_batch_size: null
+    num_buckets: 30
+    bucket_buffer_size: 20000
+    shuffle_buffer_size: 10000
+    
+[NeMo W 2026-04-04 11:23:01 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    use_lhotse: true
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    max_duration: 40.0
+    min_duration: 0.1
+    num_workers: 2
+    pin_memory: true
+    text_field: answer
+    
+[NeMo W 2026-04-04 11:23:05 modelPT:300] You tried to register an artifact under config key=tokenizer.model_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 11:23:05 modelPT:300] You tried to register an artifact under config key=tokenizer.vocab_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 11:23:05 modelPT:300] You tried to register an artifact under config key=tokenizer.spe_tokenizer_vocab but an artifact for it has already been registered.
+[NeMo W 2026-04-04 11:23:09 speech_to_text_finetune:167] The vocabulary size of the new tokenizer differs from that of the loaded model. As a result, finetuning will proceed with the new vocabulary, and the decoder will be reinitialized.
+[NeMo W 2026-04-04 11:23:10 dataloader:826] The following configuration keys are ignored by Lhotse dataloader: allow_missing_data,is_tarred

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_11-42-27/cmd-args.log

@@ -0,0 +1 @@
+./speech_to_text_finetune.py --config-path=. --config-name=speech_to_text_finetune trainer.accumulate_grad_batches=16 trainer.devices=1 trainer.accelerator=gpu +trainer.limit_train_batches=null exp_manager.create_wandb_logger=True exp_manager.wandb_logger_kwargs.name=parakeet_v3_finetune_fixed exp_manager.wandb_logger_kwargs.project=parakeet-hebrew-asr

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_11-42-27/nemo_error_log.txt

@@ -0,0 +1,45 @@
+[NeMo W 2026-04-04 11:42:26 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo W 2026-04-04 11:42:27 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000000. Please ensure that max_steps will run for at least None epochs to ensure that checkpointing will not error out.
+[NeMo W 2026-04-04 11:46:53 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    use_lhotse: true
+    skip_missing_manifest_entries: true
+    input_cfg: null
+    tarred_audio_filepaths: null
+    manifest_filepath: null
+    sample_rate: 16000
+    shuffle: true
+    num_workers: 2
+    pin_memory: true
+    max_duration: 10.0
+    min_duration: 1.0
+    text_field: answer
+    batch_duration: null
+    max_tps: null
+    use_bucketing: true
+    bucket_duration_bins: null
+    bucket_batch_size: null
+    num_buckets: 30
+    bucket_buffer_size: 20000
+    shuffle_buffer_size: 10000
+    
+[NeMo W 2026-04-04 11:46:53 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    use_lhotse: true
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    max_duration: 40.0
+    min_duration: 0.1
+    num_workers: 2
+    pin_memory: true
+    text_field: answer
+    
+[NeMo W 2026-04-04 11:46:57 modelPT:300] You tried to register an artifact under config key=tokenizer.model_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 11:46:57 modelPT:300] You tried to register an artifact under config key=tokenizer.vocab_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 11:46:57 modelPT:300] You tried to register an artifact under config key=tokenizer.spe_tokenizer_vocab but an artifact for it has already been registered.
+[NeMo W 2026-04-04 11:47:01 speech_to_text_finetune:167] The vocabulary size of the new tokenizer differs from that of the loaded model. As a result, finetuning will proceed with the new vocabulary, and the decoder will be reinitialized.
+[NeMo W 2026-04-04 11:47:02 dataloader:826] The following configuration keys are ignored by Lhotse dataloader: allow_missing_data,is_tarred

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_11-48-34/cmd-args.log

@@ -0,0 +1 @@
+./speech_to_text_finetune.py --config-path=. --config-name=speech_to_text_finetune trainer.accumulate_grad_batches=16 trainer.devices=1 trainer.accelerator=gpu +trainer.limit_train_batches=null exp_manager.create_wandb_logger=True exp_manager.wandb_logger_kwargs.name=parakeet_v3_finetune_fixed exp_manager.wandb_logger_kwargs.project=parakeet-hebrew-asr

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_11-48-34/nemo_error_log.txt

@@ -0,0 +1,45 @@
+[NeMo W 2026-04-04 11:48:34 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo W 2026-04-04 11:48:34 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000000. Please ensure that max_steps will run for at least None epochs to ensure that checkpointing will not error out.
+[NeMo W 2026-04-04 11:49:38 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    use_lhotse: true
+    skip_missing_manifest_entries: true
+    input_cfg: null
+    tarred_audio_filepaths: null
+    manifest_filepath: null
+    sample_rate: 16000
+    shuffle: true
+    num_workers: 2
+    pin_memory: true
+    max_duration: 10.0
+    min_duration: 1.0
+    text_field: answer
+    batch_duration: null
+    max_tps: null
+    use_bucketing: true
+    bucket_duration_bins: null
+    bucket_batch_size: null
+    num_buckets: 30
+    bucket_buffer_size: 20000
+    shuffle_buffer_size: 10000
+    
+[NeMo W 2026-04-04 11:49:38 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    use_lhotse: true
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    max_duration: 40.0
+    min_duration: 0.1
+    num_workers: 2
+    pin_memory: true
+    text_field: answer
+    
+[NeMo W 2026-04-04 11:49:42 modelPT:300] You tried to register an artifact under config key=tokenizer.model_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 11:49:42 modelPT:300] You tried to register an artifact under config key=tokenizer.vocab_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 11:49:42 modelPT:300] You tried to register an artifact under config key=tokenizer.spe_tokenizer_vocab but an artifact for it has already been registered.
+[NeMo W 2026-04-04 11:49:46 speech_to_text_finetune:167] The vocabulary size of the new tokenizer differs from that of the loaded model. As a result, finetuning will proceed with the new vocabulary, and the decoder will be reinitialized.
+[NeMo W 2026-04-04 11:49:47 dataloader:826] The following configuration keys are ignored by Lhotse dataloader: allow_missing_data,is_tarred

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_11-54-05/cmd-args.log

@@ -0,0 +1 @@
+./speech_to_text_finetune.py --config-path=. --config-name=speech_to_text_finetune trainer.accumulate_grad_batches=16 trainer.devices=1 trainer.accelerator=gpu +trainer.limit_train_batches=null exp_manager.create_wandb_logger=True exp_manager.wandb_logger_kwargs.name=parakeet_v3_finetune_fixed exp_manager.wandb_logger_kwargs.project=parakeet-hebrew-asr

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_11-54-05/nemo_error_log.txt

@@ -0,0 +1,45 @@
+[NeMo W 2026-04-04 11:54:04 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo W 2026-04-04 11:54:05 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000000. Please ensure that max_steps will run for at least None epochs to ensure that checkpointing will not error out.
+[NeMo W 2026-04-04 11:55:09 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    use_lhotse: true
+    skip_missing_manifest_entries: true
+    input_cfg: null
+    tarred_audio_filepaths: null
+    manifest_filepath: null
+    sample_rate: 16000
+    shuffle: true
+    num_workers: 2
+    pin_memory: true
+    max_duration: 10.0
+    min_duration: 1.0
+    text_field: answer
+    batch_duration: null
+    max_tps: null
+    use_bucketing: true
+    bucket_duration_bins: null
+    bucket_batch_size: null
+    num_buckets: 30
+    bucket_buffer_size: 20000
+    shuffle_buffer_size: 10000
+    
+[NeMo W 2026-04-04 11:55:09 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    use_lhotse: true
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    max_duration: 40.0
+    min_duration: 0.1
+    num_workers: 2
+    pin_memory: true
+    text_field: answer
+    
+[NeMo W 2026-04-04 11:55:12 modelPT:300] You tried to register an artifact under config key=tokenizer.model_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 11:55:12 modelPT:300] You tried to register an artifact under config key=tokenizer.vocab_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 11:55:12 modelPT:300] You tried to register an artifact under config key=tokenizer.spe_tokenizer_vocab but an artifact for it has already been registered.
+[NeMo W 2026-04-04 11:55:16 speech_to_text_finetune:167] The vocabulary size of the new tokenizer differs from that of the loaded model. As a result, finetuning will proceed with the new vocabulary, and the decoder will be reinitialized.
+[NeMo W 2026-04-04 11:55:17 dataloader:826] The following configuration keys are ignored by Lhotse dataloader: is_tarred,allow_missing_data

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-12-11/cmd-args.log

@@ -0,0 +1 @@
+./speech_to_text_finetune.py --config-path=. --config-name=speech_to_text_finetune trainer.accumulate_grad_batches=16 trainer.devices=1 trainer.accelerator=gpu +trainer.limit_train_batches=null exp_manager.create_wandb_logger=True exp_manager.wandb_logger_kwargs.name=parakeet_v3_finetune_fixed exp_manager.wandb_logger_kwargs.project=parakeet-hebrew-asr

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-12-11/nemo_error_log.txt

@@ -0,0 +1,45 @@
+[NeMo W 2026-04-04 12:12:11 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo W 2026-04-04 12:12:11 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000000. Please ensure that max_steps will run for at least None epochs to ensure that checkpointing will not error out.
+[NeMo W 2026-04-04 12:16:38 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    use_lhotse: true
+    skip_missing_manifest_entries: true
+    input_cfg: null
+    tarred_audio_filepaths: null
+    manifest_filepath: null
+    sample_rate: 16000
+    shuffle: true
+    num_workers: 2
+    pin_memory: true
+    max_duration: 10.0
+    min_duration: 1.0
+    text_field: answer
+    batch_duration: null
+    max_tps: null
+    use_bucketing: true
+    bucket_duration_bins: null
+    bucket_batch_size: null
+    num_buckets: 30
+    bucket_buffer_size: 20000
+    shuffle_buffer_size: 10000
+    
+[NeMo W 2026-04-04 12:16:38 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    use_lhotse: true
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    max_duration: 40.0
+    min_duration: 0.1
+    num_workers: 2
+    pin_memory: true
+    text_field: answer
+    
+[NeMo W 2026-04-04 12:16:41 modelPT:300] You tried to register an artifact under config key=tokenizer.model_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 12:16:41 modelPT:300] You tried to register an artifact under config key=tokenizer.vocab_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 12:16:41 modelPT:300] You tried to register an artifact under config key=tokenizer.spe_tokenizer_vocab but an artifact for it has already been registered.
+[NeMo W 2026-04-04 12:16:46 speech_to_text_finetune:167] The vocabulary size of the new tokenizer differs from that of the loaded model. As a result, finetuning will proceed with the new vocabulary, and the decoder will be reinitialized.
+[NeMo W 2026-04-04 12:16:47 dataloader:826] The following configuration keys are ignored by Lhotse dataloader: is_tarred,allow_missing_data

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-20-06/cmd-args.log

@@ -0,0 +1 @@
+./speech_to_text_finetune.py --config-path=. --config-name=speech_to_text_finetune trainer.accumulate_grad_batches=16 trainer.devices=1 trainer.accelerator=gpu +trainer.limit_train_batches=null exp_manager.create_wandb_logger=True exp_manager.wandb_logger_kwargs.name=parakeet_v3_finetune_fixed exp_manager.wandb_logger_kwargs.project=parakeet-hebrew-asr

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-20-06/events.out.tfevents.1775305282.d12a7902a35c.540.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:274e9dfb7ac1204275503319612ed5da784fcf5001dd5ff78a1bbeaa47750342
+size 626485

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-20-06/lightning_logs.txt

@@ -0,0 +1,19 @@
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
+
+  | Name              | Type                              | Params | Mode 
+--------------------------------------------------------------------------------
+0 | preprocessor      | AudioToMelSpectrogramPreprocessor | 0      | train
+1 | encoder           | ConformerEncoder                  | 608 M  | train
+2 | spec_augmentation | SpectrogramAugmentation           | 0      | train
+3 | wer               | WER                               | 0      | train
+4 | joint             | RNNTJoint                         | 22.1 M | train
+5 | decoder           | RNNTDecoder                       | 27.5 M | train
+6 | loss              | RNNTLoss                          | 0      | train
+7 | spec_augment      | SpectrogramAugmentation           | 0      | train
+--------------------------------------------------------------------------------
+658 M     Trainable params
+0         Non-trainable params
+658 M     Total params
+2,633.960 Total estimated model params size (MB)
+708       Modules in train mode
+0         Modules in eval mode

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-20-06/nemo_error_log.txt

@@ -0,0 +1,48 @@
+[NeMo W 2026-04-04 12:20:06 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo W 2026-04-04 12:20:06 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000000. Please ensure that max_steps will run for at least None epochs to ensure that checkpointing will not error out.
+[NeMo W 2026-04-04 12:21:10 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    use_lhotse: true
+    skip_missing_manifest_entries: true
+    input_cfg: null
+    tarred_audio_filepaths: null
+    manifest_filepath: null
+    sample_rate: 16000
+    shuffle: true
+    num_workers: 2
+    pin_memory: true
+    max_duration: 10.0
+    min_duration: 1.0
+    text_field: answer
+    batch_duration: null
+    max_tps: null
+    use_bucketing: true
+    bucket_duration_bins: null
+    bucket_batch_size: null
+    num_buckets: 30
+    bucket_buffer_size: 20000
+    shuffle_buffer_size: 10000
+    
+[NeMo W 2026-04-04 12:21:10 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    use_lhotse: true
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    max_duration: 40.0
+    min_duration: 0.1
+    num_workers: 2
+    pin_memory: true
+    text_field: answer
+    
+[NeMo W 2026-04-04 12:21:14 modelPT:300] You tried to register an artifact under config key=tokenizer.model_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 12:21:14 modelPT:300] You tried to register an artifact under config key=tokenizer.vocab_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 12:21:14 modelPT:300] You tried to register an artifact under config key=tokenizer.spe_tokenizer_vocab but an artifact for it has already been registered.
+[NeMo W 2026-04-04 12:21:18 speech_to_text_finetune:167] The vocabulary size of the new tokenizer differs from that of the loaded model. As a result, finetuning will proceed with the new vocabulary, and the decoder will be reinitialized.
+[NeMo W 2026-04-04 12:21:19 dataloader:826] The following configuration keys are ignored by Lhotse dataloader: allow_missing_data,is_tarred
+[NeMo W 2026-04-04 12:21:21 audio_to_text_dataset:833] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'sample_rate': 16000, 'batch_size': 16, 'shuffle': False, 'use_start_end_token': False, 'num_workers': 8, 'pin_memory': True}
+[NeMo W 2026-04-04 12:21:27 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/lightning/pytorch/loops/fit_loop.py:298: The number of training batches (1) is smaller than the logging interval Trainer(log_every_n_steps=10). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.
+    

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-24-26/cmd-args.log

@@ -0,0 +1 @@
+./speech_to_text_finetune.py --config-path=. --config-name=speech_to_text_finetune trainer.accumulate_grad_batches=16 trainer.devices=1 trainer.accelerator=gpu +trainer.limit_train_batches=null exp_manager.create_wandb_logger=True exp_manager.wandb_logger_kwargs.name=parakeet_v3_finetune_fixed exp_manager.wandb_logger_kwargs.project=parakeet-hebrew-asr

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-24-26/events.out.tfevents.1775305542.d12a7902a35c.1101.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e78d7b83a7c4161e082ac67c8fc8d0d3081eaef37e5cd333c5017ed9926aac6
+size 626485

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-24-26/lightning_logs.txt

@@ -0,0 +1,19 @@
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
+
+  | Name              | Type                              | Params | Mode 
+--------------------------------------------------------------------------------
+0 | preprocessor      | AudioToMelSpectrogramPreprocessor | 0      | train
+1 | encoder           | ConformerEncoder                  | 608 M  | train
+2 | spec_augmentation | SpectrogramAugmentation           | 0      | train
+3 | wer               | WER                               | 0      | train
+4 | joint             | RNNTJoint                         | 22.1 M | train
+5 | decoder           | RNNTDecoder                       | 27.5 M | train
+6 | loss              | RNNTLoss                          | 0      | train
+7 | spec_augment      | SpectrogramAugmentation           | 0      | train
+--------------------------------------------------------------------------------
+658 M     Trainable params
+0         Non-trainable params
+658 M     Total params
+2,633.960 Total estimated model params size (MB)
+708       Modules in train mode
+0         Modules in eval mode

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-24-26/nemo_error_log.txt

@@ -0,0 +1,49 @@
+[NeMo W 2026-04-04 12:24:26 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo W 2026-04-04 12:24:26 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000000. Please ensure that max_steps will run for at least None epochs to ensure that checkpointing will not error out.
+[NeMo W 2026-04-04 12:25:30 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    use_lhotse: true
+    skip_missing_manifest_entries: true
+    input_cfg: null
+    tarred_audio_filepaths: null
+    manifest_filepath: null
+    sample_rate: 16000
+    shuffle: true
+    num_workers: 2
+    pin_memory: true
+    max_duration: 10.0
+    min_duration: 1.0
+    text_field: answer
+    batch_duration: null
+    max_tps: null
+    use_bucketing: true
+    bucket_duration_bins: null
+    bucket_batch_size: null
+    num_buckets: 30
+    bucket_buffer_size: 20000
+    shuffle_buffer_size: 10000
+    
+[NeMo W 2026-04-04 12:25:30 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    use_lhotse: true
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    max_duration: 40.0
+    min_duration: 0.1
+    num_workers: 2
+    pin_memory: true
+    text_field: answer
+    
+[NeMo W 2026-04-04 12:25:33 modelPT:300] You tried to register an artifact under config key=tokenizer.model_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 12:25:33 modelPT:300] You tried to register an artifact under config key=tokenizer.vocab_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 12:25:33 modelPT:300] You tried to register an artifact under config key=tokenizer.spe_tokenizer_vocab but an artifact for it has already been registered.
+[NeMo W 2026-04-04 12:25:38 speech_to_text_finetune:167] The vocabulary size of the new tokenizer differs from that of the loaded model. As a result, finetuning will proceed with the new vocabulary, and the decoder will be reinitialized.
+[NeMo W 2026-04-04 12:25:38 dataloader:826] The following configuration keys are ignored by Lhotse dataloader: is_tarred,allow_missing_data
+[NeMo W 2026-04-04 12:25:40 audio_to_text_dataset:833] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'sample_rate': 16000, 'batch_size': 16, 'shuffle': False, 'use_start_end_token': False, 'num_workers': 8, 'pin_memory': True}
+[NeMo W 2026-04-04 12:25:46 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/lightning/pytorch/loops/fit_loop.py:298: The number of training batches (1) is smaller than the logging interval Trainer(log_every_n_steps=10). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.
+    
+[NeMo W 2026-04-04 12:25:51 audio_preprocessing:85] AudioPreprocessor received an input signal of dtype torch.bfloat16, rather than torch.float32. In sweeps across multiple datasets, we have found that the preprocessor is not robust to low precision  mathematics. As such, it runs in float32. Your input will be cast to float32, but this is not necessarily enough to recovery full accuracy. For example, simply casting input_signal from torch.float32 to torch.bfloat16, then back to torch.float32 before running AudioPreprocessor causes drops in absolute WER of up to 0.1%. torch.bfloat16 simply does not have enough mantissa bits to represent enough values in the range [-1.0,+1.0] correctly.

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-27-59/cmd-args.log

@@ -0,0 +1 @@
+./speech_to_text_finetune.py --config-path=. --config-name=speech_to_text_finetune trainer.accumulate_grad_batches=16 trainer.devices=1 trainer.accelerator=gpu +trainer.limit_train_batches=null exp_manager.create_wandb_logger=True exp_manager.wandb_logger_kwargs.name=parakeet_v3_finetune_fixed exp_manager.wandb_logger_kwargs.project=parakeet-hebrew-asr

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-27-59/nemo_error_log.txt

@@ -0,0 +1,4 @@
+[NeMo W 2026-04-04 12:27:59 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo W 2026-04-04 12:28:00 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000000. Please ensure that max_steps will run for at least None epochs to ensure that checkpointing will not error out.

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-27-59/nemo_log_globalrank-0_localrank-0.txt

@@ -0,0 +1,106 @@
+[NeMo W 2026-04-04 12:27:59 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo I 2026-04-04 12:27:59 speech_to_text_finetune:198] Hydra config: name: Speech_To_Text_Finetuning
+    init_from_pretrained_model: nvidia/parakeet-tdt-0.6b-v3
+    model:
+      sample_rate: 16000
+      activation_checkpointing: true
+      train_ds:
+        shar_path: hf://buckets/Tyl3rDrden/IvritKnessetRecordingsData
+        use_lhotse: true
+        is_tarred: true
+        batch_size: 1
+        allow_missing_data: true
+        shuffle: false
+        max_duration: 30.1
+        num_workers: 8
+        pin_memory: true
+      validation_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        batch_size: 16
+        shuffle: false
+        use_start_end_token: false
+        num_workers: 8
+        pin_memory: true
+      test_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        batch_size: 16
+        shuffle: false
+        use_start_end_token: false
+        num_workers: 8
+        pin_memory: true
+      char_labels:
+        update_labels: false
+        labels: null
+      tokenizer:
+        update_tokenizer: true
+        dir: ./tokenizer_spe_bpe_v32768_pad_bos_eos
+        type: bpe
+      spec_augment:
+        _target_: nemo.collections.asr.modules.SpectrogramAugmentation
+        freq_masks: 2
+        time_masks: 10
+        freq_width: 27
+        time_width: 0.05
+      optim:
+        name: adamw
+        lr: 0.0001
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: CosineAnnealing
+          warmup_steps: 5000
+          warmup_ratio: null
+          min_lr: 5.0e-06
+    trainer:
+      devices: 1
+      num_nodes: 1
+      max_epochs: -1
+      max_steps: 1000000
+      val_check_interval: 2000
+      accelerator: gpu
+      strategy:
+        _target_: lightning.pytorch.strategies.DDPStrategy
+        gradient_as_bucket_view: true
+      accumulate_grad_batches: 16
+      gradient_clip_val: 0.0
+      precision: bf16
+      log_every_n_steps: 10
+      enable_progress_bar: true
+      num_sanity_val_steps: 0
+      check_val_every_n_epoch: null
+      sync_batchnorm: true
+      enable_checkpointing: false
+      logger: false
+      benchmark: false
+      limit_train_batches: null
+    exp_manager:
+      exp_dir: null
+      name: ${name}
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      checkpoint_callback_params:
+        monitor: val_wer
+        mode: min
+        save_top_k: 5
+        always_save_nemo: true
+      resume_if_exists: false
+      resume_ignore_no_checkpoint: false
+      create_wandb_logger: true
+      wandb_logger_kwargs:
+        name: parakeet_v3_finetune_fixed
+        project: parakeet-hebrew-asr
+    
+[NeMo I 2026-04-04 12:27:59 exp_manager:594] ExpManager schema
+[NeMo I 2026-04-04 12:27:59 exp_manager:595] {'explicit_log_dir': None, 'exp_dir': None, 'name': None, 'version': None, 'use_datetime_version': True, 'resume_if_exists': False, 'resume_past_end': False, 'resume_ignore_no_checkpoint': False, 'resume_from_checkpoint': None, 'create_tensorboard_logger': True, 'summary_writer_kwargs': None, 'create_wandb_logger': False, 'wandb_logger_kwargs': None, 'create_mlflow_logger': False, 'mlflow_logger_kwargs': {'experiment_name': None, 'run_name': None, 'tracking_uri': None, 'tags': None, 'save_dir': './mlruns', 'prefix': '', 'artifact_location': None, 'run_id': None, 'log_model': False}, 'create_dllogger_logger': False, 'dllogger_logger_kwargs': {'verbose': False, 'stdout': False, 'json_file': './dllogger.json'}, 'create_clearml_logger': False, 'clearml_logger_kwargs': {'project': None, 'task': None, 'connect_pytorch': False, 'model_name': None, 'tags': None, 'log_model': False, 'log_cfg': False, 'log_metrics': False}, 'create_neptune_logger': False, 'neptune_logger_kwargs': None, 'create_checkpoint_callback': True, 'checkpoint_callback_params': {'filepath': None, 'dirpath': None, 'filename': None, 'monitor': 'val_loss', 'verbose': True, 'save_last': True, 'save_top_k': 3, 'save_weights_only': False, 'mode': 'min', 'auto_insert_metric_name': True, 'every_n_epochs': 1, 'every_n_train_steps': None, 'train_time_interval': None, 'prefix': None, 'postfix': '.nemo', 'save_best_model': False, 'always_save_nemo': False, 'save_nemo_on_train_end': True, 'model_parallel_size': None, 'save_on_train_epoch_end': False, 'async_save': False, 'save_last_n_optim_states': -1}, 'create_early_stopping_callback': False, 'create_ipl_epoch_stopper_callback': False, 'early_stopping_callback_params': {'monitor': 'val_loss', 'mode': 'min', 'min_delta': 0.001, 'patience': 10, 'verbose': True, 'strict': True, 'check_finite': True, 'stopping_threshold': None, 'divergence_threshold': None, 'check_on_train_epoch_end': None, 'log_rank_zero_only': False}, 'ipl_epoch_stopper_callback_params': {'enable_stop': True, 'stop_every_n_epochs': 1}, 'create_preemption_callback': True, 'files_to_copy': None, 'log_step_timing': True, 'log_delta_step_timing': False, 'step_timing_kwargs': {'reduction': 'mean', 'sync_cuda': False, 'buffer_size': 1}, 'log_local_rank_0_only': False, 'log_global_rank_0_only': False, 'disable_validation_on_resume': True, 'ema': {'enable': False, 'decay': 0.999, 'cpu_offload': False, 'validate_original_weights': False, 'every_n_steps': 1}, 'max_time_per_run': None, 'seconds_to_sleep': 5.0, 'create_straggler_detection_callback': False, 'straggler_detection_params': {'report_time_interval': 300.0, 'calc_relative_gpu_perf': True, 'calc_individual_gpu_perf': True, 'num_gpu_perf_scores_to_log': 5, 'gpu_relative_perf_threshold': 0.7, 'gpu_individual_perf_threshold': 0.7, 'stop_if_detected': False}, 'create_fault_tolerance_callback': False, 'fault_tolerance': {'workload_check_interval': 5.0, 'initial_rank_heartbeat_timeout': 3600.0, 'rank_heartbeat_timeout': 2700.0, 'calculate_timeouts': True, 'safety_factor': 5.0, 'rank_termination_signal': <Signals.SIGKILL: 9>, 'log_level': 'INFO', 'max_rank_restarts': 0, 'max_subsequent_job_failures': 0, 'additional_ft_launcher_args': '', 'simulated_fault': None}, 'log_tflops_per_sec_per_gpu': True}
+[NeMo I 2026-04-04 12:27:59 exp_manager:655] Experiments will be logged at /workspace/nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-27-59
+[NeMo I 2026-04-04 12:28:00 exp_manager:1262] TensorboardLogger has been set up
+[NeMo I 2026-04-04 12:28:00 exp_manager:1277] WandBLogger has been set up
+[NeMo W 2026-04-04 12:28:00 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000000. Please ensure that max_steps will run for at least None epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-04 12:28:00 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-04 12:28:00 speech_to_text_finetune:106] Sleeping for at least 60 seconds to wait for model download to finish.

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-29-27/cmd-args.log

@@ -0,0 +1 @@
+./speech_to_text_finetune.py --config-path=. --config-name=speech_to_text_finetune trainer.accumulate_grad_batches=16 trainer.devices=1 trainer.accelerator=gpu +trainer.limit_train_batches=null exp_manager.create_wandb_logger=True exp_manager.wandb_logger_kwargs.name=parakeet_v3_finetune_fixed exp_manager.wandb_logger_kwargs.project=parakeet-hebrew-asr

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-29-27/events.out.tfevents.1775305842.d12a7902a35c.1768.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:139ab44e948c263b99dc14d04c326745db4f67c834e9a84e841f1e97593e3923
+size 626485

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-29-27/lightning_logs.txt

@@ -0,0 +1,19 @@
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
+
+  | Name              | Type                              | Params | Mode 
+--------------------------------------------------------------------------------
+0 | preprocessor      | AudioToMelSpectrogramPreprocessor | 0      | train
+1 | encoder           | ConformerEncoder                  | 608 M  | train
+2 | spec_augmentation | SpectrogramAugmentation           | 0      | train
+3 | wer               | WER                               | 0      | train
+4 | joint             | RNNTJoint                         | 22.1 M | train
+5 | decoder           | RNNTDecoder                       | 27.5 M | train
+6 | loss              | RNNTLoss                          | 0      | train
+7 | spec_augment      | SpectrogramAugmentation           | 0      | train
+--------------------------------------------------------------------------------
+658 M     Trainable params
+0         Non-trainable params
+658 M     Total params
+2,633.960 Total estimated model params size (MB)
+708       Modules in train mode
+0         Modules in eval mode

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-29-27/nemo_error_log.txt

@@ -0,0 +1,48 @@
+[NeMo W 2026-04-04 12:29:26 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo W 2026-04-04 12:29:27 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000000. Please ensure that max_steps will run for at least None epochs to ensure that checkpointing will not error out.
+[NeMo W 2026-04-04 12:30:31 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    use_lhotse: true
+    skip_missing_manifest_entries: true
+    input_cfg: null
+    tarred_audio_filepaths: null
+    manifest_filepath: null
+    sample_rate: 16000
+    shuffle: true
+    num_workers: 2
+    pin_memory: true
+    max_duration: 10.0
+    min_duration: 1.0
+    text_field: answer
+    batch_duration: null
+    max_tps: null
+    use_bucketing: true
+    bucket_duration_bins: null
+    bucket_batch_size: null
+    num_buckets: 30
+    bucket_buffer_size: 20000
+    shuffle_buffer_size: 10000
+    
+[NeMo W 2026-04-04 12:30:31 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    use_lhotse: true
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    max_duration: 40.0
+    min_duration: 0.1
+    num_workers: 2
+    pin_memory: true
+    text_field: answer
+    
+[NeMo W 2026-04-04 12:30:34 modelPT:300] You tried to register an artifact under config key=tokenizer.model_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 12:30:34 modelPT:300] You tried to register an artifact under config key=tokenizer.vocab_path but an artifact for it has already been registered.
+[NeMo W 2026-04-04 12:30:34 modelPT:300] You tried to register an artifact under config key=tokenizer.spe_tokenizer_vocab but an artifact for it has already been registered.
+[NeMo W 2026-04-04 12:30:39 speech_to_text_finetune:167] The vocabulary size of the new tokenizer differs from that of the loaded model. As a result, finetuning will proceed with the new vocabulary, and the decoder will be reinitialized.
+[NeMo W 2026-04-04 12:30:39 dataloader:826] The following configuration keys are ignored by Lhotse dataloader: is_tarred,allow_missing_data
+[NeMo W 2026-04-04 12:30:41 audio_to_text_dataset:833] Could not load dataset as `manifest_filepath` was None. Provided config : {'manifest_filepath': None, 'sample_rate': 16000, 'batch_size': 16, 'shuffle': False, 'use_start_end_token': False, 'num_workers': 8, 'pin_memory': True}
+[NeMo W 2026-04-04 12:30:47 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/lightning/pytorch/loops/fit_loop.py:298: The number of training batches (1) is smaller than the logging interval Trainer(log_every_n_steps=10). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.
+    

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-37-16/cmd-args.log

@@ -0,0 +1 @@
+./speech_to_text_finetune.py --config-path=. --config-name=speech_to_text_finetune trainer.accumulate_grad_batches=16 trainer.devices=1 trainer.accelerator=gpu +trainer.limit_train_batches=null exp_manager.create_wandb_logger=True exp_manager.wandb_logger_kwargs.name=parakeet_v3_finetune_fixed exp_manager.wandb_logger_kwargs.project=parakeet-hebrew-asr

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-37-16/nemo_error_log.txt

@@ -0,0 +1,40 @@
+[NeMo W 2026-04-04 12:37:16 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo W 2026-04-04 12:37:16 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000000. Please ensure that max_steps will run for at least None epochs to ensure that checkpointing will not error out.
+[NeMo W 2026-04-04 12:41:44 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    use_lhotse: true
+    skip_missing_manifest_entries: true
+    input_cfg: null
+    tarred_audio_filepaths: null
+    manifest_filepath: null
+    sample_rate: 16000
+    shuffle: true
+    num_workers: 2
+    pin_memory: true
+    max_duration: 10.0
+    min_duration: 1.0
+    text_field: answer
+    batch_duration: null
+    max_tps: null
+    use_bucketing: true
+    bucket_duration_bins: null
+    bucket_batch_size: null
+    num_buckets: 30
+    bucket_buffer_size: 20000
+    shuffle_buffer_size: 10000
+    
+[NeMo W 2026-04-04 12:41:44 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    use_lhotse: true
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    max_duration: 40.0
+    min_duration: 0.1
+    num_workers: 2
+    pin_memory: true
+    text_field: answer
+    

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-37-16/nemo_log_globalrank-0_localrank-0.txt

@@ -0,0 +1,143 @@
+[NeMo W 2026-04-04 12:37:16 nemo_logging:364] /opt/venv/lib/python3.12/site-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work
+      warn("Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work", RuntimeWarning)
+    
+[NeMo I 2026-04-04 12:37:16 speech_to_text_finetune:198] Hydra config: name: Speech_To_Text_Finetuning
+    init_from_pretrained_model: nvidia/parakeet-tdt-0.6b-v3
+    model:
+      sample_rate: 16000
+      activation_checkpointing: true
+      train_ds:
+        shar_path: hf://buckets/Tyl3rDrden/IvritKnessetRecordingsData
+        use_lhotse: true
+        is_tarred: true
+        batch_size: 1
+        allow_missing_data: true
+        shuffle: false
+        max_duration: 30.1
+        num_workers: 8
+        pin_memory: true
+      validation_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        batch_size: 16
+        shuffle: false
+        use_start_end_token: false
+        num_workers: 8
+        pin_memory: true
+      test_ds:
+        manifest_filepath: null
+        sample_rate: ${model.sample_rate}
+        batch_size: 16
+        shuffle: false
+        use_start_end_token: false
+        num_workers: 8
+        pin_memory: true
+      char_labels:
+        update_labels: false
+        labels: null
+      tokenizer:
+        update_tokenizer: true
+        dir: ./tokenizer_spe_bpe_v32768_pad_bos_eos
+        type: bpe
+      spec_augment:
+        _target_: nemo.collections.asr.modules.SpectrogramAugmentation
+        freq_masks: 2
+        time_masks: 10
+        freq_width: 27
+        time_width: 0.05
+      optim:
+        name: adamw
+        lr: 0.0001
+        betas:
+        - 0.9
+        - 0.98
+        weight_decay: 0.001
+        sched:
+          name: CosineAnnealing
+          warmup_steps: 5000
+          warmup_ratio: null
+          min_lr: 5.0e-06
+    trainer:
+      devices: 1
+      num_nodes: 1
+      max_epochs: -1
+      max_steps: 1000000
+      val_check_interval: 2000
+      accelerator: gpu
+      strategy:
+        _target_: lightning.pytorch.strategies.DDPStrategy
+        gradient_as_bucket_view: true
+      accumulate_grad_batches: 16
+      gradient_clip_val: 0.0
+      precision: bf16
+      log_every_n_steps: 10
+      enable_progress_bar: true
+      num_sanity_val_steps: 0
+      check_val_every_n_epoch: null
+      sync_batchnorm: true
+      enable_checkpointing: false
+      logger: false
+      benchmark: false
+      limit_train_batches: null
+    exp_manager:
+      exp_dir: null
+      name: ${name}
+      create_tensorboard_logger: true
+      create_checkpoint_callback: true
+      checkpoint_callback_params:
+        monitor: val_wer
+        mode: min
+        save_top_k: 5
+        always_save_nemo: true
+      resume_if_exists: false
+      resume_ignore_no_checkpoint: false
+      create_wandb_logger: true
+      wandb_logger_kwargs:
+        name: parakeet_v3_finetune_fixed
+        project: parakeet-hebrew-asr
+    
+[NeMo I 2026-04-04 12:37:16 exp_manager:594] ExpManager schema
+[NeMo I 2026-04-04 12:37:16 exp_manager:595] {'explicit_log_dir': None, 'exp_dir': None, 'name': None, 'version': None, 'use_datetime_version': True, 'resume_if_exists': False, 'resume_past_end': False, 'resume_ignore_no_checkpoint': False, 'resume_from_checkpoint': None, 'create_tensorboard_logger': True, 'summary_writer_kwargs': None, 'create_wandb_logger': False, 'wandb_logger_kwargs': None, 'create_mlflow_logger': False, 'mlflow_logger_kwargs': {'experiment_name': None, 'run_name': None, 'tracking_uri': None, 'tags': None, 'save_dir': './mlruns', 'prefix': '', 'artifact_location': None, 'run_id': None, 'log_model': False}, 'create_dllogger_logger': False, 'dllogger_logger_kwargs': {'verbose': False, 'stdout': False, 'json_file': './dllogger.json'}, 'create_clearml_logger': False, 'clearml_logger_kwargs': {'project': None, 'task': None, 'connect_pytorch': False, 'model_name': None, 'tags': None, 'log_model': False, 'log_cfg': False, 'log_metrics': False}, 'create_neptune_logger': False, 'neptune_logger_kwargs': None, 'create_checkpoint_callback': True, 'checkpoint_callback_params': {'filepath': None, 'dirpath': None, 'filename': None, 'monitor': 'val_loss', 'verbose': True, 'save_last': True, 'save_top_k': 3, 'save_weights_only': False, 'mode': 'min', 'auto_insert_metric_name': True, 'every_n_epochs': 1, 'every_n_train_steps': None, 'train_time_interval': None, 'prefix': None, 'postfix': '.nemo', 'save_best_model': False, 'always_save_nemo': False, 'save_nemo_on_train_end': True, 'model_parallel_size': None, 'save_on_train_epoch_end': False, 'async_save': False, 'save_last_n_optim_states': -1}, 'create_early_stopping_callback': False, 'create_ipl_epoch_stopper_callback': False, 'early_stopping_callback_params': {'monitor': 'val_loss', 'mode': 'min', 'min_delta': 0.001, 'patience': 10, 'verbose': True, 'strict': True, 'check_finite': True, 'stopping_threshold': None, 'divergence_threshold': None, 'check_on_train_epoch_end': None, 'log_rank_zero_only': False}, 'ipl_epoch_stopper_callback_params': {'enable_stop': True, 'stop_every_n_epochs': 1}, 'create_preemption_callback': True, 'files_to_copy': None, 'log_step_timing': True, 'log_delta_step_timing': False, 'step_timing_kwargs': {'reduction': 'mean', 'sync_cuda': False, 'buffer_size': 1}, 'log_local_rank_0_only': False, 'log_global_rank_0_only': False, 'disable_validation_on_resume': True, 'ema': {'enable': False, 'decay': 0.999, 'cpu_offload': False, 'validate_original_weights': False, 'every_n_steps': 1}, 'max_time_per_run': None, 'seconds_to_sleep': 5.0, 'create_straggler_detection_callback': False, 'straggler_detection_params': {'report_time_interval': 300.0, 'calc_relative_gpu_perf': True, 'calc_individual_gpu_perf': True, 'num_gpu_perf_scores_to_log': 5, 'gpu_relative_perf_threshold': 0.7, 'gpu_individual_perf_threshold': 0.7, 'stop_if_detected': False}, 'create_fault_tolerance_callback': False, 'fault_tolerance': {'workload_check_interval': 5.0, 'initial_rank_heartbeat_timeout': 3600.0, 'rank_heartbeat_timeout': 2700.0, 'calculate_timeouts': True, 'safety_factor': 5.0, 'rank_termination_signal': <Signals.SIGKILL: 9>, 'log_level': 'INFO', 'max_rank_restarts': 0, 'max_subsequent_job_failures': 0, 'additional_ft_launcher_args': '', 'simulated_fault': None}, 'log_tflops_per_sec_per_gpu': True}
+[NeMo I 2026-04-04 12:37:16 exp_manager:655] Experiments will be logged at /workspace/nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-37-16
+[NeMo I 2026-04-04 12:37:16 exp_manager:1262] TensorboardLogger has been set up
+[NeMo I 2026-04-04 12:37:16 exp_manager:1277] WandBLogger has been set up
+[NeMo W 2026-04-04 12:37:16 exp_manager:1413] The checkpoint callback was told to monitor a validation value and trainer's max_steps was set to 1000000. Please ensure that max_steps will run for at least None epochs to ensure that checkpointing will not error out.
+[NeMo I 2026-04-04 12:37:16 exp_manager:804] TFLOPs per sec per GPU will be calculated, conditioned on supported models. Defaults to -1 upon failure.
+[NeMo I 2026-04-04 12:37:17 speech_to_text_finetune:106] Sleeping for at least 60 seconds to wait for model download to finish.
+[NeMo I 2026-04-04 12:41:43 mixins:184] Tokenizer SentencePieceTokenizer initialized with 8192 tokens
+[NeMo W 2026-04-04 12:41:44 modelPT:188] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
+    Train config : 
+    use_lhotse: true
+    skip_missing_manifest_entries: true
+    input_cfg: null
+    tarred_audio_filepaths: null
+    manifest_filepath: null
+    sample_rate: 16000
+    shuffle: true
+    num_workers: 2
+    pin_memory: true
+    max_duration: 10.0
+    min_duration: 1.0
+    text_field: answer
+    batch_duration: null
+    max_tps: null
+    use_bucketing: true
+    bucket_duration_bins: null
+    bucket_batch_size: null
+    num_buckets: 30
+    bucket_buffer_size: 20000
+    shuffle_buffer_size: 10000
+    
+[NeMo W 2026-04-04 12:41:44 modelPT:195] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
+    Validation config : 
+    use_lhotse: true
+    manifest_filepath: null
+    sample_rate: 16000
+    batch_size: 16
+    shuffle: false
+    max_duration: 40.0
+    min_duration: 0.1
+    num_workers: 2
+    pin_memory: true
+    text_field: answer
+    

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-42-01/cmd-args.log

@@ -0,0 +1 @@
+./speech_to_text_finetune.py --config-path=. --config-name=speech_to_text_finetune trainer.accumulate_grad_batches=16 trainer.devices=1 trainer.accelerator=gpu +trainer.limit_train_batches=null exp_manager.create_wandb_logger=True exp_manager.wandb_logger_kwargs.name=parakeet_v3_finetune_fixed exp_manager.wandb_logger_kwargs.project=parakeet-hebrew-asr

nemo_experiments/Speech_To_Text_Finetuning/2026-04-04_12-42-01/events.out.tfevents.1775306597.371eaa8bcdbe.372.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59663ec21cc56d873f79139f8096a37a029259323d95a283c610907f70a06dff
+size 626485