diff --git "a/training.log" "b/training.log" --- "a/training.log" +++ "b/training.log" @@ -5,618 +5,853 @@ Resolved paths: Starting training... +02/10/2026 03:16:07 - INFO - __main__ - Training/evaluation parameters CustomTrainingArguments( +accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None, 'use_configured_state': False}, +adam_beta1=0.9, +adam_beta2=0.999, +adam_epsilon=1e-08, +auto_find_batch_size=False, +average_tokens_across_devices=True, +batch_eval_metrics=False, +bf16=False, +bf16_full_eval=False, +data_seed=None, +dataloader_drop_last=False, +dataloader_num_workers=8, +dataloader_persistent_workers=False, +dataloader_pin_memory=False, +dataloader_prefetch_factor=None, +ddp_backend=None, +ddp_broadcast_buffers=None, +ddp_bucket_cap_mb=None, +ddp_find_unused_parameters=None, +ddp_timeout=1800, +debug=[], +deepspeed=None, +disable_tqdm=False, +do_eval=True, +do_predict=False, +do_train=True, +early_stopping_patience=None, +enable_jit_checkpoint=False, +eval_accumulation_steps=None, +eval_delay=0, +eval_do_concat_batches=True, +eval_on_start=True, +eval_steps=2, +eval_strategy=IntervalStrategy.STEPS, +eval_use_gather_object=False, +fp16=True, +fp16_full_eval=False, +fsdp=[], +fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, +full_determinism=False, +gradient_accumulation_steps=1, +gradient_checkpointing=False, +gradient_checkpointing_kwargs=None, +greater_is_better=None, +group_by_length=False, +hub_always_push=False, +hub_model_id=None, +hub_private_repo=None, +hub_revision=None, +hub_strategy=HubStrategy.EVERY_SAVE, +hub_token=, +ignore_data_skip=False, +include_for_metrics=[], +include_num_input_tokens_seen=no, +label_names=['labels_speech'], +label_smoothing_factor=0.0, +learning_rate=5e-05, +length_column_name=length, +liger_kernel_config=None, +load_best_model_at_end=False, +local_rank=-1, +log_level=passive, +log_level_replica=warning, +log_on_each_node=True, +logging_dir=None, +logging_first_step=False, +logging_nan_inf_filter=True, +logging_steps=1.0, +logging_strategy=IntervalStrategy.STEPS, +lr_scheduler_kwargs=None, +lr_scheduler_type=SchedulerType.LINEAR, +max_grad_norm=1.0, +max_steps=-1, +metric_for_best_model=None, +neftune_noise_alpha=None, +num_train_epochs=1.0, +optim=OptimizerNames.ADAMW_TORCH_FUSED, +optim_args=None, +optim_target_modules=None, +output_dir=/app/checkpoints/v1, +parallelism_config=None, +per_device_eval_batch_size=8, +per_device_train_batch_size=1, +prediction_loss_only=False, +project=huggingface, +push_to_hub=False, +remove_unused_columns=True, +report_to=['tensorboard'], +restore_callback_states_from_checkpoint=False, +resume_from_checkpoint=None, +run_name=None, +save_on_each_node=False, +save_only_model=False, +save_steps=1.0, +save_strategy=SaveStrategy.STEPS, +save_total_limit=1, +seed=42, +skip_memory_metrics=True, +tf32=None, +torch_compile=False, +torch_compile_backend=None, +torch_compile_mode=None, +torch_empty_cache_steps=None, +trackio_space_id=trackio, +use_cache=False, +use_cpu=False, +use_liger_kernel=False, +warmup_ratio=None, +warmup_steps=1.0, +weight_decay=0.0, +) +02/10/2026 03:16:07 - INFO - __main__ - Model parameters ModelArguments(model_name_or_path='/app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9', local_model_dir=None, cache_dir=None, freeze_voice_encoder=True, freeze_s3gen=True) +02/10/2026 03:16:07 - INFO - __main__ - Data parameters DataArguments(language='hi', dataset_dir=None, metadata_file=None, dataset_name='rahul7star/hindi-speech-dataset', dataset_config_name=None, train_split_name='train', eval_split_name='validation', text_column_name='transcript', audio_column_name='audio', max_text_len=256, max_speech_len=800, audio_prompt_duration_s=3.0, eval_split_size=0.0002, preprocessing_num_workers=None, ignore_verifications=False) +02/10/2026 03:16:07 - INFO - __main__ - Loading ChatterboxTTS model... +02/10/2026 03:16:07 - INFO - __main__ - Loading model from Hugging Face Hub: /app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9 +02/10/2026 03:16:07 - WARNING - __main__ - Could not download ve.safetensors from /app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9'. Use `repo_type` argument if needed.. +02/10/2026 03:16:07 - WARNING - __main__ - Could not download t3_mtl23ls_v2.safetensors from /app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9'. Use `repo_type` argument if needed.. +02/10/2026 03:16:07 - WARNING - __main__ - Could not download s3gen.safetensors from /app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9'. Use `repo_type` argument if needed.. +02/10/2026 03:16:07 - WARNING - __main__ - Could not download mtl_tokenizer.json from /app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9'. Use `repo_type` argument if needed.. +02/10/2026 03:16:07 - INFO - __main__ - conds.pt not found on Hub or failed to download for this model. +02/10/2026 03:16:07 - INFO - httpx - HTTP Request: GET https://huggingface.co/api/models/ResembleAI/chatterbox/revision/main "HTTP/1.1 200 OK" + + +Downloading (incomplete total...): 0.00B [00:00, ?B/s] + +Fetching 5 files: 0%| | 0/5 [00:00. Skipping. -ERROR:__main__:Unexpected audio data format for item 16: . Skipping. -ERROR:__main__:Unexpected audio data format for item 8: . Skipping. -ERROR:__main__:Unexpected audio data format for item 48: . Skipping. -ERROR:__main__:Unexpected audio data format for item 56: . Skipping. -ERROR:__main__:Unexpected audio data format for item 40: . Skipping. -ERROR:__main__:Unexpected audio data format for item 24: . Skipping. -ERROR:__main__:Unexpected audio data format for item 32: . Skipping. -ERROR:__main__:Unexpected audio data format for item 1: . Skipping. -ERROR:__main__:Unexpected audio data format for item 33: . Skipping. -ERROR:__main__:Unexpected audio data format for item 41: . Skipping. -ERROR:__main__:Unexpected audio data format for item 17: . Skipping. -ERROR:__main__:Unexpected audio data format for item 49: . Skipping. -ERROR:__main__:Unexpected audio data format for item 9: . Skipping. -ERROR:__main__:Unexpected audio data format for item 57: . Skipping. -ERROR:__main__:Unexpected audio data format for item 34: . Skipping. -ERROR:__main__:Unexpected audio data format for item 42: . Skipping. -ERROR:__main__:Unexpected audio data format for item 25: . Skipping. -ERROR:__main__:Unexpected audio data format for item 10: . Skipping. -ERROR:__main__:Unexpected audio data format for item 18: . Skipping. -ERROR:__main__:Unexpected audio data format for item 58: . Skipping. -ERROR:__main__:Unexpected audio data format for item 35: . Skipping. -ERROR:__main__:Unexpected audio data format for item 50: . Skipping. -ERROR:__main__:Unexpected audio data format for item 43: . Skipping. -ERROR:__main__:Unexpected audio data format for item 19: . Skipping. -ERROR:__main__:Unexpected audio data format for item 26: . Skipping. -ERROR:__main__:Unexpected audio data format for item 36: . Skipping. -ERROR:__main__:Unexpected audio data format for item 11: . Skipping. -ERROR:__main__:Unexpected audio data format for item 2: . Skipping. -ERROR:__main__:Unexpected audio data format for item 59: . Skipping. -ERROR:__main__:Unexpected audio data format for item 51: . Skipping. -ERROR:__main__:Unexpected audio data format for item 37: . Skipping. -ERROR:__main__:Unexpected audio data format for item 44: . Skipping. -ERROR:__main__:Unexpected audio data format for item 20: . Skipping. -ERROR:__main__:Unexpected audio data format for item 12: . Skipping. -ERROR:__main__:Unexpected audio data format for item 27: . Skipping. -ERROR:__main__:Unexpected audio data format for item 3: . Skipping. -ERROR:__main__:Unexpected audio data format for item 60: . Skipping. -ERROR:__main__:Unexpected audio data format for item 38: . Skipping. -ERROR:__main__:Unexpected audio data format for item 52: . Skipping. -ERROR:__main__:Unexpected audio data format for item 45: . Skipping. -ERROR:__main__:Unexpected audio data format for item 21: . Skipping. -ERROR:__main__:Unexpected audio data format for item 28: . Skipping. -ERROR:__main__:Unexpected audio data format for item 39: . Skipping. -ERROR:__main__:Unexpected audio data format for item 61: . Skipping. -ERROR:__main__:Unexpected audio data format for item 4: . Skipping. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -ERROR:__main__:Unexpected audio data format for item 13: . Skipping. -ERROR:__main__:Unexpected audio data format for item 53: . Skipping. -ERROR:__main__:Unexpected audio data format for item 46: . Skipping. -ERROR:__main__:Unexpected audio data format for item 22: . Skipping. -ERROR:__main__:Unexpected audio data format for item 29: . Skipping. -ERROR:__main__:Unexpected audio data format for item 62: . Skipping. -ERROR:__main__:Unexpected audio data format for item 14: . Skipping. -ERROR:__main__:Unexpected audio data format for item 5: . Skipping. -ERROR:__main__:Unexpected audio data format for item 54: . Skipping. -ERROR:__main__:Unexpected audio data format for item 47: . Skipping. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -ERROR:__main__:Unexpected audio data format for item 63: . Skipping. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -ERROR:__main__:Unexpected audio data format for item 30: . Skipping. -ERROR:__main__:Unexpected audio data format for item 15: . Skipping. -ERROR:__main__:Unexpected audio data format for item 23: . Skipping. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -ERROR:__main__:Unexpected audio data format for item 96: . Skipping. -ERROR:__main__:Unexpected audio data format for item 6: . Skipping. -ERROR:__main__:Unexpected audio data format for item 55: . Skipping. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -ERROR:__main__:Unexpected audio data format for item 31: . Skipping. -ERROR:__main__:Unexpected audio data format for item 97: . Skipping. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -ERROR:__main__:Unexpected audio data format for item 7: . Skipping. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -ERROR:__main__:Unexpected audio data format for item 104: . Skipping. -ERROR:__main__:Unexpected audio data format for item 80: . Skipping. -ERROR:__main__:Unexpected audio data format for item 72: . Skipping. -ERROR:__main__:Unexpected audio data format for item 120: . Skipping. -ERROR:__main__:Unexpected audio data format for item 105: . Skipping. -ERROR:__main__:Unexpected audio data format for item 112: . Skipping. -ERROR:__main__:Unexpected audio data format for item 73: . Skipping. -ERROR:__main__:Unexpected audio data format for item 81: . Skipping. -ERROR:__main__:Unexpected audio data format for item 121: . Skipping. -ERROR:__main__:Unexpected audio data format for item 88: . Skipping. -ERROR:__main__:Unexpected audio data format for item 64: . Skipping. -ERROR:__main__:Unexpected audio data format for item 106: . Skipping. -ERROR:__main__:Unexpected audio data format for item 122: . Skipping. -ERROR:__main__:Unexpected audio data format for item 113: . Skipping. -ERROR:__main__:Unexpected audio data format for item 98: . Skipping. -ERROR:__main__:Unexpected audio data format for item 65: . Skipping. -ERROR:__main__:Unexpected audio data format for item 89: . Skipping. -ERROR:__main__:Unexpected audio data format for item 107: . Skipping. -ERROR:__main__:Unexpected audio data format for item 123: . Skipping. -ERROR:__main__:Unexpected audio data format for item 74: . Skipping. -ERROR:__main__:Unexpected audio data format for item 99: . Skipping. -ERROR:__main__:Unexpected audio data format for item 66: . Skipping. -ERROR:__main__:Unexpected audio data format for item 108: . Skipping. -ERROR:__main__:Unexpected audio data format for item 114: . Skipping. -ERROR:__main__:Unexpected audio data format for item 90: . Skipping. -ERROR:__main__:Unexpected audio data format for item 124: . Skipping. -ERROR:__main__:Unexpected audio data format for item 82: . Skipping. -ERROR:__main__:Unexpected audio data format for item 109: . Skipping. -ERROR:__main__:Unexpected audio data format for item 67: . Skipping. -ERROR:__main__:Unexpected audio data format for item 115: . Skipping. -ERROR:__main__:Unexpected audio data format for item 75: . Skipping. -ERROR:__main__:Unexpected audio data format for item 100: . Skipping. -ERROR:__main__:Unexpected audio data format for item 91: . Skipping. -ERROR:__main__:Unexpected audio data format for item 125: . Skipping. -ERROR:__main__:Unexpected audio data format for item 110: . Skipping. -ERROR:__main__:Unexpected audio data format for item 68: . Skipping. -ERROR:__main__:Unexpected audio data format for item 83: . Skipping. -ERROR:__main__:Unexpected audio data format for item 116: . Skipping. -ERROR:__main__:Unexpected audio data format for item 76: . Skipping. -ERROR:__main__:Unexpected audio data format for item 126: . Skipping. -ERROR:__main__:Unexpected audio data format for item 92: . Skipping. -ERROR:__main__:Unexpected audio data format for item 101: . Skipping. -ERROR:__main__:Unexpected audio data format for item 111: . Skipping. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -ERROR:__main__:Unexpected audio data format for item 84: . Skipping. -ERROR:__main__:Unexpected audio data format for item 69: . Skipping. -ERROR:__main__:Unexpected audio data format for item 127: . Skipping. -ERROR:__main__:Unexpected audio data format for item 77: . Skipping. -ERROR:__main__:Unexpected audio data format for item 117: . Skipping. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -ERROR:__main__:Unexpected audio data format for item 85: . Skipping. -ERROR:__main__:Unexpected audio data format for item 102: . Skipping. -ERROR:__main__:Unexpected audio data format for item 118: . Skipping. -ERROR:__main__:Unexpected audio data format for item 78: . Skipping. -ERROR:__main__:Unexpected audio data format for item 70: . Skipping. -ERROR:__main__:Unexpected audio data format for item 86: . Skipping. -ERROR:__main__:Unexpected audio data format for item 103: . Skipping. -ERROR:__main__:Unexpected audio data format for item 79: . Skipping. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -ERROR:__main__:Unexpected audio data format for item 87: . Skipping. -ERROR:__main__:Unexpected audio data format for item 119: . Skipping. -ERROR:__main__:Unexpected audio data format for item 71: . Skipping. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. -ERROR:__main__:Unexpected audio data format for item 93: . Skipping. -ERROR:__main__:Unexpected audio data format for item 94: . Skipping. -ERROR:__main__:Unexpected audio data format for item 95: . Skipping. -WARNING:__main__:SpeechDataCollator received no valid features. Returning empty batch. +Generating test split: 48%|████▊ | 200/418 [00:01<00:01, 167.44 examples/s] +Generating test split: 100%|██████████| 418/418 [00:01<00:00, 257.34 examples/s] +02/10/2026 03:19:07 - INFO - __main__ - *** Training T3 model *** + + + 0%| | 0/145152 [00:00. Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 48: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 8: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 16: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 24: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 40: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 56: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 0: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 49: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 33: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 9: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 41: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 1: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 17: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 57: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 34: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 10: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 50: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 42: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 18: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 25: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 35: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 58: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 11: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 51: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 43: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 36: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 19: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 26: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 12: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 59: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 44: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 52: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 37: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 2: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 60: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 20: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 27: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 53: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 13: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 38: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 45: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 3: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 61: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 54: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 39: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 14: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 28: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 21: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 46: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 55: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 62: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 15: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 4: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 47: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 22: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 96: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 63: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 5: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 97: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 23: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 6: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 120: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 7: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 121: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 98: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 122: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 64: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 99: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 65: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 100: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 123: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 66: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 124: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 101: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 67: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 125: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 102: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 68: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 126: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 103: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 69: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 127: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 70: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 29: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 112: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 71: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 113: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 30: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 72: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 80: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 104: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 31: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 114: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 128: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 73: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 81: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 115: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 105: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 129: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 116: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 106: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 130: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 74: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 131: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 117: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 107: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 75: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 132: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 108: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 118: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 82: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 76: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 133: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 109: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 119: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 134: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 77: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 110: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 135: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 78: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 111: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 83: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 79: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 84: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 85: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 86: . Skipping. +02/10/2026 03:19:11 - ERROR - __main__ - Unexpected audio data format for item 87: . Skipping. +02/10/2026 03:19:11 - WARNING - __main__ - SpeechDataCollator received no valid features. Returning empty batch. Traceback (most recent call last): - File "/app/chatterbox-multilingual-finetuning/src/finetune_t3.py", line 570, in + File "/app/chatterbox-multilingual-finetuning/src/finetune_t3.py", line 848, in main() ~~~~^^ - File "/app/chatterbox-multilingual-finetuning/src/finetune_t3.py", line 566, in main - trainer.train() - ~~~~~~~~~~~~~^^ + File "/app/chatterbox-multilingual-finetuning/src/finetune_t3.py", line 796, in main + train_result = trainer_instance.train( + resume_from_checkpoint=training_args.resume_from_checkpoint + ) File "/usr/local/lib/python3.13/site-packages/transformers/trainer.py", line 2170, in train return inner_training_loop( args=args, @@ -645,4 +880,4 @@ Traceback (most recent call last): ) ValueError: The batch received was empty, your model won't be able to train on it. Double-check that your training dataset contains keys expected by the model: text_tokens,text_token_lens,speech_tokens,speech_token_lens,t3_cond_speaker_emb,t3_cond_prompt_speech_tokens,t3_cond_emotion_adv,labels_text,labels_speech,labels_speech,label,label_ids. - 0%| | 0/145152 [00:03