Chatterbox fine-tuned model + logs

training.log

@@ -5,222 +5,7 @@ Resolved paths:
 
 Starting training...
 
-/usr/local/lib/python3.13/site-packages/perth/perth_net/__init__.py:1: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
-  from pkg_resources import resource_filename
-02/08/2026 23:54:54 - INFO - __main__ - Training/evaluation parameters CustomTrainingArguments(
-accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None, 'use_configured_state': False},
-adam_beta1=0.9,
-adam_beta2=0.999,
-adam_epsilon=1e-08,
-auto_find_batch_size=False,
-average_tokens_across_devices=True,
-batch_eval_metrics=False,
-bf16=False,
-bf16_full_eval=False,
-data_seed=None,
-dataloader_drop_last=False,
-dataloader_num_workers=8,
-dataloader_persistent_workers=False,
-dataloader_pin_memory=False,
-dataloader_prefetch_factor=None,
-ddp_backend=None,
-ddp_broadcast_buffers=None,
-ddp_bucket_cap_mb=None,
-ddp_find_unused_parameters=None,
-ddp_timeout=1800,
-debug=[],
-deepspeed=None,
-disable_tqdm=False,
-do_eval=True,
-do_predict=False,
-do_train=True,
-early_stopping_patience=None,
-enable_jit_checkpoint=False,
-eval_accumulation_steps=None,
-eval_delay=0,
-eval_do_concat_batches=True,
-eval_on_start=True,
-eval_steps=2,
-eval_strategy=IntervalStrategy.STEPS,
-eval_use_gather_object=False,
-fp16=True,
-fp16_full_eval=False,
-fsdp=[],
-fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
-full_determinism=False,
-gradient_accumulation_steps=1,
-gradient_checkpointing=False,
-gradient_checkpointing_kwargs=None,
-greater_is_better=None,
-group_by_length=False,
-hub_always_push=False,
-hub_model_id=None,
-hub_private_repo=None,
-hub_revision=None,
-hub_strategy=HubStrategy.EVERY_SAVE,
-hub_token=<HUB_TOKEN>,
-ignore_data_skip=False,
-include_for_metrics=[],
-include_num_input_tokens_seen=no,
-label_names=['labels_speech'],
-label_smoothing_factor=0.0,
-learning_rate=5e-05,
-length_column_name=length,
-liger_kernel_config=None,
-load_best_model_at_end=False,
-local_rank=-1,
-log_level=passive,
-log_level_replica=warning,
-log_on_each_node=True,
-logging_dir=None,
-logging_first_step=False,
-logging_nan_inf_filter=True,
-logging_steps=1.0,
-logging_strategy=IntervalStrategy.STEPS,
-lr_scheduler_kwargs=None,
-lr_scheduler_type=SchedulerType.LINEAR,
-max_grad_norm=1.0,
-max_steps=-1,
-metric_for_best_model=None,
-neftune_noise_alpha=None,
-num_train_epochs=1.0,
-optim=OptimizerNames.ADAMW_TORCH_FUSED,
-optim_args=None,
-optim_target_modules=None,
-output_dir=/app/checkpoints/v1,
-parallelism_config=None,
-per_device_eval_batch_size=8,
-per_device_train_batch_size=1,
-prediction_loss_only=False,
-project=huggingface,
-push_to_hub=False,
-remove_unused_columns=True,
-report_to=['tensorboard'],
-restore_callback_states_from_checkpoint=False,
-resume_from_checkpoint=None,
-run_name=None,
-save_on_each_node=False,
-save_only_model=False,
-save_steps=1.0,
-save_strategy=SaveStrategy.STEPS,
-save_total_limit=1,
-seed=42,
-skip_memory_metrics=True,
-tf32=None,
-torch_compile=False,
-torch_compile_backend=None,
-torch_compile_mode=None,
-torch_empty_cache_steps=None,
-trackio_space_id=trackio,
-use_cache=False,
-use_cpu=False,
-use_liger_kernel=False,
-warmup_ratio=None,
-warmup_steps=1.0,
-weight_decay=0.0,
-)
-02/08/2026 23:54:54 - INFO - __main__ - Model parameters ModelArguments(model_name_or_path='/app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9', local_model_dir=None, cache_dir=None, freeze_voice_encoder=True, freeze_s3gen=True)
-02/08/2026 23:54:54 - INFO - __main__ - Data parameters DataArguments(language='hi', dataset_dir=None, metadata_file=None, dataset_name='rahul7star/hindi-speech-dataset', dataset_config_name=None, train_split_name='train', eval_split_name='validation', text_column_name='transcript', audio_column_name='audio', max_text_len=256, max_speech_len=800, audio_prompt_duration_s=3.0, eval_split_size=0.0002, preprocessing_num_workers=None, ignore_verifications=False)
-02/08/2026 23:54:54 - INFO - __main__ - Loading ChatterboxTTS model...
-02/08/2026 23:54:54 - INFO - __main__ - Loading model from Hugging Face Hub: /app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9
-02/08/2026 23:54:54 - WARNING - __main__ - Could not download ve.safetensors from /app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9'. Use `repo_type` argument if needed..
-02/08/2026 23:54:54 - WARNING - __main__ - Could not download t3_mtl23ls_v2.safetensors from /app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9'. Use `repo_type` argument if needed..
-02/08/2026 23:54:54 - WARNING - __main__ - Could not download s3gen.safetensors from /app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9'. Use `repo_type` argument if needed..
-02/08/2026 23:54:54 - WARNING - __main__ - Could not download mtl_tokenizer.json from /app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/app/hf_cache/models--ResembleAI--chatterbox/snapshots/05e904af2b5c7f8e482687a9d7336c5c824467d9'. Use `repo_type` argument if needed..
-02/08/2026 23:54:54 - INFO - __main__ - conds.pt not found on Hub or failed to download for this model.
-02/08/2026 23:54:54 - INFO - httpx - HTTP Request: GET https://huggingface.co/api/models/ResembleAI/chatterbox/revision/main "HTTP/1.1 200 OK"
-
-
-Downloading (incomplete total...): 0.00B [00:00, ?B/s]
-
-Fetching 6 files:   0%|          | 0/6 [00:00<?, ?it/s]02/08/2026 23:54:54 - INFO - httpx - HTTP Request: HEAD https://huggingface.co/ResembleAI/chatterbox/resolve/05e904af2b5c7f8e482687a9d7336c5c824467d9/Cangjie5_TC.json "HTTP/1.1 307 Temporary Redirect"
-02/08/2026 23:54:54 - INFO - httpx - HTTP Request: HEAD https://huggingface.co/ResembleAI/chatterbox/resolve/05e904af2b5c7f8e482687a9d7336c5c824467d9/conds.pt "HTTP/1.1 302 Found"
-02/08/2026 23:54:54 - INFO - httpx - HTTP Request: HEAD https://huggingface.co/ResembleAI/chatterbox/resolve/05e904af2b5c7f8e482687a9d7336c5c824467d9/grapheme_mtl_merged_expanded_v1.json "HTTP/1.1 307 Temporary Redirect"
-02/08/2026 23:54:54 - INFO - httpx - HTTP Request: HEAD https://huggingface.co/ResembleAI/chatterbox/resolve/05e904af2b5c7f8e482687a9d7336c5c824467d9/t3_mtl23ls_v2.safetensors "HTTP/1.1 302 Found"
-02/08/2026 23:54:54 - INFO - httpx - HTTP Request: HEAD https://huggingface.co/ResembleAI/chatterbox/resolve/05e904af2b5c7f8e482687a9d7336c5c824467d9/ve.pt "HTTP/1.1 302 Found"
-02/08/2026 23:54:54 - INFO - httpx - HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/ResembleAI/chatterbox/05e904af2b5c7f8e482687a9d7336c5c824467d9/Cangjie5_TC.json "HTTP/1.1 200 OK"
-02/08/2026 23:54:54 - INFO - httpx - HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/ResembleAI/chatterbox/05e904af2b5c7f8e482687a9d7336c5c824467d9/grapheme_mtl_merged_expanded_v1.json "HTTP/1.1 200 OK"
-02/08/2026 23:54:54 - INFO - httpx - HTTP Request: HEAD https://huggingface.co/ResembleAI/chatterbox/resolve/05e904af2b5c7f8e482687a9d7336c5c824467d9/s3gen.pt "HTTP/1.1 302 Found"
-02/08/2026 23:54:54 - INFO - httpx - HTTP Request: GET https://huggingface.co/api/models/ResembleAI/chatterbox/xet-read-token/05e904af2b5c7f8e482687a9d7336c5c824467d9 "HTTP/1.1 200 OK"
-
-
-Downloading (incomplete total...):   0%|          | 0.00/2.14G [00:00<?, ?B/s]02/08/2026 23:54:54 - INFO - httpx - HTTP Request: GET https://huggingface.co/api/models/ResembleAI/chatterbox/xet-read-token/05e904af2b5c7f8e482687a9d7336c5c824467d9 "HTTP/1.1 200 OK"
-
-
-Downloading (incomplete total...):   0%|          | 0.00/2.14G [00:00<?, ?B/s]02/08/2026 23:54:54 - INFO - httpx - HTTP Request: GET https://huggingface.co/api/models/ResembleAI/chatterbox/xet-read-token/05e904af2b5c7f8e482687a9d7336c5c824467d9 "HTTP/1.1 200 OK"
-
-
-Downloading (incomplete total...):   0%|          | 0.00/2.15G [00:00<?, ?B/s]02/08/2026 23:54:54 - INFO - httpx - HTTP Request: GET https://huggingface.co/api/models/ResembleAI/chatterbox/xet-read-token/05e904af2b5c7f8e482687a9d7336c5c824467d9 "HTTP/1.1 200 OK"
-
-
-Downloading (incomplete total...):   0%|          | 0.00/3.21G [00:00<?, ?B/s]02/08/2026 23:54:54 - INFO - httpx - HTTP Request: GET https://huggingface.co/api/resolve-cache/models/ResembleAI/chatterbox/05e904af2b5c7f8e482687a9d7336c5c824467d9/grapheme_mtl_merged_expanded_v1.json "HTTP/1.1 200 OK"
-
-
-Downloading (incomplete total...):   0%|          | 0.00/3.21G [00:00<?, ?B/s]02/08/2026 23:54:54 - INFO - httpx - HTTP Request: GET https://huggingface.co/api/resolve-cache/models/ResembleAI/chatterbox/05e904af2b5c7f8e482687a9d7336c5c824467d9/Cangjie5_TC.json "HTTP/1.1 200 OK"
-
-
-Downloading (incomplete total...):   0%|          | 70.0k/3.21G [00:00<50:21, 1.06MB/s]
-
-Downloading (incomplete total...):   0%|          | 15.4M/3.21G [00:02<07:59, 6.66MB/s]
-
-Downloading (incomplete total...):   3%|▎         | 82.5M/3.21G [00:04<02:16, 22.8MB/s]
-
-Downloading (incomplete total...):   8%|▊         | 268M/3.21G [00:05<00:49, 59.2MB/s] 
-
-Downloading (incomplete total...):  10%|█         | 335M/3.21G [00:07<00:50, 56.5MB/s]
-
-Downloading (incomplete total...):  26%|██▌       | 825M/3.21G [00:08<00:14, 166MB/s] 
-
-Downloading (incomplete total...):  41%|████      | 1.31G/3.21G [00:10<00:09, 203MB/s]
-
-Downloading (incomplete total...):  50%|████▉     | 1.59G/3.21G [00:11<00:07, 210MB/s]
-
-Fetching 6 files:  67%|██████▋   | 4/6 [00:11<00:05,  2.97s/it]
-
-Downloading (incomplete total...):  91%|█████████ | 2.92G/3.21G [00:12<00:00, 483MB/s]
-Fetching 6 files: 100%|██████████| 6/6 [00:12<00:00,  2.06s/it]
-
-
-Download complete: 100%|██████████| 3.21G/3.21G [00:12<00:00, 483MB/s]                
-Download complete: 100%|██████████| 3.21G/3.21G [00:25<00:00, 128MB/s]
-/usr/local/lib/python3.13/site-packages/diffusers/models/lora.py:393: FutureWarning: `LoRACompatibleLinear` is deprecated and will be removed in version 1.0.0. Use of `LoRACompatibleLinear` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.
-  deprecate("LoRACompatibleLinear", "1.0.0", deprecation_message)
-02/08/2026 23:55:21 - INFO - root - input frame rate=25
 Traceback (most recent call last):
-  File "/app/chatterbox-multilingual-finetuning/src/finetune_t3.py", line 848, in <module>
-    main()
-    ~~~~^^
-  File "/app/chatterbox-multilingual-finetuning/src/finetune_t3.py", line 616, in main
-    chatterbox_model = ChatterboxMultilingualTTS.from_pretrained(device="cpu")
-  File "/app/chatterbox-multilingual-finetuning/src/chatterbox/mtl_tts.py", line 203, in from_pretrained
-    return cls.from_local(ckpt_dir, device)
-           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
-  File "/app/chatterbox-multilingual-finetuning/src/chatterbox/mtl_tts.py", line 174, in from_local
-    s3gen.load_state_dict(torch.load(ckpt_dir / "s3gen.pt", weights_only=True))
-                          ~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.13/site-packages/torch/serialization.py", line 1540, in load
-    return _load(
-        opened_zipfile,
-    ...<3 lines>...
-        **pickle_load_args,
-    )
-  File "/usr/local/lib/python3.13/site-packages/torch/serialization.py", line 2143, in _load
-    result = unpickler.load()
-  File "/usr/local/lib/python3.13/site-packages/torch/_weights_only_unpickler.py", line 539, in load
-    self.append(self.persistent_load(pid))
-                ~~~~~~~~~~~~~~~~~~~~^^^^^
-  File "/usr/local/lib/python3.13/site-packages/torch/serialization.py", line 2107, in persistent_load
-    typed_storage = load_tensor(
-        dtype, nbytes, key, _maybe_decode_ascii(location)
-    )
-  File "/usr/local/lib/python3.13/site-packages/torch/serialization.py", line 2073, in load_tensor
-    wrap_storage = restore_location(storage, location)
-  File "/usr/local/lib/python3.13/site-packages/torch/serialization.py", line 707, in default_restore_location
-    result = fn(storage, location)
-  File "/usr/local/lib/python3.13/site-packages/torch/serialization.py", line 640, in _deserialize
-    device = _validate_device(location, backend_name)
-  File "/usr/local/lib/python3.13/site-packages/torch/serialization.py", line 609, in _validate_device
-    raise RuntimeError(
-    ...<5 lines>...
-    )
-RuntimeError: Attempting to deserialize object on a CUDA device but torch.cuda.is_available() is False. If you are running on a CPU-only machine, please use torch.load with map_location=torch.device('cpu') to map your storages to the CPU.
+  File "/app/chatterbox-multilingual-finetuning/src/finetune_t3.py", line 19, in <module>
+    from src.models.t3 import T3
+ModuleNotFoundError: No module named 'src'