Upload folder using huggingface_hub

finetuned_model/best_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a538c78e65ee3d69272552405032f952685de4b3e03a8b295eb8cc476b2db62d
+size 5607927701

finetuned_model/best_model_1773.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a538c78e65ee3d69272552405032f952685de4b3e03a8b295eb8cc476b2db62d
+size 5607927701

finetuned_model/best_model_3546.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8aca7f6cf414b855dcca79d8559b39666b57b482cd15cb4f453c7d3ddd4a660a
+size 5562740736

finetuned_model/colab_kernel_launcher.py

@@ -0,0 +1,37 @@
+# Copyright 2023 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Custom kernel launcher app to customize socket options."""
+
+from ipykernel import kernelapp
+import zmq
+
+
+# We want to set the high water mark on *all* sockets to 0, as we don't want
+# the backend dropping any messages. We want to set this before any calls to
+# bind or connect.
+#
+# In principle we should override `init_sockets`, but it's hard to set options
+# on the `zmq.Context` there without rewriting the entire method. Instead we
+# settle for only setting this on `iopub`, as that's the most important for our
+# use case.
+class ColabKernelApp(kernelapp.IPKernelApp):
+
+  def init_iopub(self, context):
+    context.setsockopt(zmq.RCVHWM, 0)
+    context.setsockopt(zmq.SNDHWM, 0)
+    return super().init_iopub(context)
+
+
+if __name__ == '__main__':
+  ColabKernelApp.launch_instance()

finetuned_model/config.json

@@ -0,0 +1,206 @@
+{
+    "output_path": "tts/run/",
+    "logger_uri": null,
+    "run_name": "xtts-finetuning",
+    "project_name": "VoiceCrafter",
+    "run_description": "\n        GPT XTTS training\n        ",
+    "print_step": 250,
+    "plot_step": 1000,
+    "model_param_stats": false,
+    "wandb_entity": null,
+    "dashboard_logger": "wandb",
+    "save_on_interrupt": true,
+    "log_model_step": 1000,
+    "save_step": 9999999999,
+    "save_n_checkpoints": 1,
+    "save_checkpoints": false,
+    "save_all_best": false,
+    "save_best_after": 0,
+    "target_loss": null,
+    "print_eval": false,
+    "test_delay_epochs": 0,
+    "run_eval": true,
+    "run_eval_steps": null,
+    "distributed_backend": "nccl",
+    "distributed_url": "tcp://localhost:54321",
+    "mixed_precision": false,
+    "precision": "fp16",
+    "epochs": 1000,
+    "batch_size": 1,
+    "eval_batch_size": 1,
+    "grad_clip": 0.0,
+    "scheduler_after_epoch": true,
+    "lr": 5e-06,
+    "optimizer": "AdamW",
+    "optimizer_params": {
+        "betas": [
+            0.9,
+            0.96
+        ],
+        "eps": 1e-08,
+        "weight_decay": 0.01
+    },
+    "lr_scheduler": "MultiStepLR",
+    "lr_scheduler_params": {
+        "milestones": [
+            900000,
+            2700000,
+            5400000
+        ],
+        "gamma": 0.5,
+        "last_epoch": -1
+    },
+    "use_grad_scaler": false,
+    "allow_tf32": false,
+    "cudnn_enable": true,
+    "cudnn_deterministic": false,
+    "cudnn_benchmark": false,
+    "training_seed": 1,
+    "model": "xtts",
+    "num_loader_workers": 8,
+    "num_eval_loader_workers": 0,
+    "use_noise_augment": false,
+    "audio": {
+        "sample_rate": 24000,
+        "output_sample_rate": 24000,
+        "dvae_sample_rate": 24000
+    },
+    "use_phonemes": false,
+    "phonemizer": null,
+    "phoneme_language": null,
+    "compute_input_seq_cache": false,
+    "text_cleaner": null,
+    "enable_eos_bos_chars": false,
+    "test_sentences_file": "",
+    "phoneme_cache_path": null,
+    "characters": null,
+    "add_blank": false,
+    "batch_group_size": 1,
+    "loss_masking": null,
+    "min_audio_len": 1,
+    "max_audio_len": Infinity,
+    "min_text_len": 1,
+    "max_text_len": Infinity,
+    "compute_f0": false,
+    "compute_energy": false,
+    "compute_linear_spec": false,
+    "precompute_num_workers": 0,
+    "start_by_longest": false,
+    "shuffle": false,
+    "drop_last": false,
+    "datasets": [
+        {
+            "formatter": "",
+            "dataset_name": "",
+            "path": "",
+            "meta_file_train": "",
+            "ignored_speakers": null,
+            "language": "",
+            "phonemizer": "",
+            "meta_file_val": "",
+            "meta_file_attn_mask": ""
+        }
+    ],
+    "test_sentences": [
+        {
+            "text": "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.",
+            "speaker_wav": "extracted_dataset/output/wavs/d65488ea-a4f2-4c65-98e4-0fb2d4cee3db_1.wav",
+            "language": "en"
+        },
+        {
+            "text": "This cake is great. It's so delicious and moist.",
+            "speaker_wav": "extracted_dataset/output/wavs/d65488ea-a4f2-4c65-98e4-0fb2d4cee3db_1.wav",
+            "language": "en"
+        },
+        {
+            "text": "And soon, nothing more terrible, nothing more true, and specious stuff that says no rational being can fear a thing it will not feel, not seeing that this is what we fear.",
+            "speaker_wav": "extracted_dataset/output/wavs/d65488ea-a4f2-4c65-98e4-0fb2d4cee3db_1.wav",
+            "language": "en"
+        }
+    ],
+    "eval_split_max_size": 256,
+    "eval_split_size": 0.01,
+    "use_speaker_weighted_sampler": false,
+    "speaker_weighted_sampler_alpha": 1.0,
+    "use_language_weighted_sampler": false,
+    "language_weighted_sampler_alpha": 1.0,
+    "use_length_weighted_sampler": false,
+    "length_weighted_sampler_alpha": 1.0,
+    "model_args": {
+        "gpt_batch_size": 1,
+        "enable_redaction": false,
+        "kv_cache": true,
+        "gpt_checkpoint": "",
+        "clvp_checkpoint": null,
+        "decoder_checkpoint": null,
+        "num_chars": 255,
+        "tokenizer_file": "tts/run/XTTS_v2.0_original_model_files/vocab.json",
+        "gpt_max_audio_tokens": 605,
+        "gpt_max_text_tokens": 402,
+        "gpt_max_prompt_tokens": 70,
+        "gpt_layers": 30,
+        "gpt_n_model_channels": 1024,
+        "gpt_n_heads": 16,
+        "gpt_number_text_tokens": 6681,
+        "gpt_start_text_token": 261,
+        "gpt_stop_text_token": 0,
+        "gpt_num_audio_tokens": 1026,
+        "gpt_start_audio_token": 1024,
+        "gpt_stop_audio_token": 1025,
+        "gpt_code_stride_len": 1024,
+        "gpt_use_masking_gt_prompt_approach": true,
+        "gpt_use_perceiver_resampler": true,
+        "input_sample_rate": 22050,
+        "output_sample_rate": 24000,
+        "output_hop_length": 256,
+        "decoder_input_dim": 1024,
+        "d_vector_dim": 512,
+        "cond_d_vector_in_each_upsampling_layer": true,
+        "duration_const": 102400,
+        "min_conditioning_length": 20640,
+        "max_conditioning_length": 720000,
+        "gpt_loss_text_ce_weight": 0.01,
+        "gpt_loss_mel_ce_weight": 1.0,
+        "debug_loading_failures": true,
+        "max_wav_length": 720000,
+        "max_text_length": 200,
+        "mel_norm_file": "tts/run/XTTS_v2.0_original_model_files/mel_stats.pth",
+        "dvae_checkpoint": "tts/run/XTTS_v2.0_original_model_files/dvae.pth",
+        "xtts_checkpoint": "tts/run/XTTS_v2.0_original_model_files/model.pth",
+        "vocoder": ""
+    },
+    "model_dir": null,
+    "languages": [
+        "en",
+        "es",
+        "fr",
+        "de",
+        "it",
+        "pt",
+        "pl",
+        "tr",
+        "ru",
+        "nl",
+        "cs",
+        "ar",
+        "zh-cn",
+        "hu",
+        "ko",
+        "ja",
+        "hi"
+    ],
+    "temperature": 0.85,
+    "length_penalty": 1.0,
+    "repetition_penalty": 2.0,
+    "top_k": 50,
+    "top_p": 0.85,
+    "num_gpt_outputs": 1,
+    "gpt_cond_len": 12,
+    "gpt_cond_chunk_len": 4,
+    "max_ref_len": 10,
+    "sound_norm_refs": false,
+    "optimizer_wd_only_on_weights": true,
+    "weighted_loss_attrs": null,
+    "weighted_loss_multipliers": null,
+    "github_branch": "inside_docker"
+}

finetuned_model/trainer_0_log.txt

@@ -0,0 +1,197 @@
+ > Training Environment:
+ | > Backend: Torch
+ | > Mixed precision: False
+ | > Precision: float32
+ | > Current device: 0
+ | > Num. of GPUs: 1
+ | > Num. of CPUs: 4
+ | > Num. of Torch Threads: 1
+ | > Torch seed: 1
+ | > Torch CUDNN: True
+ | > Torch CUDNN deterministic: False
+ | > Torch CUDNN benchmark: False
+ | > Torch TF32 MatMul: False
+
+ > Model has 518442047 parameters
+
+ > EPOCH: 0/1000
+ --> tts/run/xtts-finetuning-May-28-2025_05+56AM-0000000
+
+ > EVALUATION 
+
+
+  --> EVAL PERFORMANCE
+     | > avg_loader_time: 0.05358801569257464 (+0)
+     | > avg_loss_text_ce: 0.02478479796222278 (+0)
+     | > avg_loss_mel_ce: 3.741819156919207 (+0)
+     | > avg_loss: 3.76660396030971 (+0)
+
+
+ > EPOCH: 1/1000
+ --> tts/run/xtts-finetuning-May-28-2025_05+56AM-0000000
+
+ > TRAINING (2025-05-28 05:57:23) 
+
+   --> TIME: 2025-05-28 05:57:26 -- STEP: 0/1773 -- GLOBAL_STEP: 0
+     | > loss_text_ce: 0.024488264694809914  (0.024488264694809914)
+     | > loss_mel_ce: 4.036444187164307  (4.036444187164307)
+     | > loss: 0.06345207244157791  (0.06345207244157791)
+     | > current_lr: 5e-06 
+     | > step_time: 0.7383  (0.7382748126983643)
+     | > loader_time: 2.1633  (2.1632590293884277)
+
+
+   --> TIME: 2025-05-28 05:59:55 -- STEP: 250/1773 -- GLOBAL_STEP: 250
+     | > loss_text_ce: 0.022974064573645592  (0.024572087943553922)
+     | > loss_mel_ce: 4.102844715118408  (nan)
+     | > loss: 0.06446591764688492  (nan)
+     | > current_lr: 5e-06 
+     | > step_time: 0.5202  (0.5721063699722287)
+     | > loader_time: 0.0127  (0.021605122566223153)
+
+
+   --> TIME: 2025-05-28 06:02:25 -- STEP: 500/1773 -- GLOBAL_STEP: 500
+     | > loss_text_ce: 0.022191306576132774  (0.024244515029713504)
+     | > loss_mel_ce: 3.560326099395752  (nan)
+     | > loss: 0.055976834148168564  (nan)
+     | > current_lr: 5e-06 
+     | > step_time: 0.5558  (0.5743874120712283)
+     | > loader_time: 0.0145  (0.021079427242279045)
+
+
+   --> TIME: 2025-05-28 06:04:52 -- STEP: 750/1773 -- GLOBAL_STEP: 750
+     | > loss_text_ce: 0.023066112771630287  (0.024316282462328673)
+     | > loss_mel_ce: 3.4665474891662598  (nan)
+     | > loss: 0.054525211453437805  (nan)
+     | > current_lr: 5e-06 
+     | > step_time: 0.6241  (0.5720558166503911)
+     | > loader_time: 0.0226  (0.020561660766601552)
+
+
+   --> TIME: 2025-05-28 06:07:18 -- STEP: 1000/1773 -- GLOBAL_STEP: 1000
+     | > loss_text_ce: 0.023107804358005524  (0.0242976028714329)
+     | > loss_mel_ce: 3.397033214569092  (nan)
+     | > loss: 0.053439702838659286  (nan)
+     | > current_lr: 5e-06 
+     | > step_time: 0.631  (0.5697787494659426)
+     | > loader_time: 0.0242  (0.020013311624526974)
+
+
+   --> TIME: 2025-05-28 06:09:42 -- STEP: 1250/1773 -- GLOBAL_STEP: 1250
+     | > loss_text_ce: 0.019792545586824417  (0.02419562596976758)
+     | > loss_mel_ce: 5.3489155769348145  (nan)
+     | > loss: 0.08388606458902359  (nan)
+     | > current_lr: 5e-06 
+     | > step_time: 0.5923  (0.5667600589752204)
+     | > loader_time: 0.0251  (0.019566329574584956)
+
+
+   --> TIME: 2025-05-28 06:12:06 -- STEP: 1500/1773 -- GLOBAL_STEP: 1500
+     | > loss_text_ce: 0.02061852626502514  (0.024097217947865537)
+     | > loss_mel_ce: 3.454913377761841  (nan)
+     | > loss: 0.05430518463253975  (nan)
+     | > current_lr: 5e-06 
+     | > step_time: 0.5326  (0.5648289133707688)
+     | > loader_time: 0.0122  (0.019175546487172453)
+
+
+   --> TIME: 2025-05-28 06:14:31 -- STEP: 1750/1773 -- GLOBAL_STEP: 1750
+     | > loss_text_ce: 0.018680287525057793  (0.024025386422872552)
+     | > loss_mel_ce: 3.2065491676330566  (nan)
+     | > loss: 0.0503942109644413  (nan)
+     | > current_lr: 5e-06 
+     | > step_time: 0.5234  (0.5642632334572935)
+     | > loader_time: 0.0126  (0.019005961554391045)
+
+
+ > EVALUATION 
+
+
+  --> EVAL PERFORMANCE
+     | > avg_loader_time: 0.04692824227469308 (-0.0066597734178815565)
+     | > avg_loss_text_ce: 0.02401351955320154 (-0.0007712784090212407)
+     | > avg_loss_mel_ce: 3.0045580523354665 (-0.7372611045837405)
+     | > avg_loss: 3.028571551186698 (-0.7380324091230119)
+
+ > BEST MODEL : tts/run/xtts-finetuning-May-28-2025_05+56AM-0000000/best_model_1773.pth
+
+ > EPOCH: 2/1000
+ --> tts/run/xtts-finetuning-May-28-2025_05+56AM-0000000
+
+ > TRAINING (2025-05-28 06:15:44) 
+
+   --> TIME: 2025-05-28 06:17:57 -- STEP: 227/1773 -- GLOBAL_STEP: 2000
+     | > loss_text_ce: 0.02634984627366066  (0.02347405103039374)
+     | > loss_mel_ce: 2.9743587970733643  (nan)
+     | > loss: 0.04688607156276703  (nan)
+     | > current_lr: 5e-06 
+     | > step_time: 0.5701  (0.5613324474133058)
+     | > loader_time: 0.024  (0.01852047705965421)
+
+
+   --> TIME: 2025-05-28 06:20:24 -- STEP: 477/1773 -- GLOBAL_STEP: 2250
+     | > loss_text_ce: 0.019006451591849327  (0.0235798091996672)
+     | > loss_mel_ce: 2.9675517082214355  (nan)
+     | > loss: 0.046664971858263016  (nan)
+     | > current_lr: 5e-06 
+     | > step_time: 0.5472  (0.5630224965653334)
+     | > loader_time: 0.0132  (0.01841085212035749)
+
+
+   --> TIME: 2025-05-28 06:22:49 -- STEP: 727/1773 -- GLOBAL_STEP: 2500
+     | > loss_text_ce: 0.019247999414801598  (0.023683539267624253)
+     | > loss_mel_ce: 4.047469615936279  (nan)
+     | > loss: 0.06354246288537979  (nan)
+     | > current_lr: 5e-06 
+     | > step_time: 0.6674  (0.5622831017311867)
+     | > loader_time: 0.0246  (0.018256587536167733)
+
+
+   --> TIME: 2025-05-28 06:25:14 -- STEP: 977/1773 -- GLOBAL_STEP: 2750
+     | > loss_text_ce: 0.019411535933613777  (0.023644839759492817)
+     | > loss_mel_ce: 2.550281047821045  (nan)
+     | > loss: 0.040151447057724  (nan)
+     | > current_lr: 5e-06 
+     | > step_time: 0.6017  (0.5617610405265011)
+     | > loader_time: 0.0232  (0.018097628590516446)
+
+
+   --> TIME: 2025-05-28 06:27:39 -- STEP: 1227/1773 -- GLOBAL_STEP: 3000
+     | > loss_text_ce: 0.018359506502747536  (0.023614716428440572)
+     | > loss_mel_ce: 2.8278863430023193  (nan)
+     | > loss: 0.0444725900888443  (nan)
+     | > current_lr: 5e-06 
+     | > step_time: 0.5529  (0.5614671928666929)
+     | > loader_time: 0.014  (0.01804757506845438)
+
+
+   --> TIME: 2025-05-28 06:30:05 -- STEP: 1477/1773 -- GLOBAL_STEP: 3250
+     | > loss_text_ce: 0.024424398317933083  (0.02365642989756753)
+     | > loss_mel_ce: 2.85211443901062  (nan)
+     | > loss: 0.04494591802358627  (nan)
+     | > current_lr: 5e-06 
+     | > step_time: 0.5568  (0.5615530244869896)
+     | > loader_time: 0.0127  (0.01798885096225209)
+
+
+   --> TIME: 2025-05-28 06:32:30 -- STEP: 1727/1773 -- GLOBAL_STEP: 3500
+     | > loss_text_ce: 0.013657196424901485  (0.023524502163389426)
+     | > loss_mel_ce: 2.7606756687164307  (nan)
+     | > loss: 0.043348949402570724  (nan)
+     | > grad_norm: 0  (0.0)
+     | > current_lr: 5e-06 
+     | > step_time: 0.5705  (0.5614900450935659)
+     | > loader_time: 0.011  (0.01798361260282141)
+
+
+ > EVALUATION 
+
+
+  --> EVAL PERFORMANCE
+     | > avg_loader_time: 0.047074174880981444 (+0.0001459326062883612)
+     | > avg_loss_text_ce: 0.023535797904644693 (-0.0004777216485568475)
+     | > avg_loss_mel_ce: 2.8784927810941423 (-0.12606527124132416)
+     | > avg_loss: 2.9020285640444072 (-0.12654298714229073)
+
+ > BEST MODEL : tts/run/xtts-finetuning-May-28-2025_05+56AM-0000000/best_model_3546.pth
+ ! Run is kept in tts/run/xtts-finetuning-May-28-2025_05+56AM-0000000