Upload folder using huggingface_hub
Browse files- finetuned/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5/best_model.pth +3 -0
- finetuned/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5/best_model_4323.pth +3 -0
- finetuned/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5/config.json +206 -0
- finetuned/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5/events.out.tfevents.1792547554.90c29ef008dd.122.0 +3 -0
- finetuned/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5/recipe_frox_finetune.py +178 -0
- finetuned/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5/trainer_0_log.txt +828 -0
finetuned/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5/best_model.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7535e49e8e0cb01c26dd8b3eb77f7e1b8fc04c3a4a707d0ad58af148991b6943
|
| 3 |
+
size 5607856542
|
finetuned/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5/best_model_4323.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7535e49e8e0cb01c26dd8b3eb77f7e1b8fc04c3a4a707d0ad58af148991b6943
|
| 3 |
+
size 5607856542
|
finetuned/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5/config.json
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"output_path": "/kaggle/working/coqui-ai-TTS/run/training",
|
| 3 |
+
"logger_uri": null,
|
| 4 |
+
"run_name": "GPT_XTTS_v2.0_LJSpeech_FT",
|
| 5 |
+
"project_name": "XTTS_trainer",
|
| 6 |
+
"run_description": "\n GPT XTTS training\n ",
|
| 7 |
+
"print_step": 50,
|
| 8 |
+
"plot_step": 100,
|
| 9 |
+
"model_param_stats": false,
|
| 10 |
+
"wandb_entity": null,
|
| 11 |
+
"dashboard_logger": "tensorboard",
|
| 12 |
+
"save_on_interrupt": true,
|
| 13 |
+
"log_model_step": 1000,
|
| 14 |
+
"save_step": 10000,
|
| 15 |
+
"save_n_checkpoints": 1,
|
| 16 |
+
"save_checkpoints": true,
|
| 17 |
+
"save_all_best": false,
|
| 18 |
+
"save_best_after": 0,
|
| 19 |
+
"target_loss": null,
|
| 20 |
+
"print_eval": false,
|
| 21 |
+
"test_delay_epochs": 0,
|
| 22 |
+
"run_eval": true,
|
| 23 |
+
"run_eval_steps": null,
|
| 24 |
+
"distributed_backend": "nccl",
|
| 25 |
+
"distributed_url": "tcp://localhost:54321",
|
| 26 |
+
"mixed_precision": false,
|
| 27 |
+
"precision": "fp16",
|
| 28 |
+
"epochs": 2,
|
| 29 |
+
"batch_size": 3,
|
| 30 |
+
"eval_batch_size": 3,
|
| 31 |
+
"grad_clip": 0.0,
|
| 32 |
+
"scheduler_after_epoch": true,
|
| 33 |
+
"lr": 5e-06,
|
| 34 |
+
"optimizer": "AdamW",
|
| 35 |
+
"optimizer_params": {
|
| 36 |
+
"betas": [
|
| 37 |
+
0.9,
|
| 38 |
+
0.96
|
| 39 |
+
],
|
| 40 |
+
"eps": 1e-08,
|
| 41 |
+
"weight_decay": 0.01
|
| 42 |
+
},
|
| 43 |
+
"lr_scheduler": "MultiStepLR",
|
| 44 |
+
"lr_scheduler_params": {
|
| 45 |
+
"milestones": [
|
| 46 |
+
900000,
|
| 47 |
+
2700000,
|
| 48 |
+
5400000
|
| 49 |
+
],
|
| 50 |
+
"gamma": 0.5,
|
| 51 |
+
"last_epoch": -1
|
| 52 |
+
},
|
| 53 |
+
"use_grad_scaler": false,
|
| 54 |
+
"allow_tf32": false,
|
| 55 |
+
"cudnn_enable": true,
|
| 56 |
+
"cudnn_deterministic": false,
|
| 57 |
+
"cudnn_benchmark": false,
|
| 58 |
+
"training_seed": 1,
|
| 59 |
+
"model": "xtts",
|
| 60 |
+
"num_loader_workers": 8,
|
| 61 |
+
"num_eval_loader_workers": 0,
|
| 62 |
+
"use_noise_augment": false,
|
| 63 |
+
"audio": {
|
| 64 |
+
"sample_rate": 22050,
|
| 65 |
+
"output_sample_rate": 24000,
|
| 66 |
+
"dvae_sample_rate": 22050
|
| 67 |
+
},
|
| 68 |
+
"model_args": {
|
| 69 |
+
"gpt_batch_size": 1,
|
| 70 |
+
"enable_redaction": false,
|
| 71 |
+
"kv_cache": true,
|
| 72 |
+
"gpt_checkpoint": "",
|
| 73 |
+
"clvp_checkpoint": null,
|
| 74 |
+
"decoder_checkpoint": null,
|
| 75 |
+
"num_chars": 67,
|
| 76 |
+
"tokenizer_file": "/kaggle/working/coqui-ai-TTS/run/training/XTTS_v2.0_original_model_files/vocab.json",
|
| 77 |
+
"gpt_max_audio_tokens": 605,
|
| 78 |
+
"gpt_max_text_tokens": 402,
|
| 79 |
+
"gpt_max_prompt_tokens": 70,
|
| 80 |
+
"gpt_layers": 30,
|
| 81 |
+
"gpt_n_model_channels": 1024,
|
| 82 |
+
"gpt_n_heads": 16,
|
| 83 |
+
"gpt_number_text_tokens": 6681,
|
| 84 |
+
"gpt_start_text_token": 261,
|
| 85 |
+
"gpt_stop_text_token": 0,
|
| 86 |
+
"gpt_num_audio_tokens": 1026,
|
| 87 |
+
"gpt_start_audio_token": 1024,
|
| 88 |
+
"gpt_stop_audio_token": 1025,
|
| 89 |
+
"gpt_code_stride_len": 1024,
|
| 90 |
+
"gpt_use_masking_gt_prompt_approach": true,
|
| 91 |
+
"gpt_use_perceiver_resampler": true,
|
| 92 |
+
"input_sample_rate": 22050,
|
| 93 |
+
"output_sample_rate": 24000,
|
| 94 |
+
"output_hop_length": 256,
|
| 95 |
+
"decoder_input_dim": 1024,
|
| 96 |
+
"d_vector_dim": 512,
|
| 97 |
+
"cond_d_vector_in_each_upsampling_layer": true,
|
| 98 |
+
"duration_const": 102400,
|
| 99 |
+
"min_conditioning_length": 66150,
|
| 100 |
+
"max_conditioning_length": 132300,
|
| 101 |
+
"gpt_loss_text_ce_weight": 0.01,
|
| 102 |
+
"gpt_loss_mel_ce_weight": 1.0,
|
| 103 |
+
"debug_loading_failures": false,
|
| 104 |
+
"max_wav_length": 255995,
|
| 105 |
+
"max_text_length": 200,
|
| 106 |
+
"mel_norm_file": "/kaggle/working/coqui-ai-TTS/run/training/XTTS_v2.0_original_model_files/mel_stats.pth",
|
| 107 |
+
"dvae_checkpoint": "/kaggle/working/coqui-ai-TTS/run/training/XTTS_v2.0_original_model_files/dvae.pth",
|
| 108 |
+
"xtts_checkpoint": "/kaggle/working/coqui-ai-TTS/run/training/XTTS_v2.0_original_model_files/model.pth",
|
| 109 |
+
"vocoder": ""
|
| 110 |
+
},
|
| 111 |
+
"_supports_cloning": true,
|
| 112 |
+
"languages": [
|
| 113 |
+
"en",
|
| 114 |
+
"es",
|
| 115 |
+
"fr",
|
| 116 |
+
"de",
|
| 117 |
+
"it",
|
| 118 |
+
"pt",
|
| 119 |
+
"pl",
|
| 120 |
+
"tr",
|
| 121 |
+
"ru",
|
| 122 |
+
"nl",
|
| 123 |
+
"cs",
|
| 124 |
+
"ar",
|
| 125 |
+
"zh-cn",
|
| 126 |
+
"hu",
|
| 127 |
+
"ko",
|
| 128 |
+
"ja",
|
| 129 |
+
"hi"
|
| 130 |
+
],
|
| 131 |
+
"speakers": [],
|
| 132 |
+
"use_phonemes": false,
|
| 133 |
+
"phonemizer": null,
|
| 134 |
+
"phoneme_language": null,
|
| 135 |
+
"compute_input_seq_cache": false,
|
| 136 |
+
"text_cleaner": null,
|
| 137 |
+
"enable_eos_bos_chars": false,
|
| 138 |
+
"test_sentences_file": "",
|
| 139 |
+
"phoneme_cache_path": null,
|
| 140 |
+
"characters": null,
|
| 141 |
+
"add_blank": false,
|
| 142 |
+
"batch_group_size": 48,
|
| 143 |
+
"loss_masking": null,
|
| 144 |
+
"min_audio_len": 1,
|
| 145 |
+
"max_audio_len": Infinity,
|
| 146 |
+
"min_text_len": 1,
|
| 147 |
+
"max_text_len": Infinity,
|
| 148 |
+
"compute_f0": false,
|
| 149 |
+
"compute_energy": false,
|
| 150 |
+
"compute_linear_spec": false,
|
| 151 |
+
"precompute_num_workers": 0,
|
| 152 |
+
"start_by_longest": false,
|
| 153 |
+
"shuffle": false,
|
| 154 |
+
"drop_last": false,
|
| 155 |
+
"datasets": [
|
| 156 |
+
{
|
| 157 |
+
"formatter": "ljspeech",
|
| 158 |
+
"dataset_name": "ljspeech",
|
| 159 |
+
"path": "/kaggle/working/ljspeech_fmt/",
|
| 160 |
+
"meta_file_train": "/kaggle/working/ljspeech_fmt/metadata.csv",
|
| 161 |
+
"ignored_speakers": null,
|
| 162 |
+
"language": "en",
|
| 163 |
+
"phonemizer": "",
|
| 164 |
+
"meta_file_val": "",
|
| 165 |
+
"meta_file_attn_mask": ""
|
| 166 |
+
}
|
| 167 |
+
],
|
| 168 |
+
"test_sentences": [
|
| 169 |
+
{
|
| 170 |
+
"text": "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.",
|
| 171 |
+
"speaker_wav": [
|
| 172 |
+
"./tests/data/ljspeech/wavs/LJ001-0002.wav"
|
| 173 |
+
],
|
| 174 |
+
"language": "en"
|
| 175 |
+
},
|
| 176 |
+
{
|
| 177 |
+
"text": "This cake is great. It's so delicious and moist.",
|
| 178 |
+
"speaker_wav": [
|
| 179 |
+
"./tests/data/ljspeech/wavs/LJ001-0002.wav"
|
| 180 |
+
],
|
| 181 |
+
"language": "en"
|
| 182 |
+
}
|
| 183 |
+
],
|
| 184 |
+
"eval_split_max_size": 256,
|
| 185 |
+
"eval_split_size": 0.01,
|
| 186 |
+
"use_speaker_weighted_sampler": false,
|
| 187 |
+
"speaker_weighted_sampler_alpha": 1.0,
|
| 188 |
+
"use_language_weighted_sampler": false,
|
| 189 |
+
"language_weighted_sampler_alpha": 1.0,
|
| 190 |
+
"use_length_weighted_sampler": false,
|
| 191 |
+
"length_weighted_sampler_alpha": 1.0,
|
| 192 |
+
"temperature": 0.85,
|
| 193 |
+
"length_penalty": 1.0,
|
| 194 |
+
"repetition_penalty": 2.0,
|
| 195 |
+
"top_k": 50,
|
| 196 |
+
"top_p": 0.85,
|
| 197 |
+
"num_gpt_outputs": 1,
|
| 198 |
+
"gpt_cond_len": 12,
|
| 199 |
+
"gpt_cond_chunk_len": 4,
|
| 200 |
+
"max_ref_len": 10,
|
| 201 |
+
"sound_norm_refs": false,
|
| 202 |
+
"optimizer_wd_only_on_weights": true,
|
| 203 |
+
"weighted_loss_attrs": {},
|
| 204 |
+
"weighted_loss_multipliers": {},
|
| 205 |
+
"github_branch": "* dev"
|
| 206 |
+
}
|
finetuned/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5/events.out.tfevents.1792547554.90c29ef008dd.122.0
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:64033e7fa48c7b74b32f9677a57cb1846d8bea7239cd2bcbfbcb526bebb5d1ba
|
| 3 |
+
size 1017758
|
finetuned/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5/recipe_frox_finetune.py
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
from trainer import Trainer, TrainerArgs
|
| 4 |
+
|
| 5 |
+
from TTS.config.shared_configs import BaseDatasetConfig
|
| 6 |
+
from TTS.tts.configs.xtts_config import XttsAudioConfig
|
| 7 |
+
from TTS.tts.datasets import load_tts_samples
|
| 8 |
+
from TTS.tts.layers.xtts.trainer.gpt_trainer import GPTArgs, GPTTrainer, GPTTrainerConfig
|
| 9 |
+
from TTS.utils.manage import ModelManager
|
| 10 |
+
|
| 11 |
+
# Logging parameters
|
| 12 |
+
RUN_NAME = "GPT_XTTS_v2.0_LJSpeech_FT"
|
| 13 |
+
PROJECT_NAME = "XTTS_trainer"
|
| 14 |
+
DASHBOARD_LOGGER = "tensorboard"
|
| 15 |
+
LOGGER_URI = None
|
| 16 |
+
|
| 17 |
+
# Set here the path that the checkpoints will be saved. Default: ./run/training/
|
| 18 |
+
OUT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "run", "training")
|
| 19 |
+
|
| 20 |
+
# Training Parameters
|
| 21 |
+
OPTIMIZER_WD_ONLY_ON_WEIGHTS = True # for multi-gpu training please make it False
|
| 22 |
+
START_WITH_EVAL = True # if True it will star with evaluation
|
| 23 |
+
BATCH_SIZE = 3 # set here the batch size
|
| 24 |
+
GRAD_ACUMM_STEPS = 84 # set here the grad accumulation steps
|
| 25 |
+
# Note: we recommend that BATCH_SIZE * GRAD_ACUMM_STEPS need to be at least 252 for more efficient training. You can increase/decrease BATCH_SIZE but then set GRAD_ACUMM_STEPS accordingly.
|
| 26 |
+
|
| 27 |
+
# Define here the dataset(s) that you want to use for the fine-tuning.
|
| 28 |
+
DATASETS_CONFIG_LIST = [
|
| 29 |
+
BaseDatasetConfig(
|
| 30 |
+
formatter="ljspeech",
|
| 31 |
+
dataset_name="ljspeech",
|
| 32 |
+
path="/kaggle/working/ljspeech_fmt/",
|
| 33 |
+
meta_file_train="/kaggle/working/ljspeech_fmt/metadata.csv",
|
| 34 |
+
language="en",
|
| 35 |
+
)
|
| 36 |
+
]
|
| 37 |
+
|
| 38 |
+
# Define the path where XTTS v2.0.1 files will be downloaded
|
| 39 |
+
CHECKPOINTS_OUT_PATH = os.path.join(OUT_PATH, "XTTS_v2.0_original_model_files/")
|
| 40 |
+
os.makedirs(CHECKPOINTS_OUT_PATH, exist_ok=True)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
# DVAE files
|
| 44 |
+
DVAE_CHECKPOINT_LINK = "https://huggingface.co/coqui/XTTS-v2/resolve/main/dvae.pth"
|
| 45 |
+
MEL_NORM_LINK = "https://huggingface.co/coqui/XTTS-v2/resolve/main/mel_stats.pth"
|
| 46 |
+
|
| 47 |
+
# Set the path to the downloaded files
|
| 48 |
+
DVAE_CHECKPOINT = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(DVAE_CHECKPOINT_LINK))
|
| 49 |
+
MEL_NORM_FILE = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(MEL_NORM_LINK))
|
| 50 |
+
|
| 51 |
+
# download DVAE files if needed
|
| 52 |
+
if not os.path.isfile(DVAE_CHECKPOINT) or not os.path.isfile(MEL_NORM_FILE):
|
| 53 |
+
print(" > Downloading DVAE files!")
|
| 54 |
+
ModelManager._download_model_files([MEL_NORM_LINK, DVAE_CHECKPOINT_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
# Download XTTS v2.0 checkpoint if needed
|
| 58 |
+
TOKENIZER_FILE_LINK = "https://huggingface.co/coqui/XTTS-v2/resolve/main/vocab.json"
|
| 59 |
+
XTTS_CHECKPOINT_LINK = "https://huggingface.co/coqui/XTTS-v2/resolve/main/model.pth"
|
| 60 |
+
|
| 61 |
+
# XTTS transfer learning parameters: You we need to provide the paths of XTTS model checkpoint that you want to do the fine tuning.
|
| 62 |
+
TOKENIZER_FILE = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(TOKENIZER_FILE_LINK)) # vocab.json file
|
| 63 |
+
XTTS_CHECKPOINT = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(XTTS_CHECKPOINT_LINK)) # model.pth file
|
| 64 |
+
|
| 65 |
+
# download XTTS v2.0 files if needed
|
| 66 |
+
if not os.path.isfile(TOKENIZER_FILE) or not os.path.isfile(XTTS_CHECKPOINT):
|
| 67 |
+
print(" > Downloading XTTS v2.0 files!")
|
| 68 |
+
ModelManager._download_model_files(
|
| 69 |
+
[TOKENIZER_FILE_LINK, XTTS_CHECKPOINT_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
# Training sentences generations
|
| 74 |
+
SPEAKER_REFERENCE = [
|
| 75 |
+
"./tests/data/ljspeech/wavs/LJ001-0002.wav" # speaker reference to be used in training test sentences
|
| 76 |
+
]
|
| 77 |
+
LANGUAGE = DATASETS_CONFIG_LIST[0].language
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def main():
|
| 81 |
+
# init args and config
|
| 82 |
+
model_args = GPTArgs(
|
| 83 |
+
max_conditioning_length=132300, # 6 secs
|
| 84 |
+
min_conditioning_length=66150, # 3 secs
|
| 85 |
+
debug_loading_failures=False,
|
| 86 |
+
max_wav_length=255995, # ~11.6 seconds
|
| 87 |
+
max_text_length=200,
|
| 88 |
+
mel_norm_file=MEL_NORM_FILE,
|
| 89 |
+
dvae_checkpoint=DVAE_CHECKPOINT,
|
| 90 |
+
xtts_checkpoint=XTTS_CHECKPOINT, # checkpoint path of the model that you want to fine-tune
|
| 91 |
+
tokenizer_file=TOKENIZER_FILE,
|
| 92 |
+
gpt_num_audio_tokens=1026,
|
| 93 |
+
gpt_start_audio_token=1024,
|
| 94 |
+
gpt_stop_audio_token=1025,
|
| 95 |
+
gpt_use_masking_gt_prompt_approach=True,
|
| 96 |
+
gpt_use_perceiver_resampler=True,
|
| 97 |
+
)
|
| 98 |
+
# define audio config
|
| 99 |
+
audio_config = XttsAudioConfig(sample_rate=22050, dvae_sample_rate=22050, output_sample_rate=24000)
|
| 100 |
+
# training parameters config
|
| 101 |
+
config = GPTTrainerConfig(
|
| 102 |
+
epochs=2,
|
| 103 |
+
output_path=OUT_PATH,
|
| 104 |
+
model_args=model_args,
|
| 105 |
+
run_name=RUN_NAME,
|
| 106 |
+
project_name=PROJECT_NAME,
|
| 107 |
+
run_description="""
|
| 108 |
+
GPT XTTS training
|
| 109 |
+
""",
|
| 110 |
+
datasets=DATASETS_CONFIG_LIST,
|
| 111 |
+
dashboard_logger=DASHBOARD_LOGGER,
|
| 112 |
+
logger_uri=LOGGER_URI,
|
| 113 |
+
audio=audio_config,
|
| 114 |
+
batch_size=BATCH_SIZE,
|
| 115 |
+
batch_group_size=48,
|
| 116 |
+
eval_batch_size=BATCH_SIZE,
|
| 117 |
+
num_loader_workers=8,
|
| 118 |
+
eval_split_max_size=256,
|
| 119 |
+
print_step=50,
|
| 120 |
+
plot_step=100,
|
| 121 |
+
log_model_step=1000,
|
| 122 |
+
save_step=10000,
|
| 123 |
+
save_n_checkpoints=1,
|
| 124 |
+
save_checkpoints=True,
|
| 125 |
+
# target_loss="loss",
|
| 126 |
+
print_eval=False,
|
| 127 |
+
# Optimizer values like tortoise, pytorch implementation with modifications to not apply WD to non-weight parameters.
|
| 128 |
+
optimizer="AdamW",
|
| 129 |
+
optimizer_wd_only_on_weights=OPTIMIZER_WD_ONLY_ON_WEIGHTS,
|
| 130 |
+
optimizer_params={"betas": [0.9, 0.96], "eps": 1e-8, "weight_decay": 1e-2},
|
| 131 |
+
lr=5e-06, # learning rate
|
| 132 |
+
lr_scheduler="MultiStepLR",
|
| 133 |
+
# it was adjusted accordly for the new step scheme
|
| 134 |
+
lr_scheduler_params={"milestones": [50000 * 18, 150000 * 18, 300000 * 18], "gamma": 0.5, "last_epoch": -1},
|
| 135 |
+
test_sentences=[
|
| 136 |
+
{
|
| 137 |
+
"text": "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.",
|
| 138 |
+
"speaker_wav": SPEAKER_REFERENCE,
|
| 139 |
+
"language": LANGUAGE,
|
| 140 |
+
},
|
| 141 |
+
{
|
| 142 |
+
"text": "This cake is great. It's so delicious and moist.",
|
| 143 |
+
"speaker_wav": SPEAKER_REFERENCE,
|
| 144 |
+
"language": LANGUAGE,
|
| 145 |
+
},
|
| 146 |
+
],
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
# init the model from config
|
| 150 |
+
model = GPTTrainer(config)
|
| 151 |
+
|
| 152 |
+
# load training samples
|
| 153 |
+
train_samples, eval_samples = load_tts_samples(
|
| 154 |
+
config,
|
| 155 |
+
eval_split=True,
|
| 156 |
+
eval_split_max_size=config.eval_split_max_size,
|
| 157 |
+
eval_split_size=config.eval_split_size,
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
# init the trainer and 🚀
|
| 161 |
+
trainer = Trainer(
|
| 162 |
+
TrainerArgs(
|
| 163 |
+
restore_path=None, # xtts checkpoint is restored via xtts_checkpoint key so no need of restore it using Trainer restore_path parameter
|
| 164 |
+
skip_train_epoch=False,
|
| 165 |
+
start_with_eval=START_WITH_EVAL,
|
| 166 |
+
grad_accum_steps=GRAD_ACUMM_STEPS,
|
| 167 |
+
),
|
| 168 |
+
config,
|
| 169 |
+
output_path=OUT_PATH,
|
| 170 |
+
model=model,
|
| 171 |
+
train_samples=train_samples,
|
| 172 |
+
eval_samples=eval_samples,
|
| 173 |
+
)
|
| 174 |
+
trainer.fit()
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
if __name__ == "__main__":
|
| 178 |
+
main()
|
finetuned/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5/trainer_0_log.txt
ADDED
|
@@ -0,0 +1,828 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
> Training Environment:
|
| 2 |
+
| > Backend: Torch
|
| 3 |
+
| > Mixed precision: False
|
| 4 |
+
| > Precision: float32
|
| 5 |
+
| > Current device: 0
|
| 6 |
+
| > Num. of GPUs: 1
|
| 7 |
+
| > Num. of CPUs: 4
|
| 8 |
+
| > Num. of Torch Threads: 1
|
| 9 |
+
| > Torch seed: 1
|
| 10 |
+
| > Torch CUDNN: True
|
| 11 |
+
| > Torch CUDNN deterministic: False
|
| 12 |
+
| > Torch CUDNN benchmark: False
|
| 13 |
+
| > Torch TF32 MatMul: False
|
| 14 |
+
> Start Tensorboard: tensorboard --logdir=/kaggle/working/coqui-ai-TTS/run/training/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5
|
| 15 |
+
|
| 16 |
+
> Model has 518442047 parameters
|
| 17 |
+
|
| 18 |
+
[4m[1m > EPOCH: 0/1[0m
|
| 19 |
+
--> /kaggle/working/coqui-ai-TTS/run/training/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5
|
| 20 |
+
|
| 21 |
+
[1m > EVALUATION [0m
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
[1m--> EVAL PERFORMANCE[0m
|
| 25 |
+
| > avg_loader_time: 0.0775634299877078 [0m(+0.0)
|
| 26 |
+
| > avg_loss_text_ce: 0.022715861107720885 [0m(+0.0)
|
| 27 |
+
| > avg_loss_mel_ce: 3.651390269745228 [0m(+0.0)
|
| 28 |
+
| > avg_loss: 3.6741061155186143 [0m(+0.0)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
[4m[1m > EPOCH: 1/1[0m
|
| 32 |
+
--> /kaggle/working/coqui-ai-TTS/run/training/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5
|
| 33 |
+
|
| 34 |
+
[1m > TRAINING (2026-10-21 01:55:49) [0m
|
| 35 |
+
|
| 36 |
+
[1m --> TIME: 2026-10-21 01:55:54 -- STEP: 0/4322 -- GLOBAL_STEP: 0[0m
|
| 37 |
+
| > current_lr: 5e-06 (5e-06)
|
| 38 |
+
| > loss_text_ce: 0.023607278242707253 (0.023607278242707253)
|
| 39 |
+
| > loss_mel_ce: 3.562453269958496 (3.562453269958496)
|
| 40 |
+
| > loss: 0.042691197246313095 (0.042691197246313095)
|
| 41 |
+
| > step_time: 1.0577 (1.0576918125152588)
|
| 42 |
+
| > loader_time: 3.7638 (3.763838768005371)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
[1m --> TIME: 2026-10-21 01:56:41 -- STEP: 50/4322 -- GLOBAL_STEP: 50[0m
|
| 46 |
+
| > current_lr: 5e-06 (5.0000000000000055e-06)
|
| 47 |
+
| > loss_text_ce: 0.023330125957727432 (0.023871341645717623)
|
| 48 |
+
| > loss_mel_ce: 3.461495876312256 (3.6548802995681764)
|
| 49 |
+
| > loss: 0.04148602485656738 (0.0437946629524231)
|
| 50 |
+
| > step_time: 0.6104 (0.6138731288909911)
|
| 51 |
+
| > loader_time: 0.0148 (0.015347743034362793)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
[1m --> TIME: 2026-10-21 01:57:27 -- STEP: 100/4322 -- GLOBAL_STEP: 100[0m
|
| 55 |
+
| > current_lr: 5e-06 (5.000000000000009e-06)
|
| 56 |
+
| > loss_text_ce: 0.022796062752604485 (0.023814733307808637)
|
| 57 |
+
| > loss_mel_ce: 3.5732829570770264 (3.6294152784347533)
|
| 58 |
+
| > loss: 0.04281046614050865 (0.043490834161639215)
|
| 59 |
+
| > step_time: 0.5825 (0.6164441776275634)
|
| 60 |
+
| > loader_time: 0.0171 (0.014822771549224853)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
[1m --> TIME: 2026-10-21 01:58:13 -- STEP: 150/4322 -- GLOBAL_STEP: 150[0m
|
| 64 |
+
| > current_lr: 5e-06 (5.00000000000001e-06)
|
| 65 |
+
| > loss_text_ce: 0.022212712094187737 (0.02389630557348331)
|
| 66 |
+
| > loss_mel_ce: 3.4683268070220947 (3.5989029963811237)
|
| 67 |
+
| > loss: 0.0415540412068367 (0.04312856381138167)
|
| 68 |
+
| > step_time: 0.5846 (0.6149810393651326)
|
| 69 |
+
| > loader_time: 0.0157 (0.01448255221048991)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
[1m --> TIME: 2026-10-21 01:58:58 -- STEP: 200/4322 -- GLOBAL_STEP: 200[0m
|
| 73 |
+
| > current_lr: 5e-06 (5.00000000000001e-06)
|
| 74 |
+
| > loss_text_ce: 0.023157035931944847 (0.02389827018603683)
|
| 75 |
+
| > loss_mel_ce: 3.4178578853607178 (3.581750749349594)
|
| 76 |
+
| > loss: 0.04096446558833122 (0.04292439378798008)
|
| 77 |
+
| > step_time: 0.6188 (0.6102964365482325)
|
| 78 |
+
| > loader_time: 0.0123 (0.014304686784744263)
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
[1m --> TIME: 2026-10-21 01:59:41 -- STEP: 250/4322 -- GLOBAL_STEP: 250[0m
|
| 82 |
+
| > current_lr: 5e-06 (5.0000000000000106e-06)
|
| 83 |
+
| > loss_text_ce: 0.0207450482994318 (0.023818049624562267)
|
| 84 |
+
| > loss_mel_ce: 3.610269784927368 (3.5579456262588502)
|
| 85 |
+
| > loss: 0.043226368725299835 (0.042640044435858726)
|
| 86 |
+
| > step_time: 0.4397 (0.6008358507156367)
|
| 87 |
+
| > loader_time: 0.0174 (0.014198965072631835)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
[1m --> TIME: 2026-10-21 02:00:26 -- STEP: 300/4322 -- GLOBAL_STEP: 300[0m
|
| 91 |
+
| > current_lr: 5e-06 (5.0000000000000106e-06)
|
| 92 |
+
| > loss_text_ce: 0.02244749292731285 (0.02378187150383989)
|
| 93 |
+
| > loss_mel_ce: 3.2661588191986084 (3.5324360863367708)
|
| 94 |
+
| > loss: 0.03915007784962654 (0.04233592879027127)
|
| 95 |
+
| > step_time: 0.7449 (0.6004247697194415)
|
| 96 |
+
| > loader_time: 0.0105 (0.01390298843383789)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
[1m --> TIME: 2026-10-21 02:01:11 -- STEP: 350/4322 -- GLOBAL_STEP: 350[0m
|
| 100 |
+
| > current_lr: 5e-06 (5.0000000000000106e-06)
|
| 101 |
+
| > loss_text_ce: 0.023772336542606354 (0.023702249436506194)
|
| 102 |
+
| > loss_mel_ce: 3.5447685718536377 (3.518683438301086)
|
| 103 |
+
| > loss: 0.04248262941837311 (0.04217125889446053)
|
| 104 |
+
| > step_time: 0.671 (0.5991947923387797)
|
| 105 |
+
| > loader_time: 0.0131 (0.013757729530334472)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
[1m --> TIME: 2026-10-21 02:01:55 -- STEP: 400/4322 -- GLOBAL_STEP: 400[0m
|
| 109 |
+
| > current_lr: 5e-06 (5.0000000000000055e-06)
|
| 110 |
+
| > loss_text_ce: 0.024473151192069054 (0.023693136773072176)
|
| 111 |
+
| > loss_mel_ce: 3.1853315830230713 (3.49852819263935)
|
| 112 |
+
| > loss: 0.03821196407079697 (0.041931206975132224)
|
| 113 |
+
| > step_time: 0.6656 (0.5977507787942884)
|
| 114 |
+
| > loader_time: 0.014 (0.013656079173088073)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
[1m --> TIME: 2026-10-21 02:02:41 -- STEP: 450/4322 -- GLOBAL_STEP: 450[0m
|
| 118 |
+
| > current_lr: 5e-06 (4.999999999999983e-06)
|
| 119 |
+
| > loss_text_ce: 0.023743927478790283 (0.023701659581727444)
|
| 120 |
+
| > loss_mel_ce: 3.377760171890259 (3.4801506688859725)
|
| 121 |
+
| > loss: 0.04049409553408623 (0.041712528417507805)
|
| 122 |
+
| > step_time: 0.4654 (0.5993869839774233)
|
| 123 |
+
| > loader_time: 0.0092 (0.013543645540873209)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
[1m --> TIME: 2026-10-21 02:03:26 -- STEP: 500/4322 -- GLOBAL_STEP: 500[0m
|
| 127 |
+
| > current_lr: 5e-06 (4.999999999999964e-06)
|
| 128 |
+
| > loss_text_ce: 0.028117351233959198 (0.023697244830429547)
|
| 129 |
+
| > loss_mel_ce: 3.3383777141571045 (3.4620542507171628)
|
| 130 |
+
| > loss: 0.0400773249566555 (0.04149704233556986)
|
| 131 |
+
| > step_time: 0.4946 (0.5986857099533076)
|
| 132 |
+
| > loader_time: 0.0118 (0.013421703338623046)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
[1m --> TIME: 2026-10-21 02:04:12 -- STEP: 550/4322 -- GLOBAL_STEP: 550[0m
|
| 136 |
+
| > current_lr: 5e-06 (4.99999999999995e-06)
|
| 137 |
+
| > loss_text_ce: 0.024159494787454605 (0.023733612895011892)
|
| 138 |
+
| > loss_mel_ce: 3.1997478008270264 (3.445531345280734)
|
| 139 |
+
| > loss: 0.03837984800338745 (0.04130077404054731)
|
| 140 |
+
| > step_time: 0.6202 (0.5997316876324739)
|
| 141 |
+
| > loader_time: 0.0123 (0.013335978768088603)
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
[1m --> TIME: 2026-10-21 02:04:58 -- STEP: 600/4322 -- GLOBAL_STEP: 600[0m
|
| 145 |
+
| > current_lr: 5e-06 (4.9999999999999335e-06)
|
| 146 |
+
| > loss_text_ce: 0.02490082010626793 (0.02375749908698101)
|
| 147 |
+
| > loss_mel_ce: 3.2087249755859375 (3.4295414439837137)
|
| 148 |
+
| > loss: 0.03849554806947708 (0.041110702430208555)
|
| 149 |
+
| > step_time: 0.6296 (0.6005522024631497)
|
| 150 |
+
| > loader_time: 0.0105 (0.013264158566792805)
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
[1m --> TIME: 2026-10-21 02:05:43 -- STEP: 650/4322 -- GLOBAL_STEP: 650[0m
|
| 154 |
+
| > current_lr: 5e-06 (4.999999999999923e-06)
|
| 155 |
+
| > loss_text_ce: 0.02134285867214203 (0.023758635317476892)
|
| 156 |
+
| > loss_mel_ce: 3.2015085220336914 (3.4134296200825616)
|
| 157 |
+
| > loss: 0.03836727887392044 (0.04091890854904286)
|
| 158 |
+
| > step_time: 0.5952 (0.6004456068919255)
|
| 159 |
+
| > loader_time: 0.011 (0.013190783720750075)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
[1m --> TIME: 2026-10-21 02:06:29 -- STEP: 700/4322 -- GLOBAL_STEP: 700[0m
|
| 163 |
+
| > current_lr: 5e-06 (4.999999999999907e-06)
|
| 164 |
+
| > loss_text_ce: 0.020476728677749634 (0.023712386890713647)
|
| 165 |
+
| > loss_mel_ce: 3.1137752532958984 (3.398600721018655)
|
| 166 |
+
| > loss: 0.037312526255846024 (0.04074182349124122)
|
| 167 |
+
| > step_time: 0.5622 (0.6007283912386213)
|
| 168 |
+
| > loader_time: 0.008 (0.013160109519958496)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
[1m --> TIME: 2026-10-21 02:07:14 -- STEP: 750/4322 -- GLOBAL_STEP: 750[0m
|
| 172 |
+
| > current_lr: 5e-06 (4.999999999999906e-06)
|
| 173 |
+
| > loss_text_ce: 0.025615012273192406 (0.02372802059104044)
|
| 174 |
+
| > loss_mel_ce: 3.301236152648926 (3.3874267937342326)
|
| 175 |
+
| > loss: 0.03960537165403366 (0.040608986645936915)
|
| 176 |
+
| > step_time: 0.5906 (0.6008401826222738)
|
| 177 |
+
| > loader_time: 0.0138 (0.013096263885498047)
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
[1m --> TIME: 2026-10-21 02:07:59 -- STEP: 800/4322 -- GLOBAL_STEP: 800[0m
|
| 181 |
+
| > current_lr: 5e-06 (4.999999999999904e-06)
|
| 182 |
+
| > loss_text_ce: 0.022030288353562355 (0.02372636462096124)
|
| 183 |
+
| > loss_mel_ce: 3.0623843669891357 (3.3740680557489395)
|
| 184 |
+
| > loss: 0.036719225347042084 (0.04044993432238694)
|
| 185 |
+
| > step_time: 0.6571 (0.5999964663386342)
|
| 186 |
+
| > loader_time: 0.0125 (0.013091897070407867)
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
[1m --> TIME: 2026-10-21 02:08:44 -- STEP: 850/4322 -- GLOBAL_STEP: 850[0m
|
| 190 |
+
| > current_lr: 5e-06 (4.999999999999897e-06)
|
| 191 |
+
| > loss_text_ce: 0.02976416051387787 (0.023754417444853213)
|
| 192 |
+
| > loss_mel_ce: 3.136929988861084 (3.3624984926335952)
|
| 193 |
+
| > loss: 0.037698742002248764 (0.04031253539902319)
|
| 194 |
+
| > step_time: 0.4763 (0.6000622721279365)
|
| 195 |
+
| > loader_time: 0.0117 (0.013060874097487506)
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
[1m --> TIME: 2026-10-21 02:09:29 -- STEP: 900/4322 -- GLOBAL_STEP: 900[0m
|
| 199 |
+
| > current_lr: 5e-06 (4.999999999999892e-06)
|
| 200 |
+
| > loss_text_ce: 0.024680709466338158 (0.023769160101397162)
|
| 201 |
+
| > loss_mel_ce: 3.0224132537841797 (3.3509844075308908)
|
| 202 |
+
| > loss: 0.036274928599596024 (0.04017563847617967)
|
| 203 |
+
| > step_time: 0.6022 (0.6000583444701298)
|
| 204 |
+
| > loader_time: 0.0069 (0.012994143433041042)
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
[1m --> TIME: 2026-10-21 02:10:15 -- STEP: 950/4322 -- GLOBAL_STEP: 950[0m
|
| 208 |
+
| > current_lr: 5e-06 (4.999999999999887e-06)
|
| 209 |
+
| > loss_text_ce: 0.02795293740928173 (0.02378540836470691)
|
| 210 |
+
| > loss_mel_ce: 2.8190839290618896 (3.340535431912071)
|
| 211 |
+
| > loss: 0.033893294632434845 (0.04005143933782449)
|
| 212 |
+
| > step_time: 0.5454 (0.6004340179342971)
|
| 213 |
+
| > loader_time: 0.0107 (0.01297067165374756)
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
[1m --> TIME: 2026-10-21 02:10:59 -- STEP: 1000/4322 -- GLOBAL_STEP: 1000[0m
|
| 217 |
+
| > current_lr: 5e-06 (4.999999999999882e-06)
|
| 218 |
+
| > loss_text_ce: 0.020246483385562897 (0.02378400870971381)
|
| 219 |
+
| > loss_mel_ce: 3.172618865966797 (3.329640386581421)
|
| 220 |
+
| > loss: 0.03801030293107033 (0.039921719752252074)
|
| 221 |
+
| > step_time: 0.5444 (0.5995927650928495)
|
| 222 |
+
| > loader_time: 0.0122 (0.012936718463897705)
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
[1m --> TIME: 2026-10-21 02:11:45 -- STEP: 1050/4322 -- GLOBAL_STEP: 1050[0m
|
| 226 |
+
| > current_lr: 5e-06 (4.9999999999998784e-06)
|
| 227 |
+
| > loss_text_ce: 0.023922035470604897 (0.023778165232922345)
|
| 228 |
+
| > loss_mel_ce: 3.2467410564422607 (3.32012759799049)
|
| 229 |
+
| > loss: 0.038936465978622437 (0.039808402703631465)
|
| 230 |
+
| > step_time: 0.7345 (0.599874627703712)
|
| 231 |
+
| > loader_time: 0.0125 (0.012911377180190314)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
[1m --> TIME: 2026-10-21 02:12:28 -- STEP: 1100/4322 -- GLOBAL_STEP: 1100[0m
|
| 235 |
+
| > current_lr: 5e-06 (4.9999999999998725e-06)
|
| 236 |
+
| > loss_text_ce: 0.022987011820077896 (0.02377308981831778)
|
| 237 |
+
| > loss_mel_ce: 3.168280839920044 (3.3113870924169366)
|
| 238 |
+
| > loss: 0.03799128532409668 (0.03970428864048283)
|
| 239 |
+
| > step_time: 0.6301 (0.5985342764854431)
|
| 240 |
+
| > loader_time: 0.0122 (0.012879582968625155)
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
[1m --> TIME: 2026-10-21 02:13:14 -- STEP: 1150/4322 -- GLOBAL_STEP: 1150[0m
|
| 244 |
+
| > current_lr: 5e-06 (4.999999999999866e-06)
|
| 245 |
+
| > loss_text_ce: 0.024523338302969933 (0.023767024406596372)
|
| 246 |
+
| > loss_mel_ce: 3.2279915809631348 (3.3026936597409446)
|
| 247 |
+
| > loss: 0.03872041404247284 (0.039600723184969086)
|
| 248 |
+
| > step_time: 0.6668 (0.5987930737371031)
|
| 249 |
+
| > loader_time: 0.0128 (0.01285959927932076)
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
[1m --> TIME: 2026-10-21 02:13:58 -- STEP: 1200/4322 -- GLOBAL_STEP: 1200[0m
|
| 253 |
+
| > current_lr: 5e-06 (4.999999999999865e-06)
|
| 254 |
+
| > loss_text_ce: 0.024073252454400063 (0.023750210852983098)
|
| 255 |
+
| > loss_mel_ce: 3.2074012756347656 (3.294540059367815)
|
| 256 |
+
| > loss: 0.038469936698675156 (0.03950345635414121)
|
| 257 |
+
| > step_time: 0.6267 (0.5983327811956408)
|
| 258 |
+
| > loader_time: 0.0086 (0.012834071715672815)
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
[1m --> TIME: 2026-10-21 02:14:43 -- STEP: 1250/4322 -- GLOBAL_STEP: 1250[0m
|
| 262 |
+
| > current_lr: 5e-06 (4.999999999999865e-06)
|
| 263 |
+
| > loss_text_ce: 0.02361719310283661 (0.023736771930754175)
|
| 264 |
+
| > loss_mel_ce: 3.1402785778045654 (3.2860619565963747)
|
| 265 |
+
| > loss: 0.03766542673110962 (0.039402366572618454)
|
| 266 |
+
| > step_time: 0.6379 (0.5983164710998535)
|
| 267 |
+
| > loader_time: 0.0123 (0.012797017288208003)
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
[1m --> TIME: 2026-10-21 02:15:28 -- STEP: 1300/4322 -- GLOBAL_STEP: 1300[0m
|
| 271 |
+
| > current_lr: 5e-06 (4.999999999999865e-06)
|
| 272 |
+
| > loss_text_ce: 0.02002999745309353 (0.023741446633178443)
|
| 273 |
+
| > loss_mel_ce: 2.91009783744812 (3.276316642027635)
|
| 274 |
+
| > loss: 0.034882474690675735 (0.03928640657606032)
|
| 275 |
+
| > step_time: 0.5945 (0.5977295703154349)
|
| 276 |
+
| > loader_time: 0.0122 (0.01277180983470036)
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
[1m --> TIME: 2026-10-21 02:16:12 -- STEP: 1350/4322 -- GLOBAL_STEP: 1350[0m
|
| 280 |
+
| > current_lr: 5e-06 (4.999999999999865e-06)
|
| 281 |
+
| > loss_text_ce: 0.021801773458719254 (0.023740276019606304)
|
| 282 |
+
| > loss_mel_ce: 3.1130032539367676 (3.268503632898684)
|
| 283 |
+
| > loss: 0.037319108843803406 (0.039193380621296375)
|
| 284 |
+
| > step_time: 0.6306 (0.5971331243161812)
|
| 285 |
+
| > loader_time: 0.0121 (0.012747642022592045)
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
[1m --> TIME: 2026-10-21 02:16:57 -- STEP: 1400/4322 -- GLOBAL_STEP: 1400[0m
|
| 289 |
+
| > current_lr: 5e-06 (4.999999999999865e-06)
|
| 290 |
+
| > loss_text_ce: 0.024811046198010445 (0.02374142150793755)
|
| 291 |
+
| > loss_mel_ce: 3.0756020545959473 (3.2625486939294)
|
| 292 |
+
| > loss: 0.03690968081355095 (0.03912250211728469)
|
| 293 |
+
| > step_time: 0.5325 (0.597321861812047)
|
| 294 |
+
| > loader_time: 0.0066 (0.012735525199345175)
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
[1m --> TIME: 2026-10-21 02:17:43 -- STEP: 1450/4322 -- GLOBAL_STEP: 1450[0m
|
| 298 |
+
| > current_lr: 5e-06 (4.999999999999865e-06)
|
| 299 |
+
| > loss_text_ce: 0.022022727876901627 (0.023734815263285698)
|
| 300 |
+
| > loss_mel_ce: 3.0583367347717285 (3.256251094916771)
|
| 301 |
+
| > loss: 0.03667094558477402 (0.03904745205209174)
|
| 302 |
+
| > step_time: 0.6574 (0.5975845338558333)
|
| 303 |
+
| > loader_time: 0.0164 (0.012712322596845951)
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
[1m --> TIME: 2026-10-21 02:18:28 -- STEP: 1500/4322 -- GLOBAL_STEP: 1500[0m
|
| 307 |
+
| > current_lr: 5e-06 (4.999999999999865e-06)
|
| 308 |
+
| > loss_text_ce: 0.02297208644449711 (0.02374113366380332)
|
| 309 |
+
| > loss_mel_ce: 3.050678014755249 (3.2495009508132933)
|
| 310 |
+
| > loss: 0.036591075360774994 (0.038967168403168495)
|
| 311 |
+
| > step_time: 0.6333 (0.5978817100524908)
|
| 312 |
+
| > loader_time: 0.0125 (0.01268777195612589)
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
[1m --> TIME: 2026-10-21 02:19:13 -- STEP: 1550/4322 -- GLOBAL_STEP: 1550[0m
|
| 316 |
+
| > current_lr: 5e-06 (4.999999999999865e-06)
|
| 317 |
+
| > loss_text_ce: 0.02353771962225437 (0.023733780615993065)
|
| 318 |
+
| > loss_mel_ce: 2.956113815307617 (3.2425041924753497)
|
| 319 |
+
| > loss: 0.035472042858600616 (0.038883786131778085)
|
| 320 |
+
| > step_time: 0.581 (0.5976957441145371)
|
| 321 |
+
| > loader_time: 0.0111 (0.012671740593448758)
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
[1m --> TIME: 2026-10-21 02:19:57 -- STEP: 1600/4322 -- GLOBAL_STEP: 1600[0m
|
| 325 |
+
| > current_lr: 5e-06 (4.999999999999863e-06)
|
| 326 |
+
| > loss_text_ce: 0.02201055735349655 (0.02372139338636769)
|
| 327 |
+
| > loss_mel_ce: 3.0538761615753174 (3.2370670783519744)
|
| 328 |
+
| > loss: 0.0366177000105381 (0.03881891111144796)
|
| 329 |
+
| > step_time: 0.6707 (0.5972625337541095)
|
| 330 |
+
| > loader_time: 0.0087 (0.012655767202377315)
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
[1m --> TIME: 2026-10-21 02:20:43 -- STEP: 1650/4322 -- GLOBAL_STEP: 1650[0m
|
| 334 |
+
| > current_lr: 5e-06 (4.9999999999998615e-06)
|
| 335 |
+
| > loss_text_ce: 0.02265886962413788 (0.023707734198507018)
|
| 336 |
+
| > loss_mel_ce: 3.0409398078918457 (3.2308994569200458)
|
| 337 |
+
| > loss: 0.0364714115858078 (0.03874532444007469)
|
| 338 |
+
| > step_time: 0.7344 (0.5974271405826904)
|
| 339 |
+
| > loader_time: 0.017 (0.012655190410036024)
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
[1m --> TIME: 2026-10-21 02:21:28 -- STEP: 1700/4322 -- GLOBAL_STEP: 1700[0m
|
| 343 |
+
| > current_lr: 5e-06 (4.99999999999986e-06)
|
| 344 |
+
| > loss_text_ce: 0.023584647104144096 (0.02371008093304492)
|
| 345 |
+
| > loss_mel_ce: 2.9107162952423096 (3.224411839877858)
|
| 346 |
+
| > loss: 0.034932155162096024 (0.03866811884676709)
|
| 347 |
+
| > step_time: 0.622 (0.5972898818464829)
|
| 348 |
+
| > loader_time: 0.0125 (0.012630745523116164)
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
[1m --> TIME: 2026-10-21 02:22:12 -- STEP: 1750/4322 -- GLOBAL_STEP: 1750[0m
|
| 352 |
+
| > current_lr: 5e-06 (4.999999999999858e-06)
|
| 353 |
+
| > loss_text_ce: 0.020321423187851906 (0.023709888081465434)
|
| 354 |
+
| > loss_mel_ce: 2.746750593185425 (3.218360703468323)
|
| 355 |
+
| > loss: 0.03294133394956589 (0.03859607922179358)
|
| 356 |
+
| > step_time: 0.5363 (0.5970047991616375)
|
| 357 |
+
| > loader_time: 0.0083 (0.012610953739711212)
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
[1m --> TIME: 2026-10-21 02:22:57 -- STEP: 1800/4322 -- GLOBAL_STEP: 1800[0m
|
| 361 |
+
| > current_lr: 5e-06 (4.999999999999856e-06)
|
| 362 |
+
| > loss_text_ce: 0.02185886725783348 (0.023704143172750856)
|
| 363 |
+
| > loss_mel_ce: 3.109938859939575 (3.2124653157922958)
|
| 364 |
+
| > loss: 0.03728330880403519 (0.03852582764294413)
|
| 365 |
+
| > step_time: 0.561 (0.5969216158654944)
|
| 366 |
+
| > loader_time: 0.0111 (0.012595439487033417)
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
[1m --> TIME: 2026-10-21 02:23:42 -- STEP: 1850/4322 -- GLOBAL_STEP: 1850[0m
|
| 370 |
+
| > current_lr: 5e-06 (4.999999999999855e-06)
|
| 371 |
+
| > loss_text_ce: 0.024742452427744865 (0.0236988801801124)
|
| 372 |
+
| > loss_mel_ce: 2.976585865020752 (3.207316894402375)
|
| 373 |
+
| > loss: 0.03573009744286537 (0.03846447426322344)
|
| 374 |
+
| > step_time: 0.6692 (0.5968153008899163)
|
| 375 |
+
| > loader_time: 0.0153 (0.012587847194156128)
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
[1m --> TIME: 2026-10-21 02:24:25 -- STEP: 1900/4322 -- GLOBAL_STEP: 1900[0m
|
| 379 |
+
| > current_lr: 5e-06 (4.999999999999853e-06)
|
| 380 |
+
| > loss_text_ce: 0.025433124974370003 (0.023695998475151612)
|
| 381 |
+
| > loss_mel_ce: 3.161982536315918 (3.202033802333631)
|
| 382 |
+
| > loss: 0.037945423275232315 (0.03840154600574782)
|
| 383 |
+
| > step_time: 0.579 (0.5961658215522756)
|
| 384 |
+
| > loader_time: 0.0085 (0.012579835339596394)
|
| 385 |
+
|
| 386 |
+
|
| 387 |
+
[1m --> TIME: 2026-10-21 02:25:11 -- STEP: 1950/4322 -- GLOBAL_STEP: 1950[0m
|
| 388 |
+
| > current_lr: 5e-06 (4.999999999999851e-06)
|
| 389 |
+
| > loss_text_ce: 0.019193362444639206 (0.023692078514932043)
|
| 390 |
+
| > loss_mel_ce: 3.1381239891052246 (3.1976696112217047)
|
| 391 |
+
| > loss: 0.03758711367845535 (0.03834954468867718)
|
| 392 |
+
| > step_time: 0.5367 (0.5963146784366696)
|
| 393 |
+
| > loader_time: 0.0108 (0.012574802789932638)
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
[1m --> TIME: 2026-10-21 02:25:56 -- STEP: 2000/4322 -- GLOBAL_STEP: 2000[0m
|
| 397 |
+
| > current_lr: 5e-06 (4.99999999999985e-06)
|
| 398 |
+
| > loss_text_ce: 0.020343022421002388 (0.023680220687761887)
|
| 399 |
+
| > loss_mel_ce: 3.0056772232055664 (3.192688380122185)
|
| 400 |
+
| > loss: 0.036024052649736404 (0.03829010315425694)
|
| 401 |
+
| > step_time: 0.6614 (0.5964905810356131)
|
| 402 |
+
| > loader_time: 0.0088 (0.012561839818954465)
|
| 403 |
+
|
| 404 |
+
|
| 405 |
+
[1m --> TIME: 2026-10-21 02:26:42 -- STEP: 2050/4322 -- GLOBAL_STEP: 2050[0m
|
| 406 |
+
| > current_lr: 5e-06 (4.999999999999849e-06)
|
| 407 |
+
| > loss_text_ce: 0.021688129752874374 (0.023675753405181358)
|
| 408 |
+
| > loss_mel_ce: 3.111064910888672 (3.1884225934889256)
|
| 409 |
+
| > loss: 0.03729468211531639 (0.0382392667979002)
|
| 410 |
+
| > step_time: 0.661 (0.5967303099283349)
|
| 411 |
+
| > loader_time: 0.0121 (0.012545026569831658)
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
[1m --> TIME: 2026-10-21 02:27:27 -- STEP: 2100/4322 -- GLOBAL_STEP: 2100[0m
|
| 415 |
+
| > current_lr: 5e-06 (4.999999999999846e-06)
|
| 416 |
+
| > loss_text_ce: 0.02272702194750309 (0.02366948241013145)
|
| 417 |
+
| > loss_mel_ce: 2.94507098197937 (3.183759974752154)
|
| 418 |
+
| > loss: 0.03533092886209488 (0.038183684774807526)
|
| 419 |
+
| > step_time: 0.5204 (0.5966567674137289)
|
| 420 |
+
| > loader_time: 0.0239 (0.012538471562521795)
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
[1m --> TIME: 2026-10-21 02:28:12 -- STEP: 2150/4322 -- GLOBAL_STEP: 2150[0m
|
| 424 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 425 |
+
| > loss_text_ce: 0.02649291418492794 (0.023676633604390665)
|
| 426 |
+
| > loss_mel_ce: 2.8881752490997314 (3.179549894554671)
|
| 427 |
+
| > loss: 0.03469843044877052 (0.03813364991268448)
|
| 428 |
+
| > step_time: 0.5882 (0.5967577403090712)
|
| 429 |
+
| > loader_time: 0.0124 (0.012527329311814417)
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
[1m --> TIME: 2026-10-21 02:28:57 -- STEP: 2200/4322 -- GLOBAL_STEP: 2200[0m
|
| 433 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 434 |
+
| > loss_text_ce: 0.022103603929281235 (0.023670796513218767)
|
| 435 |
+
| > loss_mel_ce: 3.0208725929260254 (3.1755333759567956)
|
| 436 |
+
| > loss: 0.03622590750455856 (0.03808576472103598)
|
| 437 |
+
| > step_time: 0.6283 (0.5968182917074716)
|
| 438 |
+
| > loader_time: 0.0073 (0.012522149952975183)
|
| 439 |
+
|
| 440 |
+
|
| 441 |
+
[1m --> TIME: 2026-10-21 02:29:42 -- STEP: 2250/4322 -- GLOBAL_STEP: 2250[0m
|
| 442 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 443 |
+
| > loss_text_ce: 0.024208255112171173 (0.02366303620818589)
|
| 444 |
+
| > loss_mel_ce: 2.8740413188934326 (3.1709457555347007)
|
| 445 |
+
| > loss: 0.03450297191739082 (0.0380310577998559)
|
| 446 |
+
| > step_time: 0.5892 (0.5967026240030915)
|
| 447 |
+
| > loader_time: 0.0084 (0.012517317348056365)
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
[1m --> TIME: 2026-10-21 02:30:27 -- STEP: 2300/4322 -- GLOBAL_STEP: 2300[0m
|
| 451 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 452 |
+
| > loss_text_ce: 0.023592893034219742 (0.02365464257157368)
|
| 453 |
+
| > loss_mel_ce: 3.097675085067749 (3.166035107011379)
|
| 454 |
+
| > loss: 0.03715795278549194 (0.03797249777809437)
|
| 455 |
+
| > step_time: 0.6679 (0.596503809534984)
|
| 456 |
+
| > loader_time: 0.0145 (0.012519893127938976)
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
[1m --> TIME: 2026-10-21 02:31:12 -- STEP: 2350/4322 -- GLOBAL_STEP: 2350[0m
|
| 460 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 461 |
+
| > loss_text_ce: 0.027811063453555107 (0.023658202513259792)
|
| 462 |
+
| > loss_mel_ce: 2.979349136352539 (3.1619461951357253)
|
| 463 |
+
| > loss: 0.035799525678157806 (0.037923862631333675)
|
| 464 |
+
| > step_time: 0.5904 (0.5964998499890579)
|
| 465 |
+
| > loader_time: 0.0116 (0.012518108854902555)
|
| 466 |
+
|
| 467 |
+
|
| 468 |
+
[1m --> TIME: 2026-10-21 02:31:57 -- STEP: 2400/4322 -- GLOBAL_STEP: 2400[0m
|
| 469 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 470 |
+
| > loss_text_ce: 0.024188783019781113 (0.02365141263386856)
|
| 471 |
+
| > loss_mel_ce: 2.928410053253174 (3.1577433361609764)
|
| 472 |
+
| > loss: 0.035149987787008286 (0.03787374775623908)
|
| 473 |
+
| > step_time: 0.6712 (0.5967107424139966)
|
| 474 |
+
| > loader_time: 0.0071 (0.01251181562741597)
|
| 475 |
+
|
| 476 |
+
|
| 477 |
+
[1m --> TIME: 2026-10-21 02:32:43 -- STEP: 2450/4322 -- GLOBAL_STEP: 2450[0m
|
| 478 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 479 |
+
| > loss_text_ce: 0.023117098957300186 (0.02364511824155947)
|
| 480 |
+
| > loss_mel_ce: 2.958191156387329 (3.1539475653122873)
|
| 481 |
+
| > loss: 0.035491764545440674 (0.0378284850785015)
|
| 482 |
+
| > step_time: 0.6193 (0.5967050911455725)
|
| 483 |
+
| > loader_time: 0.0085 (0.012508083654909713)
|
| 484 |
+
|
| 485 |
+
|
| 486 |
+
[1m --> TIME: 2026-10-21 02:33:27 -- STEP: 2500/4322 -- GLOBAL_STEP: 2500[0m
|
| 487 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 488 |
+
| > loss_text_ce: 0.02223431132733822 (0.02363246348947285)
|
| 489 |
+
| > loss_mel_ce: 3.126727342605591 (3.1503854570388774)
|
| 490 |
+
| > loss: 0.037487637251615524 (0.037785928382724564)
|
| 491 |
+
| > step_time: 0.6084 (0.5966647499084458)
|
| 492 |
+
| > loader_time: 0.0132 (0.012506871509552)
|
| 493 |
+
|
| 494 |
+
|
| 495 |
+
[1m --> TIME: 2026-10-21 02:34:13 -- STEP: 2550/4322 -- GLOBAL_STEP: 2550[0m
|
| 496 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 497 |
+
| > loss_text_ce: 0.02278938516974449 (0.02361871360198537)
|
| 498 |
+
| > loss_mel_ce: 2.8958399295806885 (3.1467342854481077)
|
| 499 |
+
| > loss: 0.03474558889865875 (0.03774229835864963)
|
| 500 |
+
| > step_time: 0.6697 (0.5966710258932658)
|
| 501 |
+
| > loader_time: 0.0125 (0.012501213503818885)
|
| 502 |
+
|
| 503 |
+
|
| 504 |
+
[1m --> TIME: 2026-10-21 02:34:57 -- STEP: 2600/4322 -- GLOBAL_STEP: 2600[0m
|
| 505 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 506 |
+
| > loss_text_ce: 0.02256777510046959 (0.02361473225057121)
|
| 507 |
+
| > loss_mel_ce: 2.9899072647094727 (3.1431978268806695)
|
| 508 |
+
| > loss: 0.03586279973387718 (0.03770015026299427)
|
| 509 |
+
| > step_time: 0.6698 (0.5966223816688224)
|
| 510 |
+
| > loader_time: 0.0102 (0.012498493744776799)
|
| 511 |
+
|
| 512 |
+
|
| 513 |
+
[1m --> TIME: 2026-10-21 02:35:43 -- STEP: 2650/4322 -- GLOBAL_STEP: 2650[0m
|
| 514 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 515 |
+
| > loss_text_ce: 0.02460002340376377 (0.02361912247426101)
|
| 516 |
+
| > loss_mel_ce: 3.0308732986450195 (3.1390183373217257)
|
| 517 |
+
| > loss: 0.03637468442320824 (0.03765044669708551)
|
| 518 |
+
| > step_time: 0.6612 (0.5969012598721465)
|
| 519 |
+
| > loader_time: 0.0138 (0.0124975423093112)
|
| 520 |
+
|
| 521 |
+
|
| 522 |
+
[1m --> TIME: 2026-10-21 02:36:28 -- STEP: 2700/4322 -- GLOBAL_STEP: 2700[0m
|
| 523 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 524 |
+
| > loss_text_ce: 0.026347188279032707 (0.023611818909369087)
|
| 525 |
+
| > loss_mel_ce: 2.926762819290161 (3.1353130488925456)
|
| 526 |
+
| > loss: 0.035156071186065674 (0.037606249166721534)
|
| 527 |
+
| > step_time: 0.5838 (0.5968237059204643)
|
| 528 |
+
| > loader_time: 0.0137 (0.012485156500781023)
|
| 529 |
+
|
| 530 |
+
|
| 531 |
+
[1m --> TIME: 2026-10-21 02:37:14 -- STEP: 2750/4322 -- GLOBAL_STEP: 2750[0m
|
| 532 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 533 |
+
| > loss_text_ce: 0.02484717033803463 (0.023605261645533775)
|
| 534 |
+
| > loss_mel_ce: 2.905120372772217 (3.131761779004872)
|
| 535 |
+
| > loss: 0.03488056734204292 (0.03756389408423141)
|
| 536 |
+
| > step_time: 0.6069 (0.5968708774393233)
|
| 537 |
+
| > loader_time: 0.0082 (0.012482620152560147)
|
| 538 |
+
|
| 539 |
+
|
| 540 |
+
[1m --> TIME: 2026-10-21 02:37:58 -- STEP: 2800/4322 -- GLOBAL_STEP: 2800[0m
|
| 541 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 542 |
+
| > loss_text_ce: 0.023038901388645172 (0.0235968307126313)
|
| 543 |
+
| > loss_mel_ce: 3.145516872406006 (3.1285898637771554)
|
| 544 |
+
| > loss: 0.037720903754234314 (0.037526032818880464)
|
| 545 |
+
| > step_time: 0.6692 (0.59682412947927)
|
| 546 |
+
| > loader_time: 0.0069 (0.012476146221160889)
|
| 547 |
+
|
| 548 |
+
|
| 549 |
+
[1m --> TIME: 2026-10-21 02:38:43 -- STEP: 2850/4322 -- GLOBAL_STEP: 2850[0m
|
| 550 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 551 |
+
| > loss_text_ce: 0.028272511437535286 (0.023586630681366207)
|
| 552 |
+
| > loss_mel_ce: 2.8701202869415283 (3.1255907545591604)
|
| 553 |
+
| > loss: 0.03450467810034752 (0.03749020771172485)
|
| 554 |
+
| > step_time: 0.5 (0.5968013382794545)
|
| 555 |
+
| > loader_time: 0.0101 (0.012469556457117984)
|
| 556 |
+
|
| 557 |
+
|
| 558 |
+
[1m --> TIME: 2026-10-21 02:39:27 -- STEP: 2900/4322 -- GLOBAL_STEP: 2900[0m
|
| 559 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 560 |
+
| > loss_text_ce: 0.023247918114066124 (0.023578409356675226)
|
| 561 |
+
| > loss_mel_ce: 3.136256456375122 (3.122102714735881)
|
| 562 |
+
| > loss: 0.0376131497323513 (0.0374485855530305)
|
| 563 |
+
| > step_time: 0.5847 (0.5964668889703402)
|
| 564 |
+
| > loader_time: 0.0082 (0.012465520151730242)
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
[1m --> TIME: 2026-10-21 02:40:12 -- STEP: 2950/4322 -- GLOBAL_STEP: 2950[0m
|
| 568 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 569 |
+
| > loss_text_ce: 0.022790543735027313 (0.023573600706028694)
|
| 570 |
+
| > loss_mel_ce: 3.000525951385498 (3.118728409217571)
|
| 571 |
+
| > loss: 0.035991862416267395 (0.03740835800451244)
|
| 572 |
+
| > step_time: 0.5957 (0.596490475767748)
|
| 573 |
+
| > loader_time: 0.0086 (0.012460524106429794)
|
| 574 |
+
|
| 575 |
+
|
| 576 |
+
[1m --> TIME: 2026-10-21 02:40:57 -- STEP: 3000/4322 -- GLOBAL_STEP: 3000[0m
|
| 577 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 578 |
+
| > loss_text_ce: 0.018387440592050552 (0.023560796816522834)
|
| 579 |
+
| > loss_mel_ce: 2.9166338443756104 (3.1155400846004437)
|
| 580 |
+
| > loss: 0.03494073078036308 (0.037370249329134846)
|
| 581 |
+
| > step_time: 0.6231 (0.5963328799406666)
|
| 582 |
+
| > loader_time: 0.0142 (0.01245480227470398)
|
| 583 |
+
|
| 584 |
+
|
| 585 |
+
[1m --> TIME: 2026-10-21 02:41:43 -- STEP: 3050/4322 -- GLOBAL_STEP: 3050[0m
|
| 586 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 587 |
+
| > loss_text_ce: 0.02384212426841259 (0.023554867990925623)
|
| 588 |
+
| > loss_mel_ce: 2.994652032852173 (3.112538642883296)
|
| 589 |
+
| > loss: 0.0359344556927681 (0.03733444729484006)
|
| 590 |
+
| > step_time: 0.5883 (0.5966500426120424)
|
| 591 |
+
| > loader_time: 0.0144 (0.012449125383721024)
|
| 592 |
+
|
| 593 |
+
|
| 594 |
+
[1m --> TIME: 2026-10-21 02:42:28 -- STEP: 3100/4322 -- GLOBAL_STEP: 3100[0m
|
| 595 |
+
| > current_lr: 5e-06 (4.999999999999845e-06)
|
| 596 |
+
| > loss_text_ce: 0.023726437240839005 (0.02355175362659554)
|
| 597 |
+
| > loss_mel_ce: 3.072147846221924 (3.109775276722442)
|
| 598 |
+
| > loss: 0.03685564547777176 (0.03730151300168325)
|
| 599 |
+
| > step_time: 0.6109 (0.5965689244577942)
|
| 600 |
+
| > loader_time: 0.0235 (0.012446198078893847)
|
| 601 |
+
|
| 602 |
+
|
| 603 |
+
[1m --> TIME: 2026-10-21 02:43:12 -- STEP: 3150/4322 -- GLOBAL_STEP: 3150[0m
|
| 604 |
+
| > current_lr: 5e-06 (4.999999999999859e-06)
|
| 605 |
+
| > loss_text_ce: 0.023431098088622093 (0.02354969015256276)
|
| 606 |
+
| > loss_mel_ce: 3.0573296546936035 (3.106731427889017)
|
| 607 |
+
| > loss: 0.036675725132226944 (0.03726525213924194)
|
| 608 |
+
| > step_time: 0.4902 (0.5964068209178846)
|
| 609 |
+
| > loader_time: 0.0075 (0.012434173160129123)
|
| 610 |
+
|
| 611 |
+
|
| 612 |
+
[1m --> TIME: 2026-10-21 02:43:58 -- STEP: 3200/4322 -- GLOBAL_STEP: 3200[0m
|
| 613 |
+
| > current_lr: 5e-06 (4.999999999999885e-06)
|
| 614 |
+
| > loss_text_ce: 0.022733112797141075 (0.0235456599690951)
|
| 615 |
+
| > loss_mel_ce: 3.0735044479370117 (3.1041155190020753)
|
| 616 |
+
| > loss: 0.03685997426509857 (0.03723406239121686)
|
| 617 |
+
| > step_time: 0.7319 (0.5965762753784638)
|
| 618 |
+
| > loader_time: 0.0084 (0.01243378609418869)
|
| 619 |
+
|
| 620 |
+
|
| 621 |
+
[1m --> TIME: 2026-10-21 02:44:43 -- STEP: 3250/4322 -- GLOBAL_STEP: 3250[0m
|
| 622 |
+
| > current_lr: 5e-06 (4.999999999999911e-06)
|
| 623 |
+
| > loss_text_ce: 0.02168012410402298 (0.023540749792296114)
|
| 624 |
+
| > loss_mel_ce: 3.084882974624634 (3.1013121072328964)
|
| 625 |
+
| > loss: 0.03698289394378662 (0.037200629989115094)
|
| 626 |
+
| > step_time: 0.6666 (0.5965495873231133)
|
| 627 |
+
| > loader_time: 0.0068 (0.01242878349010761)
|
| 628 |
+
|
| 629 |
+
|
| 630 |
+
[1m --> TIME: 2026-10-21 02:45:29 -- STEP: 3300/4322 -- GLOBAL_STEP: 3300[0m
|
| 631 |
+
| > current_lr: 5e-06 (4.999999999999935e-06)
|
| 632 |
+
| > loss_text_ce: 0.021574407815933228 (0.02353459311987862)
|
| 633 |
+
| > loss_mel_ce: 2.7126429080963135 (3.0985339783899666)
|
| 634 |
+
| > loss: 0.032550208270549774 (0.037167483733571864)
|
| 635 |
+
| > step_time: 0.5793 (0.5967610989917383)
|
| 636 |
+
| > loader_time: 0.014 (0.012422258709416245)
|
| 637 |
+
|
| 638 |
+
|
| 639 |
+
[1m --> TIME: 2026-10-21 02:46:14 -- STEP: 3350/4322 -- GLOBAL_STEP: 3350[0m
|
| 640 |
+
| > current_lr: 5e-06 (4.999999999999959e-06)
|
| 641 |
+
| > loss_text_ce: 0.022662699222564697 (0.023536021248506928)
|
| 642 |
+
| > loss_mel_ce: 2.932328224182129 (3.0958801448167215)
|
| 643 |
+
| > loss: 0.03517846390604973 (0.0371359074755169)
|
| 644 |
+
| > step_time: 0.6539 (0.596746546190174)
|
| 645 |
+
| > loader_time: 0.008 (0.01241784928449944)
|
| 646 |
+
|
| 647 |
+
|
| 648 |
+
[1m --> TIME: 2026-10-21 02:46:59 -- STEP: 3400/4322 -- GLOBAL_STEP: 3400[0m
|
| 649 |
+
| > current_lr: 5e-06 (4.999999999999982e-06)
|
| 650 |
+
| > loss_text_ce: 0.02598716877400875 (0.023529283185215518)
|
| 651 |
+
| > loss_mel_ce: 2.8259429931640625 (3.0933126112993983)
|
| 652 |
+
| > loss: 0.03395155072212219 (0.037105261384893014)
|
| 653 |
+
| > step_time: 0.6138 (0.5967671413982597)
|
| 654 |
+
| > loader_time: 0.0154 (0.012412546662723317)
|
| 655 |
+
|
| 656 |
+
|
| 657 |
+
[1m --> TIME: 2026-10-21 02:47:44 -- STEP: 3450/4322 -- GLOBAL_STEP: 3450[0m
|
| 658 |
+
| > current_lr: 5e-06 (5.000000000000005e-06)
|
| 659 |
+
| > loss_text_ce: 0.024678675457835197 (0.023523903610589675)
|
| 660 |
+
| > loss_mel_ce: 2.7041141986846924 (3.0908136884716932)
|
| 661 |
+
| > loss: 0.03248563036322594 (0.03707544826241076)
|
| 662 |
+
| > step_time: 0.4569 (0.596860606013863)
|
| 663 |
+
| > loader_time: 0.0084 (0.01241255193516828)
|
| 664 |
+
|
| 665 |
+
|
| 666 |
+
[1m --> TIME: 2026-10-21 02:48:30 -- STEP: 3500/4322 -- GLOBAL_STEP: 3500[0m
|
| 667 |
+
| > current_lr: 5e-06 (5.000000000000027e-06)
|
| 668 |
+
| > loss_text_ce: 0.023256434127688408 (0.023517030794705625)
|
| 669 |
+
| > loss_mel_ce: 2.9353296756744385 (3.088597696099958)
|
| 670 |
+
| > loss: 0.03522126376628876 (0.03704898557758757)
|
| 671 |
+
| > step_time: 0.6356 (0.5969392660004735)
|
| 672 |
+
| > loader_time: 0.0078 (0.012413677215576171)
|
| 673 |
+
|
| 674 |
+
|
| 675 |
+
[1m --> TIME: 2026-10-21 02:49:15 -- STEP: 3550/4322 -- GLOBAL_STEP: 3550[0m
|
| 676 |
+
| > current_lr: 5e-06 (5.000000000000048e-06)
|
| 677 |
+
| > loss_text_ce: 0.0238086748868227 (0.023514846461024887)
|
| 678 |
+
| > loss_mel_ce: 2.7836387157440186 (3.0861410168526837)
|
| 679 |
+
| > loss: 0.033421993255615234 (0.0370197133914056)
|
| 680 |
+
| > step_time: 0.6238 (0.5969673211809599)
|
| 681 |
+
| > loader_time: 0.0079 (0.012413687101552185)
|
| 682 |
+
|
| 683 |
+
|
| 684 |
+
[1m --> TIME: 2026-10-21 02:50:00 -- STEP: 3600/4322 -- GLOBAL_STEP: 3600[0m
|
| 685 |
+
| > current_lr: 5e-06 (5.000000000000068e-06)
|
| 686 |
+
| > loss_text_ce: 0.02341790869832039 (0.023513922610630556)
|
| 687 |
+
| > loss_mel_ce: 2.905672550201416 (3.0837201721138383)
|
| 688 |
+
| > loss: 0.034870125353336334 (0.036990882810722624)
|
| 689 |
+
| > step_time: 0.5812 (0.5969302547640254)
|
| 690 |
+
| > loader_time: 0.0187 (0.0124112421936459)
|
| 691 |
+
|
| 692 |
+
|
| 693 |
+
[1m --> TIME: 2026-10-21 02:50:46 -- STEP: 3650/4322 -- GLOBAL_STEP: 3650[0m
|
| 694 |
+
| > current_lr: 5e-06 (5.0000000000000885e-06)
|
| 695 |
+
| > loss_text_ce: 0.022184323519468307 (0.023513602474066458)
|
| 696 |
+
| > loss_mel_ce: 2.865701675415039 (3.081556038987143)
|
| 697 |
+
| > loss: 0.03437959775328636 (0.03696511551074377)
|
| 698 |
+
| > step_time: 0.6591 (0.5972059504626532)
|
| 699 |
+
| > loader_time: 0.0087 (0.012405855165768975)
|
| 700 |
+
|
| 701 |
+
|
| 702 |
+
[1m --> TIME: 2026-10-21 02:51:31 -- STEP: 3700/4322 -- GLOBAL_STEP: 3700[0m
|
| 703 |
+
| > current_lr: 5e-06 (5.000000000000108e-06)
|
| 704 |
+
| > loss_text_ce: 0.023852912709116936 (0.023514605919150854)
|
| 705 |
+
| > loss_mel_ce: 2.9706547260284424 (3.07943305814588)
|
| 706 |
+
| > loss: 0.035648901015520096 (0.036939853875318895)
|
| 707 |
+
| > step_time: 0.6115 (0.5970250767630484)
|
| 708 |
+
| > loader_time: 0.0126 (0.012402766330822095)
|
| 709 |
+
|
| 710 |
+
|
| 711 |
+
[1m --> TIME: 2026-10-21 02:52:15 -- STEP: 3750/4322 -- GLOBAL_STEP: 3750[0m
|
| 712 |
+
| > current_lr: 5e-06 (5.000000000000127e-06)
|
| 713 |
+
| > loss_text_ce: 0.021222300827503204 (0.023512201261023647)
|
| 714 |
+
| > loss_mel_ce: 3.0658493041992188 (3.0769362913131677)
|
| 715 |
+
| > loss: 0.036750853061676025 (0.03691010183642308)
|
| 716 |
+
| > step_time: 0.6207 (0.5969499096552515)
|
| 717 |
+
| > loader_time: 0.0065 (0.012400638135274252)
|
| 718 |
+
|
| 719 |
+
|
| 720 |
+
[1m --> TIME: 2026-10-21 02:53:01 -- STEP: 3800/4322 -- GLOBAL_STEP: 3800[0m
|
| 721 |
+
| > current_lr: 5e-06 (5.000000000000145e-06)
|
| 722 |
+
| > loss_text_ce: 0.025848623365163803 (0.02351087316331499)
|
| 723 |
+
| > loss_mel_ce: 2.920400619506836 (3.074521592290774)
|
| 724 |
+
| > loss: 0.035074397921562195 (0.03688133960442715)
|
| 725 |
+
| > step_time: 0.54 (0.5970745817611076)
|
| 726 |
+
| > loader_time: 0.0123 (0.012395272129460386)
|
| 727 |
+
|
| 728 |
+
|
| 729 |
+
[1m --> TIME: 2026-10-21 02:53:46 -- STEP: 3850/4322 -- GLOBAL_STEP: 3850[0m
|
| 730 |
+
| > current_lr: 5e-06 (5.000000000000163e-06)
|
| 731 |
+
| > loss_text_ce: 0.02371077984571457 (0.023509672148177344)
|
| 732 |
+
| > loss_mel_ce: 2.9974076747894287 (3.072430862079963)
|
| 733 |
+
| > loss: 0.035965695977211 (0.03685643566361108)
|
| 734 |
+
| > step_time: 0.5916 (0.5971751257661085)
|
| 735 |
+
| > loader_time: 0.0102 (0.01239291153944932)
|
| 736 |
+
|
| 737 |
+
|
| 738 |
+
[1m --> TIME: 2026-10-21 02:54:32 -- STEP: 3900/4322 -- GLOBAL_STEP: 3900[0m
|
| 739 |
+
| > current_lr: 5e-06 (5.000000000000181e-06)
|
| 740 |
+
| > loss_text_ce: 0.023781029507517815 (0.023504801939408716)
|
| 741 |
+
| > loss_mel_ce: 2.8465991020202637 (3.0702009644263795)
|
| 742 |
+
| > loss: 0.03417119383811951 (0.03682983128640514)
|
| 743 |
+
| > step_time: 0.6621 (0.5972338971724862)
|
| 744 |
+
| > loader_time: 0.0108 (0.01238854414377457)
|
| 745 |
+
|
| 746 |
+
|
| 747 |
+
[1m --> TIME: 2026-10-21 02:55:18 -- STEP: 3950/4322 -- GLOBAL_STEP: 3950[0m
|
| 748 |
+
| > current_lr: 5e-06 (5.000000000000198e-06)
|
| 749 |
+
| > loss_text_ce: 0.02363649569451809 (0.023503537125225276)
|
| 750 |
+
| > loss_mel_ce: 2.942638635635376 (3.068133991277667)
|
| 751 |
+
| > loss: 0.035312801599502563 (0.036805209405720234)
|
| 752 |
+
| > step_time: 0.6292 (0.597473596259007)
|
| 753 |
+
| > loader_time: 0.0184 (0.012386093381084973)
|
| 754 |
+
|
| 755 |
+
|
| 756 |
+
[1m --> TIME: 2026-10-21 02:56:03 -- STEP: 4000/4322 -- GLOBAL_STEP: 4000[0m
|
| 757 |
+
| > current_lr: 5e-06 (5.000000000000215e-06)
|
| 758 |
+
| > loss_text_ce: 0.02388708107173443 (0.02349735677009445)
|
| 759 |
+
| > loss_mel_ce: 2.860335111618042 (3.0659711222052537)
|
| 760 |
+
| > loss: 0.03433597832918167 (0.03677938738977537)
|
| 761 |
+
| > step_time: 0.6243 (0.5974928858876214)
|
| 762 |
+
| > loader_time: 0.0123 (0.012379054605960846)
|
| 763 |
+
|
| 764 |
+
|
| 765 |
+
[1m --> TIME: 2026-10-21 02:56:48 -- STEP: 4050/4322 -- GLOBAL_STEP: 4050[0m
|
| 766 |
+
| > current_lr: 5e-06 (5.000000000000231e-06)
|
| 767 |
+
| > loss_text_ce: 0.018802084028720856 (0.02349870981303627)
|
| 768 |
+
| > loss_mel_ce: 3.0977272987365723 (3.0638770363065895)
|
| 769 |
+
| > loss: 0.037101540714502335 (0.03675447390081338)
|
| 770 |
+
| > step_time: 0.6147 (0.59731181203583)
|
| 771 |
+
| > loader_time: 0.0138 (0.012381412482556002)
|
| 772 |
+
|
| 773 |
+
|
| 774 |
+
[1m --> TIME: 2026-10-21 02:57:33 -- STEP: 4100/4322 -- GLOBAL_STEP: 4100[0m
|
| 775 |
+
| > current_lr: 5e-06 (5.000000000000246e-06)
|
| 776 |
+
| > loss_text_ce: 0.022759081795811653 (0.023493108649897107)
|
| 777 |
+
| > loss_mel_ce: 2.878800392150879 (3.0615965936823555)
|
| 778 |
+
| > loss: 0.034542378038167953 (0.036727259095031314)
|
| 779 |
+
| > step_time: 0.7327 (0.5974419159423999)
|
| 780 |
+
| > loader_time: 0.0186 (0.012378487994031209)
|
| 781 |
+
|
| 782 |
+
|
| 783 |
+
[1m --> TIME: 2026-10-21 02:58:18 -- STEP: 4150/4322 -- GLOBAL_STEP: 4150[0m
|
| 784 |
+
| > current_lr: 5e-06 (5.000000000000261e-06)
|
| 785 |
+
| > loss_text_ce: 0.02283826284110546 (0.023487839882811837)
|
| 786 |
+
| > loss_mel_ce: 2.988551616668701 (3.0595337802243483)
|
| 787 |
+
| > loss: 0.035849880427122116 (0.03670263906693962)
|
| 788 |
+
| > step_time: 0.485 (0.597323453512535)
|
| 789 |
+
| > loader_time: 0.008 (0.012371215820312502)
|
| 790 |
+
|
| 791 |
+
|
| 792 |
+
[1m --> TIME: 2026-10-21 02:59:02 -- STEP: 4200/4322 -- GLOBAL_STEP: 4200[0m
|
| 793 |
+
| > current_lr: 5e-06 (5.000000000000276e-06)
|
| 794 |
+
| > loss_text_ce: 0.027317557483911514 (0.023485163285707406)
|
| 795 |
+
| > loss_mel_ce: 2.6367311477661133 (3.0571146906557503)
|
| 796 |
+
| > loss: 0.03171486407518387 (0.03667380851249965)
|
| 797 |
+
| > step_time: 0.4114 (0.5972230173860267)
|
| 798 |
+
| > loader_time: 0.0125 (0.012365819215774537)
|
| 799 |
+
|
| 800 |
+
|
| 801 |
+
[1m --> TIME: 2026-10-21 02:59:47 -- STEP: 4250/4322 -- GLOBAL_STEP: 4250[0m
|
| 802 |
+
| > current_lr: 5e-06 (5.000000000000293e-06)
|
| 803 |
+
| > loss_text_ce: 0.02426888607442379 (0.023483077582629262)
|
| 804 |
+
| > loss_mel_ce: 2.925359010696411 (3.0548168312521504)
|
| 805 |
+
| > loss: 0.03511461988091469 (0.036646428215591366)
|
| 806 |
+
| > step_time: 0.655 (0.5971050168205698)
|
| 807 |
+
| > loader_time: 0.0173 (0.012364748113295613)
|
| 808 |
+
|
| 809 |
+
|
| 810 |
+
[1m --> TIME: 2026-10-21 03:00:32 -- STEP: 4300/4322 -- GLOBAL_STEP: 4300[0m
|
| 811 |
+
| > current_lr: 5e-06 (5.000000000000306e-06)
|
| 812 |
+
| > loss_text_ce: 0.02220184914767742 (0.02347449325891426)
|
| 813 |
+
| > loss_mel_ce: 2.9004642963409424 (3.053137297519415)
|
| 814 |
+
| > loss: 0.03479364514350891 (0.036626331571836125)
|
| 815 |
+
| > step_time: 0.6792 (0.5971289326423813)
|
| 816 |
+
| > loader_time: 0.0115 (0.012370509824087461)
|
| 817 |
+
|
| 818 |
+
|
| 819 |
+
[1m > EVALUATION [0m
|
| 820 |
+
|
| 821 |
+
|
| 822 |
+
[1m--> EVAL PERFORMANCE[0m
|
| 823 |
+
| > avg_loader_time:[92m 0.07410516849783964 [0m(-0.00345826148986815)
|
| 824 |
+
| > avg_loss_text_ce:[92m 0.021929892559730733 [0m(-0.0007859685479901513)
|
| 825 |
+
| > avg_loss_mel_ce:[92m 2.8193006404610568 [0m(-0.8320896292841713)
|
| 826 |
+
| > avg_loss:[92m 2.8412305366161257 [0m(-0.8328755789024886)
|
| 827 |
+
|
| 828 |
+
> BEST MODEL : /kaggle/working/coqui-ai-TTS/run/training/GPT_XTTS_v2.0_LJSpeech_FT-October-21-2026_01+52AM-03339e5/best_model_4323.pth
|