Moustapha91 commited on
Commit
35b6b12
·
verified ·
1 Parent(s): 7b5c43d

Add trained xTTS model

Browse files
best_model.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d13484d2e09730daa3dc3ea46b28eee10d4bc45d4dbf42d190865b99e0b81ca1
3
+ size 5640358637
best_model_11584.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d13484d2e09730daa3dc3ea46b28eee10d4bc45d4dbf42d190865b99e0b81ca1
3
+ size 5640358637
config.json ADDED
@@ -0,0 +1,193 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "output_path": "xTTS/checkpoints/",
3
+ "logger_uri": null,
4
+ "run_name": "GPT_XTTS_FT",
5
+ "project_name": "XTTS_trainer",
6
+ "run_description": [
7
+ "\n GPT XTTS training\n "
8
+ ],
9
+ "print_step": 50,
10
+ "plot_step": 100,
11
+ "model_param_stats": false,
12
+ "wandb_entity": null,
13
+ "dashboard_logger": "tensorboard",
14
+ "save_on_interrupt": true,
15
+ "log_model_step": 100,
16
+ "save_step": 9999999999,
17
+ "save_n_checkpoints": 1,
18
+ "save_checkpoints": false,
19
+ "save_all_best": false,
20
+ "save_best_after": 10000,
21
+ "target_loss": null,
22
+ "print_eval": false,
23
+ "test_delay_epochs": 0,
24
+ "run_eval": true,
25
+ "run_eval_steps": null,
26
+ "distributed_backend": "nccl",
27
+ "distributed_url": "tcp://localhost:54321",
28
+ "mixed_precision": false,
29
+ "precision": "fp16",
30
+ "epochs": 2,
31
+ "batch_size": 4,
32
+ "eval_batch_size": 16,
33
+ "grad_clip": 0.0,
34
+ "scheduler_after_epoch": true,
35
+ "lr": 5e-06,
36
+ "optimizer": "AdamW",
37
+ "optimizer_params": {
38
+ "betas": [
39
+ 0.9,
40
+ 0.96
41
+ ],
42
+ "eps": 1e-08,
43
+ "weight_decay": 0.01
44
+ },
45
+ "lr_scheduler": "MultiStepLR",
46
+ "lr_scheduler_params": {
47
+ "milestones": [
48
+ 900000,
49
+ 2700000,
50
+ 5400000
51
+ ],
52
+ "gamma": 0.5,
53
+ "last_epoch": -1
54
+ },
55
+ "use_grad_scaler": false,
56
+ "allow_tf32": false,
57
+ "cudnn_enable": true,
58
+ "cudnn_deterministic": false,
59
+ "cudnn_benchmark": false,
60
+ "training_seed": 54321,
61
+ "model": "xtts",
62
+ "num_loader_workers": 8,
63
+ "num_eval_loader_workers": 0,
64
+ "use_noise_augment": false,
65
+ "audio": {
66
+ "sample_rate": 22050,
67
+ "output_sample_rate": 24000,
68
+ "dvae_sample_rate": 22050
69
+ },
70
+ "use_phonemes": false,
71
+ "phonemizer": null,
72
+ "phoneme_language": null,
73
+ "compute_input_seq_cache": false,
74
+ "text_cleaner": null,
75
+ "enable_eos_bos_chars": false,
76
+ "test_sentences_file": "",
77
+ "phoneme_cache_path": null,
78
+ "characters": null,
79
+ "add_blank": false,
80
+ "batch_group_size": 0,
81
+ "loss_masking": null,
82
+ "min_audio_len": 1,
83
+ "max_audio_len": Infinity,
84
+ "min_text_len": 1,
85
+ "max_text_len": Infinity,
86
+ "compute_f0": false,
87
+ "compute_energy": false,
88
+ "compute_linear_spec": false,
89
+ "precompute_num_workers": 0,
90
+ "start_by_longest": false,
91
+ "shuffle": false,
92
+ "drop_last": false,
93
+ "datasets": [
94
+ {
95
+ "formatter": "",
96
+ "dataset_name": "",
97
+ "path": "",
98
+ "meta_file_train": "",
99
+ "ignored_speakers": null,
100
+ "language": "",
101
+ "phonemizer": "",
102
+ "meta_file_val": "",
103
+ "meta_file_attn_mask": ""
104
+ }
105
+ ],
106
+ "test_sentences": [],
107
+ "eval_split_max_size": 256,
108
+ "eval_split_size": 0.01,
109
+ "use_speaker_weighted_sampler": false,
110
+ "speaker_weighted_sampler_alpha": 1.0,
111
+ "use_language_weighted_sampler": false,
112
+ "language_weighted_sampler_alpha": 1.0,
113
+ "use_length_weighted_sampler": false,
114
+ "length_weighted_sampler_alpha": 1.0,
115
+ "model_args": {
116
+ "gpt_batch_size": 1,
117
+ "enable_redaction": false,
118
+ "kv_cache": true,
119
+ "gpt_checkpoint": "",
120
+ "clvp_checkpoint": null,
121
+ "decoder_checkpoint": null,
122
+ "num_chars": 255,
123
+ "tokenizer_file": "xTTS/checkpoints/XTTS_v2.0_original_model_files/vocab.json",
124
+ "gpt_max_audio_tokens": 605,
125
+ "gpt_max_text_tokens": 402,
126
+ "gpt_max_prompt_tokens": 70,
127
+ "gpt_layers": 30,
128
+ "gpt_n_model_channels": 1024,
129
+ "gpt_n_heads": 16,
130
+ "gpt_number_text_tokens": 8000,
131
+ "gpt_start_text_token": 261,
132
+ "gpt_stop_text_token": 0,
133
+ "gpt_num_audio_tokens": 1026,
134
+ "gpt_start_audio_token": 1024,
135
+ "gpt_stop_audio_token": 1025,
136
+ "gpt_code_stride_len": 1024,
137
+ "gpt_use_masking_gt_prompt_approach": true,
138
+ "gpt_use_perceiver_resampler": true,
139
+ "input_sample_rate": 22050,
140
+ "output_sample_rate": 24000,
141
+ "output_hop_length": 256,
142
+ "decoder_input_dim": 1024,
143
+ "d_vector_dim": 512,
144
+ "cond_d_vector_in_each_upsampling_layer": true,
145
+ "duration_const": 102400,
146
+ "min_conditioning_length": 11025,
147
+ "max_conditioning_length": 132300,
148
+ "gpt_loss_text_ce_weight": 0.01,
149
+ "gpt_loss_mel_ce_weight": 1.0,
150
+ "debug_loading_failures": false,
151
+ "max_wav_length": 320000,
152
+ "max_text_length": 400,
153
+ "mel_norm_file": "xTTS/checkpoints/XTTS_v2.0_original_model_files/mel_stats.pth",
154
+ "dvae_checkpoint": "xTTS/checkpoints/XTTS_v2.0_original_model_files/dvae.pth",
155
+ "xtts_checkpoint": "xTTS/checkpoints/XTTS_v2.0_original_model_files/model.pth",
156
+ "vocoder": ""
157
+ },
158
+ "model_dir": null,
159
+ "languages": [
160
+ "en",
161
+ "es",
162
+ "fr",
163
+ "de",
164
+ "it",
165
+ "pt",
166
+ "pl",
167
+ "tr",
168
+ "ru",
169
+ "nl",
170
+ "cs",
171
+ "ar",
172
+ "zh-cn",
173
+ "hu",
174
+ "ko",
175
+ "ja",
176
+ "hi",
177
+ "wo"
178
+ ],
179
+ "temperature": 0.75,
180
+ "length_penalty": 1.0,
181
+ "repetition_penalty": 5.0,
182
+ "top_k": 50,
183
+ "top_p": 0.85,
184
+ "num_gpt_outputs": 1,
185
+ "gpt_cond_len": 30,
186
+ "gpt_cond_chunk_len": 4,
187
+ "max_ref_len": 30,
188
+ "sound_norm_refs": false,
189
+ "optimizer_wd_only_on_weights": true,
190
+ "weighted_loss_attrs": null,
191
+ "weighted_loss_multipliers": null,
192
+ "github_branch": "inside_docker"
193
+ }
events.out.tfevents.1748866975.eafda0218e5b.317.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92bbd9c8257e7635c9e148d4d60f774e3b306139eaa7dbc879c9709a077dd17f
3
+ size 60608
train_gpt_xtts.py ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import gc
3
+
4
+ from trainer import Trainer, TrainerArgs
5
+
6
+ from TTS.config.shared_configs import BaseDatasetConfig
7
+ from TTS.tts.datasets import load_tts_samples
8
+ from TTS.tts.layers.xtts.trainer.gpt_trainer import GPTArgs, GPTTrainer, GPTTrainerConfig, XttsAudioConfig
9
+ from TTS.utils.manage import ModelManager
10
+
11
+ from dataclasses import dataclass, field
12
+ from typing import Optional
13
+ from transformers import HfArgumentParser
14
+
15
+ import argparse
16
+
17
+ def create_xtts_trainer_parser():
18
+ parser = argparse.ArgumentParser(description="Arguments for XTTS Trainer")
19
+
20
+ parser.add_argument("--output_path", type=str, required=True,
21
+ help="Path to pretrained + checkpoint model")
22
+ parser.add_argument("--metadatas", nargs='+', type=str, required=True,
23
+ help="train_csv_path,eval_csv_path,language")
24
+ parser.add_argument("--num_epochs", type=int, default=1,
25
+ help="Number of epochs")
26
+ parser.add_argument("--batch_size", type=int, default=1,
27
+ help="Mini batch size")
28
+ parser.add_argument("--grad_acumm", type=int, default=1,
29
+ help="Grad accumulation steps")
30
+ parser.add_argument("--max_audio_length", type=int, default=255995,
31
+ help="Max audio length")
32
+ parser.add_argument("--max_text_length", type=int, default=200,
33
+ help="Max text length")
34
+ parser.add_argument("--weight_decay", type=float, default=1e-2,
35
+ help="Weight decay")
36
+ parser.add_argument("--lr", type=float, default=5e-6,
37
+ help="Learning rate")
38
+ parser.add_argument("--save_step", type=int, default=5000,
39
+ help="Save step")
40
+
41
+ return parser
42
+
43
+
44
+
45
+ def train_gpt(metadatas, num_epochs, batch_size, grad_acumm, output_path, max_audio_length, max_text_length, lr, weight_decay, save_step):
46
+ # Logging parameters
47
+ RUN_NAME = "GPT_XTTS_FT"
48
+ PROJECT_NAME = "XTTS_trainer"
49
+ DASHBOARD_LOGGER = "tensorboard"
50
+ LOGGER_URI = None
51
+
52
+ # Set here the path that the checkpoints will be saved. Default: ./run/training/
53
+ # OUT_PATH = os.path.join(output_path, "run", "training")
54
+ OUT_PATH = output_path
55
+
56
+ # Training Parameters
57
+ OPTIMIZER_WD_ONLY_ON_WEIGHTS = True # for multi-gpu training please make it False
58
+ START_WITH_EVAL = False # if True it will star with evaluation
59
+ BATCH_SIZE = batch_size # set here the batch size
60
+ GRAD_ACUMM_STEPS = grad_acumm # set here the grad accumulation steps
61
+
62
+
63
+ # Define here the dataset that you want to use for the fine-tuning on.
64
+ DATASETS_CONFIG_LIST = []
65
+ for metadata in metadatas:
66
+ train_csv, eval_csv, language = metadata.split(",")
67
+ print(train_csv, eval_csv, language)
68
+
69
+ config_dataset = BaseDatasetConfig(
70
+ formatter="coqui",
71
+ dataset_name="ft_dataset",
72
+ path=os.path.dirname(train_csv),
73
+ meta_file_train=os.path.basename(train_csv),
74
+ meta_file_val=os.path.basename(eval_csv),
75
+ language=language,
76
+ )
77
+
78
+ DATASETS_CONFIG_LIST.append(config_dataset)
79
+
80
+ # Define the path where XTTS v2.0.1 files will be downloaded
81
+ CHECKPOINTS_OUT_PATH = os.path.join(OUT_PATH, "XTTS_v2.0_original_model_files/")
82
+ os.makedirs(CHECKPOINTS_OUT_PATH, exist_ok=True)
83
+
84
+
85
+ # DVAE files
86
+ DVAE_CHECKPOINT_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/dvae.pth"
87
+ MEL_NORM_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/mel_stats.pth"
88
+
89
+ # Set the path to the downloaded files
90
+ DVAE_CHECKPOINT = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(DVAE_CHECKPOINT_LINK))
91
+ MEL_NORM_FILE = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(MEL_NORM_LINK))
92
+
93
+ # download DVAE files if needed
94
+ if not os.path.isfile(DVAE_CHECKPOINT) or not os.path.isfile(MEL_NORM_FILE):
95
+ print(" > Downloading DVAE files!")
96
+ ModelManager._download_model_files([MEL_NORM_LINK, DVAE_CHECKPOINT_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True)
97
+
98
+
99
+ # Download XTTS v2.0 checkpoint if needed
100
+ TOKENIZER_FILE_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/vocab.json"
101
+ XTTS_CHECKPOINT_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/model.pth"
102
+ XTTS_CONFIG_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/config.json"
103
+
104
+ # XTTS transfer learning parameters: You we need to provide the paths of XTTS model checkpoint that you want to do the fine tuning.
105
+ TOKENIZER_FILE = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(TOKENIZER_FILE_LINK)) # vocab.json file
106
+ XTTS_CHECKPOINT = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(XTTS_CHECKPOINT_LINK)) # model.pth file
107
+ XTTS_CONFIG_FILE = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(XTTS_CONFIG_LINK)) # config.json file
108
+
109
+ # download XTTS v2.0 files if needed
110
+ if not os.path.isfile(TOKENIZER_FILE):
111
+ print(" > Downloading XTTS v2.0 tokenizer!")
112
+ ModelManager._download_model_files(
113
+ [TOKENIZER_FILE_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True
114
+ )
115
+ if not os.path.isfile(XTTS_CHECKPOINT):
116
+ print(" > Downloading XTTS v2.0 checkpoint!")
117
+ ModelManager._download_model_files(
118
+ [XTTS_CHECKPOINT_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True
119
+ )
120
+ if not os.path.isfile(XTTS_CONFIG_FILE):
121
+ print(" > Downloading XTTS v2.0 config!")
122
+ ModelManager._download_model_files(
123
+ [XTTS_CONFIG_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True
124
+ )
125
+
126
+ # init args and config
127
+ model_args = GPTArgs(
128
+ max_conditioning_length=132300, # 6 secs
129
+ min_conditioning_length=11025, # 0.5 secs
130
+ debug_loading_failures=False,
131
+ max_wav_length=max_audio_length, # ~11.6 seconds
132
+ max_text_length=max_text_length,
133
+ mel_norm_file=MEL_NORM_FILE,
134
+ dvae_checkpoint=DVAE_CHECKPOINT,
135
+ xtts_checkpoint=XTTS_CHECKPOINT, # checkpoint path of the model that you want to fine-tune
136
+ tokenizer_file=TOKENIZER_FILE,
137
+ gpt_num_audio_tokens=1026,
138
+ gpt_start_audio_token=1024,
139
+ gpt_stop_audio_token=1025,
140
+ gpt_use_masking_gt_prompt_approach=True,
141
+ gpt_use_perceiver_resampler=True,
142
+ )
143
+ # define audio config
144
+ audio_config = XttsAudioConfig(sample_rate=22050, dvae_sample_rate=22050, output_sample_rate=24000)
145
+ # training parameters config
146
+
147
+ config = GPTTrainerConfig()
148
+
149
+ config.load_json(XTTS_CONFIG_FILE)
150
+
151
+ config.epochs = num_epochs
152
+ config.output_path = OUT_PATH
153
+ config.model_args = model_args
154
+ config.run_name = RUN_NAME
155
+ config.project_name = PROJECT_NAME
156
+ config.run_description = """
157
+ GPT XTTS training
158
+ """,
159
+ config.dashboard_logger = DASHBOARD_LOGGER
160
+ config.logger_uri = LOGGER_URI
161
+ config.audio = audio_config
162
+ config.batch_size = BATCH_SIZE
163
+ config.num_loader_workers = 8
164
+ config.eval_split_max_size = 256
165
+ config.print_step = 50
166
+ config.plot_step = 100
167
+ config.log_model_step = 100
168
+ config.save_n_checkpoints = 1
169
+ config.save_checkpoints = False
170
+ config.print_eval = False
171
+ config.optimizer = "AdamW"
172
+ config.optimizer_wd_only_on_weights = OPTIMIZER_WD_ONLY_ON_WEIGHTS
173
+ config.optimizer_params = {"betas": [0.9, 0.96], "eps": 1e-8, "weight_decay": weight_decay}
174
+ config.lr = lr
175
+ config.lr_scheduler = "MultiStepLR"
176
+ config.lr_scheduler_params = {"milestones": [50000 * 18, 150000 * 18, 300000 * 18], "gamma": 0.5, "last_epoch": -1}
177
+ config.test_sentences = []
178
+ config.save_step = 9999999999
179
+
180
+
181
+ # init the model from config
182
+ model = GPTTrainer.init_from_config(config)
183
+
184
+ # load training samples
185
+ train_samples, eval_samples = load_tts_samples(
186
+ DATASETS_CONFIG_LIST,
187
+ eval_split=True,
188
+ eval_split_max_size=config.eval_split_max_size,
189
+ eval_split_size=config.eval_split_size,
190
+ )
191
+
192
+ # init the trainer and 🚀
193
+ trainer = Trainer(
194
+ TrainerArgs(
195
+ restore_path=None, # xtts checkpoint is restored via xtts_checkpoint key so no need of restore it using Trainer restore_path parameter
196
+ skip_train_epoch=False,
197
+ start_with_eval=START_WITH_EVAL,
198
+ grad_accum_steps=GRAD_ACUMM_STEPS
199
+ ),
200
+ config,
201
+ output_path=os.path.join(output_path, "run", "training"),
202
+ model=model,
203
+ train_samples=train_samples,
204
+ eval_samples=eval_samples,
205
+ )
206
+ trainer.fit()
207
+ trainer_out_path = trainer.output_path
208
+
209
+ # deallocate VRAM and RAM
210
+ del model, trainer, train_samples, eval_samples
211
+ gc.collect()
212
+
213
+ return trainer_out_path
214
+
215
+ if __name__ == "__main__":
216
+ parser = create_xtts_trainer_parser()
217
+ args = parser.parse_args()
218
+
219
+ trainer_out_path = train_gpt(
220
+ metadatas=args.metadatas,
221
+ output_path=args.output_path,
222
+ num_epochs=args.num_epochs,
223
+ batch_size=args.batch_size,
224
+ grad_acumm=args.grad_acumm,
225
+ weight_decay=args.weight_decay,
226
+ lr=args.lr,
227
+ max_text_length=args.max_text_length,
228
+ max_audio_length=args.max_audio_length,
229
+ save_step=args.save_step
230
+ )
231
+
232
+ print(f"Checkpoint saved in dir: {trainer_out_path}")
trainer_0_log.txt ADDED
@@ -0,0 +1,2135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ > Training Environment:
2
+ | > Backend: Torch
3
+ | > Mixed precision: False
4
+ | > Precision: float32
5
+ | > Current device: 0
6
+ | > Num. of GPUs: 1
7
+ | > Num. of CPUs: 4
8
+ | > Num. of Torch Threads: 1
9
+ | > Torch seed: 54321
10
+ | > Torch CUDNN: True
11
+ | > Torch CUDNN deterministic: False
12
+ | > Torch CUDNN benchmark: False
13
+ | > Torch TF32 MatMul: False
14
+ > Start Tensorboard: tensorboard --logdir=xTTS/checkpoints/GPT_XTTS_FT-June-02-2025_12+22PM-0000000
15
+
16
+ > Model has 521144678 parameters
17
+
18
+  > EPOCH: 0/2
19
+ --> xTTS/checkpoints/GPT_XTTS_FT-June-02-2025_12+22PM-0000000
20
+
21
+  > TRAINING (2025-06-02 12:22:56) 
22
+
23
+  --> TIME: 2025-06-02 12:23:01 -- STEP: 0/5792 -- GLOBAL_STEP: 0
24
+ | > loss_text_ce: 0.23782861232757568 (0.23782861232757568)
25
+ | > loss_mel_ce: 4.941317081451416 (4.941317081451416)
26
+ | > loss: 1.2947864532470703 (1.2947864532470703)
27
+ | > current_lr: 5e-06
28
+ | > step_time: 1.3843 (1.3842837810516357)
29
+ | > loader_time: 4.3003 (4.3003199100494385)
30
+
31
+
32
+  --> TIME: 2025-06-02 12:23:30 -- STEP: 50/5792 -- GLOBAL_STEP: 50
33
+ | > loss_text_ce: 0.23064814507961273 (0.24020873457193376)
34
+ | > loss_mel_ce: 4.853140830993652 (4.867191286087037)
35
+ | > loss: 1.2709472179412842 (1.2768500065803527)
36
+ | > current_lr: 5e-06
37
+ | > step_time: 0.2303 (0.3325099849700927)
38
+ | > loader_time: 0.0143 (0.016327280998229988)
39
+
40
+
41
+  --> TIME: 2025-06-02 12:23:59 -- STEP: 100/5792 -- GLOBAL_STEP: 100
42
+ | > loss_text_ce: 0.2179468423128128 (0.2335291735827923)
43
+ | > loss_mel_ce: 4.29183292388916 (4.678085584640503)
44
+ | > loss: 1.1274449825286865 (1.2279036939144128)
45
+ | > current_lr: 5e-06
46
+ | > step_time: 0.2657 (0.3300197696685791)
47
+ | > loader_time: 0.0095 (0.015608358383178715)
48
+
49
+
50
+  --> TIME: 2025-06-02 12:24:28 -- STEP: 150/5792 -- GLOBAL_STEP: 150
51
+ | > loss_text_ce: 0.21278369426727295 (0.22793374240398406)
52
+ | > loss_mel_ce: 4.24708890914917 (4.557877604166666)
53
+ | > loss: 1.114968180656433 (1.1964528389771778)
54
+ | > current_lr: 5e-06
55
+ | > step_time: 0.3034 (0.33196935812632256)
56
+ | > loader_time: 0.0147 (0.0150977897644043)
57
+
58
+
59
+  --> TIME: 2025-06-02 12:24:57 -- STEP: 200/5792 -- GLOBAL_STEP: 200
60
+ | > loss_text_ce: 0.21909523010253906 (0.22294333018362522)
61
+ | > loss_mel_ce: 3.632870674133301 (4.465021697282792)
62
+ | > loss: 0.96299147605896 (1.1719912576675409)
63
+ | > current_lr: 5e-06
64
+ | > step_time: 0.137 (0.33124068975448623)
65
+ | > loader_time: 0.0134 (0.014897105693817142)
66
+
67
+
68
+  --> TIME: 2025-06-02 12:25:28 -- STEP: 250/5792 -- GLOBAL_STEP: 250
69
+ | > loss_text_ce: 0.20100712776184082 (0.21740962487459184)
70
+ | > loss_mel_ce: 4.115545749664307 (4.400661633491516)
71
+ | > loss: 1.0791382789611816 (1.154517814159393)
72
+ | > current_lr: 5e-06
73
+ | > step_time: 0.4054 (0.33776610374450694)
74
+ | > loader_time: 0.0126 (0.014648478507995607)
75
+
76
+
77
+  --> TIME: 2025-06-02 12:25:57 -- STEP: 300/5792 -- GLOBAL_STEP: 300
78
+ | > loss_text_ce: 0.1881532222032547 (0.21308185448249173)
79
+ | > loss_mel_ce: 4.062322616577148 (4.338319851557412)
80
+ | > loss: 1.0626189708709717 (1.1378504260381062)
81
+ | > current_lr: 5e-06
82
+ | > step_time: 0.3084 (0.33573377132415777)
83
+ | > loader_time: 0.0168 (0.014551755587259931)
84
+
85
+
86
+  --> TIME: 2025-06-02 12:26:25 -- STEP: 350/5792 -- GLOBAL_STEP: 350
87
+ | > loss_text_ce: 0.18258750438690186 (0.20880227374179014)
88
+ | > loss_mel_ce: 3.5383872985839844 (4.286162930216108)
89
+ | > loss: 0.930243730545044 (1.1237413001060488)
90
+ | > current_lr: 5e-06
91
+ | > step_time: 0.2231 (0.3339259896959578)
92
+ | > loader_time: 0.0132 (0.014371032714843752)
93
+
94
+
95
+  --> TIME: 2025-06-02 12:26:54 -- STEP: 400/5792 -- GLOBAL_STEP: 400
96
+ | > loss_text_ce: 0.17027834057807922 (0.20482054084539406)
97
+ | > loss_mel_ce: 3.622598648071289 (4.2505981731414835)
98
+ | > loss: 0.9482192397117615 (1.113854677826166)
99
+ | > current_lr: 5e-06
100
+ | > step_time: 0.3104 (0.3339052683115007)
101
+ | > loader_time: 0.0161 (0.014224189519882205)
102
+
103
+
104
+  --> TIME: 2025-06-02 12:27:23 -- STEP: 450/5792 -- GLOBAL_STEP: 450
105
+ | > loss_text_ce: 0.1692354679107666 (0.20091567562686066)
106
+ | > loss_mel_ce: 3.857461452484131 (4.213876393636072)
107
+ | > loss: 1.0066742897033691 (1.1036980164051056)
108
+ | > current_lr: 5e-06
109
+ | > step_time: 0.3746 (0.33324571874406617)
110
+ | > loader_time: 0.0117 (0.014043218294779461)
111
+
112
+
113
+  --> TIME: 2025-06-02 12:27:52 -- STEP: 500/5792 -- GLOBAL_STEP: 500
114
+ | > loss_text_ce: 0.15726038813591003 (0.19705737382173533)
115
+ | > loss_mel_ce: 3.882636547088623 (4.188746206283573)
116
+ | > loss: 1.0099742412567139 (1.0964508949518204)
117
+ | > current_lr: 5e-06
118
+ | > step_time: 0.2675 (0.3323551220893861)
119
+ | > loader_time: 0.0094 (0.01385477685928345)
120
+
121
+
122
+  --> TIME: 2025-06-02 12:28:21 -- STEP: 550/5792 -- GLOBAL_STEP: 550
123
+ | > loss_text_ce: 0.14702577888965607 (0.1931881040605631)
124
+ | > loss_mel_ce: 3.913630723953247 (4.1642507709156416)
125
+ | > loss: 1.0151641368865967 (1.0893597186695445)
126
+ | > current_lr: 5e-06
127
+ | > step_time: 0.4815 (0.33244910023429186)
128
+ | > loader_time: 0.0126 (0.013701290217312902)
129
+
130
+
131
+  --> TIME: 2025-06-02 12:28:51 -- STEP: 600/5792 -- GLOBAL_STEP: 600
132
+ | > loss_text_ce: 0.13721345365047455 (0.18944158929089705)
133
+ | > loss_mel_ce: 4.101404190063477 (4.144896006981534)
134
+ | > loss: 1.0596543550491333 (1.0835843993226688)
135
+ | > current_lr: 5e-06
136
+ | > step_time: 0.4056 (0.3338357198238373)
137
+ | > loader_time: 0.0101 (0.013622902234395351)
138
+
139
+
140
+  --> TIME: 2025-06-02 12:29:20 -- STEP: 650/5792 -- GLOBAL_STEP: 650
141
+ | > loss_text_ce: 0.142731174826622 (0.18570196391298224)
142
+ | > loss_mel_ce: 3.8508477210998535 (4.123029582683858)
143
+ | > loss: 0.9983947277069092 (1.0771828870589912)
144
+ | > current_lr: 5e-06
145
+ | > step_time: 0.2568 (0.33367859180156995)
146
+ | > loader_time: 0.0125 (0.01352635603684646)
147
+
148
+
149
+  --> TIME: 2025-06-02 12:29:47 -- STEP: 700/5792 -- GLOBAL_STEP: 700
150
+ | > loss_text_ce: 0.13017334043979645 (0.18195634870656907)
151
+ | > loss_mel_ce: 3.6967780590057373 (4.096020159721375)
152
+ | > loss: 0.9567378759384155 (1.069494128142085)
153
+ | > current_lr: 5e-06
154
+ | > step_time: 0.2121 (0.3316369649342126)
155
+ | > loader_time: 0.0094 (0.0133876953806196)
156
+
157
+
158
+  --> TIME: 2025-06-02 12:30:16 -- STEP: 750/5792 -- GLOBAL_STEP: 750
159
+ | > loss_text_ce: 0.1326562911272049 (0.17836018381516147)
160
+ | > loss_mel_ce: 3.863213300704956 (4.077398623466496)
161
+ | > loss: 0.9989674091339111 (1.0639397028287254)
162
+ | > current_lr: 5e-06
163
+ | > step_time: 0.3401 (0.33136590449015263)
164
+ | > loader_time: 0.0089 (0.013331648508707687)
165
+
166
+
167
+  --> TIME: 2025-06-02 12:30:45 -- STEP: 800/5792 -- GLOBAL_STEP: 800
168
+ | > loss_text_ce: 0.11892132461071014 (0.17485223358497032)
169
+ | > loss_mel_ce: 3.5116255283355713 (4.059487291574481)
170
+ | > loss: 0.9076367020606995 (1.0585848820954558)
171
+ | > current_lr: 5e-06
172
+ | > step_time: 0.1597 (0.331714543998241)
173
+ | > loader_time: 0.0106 (0.013222561478614812)
174
+
175
+
176
+  --> TIME: 2025-06-02 12:31:14 -- STEP: 850/5792 -- GLOBAL_STEP: 850
177
+ | > loss_text_ce: 0.11038005352020264 (0.1714906403685318)
178
+ | > loss_mel_ce: 3.8610212802886963 (4.046719538183775)
179
+ | > loss: 0.9928503036499023 (1.0545525454072382)
180
+ | > current_lr: 5e-06
181
+ | > step_time: 0.3977 (0.3319235268761126)
182
+ | > loader_time: 0.012 (0.013164884623359236)
183
+
184
+
185
+  --> TIME: 2025-06-02 12:31:42 -- STEP: 900/5792 -- GLOBAL_STEP: 900
186
+ | > loss_text_ce: 0.11592362821102142 (0.1682853105333117)
187
+ | > loss_mel_ce: 4.2247114181518555 (4.031507659223343)
188
+ | > loss: 1.0851587057113647 (1.0499482430352092)
189
+ | > current_lr: 5e-06
190
+ | > step_time: 0.3718 (0.3308940598699778)
191
+ | > loader_time: 0.0134 (0.01308101230197483)
192
+
193
+
194
+  --> TIME: 2025-06-02 12:32:10 -- STEP: 950/5792 -- GLOBAL_STEP: 950
195
+ | > loss_text_ce: 0.10771063715219498 (0.16517683673061828)
196
+ | > loss_mel_ce: 3.5032119750976562 (4.01872667036558)
197
+ | > loss: 0.9027306437492371 (1.045975877674001)
198
+ | > current_lr: 5e-06
199
+ | > step_time: 0.1779 (0.33019118836051503)
200
+ | > loader_time: 0.0148 (0.01302782359876131)
201
+
202
+
203
+  --> TIME: 2025-06-02 12:32:38 -- STEP: 1000/5792 -- GLOBAL_STEP: 1000
204
+ | > loss_text_ce: 0.12572963535785675 (0.16233218172937638)
205
+ | > loss_mel_ce: 3.0970044136047363 (4.003747127771376)
206
+ | > loss: 0.8056834936141968 (1.041519828259944)
207
+ | > current_lr: 5e-06
208
+ | > step_time: 0.1609 (0.32947225642204253)
209
+ | > loader_time: 0.0112 (0.01298054933547974)
210
+
211
+
212
+  --> TIME: 2025-06-02 12:33:07 -- STEP: 1050/5792 -- GLOBAL_STEP: 1050
213
+ | > loss_text_ce: 0.10119885206222534 (0.1596139285252208)
214
+ | > loss_mel_ce: 3.9164676666259766 (3.993313400404794)
215
+ | > loss: 1.004416584968567 (1.0382318332649403)
216
+ | > current_lr: 5e-06
217
+ | > step_time: 0.4415 (0.33021083627428294)
218
+ | > loader_time: 0.0149 (0.01293485868544806)
219
+
220
+
221
+  --> TIME: 2025-06-02 12:33:36 -- STEP: 1100/5792 -- GLOBAL_STEP: 1100
222
+ | > loss_text_ce: 0.09815080463886261 (0.1569338711080226)
223
+ | > loss_mel_ce: 3.761666774749756 (3.9810155831683764)
224
+ | > loss: 0.9649543762207031 (1.0344873643463295)
225
+ | > current_lr: 5e-06
226
+ | > step_time: 0.2093 (0.33017331686886836)
227
+ | > loader_time: 0.0095 (0.012836727445775815)
228
+
229
+
230
+  --> TIME: 2025-06-02 12:34:03 -- STEP: 1150/5792 -- GLOBAL_STEP: 1150
231
+ | > loss_text_ce: 0.09583915770053864 (0.15432746619634008)
232
+ | > loss_mel_ce: 3.7277543544769287 (3.967495829955391)
233
+ | > loss: 0.9558984041213989 (1.0304558247586937)
234
+ | > current_lr: 5e-06
235
+ | > step_time: 0.5857 (0.3293305108858188)
236
+ | > loader_time: 0.0121 (0.01275255389835524)
237
+
238
+
239
+  --> TIME: 2025-06-02 12:34:32 -- STEP: 1200/5792 -- GLOBAL_STEP: 1200
240
+ | > loss_text_ce: 0.09167931973934174 (0.15185882307589063)
241
+ | > loss_mel_ce: 3.893519163131714 (3.9543634712696076)
242
+ | > loss: 0.9962996244430542 (1.0265555742382992)
243
+ | > current_lr: 5e-06
244
+ | > step_time: 0.2053 (0.32926277975241314)
245
+ | > loader_time: 0.0085 (0.01267102936903636)
246
+
247
+
248
+  --> TIME: 2025-06-02 12:35:03 -- STEP: 1250/5792 -- GLOBAL_STEP: 1250
249
+ | > loss_text_ce: 0.09783565998077393 (0.14956970821619048)
250
+ | > loss_mel_ce: 3.702324867248535 (3.949050336265564)
251
+ | > loss: 0.9500401020050049 (1.024655011558532)
252
+ | > current_lr: 5e-06
253
+ | > step_time: 0.4952 (0.3306157491683954)
254
+ | > loader_time: 0.0104 (0.012619953727722172)
255
+
256
+
257
+  --> TIME: 2025-06-02 12:35:31 -- STEP: 1300/5792 -- GLOBAL_STEP: 1300
258
+ | > loss_text_ce: 0.08431262522935867 (0.14732415511057936)
259
+ | > loss_mel_ce: 3.927443027496338 (3.9411918537433333)
260
+ | > loss: 1.0029388666152954 (1.0221290025344256)
261
+ | > current_lr: 5e-06
262
+ | > step_time: 0.2377 (0.3301881036391619)
263
+ | > loader_time: 0.0087 (0.012530116668114296)
264
+
265
+
266
+  --> TIME: 2025-06-02 12:36:01 -- STEP: 1350/5792 -- GLOBAL_STEP: 1350
267
+ | > loss_text_ce: 0.08895734697580338 (0.14518685861870098)
268
+ | > loss_mel_ce: 3.606088638305664 (3.93310096723062)
269
+ | > loss: 0.9237614870071411 (1.0195719568376187)
270
+ | > current_lr: 5e-06
271
+ | > step_time: 0.5945 (0.33128830627158784)
272
+ | > loader_time: 0.0219 (0.012501915649131492)
273
+
274
+
275
+  --> TIME: 2025-06-02 12:36:30 -- STEP: 1400/5792 -- GLOBAL_STEP: 1400
276
+ | > loss_text_ce: 0.08065199851989746 (0.14312162278486157)
277
+ | > loss_mel_ce: 3.0727732181549072 (3.924280370473862)
278
+ | > loss: 0.7883563041687012 (1.0168504986166955)
279
+ | > current_lr: 5e-06
280
+ | > step_time: 0.1569 (0.3313496242250711)
281
+ | > loader_time: 0.0119 (0.012489057098116194)
282
+
283
+
284
+  --> TIME: 2025-06-02 12:37:00 -- STEP: 1450/5792 -- GLOBAL_STEP: 1450
285
+ | > loss_text_ce: 0.079571932554245 (0.14114773946075607)
286
+ | > loss_mel_ce: 3.952150821685791 (3.91605803686997)
287
+ | > loss: 1.0079306364059448 (1.0143014442509617)
288
+ | > current_lr: 5e-06
289
+ | > step_time: 0.3056 (0.33169992216702143)
290
+ | > loader_time: 0.0124 (0.012441920247571221)
291
+
292
+
293
+  --> TIME: 2025-06-02 12:37:29 -- STEP: 1500/5792 -- GLOBAL_STEP: 1500
294
+ | > loss_text_ce: 0.08127571642398834 (0.1392633452763161)
295
+ | > loss_mel_ce: 3.6380221843719482 (3.905871781508128)
296
+ | > loss: 0.9298244714736938 (1.0112837819655731)
297
+ | > current_lr: 5e-06
298
+ | > step_time: 0.2384 (0.33202595345179226)
299
+ | > loader_time: 0.0098 (0.012403525034586589)
300
+
301
+
302
+  --> TIME: 2025-06-02 12:37:59 -- STEP: 1550/5792 -- GLOBAL_STEP: 1550
303
+ | > loss_text_ce: 0.08357858657836914 (0.13743022495700477)
304
+ | > loss_mel_ce: 3.2771170139312744 (3.898509816354321)
305
+ | > loss: 0.8401739001274109 (1.0089850104624223)
306
+ | > current_lr: 5e-06
307
+ | > step_time: 0.1768 (0.33223780908892203)
308
+ | > loader_time: 0.0085 (0.012367438347108902)
309
+
310
+
311
+  --> TIME: 2025-06-02 12:38:28 -- STEP: 1600/5792 -- GLOBAL_STEP: 1600
312
+ | > loss_text_ce: 0.07955673336982727 (0.13564207764342429)
313
+ | > loss_mel_ce: 3.7596960067749023 (3.892578792124987)
314
+ | > loss: 0.9598131775856018 (1.007055217660963)
315
+ | > current_lr: 5e-06
316
+ | > step_time: 0.3524 (0.33262646302580806)
317
+ | > loader_time: 0.0101 (0.012325592041015625)
318
+
319
+
320
+  --> TIME: 2025-06-02 12:38:57 -- STEP: 1650/5792 -- GLOBAL_STEP: 1650
321
+ | > loss_text_ce: 0.08837619423866272 (0.13392942287253612)
322
+ | > loss_mel_ce: 3.777754068374634 (3.885421214681683)
323
+ | > loss: 0.9665325880050659 (1.0048376595973967)
324
+ | > current_lr: 5e-06
325
+ | > step_time: 0.3983 (0.3327004941304522)
326
+ | > loader_time: 0.0101 (0.012279371637286561)
327
+
328
+
329
+  --> TIME: 2025-06-02 12:39:25 -- STEP: 1700/5792 -- GLOBAL_STEP: 1700
330
+ | > loss_text_ce: 0.07985109090805054 (0.13228617378455745)
331
+ | > loss_mel_ce: 3.664367198944092 (3.8788135032092823)
332
+ | > loss: 0.9360545873641968 (1.0027749196922073)
333
+ | > current_lr: 5e-06
334
+ | > step_time: 0.1903 (0.33247794515946305)
335
+ | > loader_time: 0.0099 (0.012235631241517908)
336
+
337
+
338
+  --> TIME: 2025-06-02 12:39:53 -- STEP: 1750/5792 -- GLOBAL_STEP: 1750
339
+ | > loss_text_ce: 0.07385779917240143 (0.13067971873496242)
340
+ | > loss_mel_ce: 3.792656421661377 (3.8709569222586495)
341
+ | > loss: 0.9666285514831543 (1.0004091606821328)
342
+ | > current_lr: 5e-06
343
+ | > step_time: 0.2969 (0.3322374545506066)
344
+ | > loader_time: 0.0094 (0.012205468041556223)
345
+
346
+
347
+  --> TIME: 2025-06-02 12:40:23 -- STEP: 1800/5792 -- GLOBAL_STEP: 1800
348
+ | > loss_text_ce: 0.07535833865404129 (0.12911513355664064)
349
+ | > loss_mel_ce: 3.611762046813965 (3.8656086026297674)
350
+ | > loss: 0.9217801094055176 (0.9986809344093001)
351
+ | > current_lr: 5e-06
352
+ | > step_time: 0.4042 (0.3324654255972966)
353
+ | > loader_time: 0.014 (0.012192738983366225)
354
+
355
+
356
+  --> TIME: 2025-06-02 12:40:51 -- STEP: 1850/5792 -- GLOBAL_STEP: 1850
357
+ | > loss_text_ce: 0.0778559073805809 (0.12761117409814027)
358
+ | > loss_mel_ce: 3.0250251293182373 (3.857865214734464)
359
+ | > loss: 0.7757202386856079 (0.9963690974583493)
360
+ | > current_lr: 5e-06
361
+ | > step_time: 0.2565 (0.3320673637132384)
362
+ | > loader_time: 0.0119 (0.012154008891131426)
363
+
364
+
365
+  --> TIME: 2025-06-02 12:41:21 -- STEP: 1900/5792 -- GLOBAL_STEP: 1900
366
+ | > loss_text_ce: 0.07138916105031967 (0.12619867451096817)
367
+ | > loss_mel_ce: 3.775388717651367 (3.851701809858021)
368
+ | > loss: 0.9616944789886475 (0.9944751213726241)
369
+ | > current_lr: 5e-06
370
+ | > step_time: 0.2679 (0.3323917353780644)
371
+ | > loader_time: 0.0108 (0.012138623438383404)
372
+
373
+
374
+  --> TIME: 2025-06-02 12:41:50 -- STEP: 1950/5792 -- GLOBAL_STEP: 1950
375
+ | > loss_text_ce: 0.06446552276611328 (0.12474528638980308)
376
+ | > loss_mel_ce: 3.478724956512451 (3.8443781879620675)
377
+ | > loss: 0.8857976198196411 (0.9922808688114847)
378
+ | > current_lr: 5e-06
379
+ | > step_time: 0.267 (0.3323717740865852)
380
+ | > loader_time: 0.0102 (0.012119236114697578)
381
+
382
+
383
+  --> TIME: 2025-06-02 12:42:16 -- STEP: 2000/5792 -- GLOBAL_STEP: 2000
384
+ | > loss_text_ce: 0.07915454357862473 (0.12336141603440026)
385
+ | > loss_mel_ce: 3.8885855674743652 (3.8369900035858153)
386
+ | > loss: 0.9919350147247314 (0.9900878550410267)
387
+ | > current_lr: 5e-06
388
+ | > step_time: 0.3543 (0.3317557517290113)
389
+ | > loader_time: 0.0126 (0.012084858655929566)
390
+
391
+
392
+  --> TIME: 2025-06-02 12:42:46 -- STEP: 2050/5792 -- GLOBAL_STEP: 2050
393
+ | > loss_text_ce: 0.06675997376441956 (0.1220419639494359)
394
+ | > loss_mel_ce: 3.372281312942505 (3.8299568820581205)
395
+ | > loss: 0.8597603440284729 (0.9879997116181906)
396
+ | > current_lr: 5e-06
397
+ | > step_time: 0.3444 (0.3319971689363803)
398
+ | > loader_time: 0.0105 (0.01205671554658471)
399
+
400
+
401
+  --> TIME: 2025-06-02 12:43:14 -- STEP: 2100/5792 -- GLOBAL_STEP: 2100
402
+ | > loss_text_ce: 0.0644061341881752 (0.12074772182142432)
403
+ | > loss_mel_ce: 3.0573296546936035 (3.823917001769656)
404
+ | > loss: 0.7804339528083801 (0.9861661809682841)
405
+ | > current_lr: 5e-06
406
+ | > step_time: 0.1591 (0.33194411834080995)
407
+ | > loader_time: 0.0112 (0.012042945112500873)
408
+
409
+
410
+  --> TIME: 2025-06-02 12:43:45 -- STEP: 2150/5792 -- GLOBAL_STEP: 2150
411
+ | > loss_text_ce: 0.0625600665807724 (0.11952382788062077)
412
+ | > loss_mel_ce: 3.1828041076660156 (3.819017451530279)
413
+ | > loss: 0.8113410472869873 (0.984635319903839)
414
+ | > current_lr: 5e-06
415
+ | > step_time: 0.2213 (0.33232841746751635)
416
+ | > loader_time: 0.0153 (0.012043467122454977)
417
+
418
+
419
+  --> TIME: 2025-06-02 12:44:12 -- STEP: 2200/5792 -- GLOBAL_STEP: 2200
420
+ | > loss_text_ce: 0.07580503076314926 (0.11829185286367463)
421
+ | > loss_mel_ce: 3.471283435821533 (3.8122500800002705)
422
+ | > loss: 0.886772096157074 (0.9826354832540855)
423
+ | > current_lr: 5e-06
424
+ | > step_time: 0.2548 (0.33183346889235754)
425
+ | > loader_time: 0.0097 (0.012037343978881839)
426
+
427
+
428
+  --> TIME: 2025-06-02 12:44:41 -- STEP: 2250/5792 -- GLOBAL_STEP: 2250
429
+ | > loss_text_ce: 0.06772749871015549 (0.11713893160720648)
430
+ | > loss_mel_ce: 3.4121651649475098 (3.8072973964479235)
431
+ | > loss: 0.8699731826782227 (0.9811090819570749)
432
+ | > current_lr: 5e-06
433
+ | > step_time: 0.196 (0.33194303947024884)
434
+ | > loader_time: 0.0094 (0.012012425316704644)
435
+
436
+
437
+  --> TIME: 2025-06-02 12:45:10 -- STEP: 2300/5792 -- GLOBAL_STEP: 2300
438
+ | > loss_text_ce: 0.061648476868867874 (0.11599788975942377)
439
+ | > loss_mel_ce: 3.568927764892578 (3.802683669069539)
440
+ | > loss: 0.9076440334320068 (0.9796703896574348)
441
+ | > current_lr: 5e-06
442
+ | > step_time: 0.3098 (0.33178884682448023)
443
+ | > loader_time: 0.0146 (0.012008188081824262)
444
+
445
+
446
+  --> TIME: 2025-06-02 12:45:40 -- STEP: 2350/5792 -- GLOBAL_STEP: 2350
447
+ | > loss_text_ce: 0.06095200777053833 (0.11489018085947679)
448
+ | > loss_mel_ce: 3.651190996170044 (3.798658301576655)
449
+ | > loss: 0.9280357360839844 (0.9783871206324147)
450
+ | > current_lr: 5e-06
451
+ | > step_time: 0.34 (0.3323820060364746)
452
+ | > loader_time: 0.0123 (0.011982986064667397)
453
+
454
+
455
+  --> TIME: 2025-06-02 12:46:08 -- STEP: 2400/5792 -- GLOBAL_STEP: 2400
456
+ | > loss_text_ce: 0.06559105217456818 (0.11383588070515527)
457
+ | > loss_mel_ce: 3.4982616901397705 (3.793631868362427)
458
+ | > loss: 0.8909631967544556 (0.9768669372300303)
459
+ | > current_lr: 5e-06
460
+ | > step_time: 0.3963 (0.3320956921577454)
461
+ | > loader_time: 0.015 (0.011968173583348597)
462
+
463
+
464
+  --> TIME: 2025-06-02 12:46:40 -- STEP: 2450/5792 -- GLOBAL_STEP: 2450
465
+ | > loss_text_ce: 0.06471642851829529 (0.11277475817471118)
466
+ | > loss_mel_ce: 3.9599180221557617 (3.790205652275864)
467
+ | > loss: 1.0061585903167725 (0.9757451025807123)
468
+ | > current_lr: 5e-06
469
+ | > step_time: 0.43 (0.3328273964901357)
470
+ | > loader_time: 0.0128 (0.011948673481843914)
471
+
472
+
473
+  --> TIME: 2025-06-02 12:47:08 -- STEP: 2500/5792 -- GLOBAL_STEP: 2500
474
+ | > loss_text_ce: 0.058090440928936005 (0.11174730639308675)
475
+ | > loss_mel_ce: 3.680204153060913 (3.786430072784424)
476
+ | > loss: 0.9345736503601074 (0.9745443446636196)
477
+ | > current_lr: 5e-06
478
+ | > step_time: 0.4192 (0.3327032661437986)
479
+ | > loader_time: 0.014 (0.011920587730407725)
480
+
481
+
482
+  --> TIME: 2025-06-02 12:47:37 -- STEP: 2550/5792 -- GLOBAL_STEP: 2550
483
+ | > loss_text_ce: 0.06345102936029434 (0.11076159270224602)
484
+ | > loss_mel_ce: 3.3983774185180664 (3.7819082406922884)
485
+ | > loss: 0.8654571175575256 (0.9731674582584228)
486
+ | > current_lr: 5e-06
487
+ | > step_time: 0.3032 (0.3327015813191728)
488
+ | > loader_time: 0.0101 (0.01190789082471062)
489
+
490
+
491
+  --> TIME: 2025-06-02 12:48:06 -- STEP: 2600/5792 -- GLOBAL_STEP: 2600
492
+ | > loss_text_ce: 0.05978523939847946 (0.10978607040041899)
493
+ | > loss_mel_ce: 3.5986759662628174 (3.7784423384299646)
494
+ | > loss: 0.914615273475647 (0.9720571020704046)
495
+ | > current_lr: 5e-06
496
+ | > step_time: 0.2067 (0.33279164598538297)
497
+ | > loader_time: 0.0095 (0.01188810458550087)
498
+
499
+
500
+  --> TIME: 2025-06-02 12:48:38 -- STEP: 2650/5792 -- GLOBAL_STEP: 2650
501
+ | > loss_text_ce: 0.058306336402893066 (0.1088496894923582)
502
+ | > loss_mel_ce: 2.9056618213653564 (3.7759509926022226)
503
+ | > loss: 0.7409920692443848 (0.9712001704045058)
504
+ | > current_lr: 5e-06
505
+ | > step_time: 0.2274 (0.3335093621487883)
506
+ | > loader_time: 0.0143 (0.01186871078779113)
507
+
508
+
509
+  --> TIME: 2025-06-02 12:49:06 -- STEP: 2700/5792 -- GLOBAL_STEP: 2700
510
+ | > loss_text_ce: 0.06381174176931381 (0.1079658736467912)
511
+ | > loss_mel_ce: 3.7307090759277344 (3.7717624486817254)
512
+ | > loss: 0.9486302137374878 (0.9699320805293539)
513
+ | > current_lr: 5e-06
514
+ | > step_time: 0.3863 (0.33329126207916787)
515
+ | > loader_time: 0.0102 (0.01184851937823826)
516
+
517
+
518
+  --> TIME: 2025-06-02 12:49:37 -- STEP: 2750/5792 -- GLOBAL_STEP: 2750
519
+ | > loss_text_ce: 0.06515434384346008 (0.1070680166618389)
520
+ | > loss_mel_ce: 3.7349181175231934 (3.76892153228413)
521
+ | > loss: 0.9500181078910828 (0.9689973871707913)
522
+ | > current_lr: 5e-06
523
+ | > step_time: 0.1943 (0.33362000248648843)
524
+ | > loader_time: 0.0092 (0.011837096214294438)
525
+
526
+
527
+  --> TIME: 2025-06-02 12:50:06 -- STEP: 2800/5792 -- GLOBAL_STEP: 2800
528
+ | > loss_text_ce: 0.0626336857676506 (0.10621006238939493)
529
+ | > loss_mel_ce: 3.619314193725586 (3.7653382179566792)
530
+ | > loss: 0.9204869866371155 (0.9678870699873989)
531
+ | > current_lr: 5e-06
532
+ | > step_time: 0.3534 (0.3337795458521157)
533
+ | > loader_time: 0.0102 (0.011815384626388556)
534
+
535
+
536
+  --> TIME: 2025-06-02 12:50:34 -- STEP: 2850/5792 -- GLOBAL_STEP: 2850
537
+ | > loss_text_ce: 0.05888611823320389 (0.10537111821273949)
538
+ | > loss_mel_ce: 3.1223511695861816 (3.763010348939059)
539
+ | > loss: 0.79530930519104 (0.9670953666536428)
540
+ | > current_lr: 5e-06
541
+ | > step_time: 0.2558 (0.33350726955815335)
542
+ | > loader_time: 0.0118 (0.011799037080062068)
543
+
544
+
545
+  --> TIME: 2025-06-02 12:51:03 -- STEP: 2900/5792 -- GLOBAL_STEP: 2900
546
+ | > loss_text_ce: 0.0631546750664711 (0.1045509203009563)
547
+ | > loss_mel_ce: 3.269371271133423 (3.7583381495804624)
548
+ | > loss: 0.8331314921379089 (0.9657222673399692)
549
+ | > current_lr: 5e-06
550
+ | > step_time: 0.2418 (0.33377023384488846)
551
+ | > loader_time: 0.0097 (0.011781094896382304)
552
+
553
+
554
+  --> TIME: 2025-06-02 12:51:33 -- STEP: 2950/5792 -- GLOBAL_STEP: 2950
555
+ | > loss_text_ce: 0.054032497107982635 (0.10374996635999706)
556
+ | > loss_mel_ce: 3.7166588306427 (3.754722292625298)
557
+ | > loss: 0.942672848701477 (0.9646180645894193)
558
+ | > current_lr: 5e-06
559
+ | > step_time: 0.4931 (0.3338233034489517)
560
+ | > loader_time: 0.0146 (0.01176468857264115)
561
+
562
+
563
+  --> TIME: 2025-06-02 12:52:03 -- STEP: 3000/5792 -- GLOBAL_STEP: 3000
564
+ | > loss_text_ce: 0.04497094824910164 (0.10296552998696751)
565
+ | > loss_mel_ce: 3.8430159091949463 (3.7516442841688793)
566
+ | > loss: 0.9719967246055603 (0.9636524533629415)
567
+ | > current_lr: 5e-06
568
+ | > step_time: 0.2344 (0.3341527958710988)
569
+ | > loader_time: 0.0093 (0.011756853898366297)
570
+
571
+
572
+  --> TIME: 2025-06-02 12:52:32 -- STEP: 3050/5792 -- GLOBAL_STEP: 3050
573
+ | > loss_text_ce: 0.05044661462306976 (0.10219378712480172)
574
+ | > loss_mel_ce: 3.3637678623199463 (3.7479482764103373)
575
+ | > loss: 0.8535535931587219 (0.962535515730498)
576
+ | > current_lr: 5e-06
577
+ | > step_time: 0.3357 (0.33412718593097107)
578
+ | > loader_time: 0.0094 (0.011745000745429371)
579
+
580
+
581
+  --> TIME: 2025-06-02 12:53:01 -- STEP: 3100/5792 -- GLOBAL_STEP: 3100
582
+ | > loss_text_ce: 0.0588652640581131 (0.10145230594421574)
583
+ | > loss_mel_ce: 3.9438345432281494 (3.7447509037294697)
584
+ | > loss: 1.0006749629974365 (0.9615508023769622)
585
+ | > current_lr: 5e-06
586
+ | > step_time: 0.2943 (0.3342037938487145)
587
+ | > loader_time: 0.0097 (0.011726075987662043)
588
+
589
+
590
+  --> TIME: 2025-06-02 12:53:30 -- STEP: 3150/5792 -- GLOBAL_STEP: 3150
591
+ | > loss_text_ce: 0.051031917333602905 (0.10072415719902691)
592
+ | > loss_mel_ce: 3.401136636734009 (3.7408774338828192)
593
+ | > loss: 0.8630421161651611 (0.9604003977207908)
594
+ | > current_lr: 5e-06
595
+ | > step_time: 0.2545 (0.334141366822379)
596
+ | > loader_time: 0.0144 (0.011702652280292818)
597
+
598
+
599
+  --> TIME: 2025-06-02 12:53:57 -- STEP: 3200/5792 -- GLOBAL_STEP: 3200
600
+ | > loss_text_ce: 0.06074317917227745 (0.10001451563904981)
601
+ | > loss_mel_ce: 3.513090133666992 (3.7376667942106723)
602
+ | > loss: 0.8934583067893982 (0.9594203274324533)
603
+ | > current_lr: 5e-06
604
+ | > step_time: 0.2398 (0.3338389614969492)
605
+ | > loader_time: 0.012 (0.011692634075880054)
606
+
607
+
608
+  --> TIME: 2025-06-02 12:54:24 -- STEP: 3250/5792 -- GLOBAL_STEP: 3250
609
+ | > loss_text_ce: 0.05713744834065437 (0.0993218669753807)
610
+ | > loss_mel_ce: 3.3079988956451416 (3.7339409486330473)
611
+ | > loss: 0.8412840962409973 (0.9583157039055454)
612
+ | > current_lr: 5e-06
613
+ | > step_time: 0.2647 (0.3333787508010864)
614
+ | > loader_time: 0.01 (0.011673195472130414)
615
+
616
+
617
+  --> TIME: 2025-06-02 12:54:53 -- STEP: 3300/5792 -- GLOBAL_STEP: 3300
618
+ | > loss_text_ce: 0.049549344927072525 (0.09864641397401226)
619
+ | > loss_mel_ce: 3.5223379135131836 (3.731030137611158)
620
+ | > loss: 0.8929718136787415 (0.9574191379005256)
621
+ | > current_lr: 5e-06
622
+ | > step_time: 0.2054 (0.33343946623079707)
623
+ | > loader_time: 0.0094 (0.011662811438242599)
624
+
625
+
626
+  --> TIME: 2025-06-02 12:55:21 -- STEP: 3350/5792 -- GLOBAL_STEP: 3350
627
+ | > loss_text_ce: 0.053567271679639816 (0.09798200751974499)
628
+ | > loss_mel_ce: 3.276874542236328 (3.7281440085795388)
629
+ | > loss: 0.8326104283332825 (0.9565315039656053)
630
+ | > current_lr: 5e-06
631
+ | > step_time: 0.5258 (0.3332891092015736)
632
+ | > loader_time: 0.0106 (0.011652176344572612)
633
+
634
+
635
+  --> TIME: 2025-06-02 12:55:52 -- STEP: 3400/5792 -- GLOBAL_STEP: 3400
636
+ | > loss_text_ce: 0.05310482531785965 (0.09735800301765683)
637
+ | > loss_mel_ce: 3.462460517883301 (3.7253227573282577)
638
+ | > loss: 0.8788913488388062 (0.9556701900503212)
639
+ | > current_lr: 5e-06
640
+ | > step_time: 0.1824 (0.3334693743902094)
641
+ | > loader_time: 0.0087 (0.011638146919362689)
642
+
643
+
644
+  --> TIME: 2025-06-02 12:56:21 -- STEP: 3450/5792 -- GLOBAL_STEP: 3450
645
+ | > loss_text_ce: 0.05677446722984314 (0.0967331422789804)
646
+ | > loss_mel_ce: 3.994664430618286 (3.722395037084386)
647
+ | > loss: 1.0128597021102905 (0.954782044852989)
648
+ | > current_lr: 5e-06
649
+ | > step_time: 0.3058 (0.333492807927339)
650
+ | > loader_time: 0.0099 (0.011624124844868982)
651
+
652
+
653
+  --> TIME: 2025-06-02 12:56:51 -- STEP: 3500/5792 -- GLOBAL_STEP: 3500
654
+ | > loss_text_ce: 0.058832015842199326 (0.0961081453689506)
655
+ | > loss_mel_ce: 3.7743239402770996 (3.7200029888153074)
656
+ | > loss: 0.9582889676094055 (0.9540277835471287)
657
+ | > current_lr: 5e-06
658
+ | > step_time: 0.4594 (0.3338103825705392)
659
+ | > loader_time: 0.0135 (0.011612459046500073)
660
+
661
+
662
+  --> TIME: 2025-06-02 12:57:20 -- STEP: 3550/5792 -- GLOBAL_STEP: 3550
663
+ | > loss_text_ce: 0.050712406635284424 (0.09550555716832747)
664
+ | > loss_mel_ce: 3.5286216735839844 (3.7171594685567935)
665
+ | > loss: 0.894833505153656 (0.9531662564546287)
666
+ | > current_lr: 5e-06
667
+ | > step_time: 0.3831 (0.33380109598938845)
668
+ | > loader_time: 0.0088 (0.011606321603479524)
669
+
670
+
671
+  --> TIME: 2025-06-02 12:57:48 -- STEP: 3600/5792 -- GLOBAL_STEP: 3600
672
+ | > loss_text_ce: 0.0556088350713253 (0.09492813276644371)
673
+ | > loss_mel_ce: 3.395554780960083 (3.713761396010717)
674
+ | > loss: 0.8627908825874329 (0.9521723822421495)
675
+ | > current_lr: 5e-06
676
+ | > step_time: 0.2089 (0.3337154185109668)
677
+ | > loader_time: 0.0126 (0.01158974859449599)
678
+
679
+
680
+  --> TIME: 2025-06-02 12:58:17 -- STEP: 3650/5792 -- GLOBAL_STEP: 3650
681
+ | > loss_text_ce: 0.05382274463772774 (0.0943567333778697)
682
+ | > loss_mel_ce: 3.8336167335510254 (3.7102085386563655)
683
+ | > loss: 0.971859872341156 (0.9511413181317994)
684
+ | > current_lr: 5e-06
685
+ | > step_time: 0.3718 (0.33372791290283205)
686
+ | > loader_time: 0.0094 (0.011583965510538183)
687
+
688
+
689
+  --> TIME: 2025-06-02 12:58:46 -- STEP: 3700/5792 -- GLOBAL_STEP: 3700
690
+ | > loss_text_ce: 0.054553646594285965 (0.09378471502983882)
691
+ | > loss_mel_ce: 3.3460400104522705 (3.707569275224531)
692
+ | > loss: 0.8501484394073486 (0.9503384976612552)
693
+ | > current_lr: 5e-06
694
+ | > step_time: 0.239 (0.333674537491154)
695
+ | > loader_time: 0.0098 (0.01157456649316324)
696
+
697
+
698
+  --> TIME: 2025-06-02 12:59:16 -- STEP: 3750/5792 -- GLOBAL_STEP: 3750
699
+ | > loss_text_ce: 0.04969848692417145 (0.0932255767772594)
700
+ | > loss_mel_ce: 3.773103713989258 (3.705821061897278)
701
+ | > loss: 0.9557005763053894 (0.9497616597811379)
702
+ | > current_lr: 5e-06
703
+ | > step_time: 0.4401 (0.33399006907145184)
704
+ | > loader_time: 0.0105 (0.011564516321818038)
705
+
706
+
707
+  --> TIME: 2025-06-02 12:59:45 -- STEP: 3800/5792 -- GLOBAL_STEP: 3800
708
+ | > loss_text_ce: 0.05216987058520317 (0.09270269069232429)
709
+ | > loss_mel_ce: 4.144445419311523 (3.703308340373792)
710
+ | > loss: 1.0491538047790527 (0.9490027578567202)
711
+ | > current_lr: 5e-06
712
+ | > step_time: 0.3106 (0.3339822284171456)
713
+ | > loader_time: 0.0101 (0.011551208119643365)
714
+
715
+
716
+  --> TIME: 2025-06-02 13:00:15 -- STEP: 3850/5792 -- GLOBAL_STEP: 3850
717
+ | > loss_text_ce: 0.051087573170661926 (0.09218463780527755)
718
+ | > loss_mel_ce: 3.5129377841949463 (3.7008065200780895)
719
+ | > loss: 0.891006350517273 (0.9482477895625223)
720
+ | > current_lr: 5e-06
721
+ | > step_time: 0.4407 (0.33412671906607494)
722
+ | > loader_time: 0.0109 (0.011542596259674472)
723
+
724
+
725
+  --> TIME: 2025-06-02 13:00:46 -- STEP: 3900/5792 -- GLOBAL_STEP: 3900
726
+ | > loss_text_ce: 0.05810575187206268 (0.09166618832888501)
727
+ | > loss_mel_ce: 3.621946334838867 (3.698486832961058)
728
+ | > loss: 0.9200130105018616 (0.9475382554378262)
729
+ | > current_lr: 5e-06
730
+ | > step_time: 0.3832 (0.3345049955906012)
731
+ | > loader_time: 0.0108 (0.011529551591628641)
732
+
733
+
734
+  --> TIME: 2025-06-02 13:01:15 -- STEP: 3950/5792 -- GLOBAL_STEP: 3950
735
+ | > loss_text_ce: 0.05320081114768982 (0.09115826475186428)
736
+ | > loss_mel_ce: 3.2360873222351074 (3.695133606451976)
737
+ | > loss: 0.8223220109939575 (0.946572967918613)
738
+ | > current_lr: 5e-06
739
+ | > step_time: 0.2365 (0.33447549988951863)
740
+ | > loader_time: 0.0092 (0.011514630860920198)
741
+
742
+
743
+  --> TIME: 2025-06-02 13:01:44 -- STEP: 4000/5792 -- GLOBAL_STEP: 4000
744
+ | > loss_text_ce: 0.05098037049174309 (0.09065508700720956)
745
+ | > loss_mel_ce: 3.636732578277588 (3.6921278865337372)
746
+ | > loss: 0.9219282269477844 (0.9456957434713839)
747
+ | > current_lr: 5e-06
748
+ | > step_time: 0.3267 (0.33452891433238985)
749
+ | > loader_time: 0.0102 (0.01150596791505814)
750
+
751
+
752
+  --> TIME: 2025-06-02 13:02:12 -- STEP: 4050/5792 -- GLOBAL_STEP: 4050
753
+ | > loss_text_ce: 0.0510944202542305 (0.09017181752640514)
754
+ | > loss_mel_ce: 3.851416826248169 (3.6895573917436013)
755
+ | > loss: 0.9756278395652771 (0.9449323024131632)
756
+ | > current_lr: 5e-06
757
+ | > step_time: 0.3061 (0.33426993028617197)
758
+ | > loader_time: 0.0091 (0.01149895055794422)
759
+
760
+
761
+  --> TIME: 2025-06-02 13:02:41 -- STEP: 4100/5792 -- GLOBAL_STEP: 4100
762
+ | > loss_text_ce: 0.059732966125011444 (0.0897003380263723)
763
+ | > loss_mel_ce: 4.043025016784668 (3.6875316979245443)
764
+ | > loss: 1.0256894826889038 (0.9443080091040307)
765
+ | > current_lr: 5e-06
766
+ | > step_time: 0.3286 (0.33428408331987336)
767
+ | > loader_time: 0.0132 (0.011492944345241642)
768
+
769
+
770
+  --> TIME: 2025-06-02 13:03:11 -- STEP: 4150/5792 -- GLOBAL_STEP: 4150
771
+ | > loss_text_ce: 0.046467892825603485 (0.08922771856817126)
772
+ | > loss_mel_ce: 3.8112823963165283 (3.6847478388590984)
773
+ | > loss: 0.9644375443458557 (0.9434938894840605)
774
+ | > current_lr: 5e-06
775
+ | > step_time: 0.3452 (0.3344374258546944)
776
+ | > loader_time: 0.0125 (0.011488211528364437)
777
+
778
+
779
+  --> TIME: 2025-06-02 13:03:41 -- STEP: 4200/5792 -- GLOBAL_STEP: 4200
780
+ | > loss_text_ce: 0.05000675097107887 (0.08878737421174125)
781
+ | > loss_mel_ce: 3.3880319595336914 (3.6820667290119897)
782
+ | > loss: 0.8595097064971924 (0.9427135259338786)
783
+ | > current_lr: 5e-06
784
+ | > step_time: 0.2624 (0.33464160896482914)
785
+ | > loader_time: 0.0116 (0.011479487986791706)
786
+
787
+
788
+  --> TIME: 2025-06-02 13:04:10 -- STEP: 4250/5792 -- GLOBAL_STEP: 4250
789
+ | > loss_text_ce: 0.0450843945145607 (0.08833793204146267)
790
+ | > loss_mel_ce: 3.3565142154693604 (3.6793924145418058)
791
+ | > loss: 0.8503996729850769 (0.9419325868522416)
792
+ | > current_lr: 5e-06
793
+ | > step_time: 0.5804 (0.3347087397855871)
794
+ | > loader_time: 0.0107 (0.011472317415125237)
795
+
796
+
797
+  --> TIME: 2025-06-02 13:04:41 -- STEP: 4300/5792 -- GLOBAL_STEP: 4300
798
+ | > loss_text_ce: 0.04820011556148529 (0.08789586786960442)
799
+ | > loss_mel_ce: 3.48346209526062 (3.677155670066213)
800
+ | > loss: 0.8829155564308167 (0.9412628846806143)
801
+ | > current_lr: 5e-06
802
+ | > step_time: 0.2393 (0.33497945025909787)
803
+ | > loader_time: 0.0126 (0.011467096805572514)
804
+
805
+
806
+  --> TIME: 2025-06-02 13:05:08 -- STEP: 4350/5792 -- GLOBAL_STEP: 4350
807
+ | > loss_text_ce: 0.05529556795954704 (0.08746406216474098)
808
+ | > loss_mel_ce: 3.5293896198272705 (3.674542185136642)
809
+ | > loss: 0.8961712718009949 (0.9405015619869881)
810
+ | > current_lr: 5e-06
811
+ | > step_time: 0.3449 (0.3346871621033242)
812
+ | > loader_time: 0.0123 (0.011460946072107077)
813
+
814
+
815
+  --> TIME: 2025-06-02 13:05:40 -- STEP: 4400/5792 -- GLOBAL_STEP: 4400
816
+ | > loss_text_ce: 0.0525493361055851 (0.08705084911886261)
817
+ | > loss_mel_ce: 3.2084221839904785 (3.6720786437663175)
818
+ | > loss: 0.8152428865432739 (0.9397823734175065)
819
+ | > current_lr: 5e-06
820
+ | > step_time: 0.2394 (0.3350362862240186)
821
+ | > loader_time: 0.0123 (0.011455193866382943)
822
+
823
+
824
+  --> TIME: 2025-06-02 13:06:08 -- STEP: 4450/5792 -- GLOBAL_STEP: 4450
825
+ | > loss_text_ce: 0.048889096826314926 (0.0866304319134254)
826
+ | > loss_mel_ce: 3.5298917293548584 (3.670149624642362)
827
+ | > loss: 0.8946952223777771 (0.9391950143455109)
828
+ | > current_lr: 5e-06
829
+ | > step_time: 0.5265 (0.33503451711676124)
830
+ | > loader_time: 0.0111 (0.011446116265286221)
831
+
832
+
833
+  --> TIME: 2025-06-02 13:06:37 -- STEP: 4500/5792 -- GLOBAL_STEP: 4500
834
+ | > loss_text_ce: 0.05733109265565872 (0.08622805037432246)
835
+ | > loss_mel_ce: 3.3458251953125 (3.666845300144621)
836
+ | > loss: 0.8507890701293945 (0.9382683378458013)
837
+ | > current_lr: 5e-06
838
+ | > step_time: 0.2417 (0.33490972476535413)
839
+ | > loader_time: 0.0092 (0.011438722981346978)
840
+
841
+
842
+  --> TIME: 2025-06-02 13:07:05 -- STEP: 4550/5792 -- GLOBAL_STEP: 4550
843
+ | > loss_text_ce: 0.042130474001169205 (0.08583367971437321)
844
+ | > loss_mel_ce: 3.6195030212402344 (3.664244420135416)
845
+ | > loss: 0.9154083728790283 (0.9375195251716352)
846
+ | > current_lr: 5e-06
847
+ | > step_time: 0.492 (0.33481265240973224)
848
+ | > loader_time: 0.0104 (0.011426460035554658)
849
+
850
+
851
+  --> TIME: 2025-06-02 13:07:33 -- STEP: 4600/5792 -- GLOBAL_STEP: 4600
852
+ | > loss_text_ce: 0.0529852919280529 (0.08543671885748273)
853
+ | > loss_mel_ce: 3.0790507793426514 (3.6618555622515485)
854
+ | > loss: 0.7830089926719666 (0.9368230704898409)
855
+ | > current_lr: 5e-06
856
+ | > step_time: 0.2388 (0.3346719070102861)
857
+ | > loader_time: 0.0106 (0.011421545173810875)
858
+
859
+
860
+  --> TIME: 2025-06-02 13:08:05 -- STEP: 4650/5792 -- GLOBAL_STEP: 4650
861
+ | > loss_text_ce: 0.047616079449653625 (0.08505546001497138)
862
+ | > loss_mel_ce: 3.2773044109344482 (3.659559770348253)
863
+ | > loss: 0.831230103969574 (0.9361538077938931)
864
+ | > current_lr: 5e-06
865
+ | > step_time: 0.406 (0.3350888223032801)
866
+ | > loader_time: 0.0104 (0.011418454980337488)
867
+
868
+
869
+  --> TIME: 2025-06-02 13:08:33 -- STEP: 4700/5792 -- GLOBAL_STEP: 4700
870
+ | > loss_text_ce: 0.04613576456904411 (0.08467220809072895)
871
+ | > loss_mel_ce: 3.406162977218628 (3.657190910603139)
872
+ | > loss: 0.8630746603012085 (0.9354657798751862)
873
+ | > current_lr: 5e-06
874
+ | > step_time: 0.2167 (0.33492715972535186)
875
+ | > loader_time: 0.0128 (0.011416076599283414)
876
+
877
+
878
+  --> TIME: 2025-06-02 13:09:03 -- STEP: 4750/5792 -- GLOBAL_STEP: 4750
879
+ | > loss_text_ce: 0.052280884236097336 (0.08429848386896287)
880
+ | > loss_mel_ce: 3.40238356590271 (3.655417844571567)
881
+ | > loss: 0.8636661171913147 (0.9349290823058068)
882
+ | > current_lr: 5e-06
883
+ | > step_time: 0.2554 (0.3351128557104816)
884
+ | > loader_time: 0.0099 (0.011411911663256183)
885
+
886
+
887
+  --> TIME: 2025-06-02 13:09:32 -- STEP: 4800/5792 -- GLOBAL_STEP: 4800
888
+ | > loss_text_ce: 0.042021121829748154 (0.08392824066414815)
889
+ | > loss_mel_ce: 3.1501052379608154 (3.653419889857373)
890
+ | > loss: 0.7980315685272217 (0.9343370328471056)
891
+ | > current_lr: 5e-06
892
+ | > step_time: 0.2071 (0.33511566052834235)
893
+ | > loader_time: 0.0091 (0.011409775366385762)
894
+
895
+
896
+  --> TIME: 2025-06-02 13:10:01 -- STEP: 4850/5792 -- GLOBAL_STEP: 4850
897
+ | > loss_text_ce: 0.05289650708436966 (0.08356028458122745)
898
+ | > loss_mel_ce: 3.2315640449523926 (3.6508835728143922)
899
+ | > loss: 0.8211151361465454 (0.9336109645710768)
900
+ | > current_lr: 5e-06
901
+ | > step_time: 0.335 (0.3350185093437278)
902
+ | > loader_time: 0.0092 (0.011409686619473473)
903
+
904
+
905
+  --> TIME: 2025-06-02 13:10:29 -- STEP: 4900/5792 -- GLOBAL_STEP: 4900
906
+ | > loss_text_ce: 0.04954209551215172 (0.08320543044683895)
907
+ | > loss_mel_ce: 3.3401761054992676 (3.6483470648648804)
908
+ | > loss: 0.8474295735359192 (0.9328881240499254)
909
+ | > current_lr: 5e-06
910
+ | > step_time: 0.245 (0.3349105890916322)
911
+ | > loader_time: 0.01 (0.011410678065552989)
912
+
913
+
914
+  --> TIME: 2025-06-02 13:11:00 -- STEP: 4950/5792 -- GLOBAL_STEP: 4950
915
+ | > loss_text_ce: 0.04523472115397453 (0.08286285736118316)
916
+ | > loss_mel_ce: 3.421116352081299 (3.646339290888624)
917
+ | > loss: 0.86658775806427 (0.9323005372586868)
918
+ | > current_lr: 5e-06
919
+ | > step_time: 0.267 (0.33518396363113906)
920
+ | > loader_time: 0.0095 (0.011406980524159427)
921
+
922
+
923
+  --> TIME: 2025-06-02 13:11:30 -- STEP: 5000/5792 -- GLOBAL_STEP: 5000
924
+ | > loss_text_ce: 0.051856279373168945 (0.08252262841463104)
925
+ | > loss_mel_ce: 3.7696542739868164 (3.64465701274872)
926
+ | > loss: 0.9553776383399963 (0.931794910478591)
927
+ | > current_lr: 5e-06
928
+ | > step_time: 0.4771 (0.3353376725673679)
929
+ | > loader_time: 0.0107 (0.011403836870193468)
930
+
931
+
932
+  --> TIME: 2025-06-02 13:12:00 -- STEP: 5050/5792 -- GLOBAL_STEP: 5050
933
+ | > loss_text_ce: 0.048051297664642334 (0.08218267903026988)
934
+ | > loss_mel_ce: 3.2416117191314697 (3.6419559119007388)
935
+ | > loss: 0.8224157691001892 (0.9310346478990981)
936
+ | > current_lr: 5e-06
937
+ | > step_time: 0.2594 (0.33541135504694297)
938
+ | > loader_time: 0.0094 (0.011395886864992641)
939
+
940
+
941
+  --> TIME: 2025-06-02 13:12:30 -- STEP: 5100/5792 -- GLOBAL_STEP: 5100
942
+ | > loss_text_ce: 0.04729149118065834 (0.0818471597763256)
943
+ | > loss_mel_ce: 3.463068962097168 (3.6400194923083)
944
+ | > loss: 0.8775901198387146 (0.9304666631712624)
945
+ | > current_lr: 5e-06
946
+ | > step_time: 0.2577 (0.3356010669352963)
947
+ | > loader_time: 0.0093 (0.011388069274378751)
948
+
949
+
950
+  --> TIME: 2025-06-02 13:12:59 -- STEP: 5150/5792 -- GLOBAL_STEP: 5150
951
+ | > loss_text_ce: 0.049754828214645386 (0.08151657056822972)
952
+ | > loss_mel_ce: 3.3020455837249756 (3.638027641333424)
953
+ | > loss: 0.8379501104354858 (0.9298860531408801)
954
+ | > current_lr: 5e-06
955
+ | > step_time: 0.1726 (0.33566474298829063)
956
+ | > loader_time: 0.0116 (0.011384222206560154)
957
+
958
+
959
+  --> TIME: 2025-06-02 13:13:28 -- STEP: 5200/5792 -- GLOBAL_STEP: 5200
960
+ | > loss_text_ce: 0.05030031129717827 (0.08118891543852057)
961
+ | > loss_mel_ce: 3.18542742729187 (3.635307474182204)
962
+ | > loss: 0.8089319467544556 (0.9291240975948472)
963
+ | > current_lr: 5e-06
964
+ | > step_time: 0.3433 (0.3355758514771101)
965
+ | > loader_time: 0.0094 (0.011372857231360218)
966
+
967
+
968
+  --> TIME: 2025-06-02 13:13:59 -- STEP: 5250/5792 -- GLOBAL_STEP: 5250
969
+ | > loss_text_ce: 0.05008910596370697 (0.08087093743681945)
970
+ | > loss_mel_ce: 3.088693857192993 (3.6334633124669407)
971
+ | > loss: 0.7846957445144653 (0.9285835626579457)
972
+ | > current_lr: 5e-06
973
+ | > step_time: 0.3717 (0.33582283274332736)
974
+ | > loader_time: 0.0095 (0.011365627288818372)
975
+
976
+
977
+  --> TIME: 2025-06-02 13:14:29 -- STEP: 5300/5792 -- GLOBAL_STEP: 5300
978
+ | > loss_text_ce: 0.0471770241856575 (0.08056517972277057)
979
+ | > loss_mel_ce: 3.359286069869995 (3.6318132130155036)
980
+ | > loss: 0.8516157865524292 (0.9280945984035159)
981
+ | > current_lr: 5e-06
982
+ | > step_time: 0.3564 (0.33594532296342744)
983
+ | > loader_time: 0.0105 (0.011357401227051363)
984
+
985
+
986
+  --> TIME: 2025-06-02 13:14:58 -- STEP: 5350/5792 -- GLOBAL_STEP: 5350
987
+ | > loss_text_ce: 0.047488246113061905 (0.08026272471123812)
988
+ | > loss_mel_ce: 3.0119431018829346 (3.6297121964659658)
989
+ | > loss: 0.764857828617096 (0.9274937304937942)
990
+ | > current_lr: 5e-06
991
+ | > step_time: 0.2594 (0.33586654542762623)
992
+ | > loader_time: 0.0141 (0.011354769679987546)
993
+
994
+
995
+  --> TIME: 2025-06-02 13:15:27 -- STEP: 5400/5792 -- GLOBAL_STEP: 5400
996
+ | > loss_text_ce: 0.047491975128650665 (0.079949514229127)
997
+ | > loss_mel_ce: 3.455355405807495 (3.627370389611634)
998
+ | > loss: 0.8757118582725525 (0.9268299761966414)
999
+ | > current_lr: 5e-06
1000
+ | > step_time: 0.2422 (0.3358160152258706)
1001
+ | > loader_time: 0.0092 (0.011353673890784925)
1002
+
1003
+
1004
+  --> TIME: 2025-06-02 13:15:54 -- STEP: 5450/5792 -- GLOBAL_STEP: 5450
1005
+ | > loss_text_ce: 0.046924345195293427 (0.07964956542562501)
1006
+ | > loss_mel_ce: 3.266174554824829 (3.625191910944949)
1007
+ | > loss: 0.8282747268676758 (0.9262103693200899)
1008
+ | > current_lr: 5e-06
1009
+ | > step_time: 0.4257 (0.3356467504238875)
1010
+ | > loader_time: 0.0097 (0.011349368707849337)
1011
+
1012
+
1013
+  --> TIME: 2025-06-02 13:16:25 -- STEP: 5500/5792 -- GLOBAL_STEP: 5500
1014
+ | > loss_text_ce: 0.039876487106084824 (0.07935999841446274)
1015
+ | > loss_mel_ce: 3.372520685195923 (3.62339421679757)
1016
+ | > loss: 0.8530992865562439 (0.9256885540377001)
1017
+ | > current_lr: 5e-06
1018
+ | > step_time: 0.2624 (0.3357687810984538)
1019
+ | > loader_time: 0.0098 (0.011344045855782236)
1020
+
1021
+
1022
+  --> TIME: 2025-06-02 13:16:53 -- STEP: 5550/5792 -- GLOBAL_STEP: 5550
1023
+ | > loss_text_ce: 0.05153404176235199 (0.07907796377802799)
1024
+ | > loss_mel_ce: 3.4474453926086426 (3.6210933367411307)
1025
+ | > loss: 0.8747448325157166 (0.9250428253680728)
1026
+ | > current_lr: 5e-06
1027
+ | > step_time: 0.2201 (0.3357482298000451)
1028
+ | > loader_time: 0.0085 (0.011339900300309447)
1029
+
1030
+
1031
+  --> TIME: 2025-06-02 13:17:23 -- STEP: 5600/5792 -- GLOBAL_STEP: 5600
1032
+ | > loss_text_ce: 0.05078967288136482 (0.07880074134562218)
1033
+ | > loss_mel_ce: 3.4180071353912354 (3.6193902756060887)
1034
+ | > loss: 0.867199182510376 (0.9245477544835627)
1035
+ | > current_lr: 5e-06
1036
+ | > step_time: 0.2526 (0.33589044515575817)
1037
+ | > loader_time: 0.0093 (0.011337815778596043)
1038
+
1039
+
1040
+  --> TIME: 2025-06-02 13:17:51 -- STEP: 5650/5792 -- GLOBAL_STEP: 5650
1041
+ | > loss_text_ce: 0.053694888949394226 (0.07851750885227134)
1042
+ | > loss_mel_ce: 3.102844476699829 (3.6172207505724083)
1043
+ | > loss: 0.7891348600387573 (0.9239345651073785)
1044
+ | > current_lr: 5e-06
1045
+ | > step_time: 0.3004 (0.33572653774666705)
1046
+ | > loader_time: 0.0096 (0.011334677755305183)
1047
+
1048
+
1049
+  --> TIME: 2025-06-02 13:18:20 -- STEP: 5700/5792 -- GLOBAL_STEP: 5700
1050
+ | > loss_text_ce: 0.0459514856338501 (0.07823847596974794)
1051
+ | > loss_mel_ce: 3.5479061603546143 (3.615112853050233)
1052
+ | > loss: 0.8984644412994385 (0.9233378325533441)
1053
+ | > current_lr: 5e-06
1054
+ | > step_time: 0.3254 (0.3357153525687113)
1055
+ | > loader_time: 0.0106 (0.011330316192225392)
1056
+
1057
+
1058
+  --> TIME: 2025-06-02 13:18:49 -- STEP: 5750/5792 -- GLOBAL_STEP: 5750
1059
+ | > loss_text_ce: 0.04507908225059509 (0.07796999881967298)
1060
+ | > loss_mel_ce: 3.325566530227661 (3.613506336834121)
1061
+ | > loss: 0.8426613807678223 (0.9228690842441882)
1062
+ | > current_lr: 5e-06
1063
+ | > step_time: 0.2647 (0.3357090207804818)
1064
+ | > loader_time: 0.0132 (0.011328426651332674)
1065
+
1066
+
1067
+  > EVALUATION 
1068
+
1069
+
1070
+ --> EVAL PERFORMANCE
1071
+ | > avg_loader_time: 0.2016423004495222 (+0)
1072
+ | > avg_loss_text_ce: 0.04477991773036578 (+0)
1073
+ | > avg_loss_mel_ce: 3.199991709768435 (+0)
1074
+ | > avg_loss: 3.2447716307505363 (+0)
1075
+
1076
+
1077
+  > EPOCH: 1/2
1078
+ --> xTTS/checkpoints/GPT_XTTS_FT-June-02-2025_12+22PM-0000000
1079
+
1080
+  > TRAINING (2025-06-02 13:26:46) 
1081
+
1082
+  --> TIME: 2025-06-02 13:26:52 -- STEP: 8/5792 -- GLOBAL_STEP: 5800
1083
+ | > loss_text_ce: 0.04962650686502457 (0.04589929943904281)
1084
+ | > loss_mel_ce: 3.476445436477661 (3.366139441728592)
1085
+ | > loss: 0.8815180063247681 (0.8530096858739853)
1086
+ | > current_lr: 5e-06
1087
+ | > step_time: 0.2652 (0.30354249477386475)
1088
+ | > loader_time: 0.0149 (0.011465787887573242)
1089
+
1090
+
1091
+  --> TIME: 2025-06-02 13:27:22 -- STEP: 58/5792 -- GLOBAL_STEP: 5850
1092
+ | > loss_text_ce: 0.043131984770298004 (0.046852813262877796)
1093
+ | > loss_mel_ce: 3.372774600982666 (3.4303042765321403)
1094
+ | > loss: 0.8539766669273376 (0.8692892707627395)
1095
+ | > current_lr: 5e-06
1096
+ | > step_time: 0.196 (0.3431942175174581)
1097
+ | > loader_time: 0.0127 (0.01153468263560328)
1098
+
1099
+
1100
+  --> TIME: 2025-06-02 13:27:52 -- STEP: 108/5792 -- GLOBAL_STEP: 5900
1101
+ | > loss_text_ce: 0.04623452201485634 (0.04628198635247019)
1102
+ | > loss_mel_ce: 2.833378553390503 (3.3930641324431807)
1103
+ | > loss: 0.719903290271759 (0.8598365292504981)
1104
+ | > current_lr: 5e-06
1105
+ | > step_time: 0.1939 (0.3511013057496811)
1106
+ | > loader_time: 0.0098 (0.011563482107939544)
1107
+
1108
+
1109
+  --> TIME: 2025-06-02 13:28:22 -- STEP: 158/5792 -- GLOBAL_STEP: 5950
1110
+ | > loss_text_ce: 0.04444996267557144 (0.04651183649143084)
1111
+ | > loss_mel_ce: 3.0688369274139404 (3.396478888354724)
1112
+ | > loss: 0.7783217430114746 (0.8607476810111275)
1113
+ | > current_lr: 5e-06
1114
+ | > step_time: 0.3297 (0.34766890278345425)
1115
+ | > loader_time: 0.0103 (0.01131993305834034)
1116
+
1117
+
1118
+  --> TIME: 2025-06-02 13:28:52 -- STEP: 208/5792 -- GLOBAL_STEP: 6000
1119
+ | > loss_text_ce: 0.05267365276813507 (0.04660090401123921)
1120
+ | > loss_mel_ce: 3.509939670562744 (3.395385509500137)
1121
+ | > loss: 0.8906533122062683 (0.8604966015196763)
1122
+ | > current_lr: 5e-06
1123
+ | > step_time: 0.2086 (0.34758124213952296)
1124
+ | > loader_time: 0.0099 (0.01136978314473079)
1125
+
1126
+
1127
+  --> TIME: 2025-06-02 13:29:21 -- STEP: 258/5792 -- GLOBAL_STEP: 6050
1128
+ | > loss_text_ce: 0.042573895305395126 (0.04650543012129243)
1129
+ | > loss_mel_ce: 3.198530673980713 (3.406408476275067)
1130
+ | > loss: 0.8102761507034302 (0.8632284749385922)
1131
+ | > current_lr: 5e-06
1132
+ | > step_time: 0.2678 (0.34636553313380997)
1133
+ | > loader_time: 0.0093 (0.011229625968045968)
1134
+
1135
+
1136
+  --> TIME: 2025-06-02 13:29:52 -- STEP: 308/5792 -- GLOBAL_STEP: 6100
1137
+ | > loss_text_ce: 0.04070745408535004 (0.04644479884391093)
1138
+ | > loss_mel_ce: 3.161195993423462 (3.409244562124277)
1139
+ | > loss: 0.8004758358001709 (0.8639223389811327)
1140
+ | > current_lr: 5e-06
1141
+ | > step_time: 0.2063 (0.34847222055707666)
1142
+ | > loader_time: 0.0096 (0.0111765776361738)
1143
+
1144
+
1145
+  --> TIME: 2025-06-02 13:30:21 -- STEP: 358/5792 -- GLOBAL_STEP: 6150
1146
+ | > loss_text_ce: 0.04625903442502022 (0.04643882370902503)
1147
+ | > loss_mel_ce: 3.097196578979492 (3.4112956583832896)
1148
+ | > loss: 0.7858638763427734 (0.8644336197296332)
1149
+ | > current_lr: 5e-06
1150
+ | > step_time: 0.2922 (0.3459169138743227)
1151
+ | > loader_time: 0.01 (0.011164890321273387)
1152
+
1153
+
1154
+  --> TIME: 2025-06-02 13:30:50 -- STEP: 408/5792 -- GLOBAL_STEP: 6200
1155
+ | > loss_text_ce: 0.048108890652656555 (0.04640357185374285)
1156
+ | > loss_mel_ce: 3.2346646785736084 (3.406046163217694)
1157
+ | > loss: 0.8206933736801147 (0.8631124335761161)
1158
+ | > current_lr: 5e-06
1159
+ | > step_time: 0.3294 (0.3447357386934993)
1160
+ | > loader_time: 0.0094 (0.01113292983933992)
1161
+
1162
+
1163
+  --> TIME: 2025-06-02 13:31:20 -- STEP: 458/5792 -- GLOBAL_STEP: 6250
1164
+ | > loss_text_ce: 0.048285942524671555 (0.046448883360585246)
1165
+ | > loss_mel_ce: 3.597545862197876 (3.4110943599559334)
1166
+ | > loss: 0.9114579558372498 (0.8643858104814085)
1167
+ | > current_lr: 5e-06
1168
+ | > step_time: 0.5765 (0.34515097307846554)
1169
+ | > loader_time: 0.0139 (0.011161539752410497)
1170
+
1171
+
1172
+  --> TIME: 2025-06-02 13:31:50 -- STEP: 508/5792 -- GLOBAL_STEP: 6300
1173
+ | > loss_text_ce: 0.05213044583797455 (0.04643937362372638)
1174
+ | > loss_mel_ce: 3.4266021251678467 (3.4056746645236577)
1175
+ | > loss: 0.8696831464767456 (0.8630285095276793)
1176
+ | > current_lr: 5e-06
1177
+ | > step_time: 0.3125 (0.3454259840522227)
1178
+ | > loader_time: 0.0139 (0.011176920781924038)
1179
+
1180
+
1181
+  --> TIME: 2025-06-02 13:32:18 -- STEP: 558/5792 -- GLOBAL_STEP: 6350
1182
+ | > loss_text_ce: 0.04106414318084717 (0.046370733649499944)
1183
+ | > loss_mel_ce: 3.4639627933502197 (3.397692242830885)
1184
+ | > loss: 0.8762567043304443 (0.8610157443203804)
1185
+ | > current_lr: 5e-06
1186
+ | > step_time: 0.1901 (0.3433396858981008)
1187
+ | > loader_time: 0.0088 (0.011147387138831576)
1188
+
1189
+
1190
+  --> TIME: 2025-06-02 13:32:45 -- STEP: 608/5792 -- GLOBAL_STEP: 6400
1191
+ | > loss_text_ce: 0.04276609793305397 (0.04636223264969883)
1192
+ | > loss_mel_ce: 3.2456772327423096 (3.394306463630576)
1193
+ | > loss: 0.8221108317375183 (0.8601671744530135)
1194
+ | > current_lr: 5e-06
1195
+ | > step_time: 0.2159 (0.3403207224450615)
1196
+ | > loader_time: 0.0096 (0.011096751611483725)
1197
+
1198
+
1199
+  --> TIME: 2025-06-02 13:33:15 -- STEP: 658/5792 -- GLOBAL_STEP: 6450
1200
+ | > loss_text_ce: 0.04221029207110405 (0.046298709672544465)
1201
+ | > loss_mel_ce: 3.376603364944458 (3.3917605297181384)
1202
+ | > loss: 0.854703426361084 (0.8595148100316702)
1203
+ | > current_lr: 5e-06
1204
+ | > step_time: 0.2959 (0.3403355002765601)
1205
+ | > loader_time: 0.0123 (0.01107824609634724)
1206
+
1207
+
1208
+  --> TIME: 2025-06-02 13:33:45 -- STEP: 708/5792 -- GLOBAL_STEP: 6500
1209
+ | > loss_text_ce: 0.0455298125743866 (0.04620274585161333)
1210
+ | > loss_mel_ce: 3.368074655532837 (3.3912849278099793)
1211
+ | > loss: 0.8534011244773865 (0.8593719188271266)
1212
+ | > current_lr: 5e-06
1213
+ | > step_time: 0.2983 (0.34120621048124494)
1214
+ | > loader_time: 0.0097 (0.011063059507790253)
1215
+
1216
+
1217
+  --> TIME: 2025-06-02 13:34:16 -- STEP: 758/5792 -- GLOBAL_STEP: 6550
1218
+ | > loss_text_ce: 0.04287204146385193 (0.046180617649078645)
1219
+ | > loss_mel_ce: 3.8676905632019043 (3.3931626326805056)
1220
+ | > loss: 0.9776406288146973 (0.8598358130234839)
1221
+ | > current_lr: 5e-06
1222
+ | > step_time: 0.4669 (0.3418154071692112)
1223
+ | > loader_time: 0.0107 (0.011052280743078057)
1224
+
1225
+
1226
+  --> TIME: 2025-06-02 13:34:44 -- STEP: 808/5792 -- GLOBAL_STEP: 6600
1227
+ | > loss_text_ce: 0.05415653437376022 (0.046192656469411446)
1228
+ | > loss_mel_ce: 3.0190370082855225 (3.3884078290792976)
1229
+ | > loss: 0.7682983875274658 (0.8586501218747382)
1230
+ | > current_lr: 5e-06
1231
+ | > step_time: 0.2155 (0.3408614199350376)
1232
+ | > loader_time: 0.0131 (0.011081166196577612)
1233
+
1234
+
1235
+  --> TIME: 2025-06-02 13:35:13 -- STEP: 858/5792 -- GLOBAL_STEP: 6650
1236
+ | > loss_text_ce: 0.0525716096162796 (0.04613834280425154)
1237
+ | > loss_mel_ce: 3.157501459121704 (3.3880747826227218)
1238
+ | > loss: 0.8025182485580444 (0.8585532816834658)
1239
+ | > current_lr: 5e-06
1240
+ | > step_time: 0.3441 (0.34018004690850534)
1241
+ | > loader_time: 0.0116 (0.01113530639168265)
1242
+
1243
+
1244
+  --> TIME: 2025-06-02 13:35:43 -- STEP: 908/5792 -- GLOBAL_STEP: 6700
1245
+ | > loss_text_ce: 0.04388509690761566 (0.04611085562060055)
1246
+ | > loss_mel_ce: 3.5000290870666504 (3.382610232031818)
1247
+ | > loss: 0.8859785199165344 (0.8571802720361866)
1248
+ | > current_lr: 5e-06
1249
+ | > step_time: 0.2732 (0.3402811882254311)
1250
+ | > loader_time: 0.0098 (0.01113482561405534)
1251
+
1252
+
1253
+  --> TIME: 2025-06-02 13:36:12 -- STEP: 958/5792 -- GLOBAL_STEP: 6750
1254
+ | > loss_text_ce: 0.04200979322195053 (0.046016689794359696)
1255
+ | > loss_mel_ce: 3.2025482654571533 (3.386244950812147)
1256
+ | > loss: 0.8111395239830017 (0.858065410110796)
1257
+ | > current_lr: 5e-06
1258
+ | > step_time: 0.3037 (0.3404792111701408)
1259
+ | > loader_time: 0.013 (0.011161978409036457)
1260
+
1261
+
1262
+  --> TIME: 2025-06-02 13:36:43 -- STEP: 1008/5792 -- GLOBAL_STEP: 6800
1263
+ | > loss_text_ce: 0.04656248539686203 (0.045978479368227784)
1264
+ | > loss_mel_ce: 3.424415349960327 (3.3894127905368805)
1265
+ | > loss: 0.8677444458007812 (0.8588478174947555)
1266
+ | > current_lr: 5e-06
1267
+ | > step_time: 0.2899 (0.34167367078009103)
1268
+ | > loader_time: 0.0134 (0.011164178214375927)
1269
+
1270
+
1271
+  --> TIME: 2025-06-02 13:37:12 -- STEP: 1058/5792 -- GLOBAL_STEP: 6850
1272
+ | > loss_text_ce: 0.044847793877124786 (0.04588977389727736)
1273
+ | > loss_mel_ce: 3.753209352493286 (3.3859440919807593)
1274
+ | > loss: 0.9495142698287964 (0.8579584664342983)
1275
+ | > current_lr: 5e-06
1276
+ | > step_time: 0.4274 (0.3409366860056194)
1277
+ | > loader_time: 0.0106 (0.01115854092942293)
1278
+
1279
+
1280
+  --> TIME: 2025-06-02 13:37:40 -- STEP: 1108/5792 -- GLOBAL_STEP: 6900
1281
+ | > loss_text_ce: 0.04726967215538025 (0.04582478893443348)
1282
+ | > loss_mel_ce: 3.5899248123168945 (3.3846601099313816)
1283
+ | > loss: 0.9092985987663269 (0.8576212246172693)
1284
+ | > current_lr: 5e-06
1285
+ | > step_time: 0.2437 (0.34045616211873964)
1286
+ | > loader_time: 0.0129 (0.01116837792448188)
1287
+
1288
+
1289
+  --> TIME: 2025-06-02 13:38:07 -- STEP: 1158/5792 -- GLOBAL_STEP: 6950
1290
+ | > loss_text_ce: 0.04457278549671173 (0.04580180395864244)
1291
+ | > loss_mel_ce: 3.8512332439422607 (3.3820696817984452)
1292
+ | > loss: 0.973951518535614 (0.8569678713620632)
1293
+ | > current_lr: 5e-06
1294
+ | > step_time: 0.4823 (0.33921163193302456)
1295
+ | > loader_time: 0.0142 (0.011172776601063361)
1296
+
1297
+
1298
+  --> TIME: 2025-06-02 13:38:35 -- STEP: 1208/5792 -- GLOBAL_STEP: 7000
1299
+ | > loss_text_ce: 0.04959622770547867 (0.04575508566178527)
1300
+ | > loss_mel_ce: 3.364575147628784 (3.3837944628387104)
1301
+ | > loss: 0.8535428643226624 (0.8573873870518821)
1302
+ | > current_lr: 5e-06
1303
+ | > step_time: 0.2847 (0.3381390390017176)
1304
+ | > loader_time: 0.0111 (0.011142225849707395)
1305
+
1306
+
1307
+  --> TIME: 2025-06-02 13:39:03 -- STEP: 1258/5792 -- GLOBAL_STEP: 7050
1308
+ | > loss_text_ce: 0.04021007567644119 (0.04573283367309072)
1309
+ | > loss_mel_ce: 3.0035552978515625 (3.3824539863058805)
1310
+ | > loss: 0.7609413266181946 (0.857046704982157)
1311
+ | > current_lr: 5e-06
1312
+ | > step_time: 0.2549 (0.33753872296768467)
1313
+ | > loader_time: 0.0098 (0.011134225537553312)
1314
+
1315
+
1316
+  --> TIME: 2025-06-02 13:39:33 -- STEP: 1308/5792 -- GLOBAL_STEP: 7100
1317
+ | > loss_text_ce: 0.04839656129479408 (0.04570206533263035)
1318
+ | > loss_mel_ce: 3.5808684825897217 (3.3811058704641623)
1319
+ | > loss: 0.907316267490387 (0.8567019838623313)
1320
+ | > current_lr: 5e-06
1321
+ | > step_time: 0.2648 (0.33785488281045833)
1322
+ | > loader_time: 0.0126 (0.01116103261982629)
1323
+
1324
+
1325
+  --> TIME: 2025-06-02 13:40:03 -- STEP: 1358/5792 -- GLOBAL_STEP: 7150
1326
+ | > loss_text_ce: 0.04666044935584068 (0.04564322057743835)
1327
+ | > loss_mel_ce: 3.6217894554138184 (3.380934479310341)
1328
+ | > loss: 0.9171124696731567 (0.8566444249321679)
1329
+ | > current_lr: 5e-06
1330
+ | > step_time: 0.3762 (0.33808904807241025)
1331
+ | > loader_time: 0.0109 (0.011158927726464696)
1332
+
1333
+
1334
+  --> TIME: 2025-06-02 13:40:32 -- STEP: 1408/5792 -- GLOBAL_STEP: 7200
1335
+ | > loss_text_ce: 0.04733112081885338 (0.04561400014790704)
1336
+ | > loss_mel_ce: 3.2079885005950928 (3.3785352163355467)
1337
+ | > loss: 0.8138298988342285 (0.8560373041440138)
1338
+ | > current_lr: 5e-06
1339
+ | > step_time: 0.3122 (0.33810166117142526)
1340
+ | > loader_time: 0.0126 (0.01115033944899386)
1341
+
1342
+
1343
+  --> TIME: 2025-06-02 13:41:01 -- STEP: 1458/5792 -- GLOBAL_STEP: 7250
1344
+ | > loss_text_ce: 0.04578738287091255 (0.04558497288140411)
1345
+ | > loss_mel_ce: 3.535698652267456 (3.3770085259229563)
1346
+ | > loss: 0.8953714966773987 (0.8556483748109578)
1347
+ | > current_lr: 5e-06
1348
+ | > step_time: 0.4838 (0.3378712419619776)
1349
+ | > loader_time: 0.0114 (0.011147373661255806)
1350
+
1351
+
1352
+  --> TIME: 2025-06-02 13:41:31 -- STEP: 1508/5792 -- GLOBAL_STEP: 7300
1353
+ | > loss_text_ce: 0.04656374827027321 (0.0455558022918965)
1354
+ | > loss_mel_ce: 3.0295422077178955 (3.3743492266859874)
1355
+ | > loss: 0.769026517868042 (0.8549762572944953)
1356
+ | > current_lr: 5e-06
1357
+ | > step_time: 0.1911 (0.3378510261720308)
1358
+ | > loader_time: 0.0125 (0.011148763430213425)
1359
+
1360
+
1361
+  --> TIME: 2025-06-02 13:42:00 -- STEP: 1558/5792 -- GLOBAL_STEP: 7350
1362
+ | > loss_text_ce: 0.04521656408905983 (0.04553002290743304)
1363
+ | > loss_mel_ce: 3.1920225620269775 (3.3758582573034954)
1364
+ | > loss: 0.8093097805976868 (0.8553470702165203)
1365
+ | > current_lr: 5e-06
1366
+ | > step_time: 0.2611 (0.33788349662114764)
1367
+ | > loader_time: 0.0099 (0.011138344300726721)
1368
+
1369
+
1370
+  --> TIME: 2025-06-02 13:42:28 -- STEP: 1608/5792 -- GLOBAL_STEP: 7400
1371
+ | > loss_text_ce: 0.04438360407948494 (0.04550991758741251)
1372
+ | > loss_mel_ce: 3.5675711631774902 (3.3738133661189487)
1373
+ | > loss: 0.9029886722564697 (0.8548308210215757)
1374
+ | > current_lr: 5e-06
1375
+ | > step_time: 0.2961 (0.33742392522778664)
1376
+ | > loader_time: 0.0104 (0.011117274488382674)
1377
+
1378
+
1379
+  --> TIME: 2025-06-02 13:42:59 -- STEP: 1658/5792 -- GLOBAL_STEP: 7450
1380
+ | > loss_text_ce: 0.04301420599222183 (0.045442521491226136)
1381
+ | > loss_mel_ce: 3.6960628032684326 (3.37488229901713)
1382
+ | > loss: 0.9347692728042603 (0.8550812052371849)
1383
+ | > current_lr: 5e-06
1384
+ | > step_time: 0.3433 (0.3380332888970184)
1385
+ | > loader_time: 0.0102 (0.011103294151683483)
1386
+
1387
+
1388
+  --> TIME: 2025-06-02 13:43:27 -- STEP: 1708/5792 -- GLOBAL_STEP: 7500
1389
+ | > loss_text_ce: 0.043877944350242615 (0.0453859308027865)
1390
+ | > loss_mel_ce: 2.9800140857696533 (3.373501475298433)
1391
+ | > loss: 0.7559729814529419 (0.854721851594554)
1392
+ | > current_lr: 5e-06
1393
+ | > step_time: 0.2149 (0.3374889004425923)
1394
+ | > loader_time: 0.0099 (0.011108080732180309)
1395
+
1396
+
1397
+  --> TIME: 2025-06-02 13:43:55 -- STEP: 1758/5792 -- GLOBAL_STEP: 7550
1398
+ | > loss_text_ce: 0.03403538465499878 (0.04534231755878262)
1399
+ | > loss_mel_ce: 2.907336711883545 (3.3728728446266083)
1400
+ | > loss: 0.7353430390357971 (0.8545537905956154)
1401
+ | > current_lr: 5e-06
1402
+ | > step_time: 0.2952 (0.33705969068377484)
1403
+ | > loader_time: 0.0126 (0.011120306347010488)
1404
+
1405
+
1406
+  --> TIME: 2025-06-02 13:44:24 -- STEP: 1808/5792 -- GLOBAL_STEP: 7600
1407
+ | > loss_text_ce: 0.036792825907468796 (0.04528982559552794)
1408
+ | > loss_mel_ce: 3.696239948272705 (3.3725063265424917)
1409
+ | > loss: 0.9332581758499146 (0.8544490379569804)
1410
+ | > current_lr: 5e-06
1411
+ | > step_time: 0.2613 (0.33688090253720226)
1412
+ | > loader_time: 0.0099 (0.011115989721981826)
1413
+
1414
+
1415
+  --> TIME: 2025-06-02 13:44:52 -- STEP: 1858/5792 -- GLOBAL_STEP: 7650
1416
+ | > loss_text_ce: 0.0398641973733902 (0.04524051058893085)
1417
+ | > loss_mel_ce: 3.4899098873138428 (3.3701283141280918)
1418
+ | > loss: 0.8824435472488403 (0.8538422060025648)
1419
+ | > current_lr: 5e-06
1420
+ | > step_time: 0.2554 (0.3364653570659706)
1421
+ | > loader_time: 0.0106 (0.011119012812110403)
1422
+
1423
+
1424
+  --> TIME: 2025-06-02 13:45:21 -- STEP: 1908/5792 -- GLOBAL_STEP: 7700
1425
+ | > loss_text_ce: 0.041133489459753036 (0.04522936787179024)
1426
+ | > loss_mel_ce: 3.302356481552124 (3.369759842034906)
1427
+ | > loss: 0.83587247133255 (0.8537473023187688)
1428
+ | > current_lr: 5e-06
1429
+ | > step_time: 0.2272 (0.33635687903038347)
1430
+ | > loader_time: 0.0118 (0.011123857283242345)
1431
+
1432
+
1433
+  --> TIME: 2025-06-02 13:45:49 -- STEP: 1958/5792 -- GLOBAL_STEP: 7750
1434
+ | > loss_text_ce: 0.04508072882890701 (0.045202246298242925)
1435
+ | > loss_mel_ce: 4.015277862548828 (3.3691428928988953)
1436
+ | > loss: 1.0150896310806274 (0.8535862845712101)
1437
+ | > current_lr: 5e-06
1438
+ | > step_time: 0.2578 (0.33611000692272075)
1439
+ | > loader_time: 0.01 (0.01110899363158307)
1440
+
1441
+
1442
+  --> TIME: 2025-06-02 13:46:17 -- STEP: 2008/5792 -- GLOBAL_STEP: 7800
1443
+ | > loss_text_ce: 0.040393561124801636 (0.04518387101839232)
1444
+ | > loss_mel_ce: 3.0478756427764893 (3.367659538628096)
1445
+ | > loss: 0.7720673084259033 (0.8532108522625083)
1446
+ | > current_lr: 5e-06
1447
+ | > step_time: 0.2072 (0.33570773442427937)
1448
+ | > loader_time: 0.0096 (0.011115385597920515)
1449
+
1450
+
1451
+  --> TIME: 2025-06-02 13:46:46 -- STEP: 2058/5792 -- GLOBAL_STEP: 7850
1452
+ | > loss_text_ce: 0.04632216691970825 (0.04517081452024949)
1453
+ | > loss_mel_ce: 3.593982458114624 (3.366837971527908)
1454
+ | > loss: 0.9100761413574219 (0.8530021963534248)
1455
+ | > current_lr: 5e-06
1456
+ | > step_time: 0.6081 (0.3358027338402379)
1457
+ | > loader_time: 0.0109 (0.011119001460839988)
1458
+
1459
+
1460
+  --> TIME: 2025-06-02 13:47:17 -- STEP: 2108/5792 -- GLOBAL_STEP: 7900
1461
+ | > loss_text_ce: 0.046658266335725784 (0.04515279076756657)
1462
+ | > loss_mel_ce: 3.4903433322906494 (3.3665078355872438)
1463
+ | > loss: 0.8842504024505615 (0.8529151564983534)
1464
+ | > current_lr: 5e-06
1465
+ | > step_time: 0.4083 (0.3364675910242141)
1466
+ | > loader_time: 0.0108 (0.011129756353837944)
1467
+
1468
+
1469
+  --> TIME: 2025-06-02 13:47:46 -- STEP: 2158/5792 -- GLOBAL_STEP: 7950
1470
+ | > loss_text_ce: 0.04326501116156578 (0.04511675107844374)
1471
+ | > loss_mel_ce: 3.7370195388793945 (3.365736954072983)
1472
+ | > loss: 0.9450711607933044 (0.8527134262211792)
1473
+ | > current_lr: 5e-06
1474
+ | > step_time: 0.3271 (0.3362241120554982)
1475
+ | > loader_time: 0.0134 (0.011123432065735718)
1476
+
1477
+
1478
+  --> TIME: 2025-06-02 13:48:15 -- STEP: 2208/5792 -- GLOBAL_STEP: 8000
1479
+ | > loss_text_ce: 0.04933100938796997 (0.04507167210238918)
1480
+ | > loss_mel_ce: 3.107172966003418 (3.364644525707631)
1481
+ | > loss: 0.7891259789466858 (0.8524290493594995)
1482
+ | > current_lr: 5e-06
1483
+ | > step_time: 0.2153 (0.33627322402553284)
1484
+ | > loader_time: 0.0104 (0.011131968619166944)
1485
+
1486
+
1487
+  --> TIME: 2025-06-02 13:48:43 -- STEP: 2258/5792 -- GLOBAL_STEP: 8050
1488
+ | > loss_text_ce: 0.0509863905608654 (0.04505748897576433)
1489
+ | > loss_mel_ce: 3.511996269226074 (3.3630264050354457)
1490
+ | > loss: 0.8907456398010254 (0.8520209733578247)
1491
+ | > current_lr: 5e-06
1492
+ | > step_time: 0.4846 (0.3359105811908882)
1493
+ | > loader_time: 0.0105 (0.011143911930182215)
1494
+
1495
+
1496
+  --> TIME: 2025-06-02 13:49:15 -- STEP: 2308/5792 -- GLOBAL_STEP: 8100
1497
+ | > loss_text_ce: 0.04393137991428375 (0.045034569963341514)
1498
+ | > loss_mel_ce: 2.9540281295776367 (3.362665162962478)
1499
+ | > loss: 0.7494899034500122 (0.8519249330411955)
1500
+ | > current_lr: 5e-06
1501
+ | > step_time: 0.2141 (0.33653108003441157)
1502
+ | > loader_time: 0.0099 (0.011160700325329545)
1503
+
1504
+
1505
+  --> TIME: 2025-06-02 13:49:45 -- STEP: 2358/5792 -- GLOBAL_STEP: 8150
1506
+ | > loss_text_ce: 0.04599863290786743 (0.04497464137914904)
1507
+ | > loss_mel_ce: 3.3983516693115234 (3.363921059098053)
1508
+ | > loss: 0.8610875606536865 (0.8522239249070076)
1509
+ | > current_lr: 5e-06
1510
+ | > step_time: 0.3547 (0.3368334209039317)
1511
+ | > loader_time: 0.0101 (0.011168763332188992)
1512
+
1513
+
1514
+  --> TIME: 2025-06-02 13:50:13 -- STEP: 2408/5792 -- GLOBAL_STEP: 8200
1515
+ | > loss_text_ce: 0.0410335548222065 (0.04494161312808699)
1516
+ | > loss_mel_ce: 2.8338351249694824 (3.362981556558925)
1517
+ | > loss: 0.7187171578407288 (0.8519807922285653)
1518
+ | > current_lr: 5e-06
1519
+ | > step_time: 0.1891 (0.33666275852542393)
1520
+ | > loader_time: 0.0106 (0.011154194111839877)
1521
+
1522
+
1523
+  --> TIME: 2025-06-02 13:50:44 -- STEP: 2458/5792 -- GLOBAL_STEP: 8250
1524
+ | > loss_text_ce: 0.04467436298727989 (0.04491210582857818)
1525
+ | > loss_mel_ce: 3.4023947715759277 (3.362881237942576)
1526
+ | > loss: 0.8617672920227051 (0.8519483357546673)
1527
+ | > current_lr: 5e-06
1528
+ | > step_time: 0.2659 (0.3370825775858832)
1529
+ | > loader_time: 0.0133 (0.011155853725430246)
1530
+
1531
+
1532
+  --> TIME: 2025-06-02 13:51:14 -- STEP: 2508/5792 -- GLOBAL_STEP: 8300
1533
+ | > loss_text_ce: 0.0431218296289444 (0.04488258450210355)
1534
+ | > loss_mel_ce: 3.392765998840332 (3.3614071411759463)
1535
+ | > loss: 0.8589719533920288 (0.8515724312793311)
1536
+ | > current_lr: 5e-06
1537
+ | > step_time: 0.3281 (0.33738990387087664)
1538
+ | > loader_time: 0.0105 (0.011148617028049297)
1539
+
1540
+
1541
+  --> TIME: 2025-06-02 13:51:43 -- STEP: 2558/5792 -- GLOBAL_STEP: 8350
1542
+ | > loss_text_ce: 0.042658887803554535 (0.0448320767120622)
1543
+ | > loss_mel_ce: 3.3791749477386475 (3.3615849678920893)
1544
+ | > loss: 0.8554584383964539 (0.8516042609993973)
1545
+ | > current_lr: 5e-06
1546
+ | > step_time: 0.2925 (0.3374677144334692)
1547
+ | > loader_time: 0.0098 (0.011138459217557677)
1548
+
1549
+
1550
+  --> TIME: 2025-06-02 13:52:12 -- STEP: 2608/5792 -- GLOBAL_STEP: 8400
1551
+ | > loss_text_ce: 0.03552236780524254 (0.04479173863623748)
1552
+ | > loss_mel_ce: 2.71285343170166 (3.360946973682912)
1553
+ | > loss: 0.68709397315979 (0.851434677976406)
1554
+ | > current_lr: 5e-06
1555
+ | > step_time: 0.1824 (0.33746330266349933)
1556
+ | > loader_time: 0.012 (0.01112092354911966)
1557
+
1558
+
1559
+  --> TIME: 2025-06-02 13:52:40 -- STEP: 2658/5792 -- GLOBAL_STEP: 8450
1560
+ | > loss_text_ce: 0.045843303203582764 (0.0447824644737725)
1561
+ | > loss_mel_ce: 3.9877352714538574 (3.3594930995398657)
1562
+ | > loss: 1.0083945989608765 (0.8510688909289527)
1563
+ | > current_lr: 5e-06
1564
+ | > step_time: 0.2632 (0.337073947003769)
1565
+ | > loader_time: 0.0093 (0.011125095810900938)
1566
+
1567
+
1568
+  --> TIME: 2025-06-02 13:53:08 -- STEP: 2708/5792 -- GLOBAL_STEP: 8500
1569
+ | > loss_text_ce: 0.04365628957748413 (0.04472194665195517)
1570
+ | > loss_mel_ce: 3.502896308898926 (3.358532962189819)
1571
+ | > loss: 0.8866381645202637 (0.8508137271710651)
1572
+ | > current_lr: 5e-06
1573
+ | > step_time: 0.3144 (0.3368678473299331)
1574
+ | > loader_time: 0.014 (0.011116211136123433)
1575
+
1576
+
1577
+  --> TIME: 2025-06-02 13:53:36 -- STEP: 2758/5792 -- GLOBAL_STEP: 8550
1578
+ | > loss_text_ce: 0.044692471623420715 (0.044700241800292285)
1579
+ | > loss_mel_ce: 3.10170578956604 (3.356952504717499)
1580
+ | > loss: 0.7865995764732361 (0.8504131865769383)
1581
+ | > current_lr: 5e-06
1582
+ | > step_time: 0.1751 (0.33657982109075396)
1583
+ | > loader_time: 0.0096 (0.011113227234965547)
1584
+
1585
+
1586
+  --> TIME: 2025-06-02 13:54:05 -- STEP: 2808/5792 -- GLOBAL_STEP: 8600
1587
+ | > loss_text_ce: 0.04175104200839996 (0.044659487993995044)
1588
+ | > loss_mel_ce: 3.7599270343780518 (3.3555039678543714)
1589
+ | > loss: 0.950419545173645 (0.8500408639340655)
1590
+ | > current_lr: 5e-06
1591
+ | > step_time: 0.3366 (0.3363867896234893)
1592
+ | > loader_time: 0.0102 (0.011102559050263819)
1593
+
1594
+
1595
+  --> TIME: 2025-06-02 13:54:33 -- STEP: 2858/5792 -- GLOBAL_STEP: 8650
1596
+ | > loss_text_ce: 0.04298544302582741 (0.04463345923137111)
1597
+ | > loss_mel_ce: 3.1992127895355225 (3.3554093194558448)
1598
+ | > loss: 0.8105495572090149 (0.8500106946582804)
1599
+ | > current_lr: 5e-06
1600
+ | > step_time: 0.293 (0.3361897283371222)
1601
+ | > loader_time: 0.0101 (0.011095891055566914)
1602
+
1603
+
1604
+  --> TIME: 2025-06-02 13:55:02 -- STEP: 2908/5792 -- GLOBAL_STEP: 8700
1605
+ | > loss_text_ce: 0.04670070484280586 (0.044607090496649)
1606
+ | > loss_mel_ce: 3.638002872467041 (3.354981057827541)
1607
+ | > loss: 0.9211758971214294 (0.8498970369771223)
1608
+ | > current_lr: 5e-06
1609
+ | > step_time: 0.2085 (0.33621489690127415)
1610
+ | > loader_time: 0.0093 (0.011086543947663218)
1611
+
1612
+
1613
+  --> TIME: 2025-06-02 13:55:31 -- STEP: 2958/5792 -- GLOBAL_STEP: 8750
1614
+ | > loss_text_ce: 0.04170145094394684 (0.044566551267476086)
1615
+ | > loss_mel_ce: 3.448326826095581 (3.35389118762013)
1616
+ | > loss: 0.8725070953369141 (0.849614434604148)
1617
+ | > current_lr: 5e-06
1618
+ | > step_time: 0.4418 (0.3360969547970708)
1619
+ | > loader_time: 0.0106 (0.01107192184572044)
1620
+
1621
+
1622
+  --> TIME: 2025-06-02 13:55:59 -- STEP: 3008/5792 -- GLOBAL_STEP: 8800
1623
+ | > loss_text_ce: 0.04650944098830223 (0.04452610574786532)
1624
+ | > loss_mel_ce: 3.232806921005249 (3.3524444404434646)
1625
+ | > loss: 0.8198291063308716 (0.8492426364781689)
1626
+ | > current_lr: 5e-06
1627
+ | > step_time: 0.4633 (0.33591527674109456)
1628
+ | > loader_time: 0.0122 (0.011064917958797276)
1629
+
1630
+
1631
+  --> TIME: 2025-06-02 13:56:28 -- STEP: 3058/5792 -- GLOBAL_STEP: 8850
1632
+ | > loss_text_ce: 0.04007389768958092 (0.04448877311780065)
1633
+ | > loss_mel_ce: 3.012786626815796 (3.3516998684741526)
1634
+ | > loss: 0.7632151246070862 (0.8490471603669237)
1635
+ | > current_lr: 5e-06
1636
+ | > step_time: 0.2232 (0.33596821498683804)
1637
+ | > loader_time: 0.0119 (0.01105607171866376)
1638
+
1639
+
1640
+  --> TIME: 2025-06-02 13:56:56 -- STEP: 3108/5792 -- GLOBAL_STEP: 8900
1641
+ | > loss_text_ce: 0.03358534350991249 (0.04445769784173796)
1642
+ | > loss_mel_ce: 3.2545511722564697 (3.3518611339221738)
1643
+ | > loss: 0.8220341205596924 (0.8490797078555766)
1644
+ | > current_lr: 5e-06
1645
+ | > step_time: 0.2089 (0.33573530938481194)
1646
+ | > loader_time: 0.0094 (0.011041443857293623)
1647
+
1648
+
1649
+  --> TIME: 2025-06-02 13:57:26 -- STEP: 3158/5792 -- GLOBAL_STEP: 8950
1650
+ | > loss_text_ce: 0.037105049937963486 (0.04442861938803087)
1651
+ | > loss_mel_ce: 3.5840868949890137 (3.351841525762725)
1652
+ | > loss: 0.9052979946136475 (0.8490675361832911)
1653
+ | > current_lr: 5e-06
1654
+ | > step_time: 0.4385 (0.33593913925985125)
1655
+ | > loader_time: 0.0127 (0.011041003590222006)
1656
+
1657
+
1658
+  --> TIME: 2025-06-02 13:57:54 -- STEP: 3208/5792 -- GLOBAL_STEP: 9000
1659
+ | > loss_text_ce: 0.04580637440085411 (0.04439754502504062)
1660
+ | > loss_mel_ce: 3.0522005558013916 (3.351788088567833)
1661
+ | > loss: 0.7745017409324646 (0.8490464082754161)
1662
+ | > current_lr: 5e-06
1663
+ | > step_time: 0.1902 (0.33578265449055483)
1664
+ | > loader_time: 0.0086 (0.011037279245562075)
1665
+
1666
+
1667
+  --> TIME: 2025-06-02 13:58:22 -- STEP: 3258/5792 -- GLOBAL_STEP: 9050
1668
+ | > loss_text_ce: 0.03635700047016144 (0.044366618606934825)
1669
+ | > loss_mel_ce: 3.045945167541504 (3.3514977041529024)
1670
+ | > loss: 0.7705755233764648 (0.8489660805467447)
1671
+ | > current_lr: 5e-06
1672
+ | > step_time: 0.2663 (0.3356093359256983)
1673
+ | > loader_time: 0.0145 (0.011036152924548766)
1674
+
1675
+
1676
+  --> TIME: 2025-06-02 13:58:52 -- STEP: 3308/5792 -- GLOBAL_STEP: 9100
1677
+ | > loss_text_ce: 0.0383525975048542 (0.04434114113729335)
1678
+ | > loss_mel_ce: 3.439059257507324 (3.3502290691401098)
1679
+ | > loss: 0.8693529367446899 (0.848642552417884)
1680
+ | > current_lr: 5e-06
1681
+ | > step_time: 0.2088 (0.33574586695886793)
1682
+ | > loader_time: 0.0124 (0.011047434129334404)
1683
+
1684
+
1685
+  --> TIME: 2025-06-02 13:59:20 -- STEP: 3358/5792 -- GLOBAL_STEP: 9150
1686
+ | > loss_text_ce: 0.0411563478410244 (0.04431965041788048)
1687
+ | > loss_mel_ce: 3.5818145275115967 (3.3501473820017798)
1688
+ | > loss: 0.9057427048683167 (0.8486167579587544)
1689
+ | > current_lr: 5e-06
1690
+ | > step_time: 0.333 (0.3356304534352067)
1691
+ | > loader_time: 0.0102 (0.01104049429856572)
1692
+
1693
+
1694
+  --> TIME: 2025-06-02 13:59:49 -- STEP: 3408/5792 -- GLOBAL_STEP: 9200
1695
+ | > loss_text_ce: 0.04104316234588623 (0.044287355214427344)
1696
+ | > loss_mel_ce: 3.3351967334747314 (3.349034378436249)
1697
+ | > loss: 0.844059944152832 (0.8483304332224694)
1698
+ | > current_lr: 5e-06
1699
+ | > step_time: 0.3537 (0.3354787281561352)
1700
+ | > loader_time: 0.0143 (0.01104401031010586)
1701
+
1702
+
1703
+  --> TIME: 2025-06-02 14:00:17 -- STEP: 3458/5792 -- GLOBAL_STEP: 9250
1704
+ | > loss_text_ce: 0.04712622985243797 (0.04426270988896412)
1705
+ | > loss_mel_ce: 3.5113916397094727 (3.3474128444874607)
1706
+ | > loss: 0.8896294832229614 (0.8479188883352856)
1707
+ | > current_lr: 5e-06
1708
+ | > step_time: 0.4075 (0.3352865505383985)
1709
+ | > loader_time: 0.0109 (0.011042501417593558)
1710
+
1711
+
1712
+  --> TIME: 2025-06-02 14:00:47 -- STEP: 3508/5792 -- GLOBAL_STEP: 9300
1713
+ | > loss_text_ce: 0.0433344691991806 (0.04423293619052409)
1714
+ | > loss_mel_ce: 3.186034679412842 (3.3471053151611314)
1715
+ | > loss: 0.8073422908782959 (0.8478345625960566)
1716
+ | > current_lr: 5e-06
1717
+ | > step_time: 0.3399 (0.33543082470225155)
1718
+ | > loader_time: 0.0136 (0.011043719921851497)
1719
+
1720
+
1721
+  --> TIME: 2025-06-02 14:01:15 -- STEP: 3558/5792 -- GLOBAL_STEP: 9350
1722
+ | > loss_text_ce: 0.0385572724044323 (0.04420625756611444)
1723
+ | > loss_mel_ce: 3.43131160736084 (3.3458053263604737)
1724
+ | > loss: 0.8674672245979309 (0.8475028957591555)
1725
+ | > current_lr: 5e-06
1726
+ | > step_time: 0.2685 (0.3353362489510554)
1727
+ | > loader_time: 0.0128 (0.011042438242259877)
1728
+
1729
+
1730
+  --> TIME: 2025-06-02 14:01:45 -- STEP: 3608/5792 -- GLOBAL_STEP: 9400
1731
+ | > loss_text_ce: 0.04384002462029457 (0.04419183030212402)
1732
+ | > loss_mel_ce: 3.5863473415374756 (3.3455215218722683)
1733
+ | > loss: 0.9075468182563782 (0.8474283378463888)
1734
+ | > current_lr: 5e-06
1735
+ | > step_time: 0.3275 (0.33544651070349557)
1736
+ | > loader_time: 0.01 (0.011052065572294573)
1737
+
1738
+
1739
+  --> TIME: 2025-06-02 14:02:12 -- STEP: 3658/5792 -- GLOBAL_STEP: 9450
1740
+ | > loss_text_ce: 0.04530433937907219 (0.04417096679045439)
1741
+ | > loss_mel_ce: 3.4153072834014893 (3.3448606153991842)
1742
+ | > loss: 0.865152895450592 (0.8472578953595157)
1743
+ | > current_lr: 5e-06
1744
+ | > step_time: 0.314 (0.3351731634061808)
1745
+ | > loader_time: 0.0125 (0.011047935929045499)
1746
+
1747
+
1748
+  --> TIME: 2025-06-02 14:02:42 -- STEP: 3708/5792 -- GLOBAL_STEP: 9500
1749
+ | > loss_text_ce: 0.0346110537648201 (0.04413479716469813)
1750
+ | > loss_mel_ce: 3.222661018371582 (3.3444809126622466)
1751
+ | > loss: 0.8143180012702942 (0.8471539272588174)
1752
+ | > current_lr: 5e-06
1753
+ | > step_time: 0.3003 (0.33524002194276065)
1754
+ | > loader_time: 0.011 (0.011049625685814244)
1755
+
1756
+
1757
+  --> TIME: 2025-06-02 14:03:09 -- STEP: 3758/5792 -- GLOBAL_STEP: 9550
1758
+ | > loss_text_ce: 0.04709426686167717 (0.04409706911509297)
1759
+ | > loss_mel_ce: 3.4125912189483643 (3.343986088002597)
1760
+ | > loss: 0.8649213910102844 (0.8470207890213136)
1761
+ | > current_lr: 5e-06
1762
+ | > step_time: 0.4286 (0.33501876009595527)
1763
+ | > loader_time: 0.0134 (0.011049652048855007)
1764
+
1765
+
1766
+  --> TIME: 2025-06-02 14:03:38 -- STEP: 3808/5792 -- GLOBAL_STEP: 9600
1767
+ | > loss_text_ce: 0.041285887360572815 (0.044069222885305306)
1768
+ | > loss_mel_ce: 3.439079523086548 (3.3440659681037688)
1769
+ | > loss: 0.8700913786888123 (0.8470337975676318)
1770
+ | > current_lr: 5e-06
1771
+ | > step_time: 0.2723 (0.334857460267905)
1772
+ | > loader_time: 0.0094 (0.011047247202456485)
1773
+
1774
+
1775
+  --> TIME: 2025-06-02 14:04:07 -- STEP: 3858/5792 -- GLOBAL_STEP: 9650
1776
+ | > loss_text_ce: 0.0423085018992424 (0.04404395498178935)
1777
+ | > loss_mel_ce: 3.5037617683410645 (3.343483012102007)
1778
+ | > loss: 0.8865175843238831 (0.8468817415912264)
1779
+ | > current_lr: 5e-06
1780
+ | > step_time: 0.41 (0.33492562775787443)
1781
+ | > loader_time: 0.0131 (0.011050497329933387)
1782
+
1783
+
1784
+  --> TIME: 2025-06-02 14:04:37 -- STEP: 3908/5792 -- GLOBAL_STEP: 9700
1785
+ | > loss_text_ce: 0.04542999714612961 (0.04403033828412392)
1786
+ | > loss_mel_ce: 2.891977071762085 (3.342988579214656)
1787
+ | > loss: 0.7343517541885376 (0.8467547291987015)
1788
+ | > current_lr: 5e-06
1789
+ | > step_time: 0.1907 (0.335096147006708)
1790
+ | > loader_time: 0.0098 (0.011049572435591811)
1791
+
1792
+
1793
+  --> TIME: 2025-06-02 14:05:07 -- STEP: 3958/5792 -- GLOBAL_STEP: 9750
1794
+ | > loss_text_ce: 0.0466097928583622 (0.044010729583553475)
1795
+ | > loss_mel_ce: 3.508066177368164 (3.3432548332238445)
1796
+ | > loss: 0.8886690139770508 (0.8468163905419621)
1797
+ | > current_lr: 5e-06
1798
+ | > step_time: 0.3011 (0.3353043986788423)
1799
+ | > loader_time: 0.013 (0.011049168347228097)
1800
+
1801
+
1802
+  --> TIME: 2025-06-02 14:05:38 -- STEP: 4008/5792 -- GLOBAL_STEP: 9800
1803
+ | > loss_text_ce: 0.04837203770875931 (0.04398115760806629)
1804
+ | > loss_mel_ce: 3.3902106285095215 (3.3430067203834857)
1805
+ | > loss: 0.859645664691925 (0.8467469693955307)
1806
+ | > current_lr: 5e-06
1807
+ | > step_time: 0.313 (0.33554493548151554)
1808
+ | > loader_time: 0.01 (0.011050231026080303)
1809
+
1810
+
1811
+  --> TIME: 2025-06-02 14:06:08 -- STEP: 4058/5792 -- GLOBAL_STEP: 9850
1812
+ | > loss_text_ce: 0.04397377371788025 (0.043955045121553474)
1813
+ | > loss_mel_ce: 3.5290706157684326 (3.342233718645989)
1814
+ | > loss: 0.8932610750198364 (0.8465471908251834)
1815
+ | > current_lr: 5e-06
1816
+ | > step_time: 0.4423 (0.3356970511615546)
1817
+ | > loader_time: 0.0139 (0.01104889202728947)
1818
+
1819
+
1820
+  --> TIME: 2025-06-02 14:06:36 -- STEP: 4108/5792 -- GLOBAL_STEP: 9900
1821
+ | > loss_text_ce: 0.034252800047397614 (0.04393632656793966)
1822
+ | > loss_mel_ce: 3.536194086074829 (3.3410506406916842)
1823
+ | > loss: 0.8926117420196533 (0.846246741690329)
1824
+ | > current_lr: 5e-06
1825
+ | > step_time: 0.2459 (0.33544389101175215)
1826
+ | > loader_time: 0.0095 (0.011051703973545407)
1827
+
1828
+
1829
+  --> TIME: 2025-06-02 14:07:05 -- STEP: 4158/5792 -- GLOBAL_STEP: 9950
1830
+ | > loss_text_ce: 0.03575450927019119 (0.04390224255218264)
1831
+ | > loss_mel_ce: 2.652836799621582 (3.3396065647983955)
1832
+ | > loss: 0.672147810459137 (0.8458772017268847)
1833
+ | > current_lr: 5e-06
1834
+ | > step_time: 0.3049 (0.33547147149475504)
1835
+ | > loader_time: 0.0129 (0.01104814040655123)
1836
+
1837
+
1838
+  --> TIME: 2025-06-02 14:07:34 -- STEP: 4208/5792 -- GLOBAL_STEP: 10000
1839
+ | > loss_text_ce: 0.044553492218256 (0.04387786551847873)
1840
+ | > loss_mel_ce: 3.6881346702575684 (3.3393874958780767)
1841
+ | > loss: 0.9331720471382141 (0.8458163402549665)
1842
+ | > current_lr: 5e-06
1843
+ | > step_time: 0.24 (0.33554055975190633)
1844
+ | > loader_time: 0.0136 (0.011045128107070909)
1845
+
1846
+
1847
+  --> TIME: 2025-06-02 14:08:05 -- STEP: 4258/5792 -- GLOBAL_STEP: 10050
1848
+ | > loss_text_ce: 0.04713189974427223 (0.043857144453016465)
1849
+ | > loss_mel_ce: 3.250263214111328 (3.3402409040631116)
1850
+ | > loss: 0.8243488073348999 (0.8460245119847841)
1851
+ | > current_lr: 5e-06
1852
+ | > step_time: 0.4278 (0.33582793624818846)
1853
+ | > loader_time: 0.0109 (0.011045684160453815)
1854
+
1855
+
1856
+  --> TIME: 2025-06-02 14:08:36 -- STEP: 4308/5792 -- GLOBAL_STEP: 10100
1857
+ | > loss_text_ce: 0.04142797365784645 (0.04383900594753162)
1858
+ | > loss_mel_ce: 3.029611110687256 (3.339805153861793)
1859
+ | > loss: 0.7677597999572754 (0.8459110398229447)
1860
+ | > current_lr: 5e-06
1861
+ | > step_time: 0.2426 (0.3360530724653853)
1862
+ | > loader_time: 0.0125 (0.01104891161843372)
1863
+
1864
+
1865
+  --> TIME: 2025-06-02 14:09:06 -- STEP: 4358/5792 -- GLOBAL_STEP: 10150
1866
+ | > loss_text_ce: 0.04575802758336067 (0.04381198164147425)
1867
+ | > loss_mel_ce: 3.206411838531494 (3.3394206824681243)
1868
+ | > loss: 0.8130424618721008 (0.8458081659180897)
1869
+ | > current_lr: 5e-06
1870
+ | > step_time: 0.4958 (0.33611179339114383)
1871
+ | > loader_time: 0.0112 (0.011051389699465848)
1872
+
1873
+
1874
+  --> TIME: 2025-06-02 14:09:35 -- STEP: 4408/5792 -- GLOBAL_STEP: 10200
1875
+ | > loss_text_ce: 0.04181518405675888 (0.04377960568140123)
1876
+ | > loss_mel_ce: 2.947584629058838 (3.3390845118763206)
1877
+ | > loss: 0.7473499774932861 (0.8457160292703737)
1878
+ | > current_lr: 5e-06
1879
+ | > step_time: 0.2769 (0.33606701202271466)
1880
+ | > loader_time: 0.0094 (0.011053307681680795)
1881
+
1882
+
1883
+  --> TIME: 2025-06-02 14:10:06 -- STEP: 4458/5792 -- GLOBAL_STEP: 10250
1884
+ | > loss_text_ce: 0.04814762994647026 (0.0437453800015162)
1885
+ | > loss_mel_ce: 2.8046014308929443 (3.3391340181077727)
1886
+ | > loss: 0.7131872773170471 (0.8457198493841137)
1887
+ | > current_lr: 5e-06
1888
+ | > step_time: 0.2371 (0.3363838041240513)
1889
+ | > loader_time: 0.0094 (0.011047967242895073)
1890
+
1891
+
1892
+  --> TIME: 2025-06-02 14:10:36 -- STEP: 4508/5792 -- GLOBAL_STEP: 10300
1893
+ | > loss_text_ce: 0.045141834765672684 (0.043721168837076246)
1894
+ | > loss_mel_ce: 3.2513136863708496 (3.3381951364036566)
1895
+ | > loss: 0.8241139054298401 (0.8454790762043034)
1896
+ | > current_lr: 5e-06
1897
+ | > step_time: 0.2119 (0.3364703716063989)
1898
+ | > loader_time: 0.0097 (0.011043780920456853)
1899
+
1900
+
1901
+  --> TIME: 2025-06-02 14:11:04 -- STEP: 4558/5792 -- GLOBAL_STEP: 10350
1902
+ | > loss_text_ce: 0.046194713562726974 (0.04369950713651088)
1903
+ | > loss_mel_ce: 3.618640184402466 (3.3370097836364305)
1904
+ | > loss: 0.9162087440490723 (0.8451773226261132)
1905
+ | > current_lr: 5e-06
1906
+ | > step_time: 0.4156 (0.3364097547405593)
1907
+ | > loader_time: 0.013 (0.011043213553049927)
1908
+
1909
+
1910
+  --> TIME: 2025-06-02 14:11:34 -- STEP: 4608/5792 -- GLOBAL_STEP: 10400
1911
+ | > loss_text_ce: 0.044357966631650925 (0.0436816014767424)
1912
+ | > loss_mel_ce: 3.656270742416382 (3.336337390355765)
1913
+ | > loss: 0.9251571893692017 (0.8450047479062854)
1914
+ | > current_lr: 5e-06
1915
+ | > step_time: 0.2739 (0.3365074931126504)
1916
+ | > loader_time: 0.0125 (0.011044854091273402)
1917
+
1918
+
1919
+  --> TIME: 2025-06-02 14:12:03 -- STEP: 4658/5792 -- GLOBAL_STEP: 10450
1920
+ | > loss_text_ce: 0.03670710697770119 (0.04365839876879141)
1921
+ | > loss_mel_ce: 2.583122968673706 (3.335039994193535)
1922
+ | > loss: 0.6549575328826904 (0.8446745981777001)
1923
+ | > current_lr: 5e-06
1924
+ | > step_time: 0.1691 (0.336374844487208)
1925
+ | > loader_time: 0.0109 (0.011051791632518373)
1926
+
1927
+
1928
+  --> TIME: 2025-06-02 14:12:31 -- STEP: 4708/5792 -- GLOBAL_STEP: 10500
1929
+ | > loss_text_ce: 0.04842893406748772 (0.04363689536684546)
1930
+ | > loss_mel_ce: 3.4084410667419434 (3.33495435723837)
1931
+ | > loss: 0.8642175197601318 (0.8446478130788037)
1932
+ | > current_lr: 5e-06
1933
+ | > step_time: 0.1822 (0.33627098611379563)
1934
+ | > loader_time: 0.0117 (0.011046174631953332)
1935
+
1936
+
1937
+  --> TIME: 2025-06-02 14:13:01 -- STEP: 4758/5792 -- GLOBAL_STEP: 10550
1938
+ | > loss_text_ce: 0.042808979749679565 (0.043614106523299764)
1939
+ | > loss_mel_ce: 3.381239652633667 (3.3350225664477926)
1940
+ | > loss: 0.8560121655464172 (0.8446591681745581)
1941
+ | > current_lr: 5e-06
1942
+ | > step_time: 0.293 (0.3363845983740162)
1943
+ | > loader_time: 0.0131 (0.011052533276394754)
1944
+
1945
+
1946
+  --> TIME: 2025-06-02 14:13:31 -- STEP: 4808/5792 -- GLOBAL_STEP: 10600
1947
+ | > loss_text_ce: 0.04915136843919754 (0.04358621223631336)
1948
+ | > loss_mel_ce: 2.8472352027893066 (3.3347837737515045)
1949
+ | > loss: 0.7240966558456421 (0.8445924964180201)
1950
+ | > current_lr: 5e-06
1951
+ | > step_time: 0.1576 (0.3364169633329015)
1952
+ | > loader_time: 0.0093 (0.011056910884162139)
1953
+
1954
+
1955
+  --> TIME: 2025-06-02 14:14:02 -- STEP: 4858/5792 -- GLOBAL_STEP: 10650
1956
+ | > loss_text_ce: 0.038126781582832336 (0.0435614236114802)
1957
+ | > loss_mel_ce: 3.1234774589538574 (3.3353497294939896)
1958
+ | > loss: 0.790401041507721 (0.8447277882109941)
1959
+ | > current_lr: 5e-06
1960
+ | > step_time: 0.2953 (0.3366490865548782)
1961
+ | > loader_time: 0.0091 (0.01105543447159208)
1962
+
1963
+
1964
+  --> TIME: 2025-06-02 14:14:32 -- STEP: 4908/5792 -- GLOBAL_STEP: 10700
1965
+ | > loss_text_ce: 0.04095133766531944 (0.043551660765238684)
1966
+ | > loss_mel_ce: 3.398160457611084 (3.335550274903345)
1967
+ | > loss: 0.8597779273986816 (0.8447754838623056)
1968
+ | > current_lr: 5e-06
1969
+ | > step_time: 0.2428 (0.33676366579658956)
1970
+ | > loader_time: 0.0128 (0.011054953036506476)
1971
+
1972
+
1973
+  --> TIME: 2025-06-02 14:15:02 -- STEP: 4958/5792 -- GLOBAL_STEP: 10750
1974
+ | > loss_text_ce: 0.04319563880562782 (0.0435207473597263)
1975
+ | > loss_mel_ce: 2.871285915374756 (3.335179869231316)
1976
+ | > loss: 0.7286204099655151 (0.844675154106998)
1977
+ | > current_lr: 5e-06
1978
+ | > step_time: 0.1926 (0.33679866516671897)
1979
+ | > loader_time: 0.0087 (0.011052699087126775)
1980
+
1981
+
1982
+  --> TIME: 2025-06-02 14:15:30 -- STEP: 5008/5792 -- GLOBAL_STEP: 10800
1983
+ | > loss_text_ce: 0.04007870703935623 (0.043486832795789604)
1984
+ | > loss_mel_ce: 3.0385284423828125 (3.3343036392340646)
1985
+ | > loss: 0.7696517705917358 (0.8444476179516727)
1986
+ | > current_lr: 5e-06
1987
+ | > step_time: 0.3957 (0.3367528500267494)
1988
+ | > loader_time: 0.011 (0.011056139684332816)
1989
+
1990
+
1991
+  --> TIME: 2025-06-02 14:16:00 -- STEP: 5058/5792 -- GLOBAL_STEP: 10850
1992
+ | > loss_text_ce: 0.04264426231384277 (0.04346949416229308)
1993
+ | > loss_mel_ce: 3.440608263015747 (3.333787598596726)
1994
+ | > loss: 0.8708131313323975 (0.8443142731536648)
1995
+ | > current_lr: 5e-06
1996
+ | > step_time: 0.3941 (0.3367869499032324)
1997
+ | > loader_time: 0.0124 (0.01105723639352842)
1998
+
1999
+
2000
+  --> TIME: 2025-06-02 14:16:28 -- STEP: 5108/5792 -- GLOBAL_STEP: 10900
2001
+ | > loss_text_ce: 0.034066058695316315 (0.043442419822748615)
2002
+ | > loss_mel_ce: 3.139413595199585 (3.3330060827050625)
2003
+ | > loss: 0.7933698892593384 (0.8441121256388804)
2004
+ | > current_lr: 5e-06
2005
+ | > step_time: 0.1858 (0.33673182608284047)
2006
+ | > loader_time: 0.0091 (0.011057687393836698)
2007
+
2008
+
2009
+  --> TIME: 2025-06-02 14:16:56 -- STEP: 5158/5792 -- GLOBAL_STEP: 10950
2010
+ | > loss_text_ce: 0.04518938437104225 (0.04340843048297699)
2011
+ | > loss_mel_ce: 3.1620471477508545 (3.3323658000780383)
2012
+ | > loss: 0.8018091320991516 (0.843943557682865)
2013
+ | > current_lr: 5e-06
2014
+ | > step_time: 0.2567 (0.3365875314307241)
2015
+ | > loader_time: 0.0094 (0.011058349256230763)
2016
+
2017
+
2018
+  --> TIME: 2025-06-02 14:17:24 -- STEP: 5208/5792 -- GLOBAL_STEP: 11000
2019
+ | > loss_text_ce: 0.04442836344242096 (0.04338480853733139)
2020
+ | > loss_mel_ce: 3.0665812492370605 (3.3311940293799163)
2021
+ | > loss: 0.7777523994445801 (0.8436447095280412)
2022
+ | > current_lr: 5e-06
2023
+ | > step_time: 0.2419 (0.3364083989882436)
2024
+ | > loader_time: 0.0102 (0.011055767169928952)
2025
+
2026
+
2027
+  --> TIME: 2025-06-02 14:17:53 -- STEP: 5258/5792 -- GLOBAL_STEP: 11050
2028
+ | > loss_text_ce: 0.04343422129750252 (0.04335219707671583)
2029
+ | > loss_mel_ce: 2.8784594535827637 (3.3301943961365135)
2030
+ | > loss: 0.7304733991622925 (0.8433866483510418)
2031
+ | > current_lr: 5e-06
2032
+ | > step_time: 0.3317 (0.3363045960517387)
2033
+ | > loader_time: 0.0101 (0.011054212260763195)
2034
+
2035
+
2036
+  --> TIME: 2025-06-02 14:18:21 -- STEP: 5308/5792 -- GLOBAL_STEP: 11100
2037
+ | > loss_text_ce: 0.04893643409013748 (0.04332292118490856)
2038
+ | > loss_mel_ce: 3.3955016136169434 (3.3295369724986603)
2039
+ | > loss: 0.8611094951629639 (0.8432149734929163)
2040
+ | > current_lr: 5e-06
2041
+ | > step_time: 0.326 (0.33622330490053426)
2042
+ | > loader_time: 0.0105 (0.011051318837721364)
2043
+
2044
+
2045
+  --> TIME: 2025-06-02 14:18:52 -- STEP: 5358/5792 -- GLOBAL_STEP: 11150
2046
+ | > loss_text_ce: 0.044002778828144073 (0.04330665982714582)
2047
+ | > loss_mel_ce: 4.1681904792785645 (3.3292624402642472)
2048
+ | > loss: 1.0530483722686768 (0.8431422751134073)
2049
+ | > current_lr: 5e-06
2050
+ | > step_time: 0.5552 (0.3363442467237067)
2051
+ | > loader_time: 0.0151 (0.011057644159146863)
2052
+
2053
+
2054
+  --> TIME: 2025-06-02 14:19:20 -- STEP: 5408/5792 -- GLOBAL_STEP: 11200
2055
+ | > loss_text_ce: 0.038751423358917236 (0.043278468308141)
2056
+ | > loss_mel_ce: 3.3097198009490967 (3.3285519173864784)
2057
+ | > loss: 0.8371177911758423 (0.8429575964764371)
2058
+ | > current_lr: 5e-06
2059
+ | > step_time: 0.2749 (0.3362092402647945)
2060
+ | > loader_time: 0.0135 (0.011058031013731425)
2061
+
2062
+
2063
+  --> TIME: 2025-06-02 14:19:48 -- STEP: 5458/5792 -- GLOBAL_STEP: 11250
2064
+ | > loss_text_ce: 0.041902702301740646 (0.04325453944254458)
2065
+ | > loss_mel_ce: 2.8589980602264404 (3.3282593212620424)
2066
+ | > loss: 0.7252252101898193 (0.8428784652632549)
2067
+ | > current_lr: 5e-06
2068
+ | > step_time: 0.2551 (0.33609242695248986)
2069
+ | > loader_time: 0.0099 (0.011059924594296262)
2070
+
2071
+
2072
+  --> TIME: 2025-06-02 14:20:16 -- STEP: 5508/5792 -- GLOBAL_STEP: 11300
2073
+ | > loss_text_ce: 0.036019548773765564 (0.0432351567869034)
2074
+ | > loss_mel_ce: 2.981379270553589 (3.3276002054294125)
2075
+ | > loss: 0.7543497085571289 (0.842708840619937)
2076
+ | > current_lr: 5e-06
2077
+ | > step_time: 0.1859 (0.3359816792483949)
2078
+ | > loader_time: 0.0119 (0.011056781118125613)
2079
+
2080
+
2081
+  --> TIME: 2025-06-02 14:20:47 -- STEP: 5558/5792 -- GLOBAL_STEP: 11350
2082
+ | > loss_text_ce: 0.04409126192331314 (0.04321853997813861)
2083
+ | > loss_mel_ce: 3.3099846839904785 (3.3272555071314964)
2084
+ | > loss: 0.8385189771652222 (0.842618511869736)
2085
+ | > current_lr: 5e-06
2086
+ | > step_time: 0.3187 (0.33623436845091076)
2087
+ | > loader_time: 0.0121 (0.011060114171102605)
2088
+
2089
+
2090
+  --> TIME: 2025-06-02 14:21:16 -- STEP: 5608/5792 -- GLOBAL_STEP: 11400
2091
+ | > loss_text_ce: 0.03766907751560211 (0.043189963759633916)
2092
+ | > loss_mel_ce: 3.2820019721984863 (3.325929513628915)
2093
+ | > loss: 0.8299177885055542 (0.8422798694412984)
2094
+ | > current_lr: 5e-06
2095
+ | > step_time: 0.3434 (0.3362132219037052)
2096
+ | > loader_time: 0.0099 (0.011059219843990964)
2097
+
2098
+
2099
+  --> TIME: 2025-06-02 14:21:45 -- STEP: 5658/5792 -- GLOBAL_STEP: 11450
2100
+ | > loss_text_ce: 0.0405101515352726 (0.04316613374270707)
2101
+ | > loss_mel_ce: 3.357116937637329 (3.325798077269804)
2102
+ | > loss: 0.8494067788124084 (0.8422410528686782)
2103
+ | > current_lr: 5e-06
2104
+ | > step_time: 0.2667 (0.3361453908035056)
2105
+ | > loader_time: 0.0096 (0.011055887103628554)
2106
+
2107
+
2108
+  --> TIME: 2025-06-02 14:22:11 -- STEP: 5708/5792 -- GLOBAL_STEP: 11500
2109
+ | > loss_text_ce: 0.033331647515296936 (0.04313662911477462)
2110
+ | > loss_mel_ce: 2.8956968784332275 (3.3249534104951377)
2111
+ | > loss: 0.7322571277618408 (0.8420225100117953)
2112
+ | > current_lr: 5e-06
2113
+ | > step_time: 0.1596 (0.3358227129406929)
2114
+ | > loader_time: 0.0095 (0.011055266556409003)
2115
+
2116
+
2117
+  --> TIME: 2025-06-02 14:22:39 -- STEP: 5758/5792 -- GLOBAL_STEP: 11550
2118
+ | > loss_text_ce: 0.042040519416332245 (0.04311639427965342)
2119
+ | > loss_mel_ce: 2.8961079120635986 (3.3243683234152175)
2120
+ | > loss: 0.7345371246337891 (0.8418711795600674)
2121
+ | > current_lr: 5e-06
2122
+ | > step_time: 0.2629 (0.33571524755041415)
2123
+ | > loader_time: 0.0125 (0.011058961322382958)
2124
+
2125
+
2126
+  > EVALUATION 
2127
+
2128
+
2129
+ --> EVAL PERFORMANCE
2130
+ | > avg_loader_time: 0.20065663089859956 (-0.0009856695509226354)
2131
+ | > avg_loss_text_ce: 0.0388087550504396 (-0.00597116267992618)
2132
+ | > avg_loss_mel_ce: 3.061407669115872 (-0.1385840406525629)
2133
+ | > avg_loss: 3.100216426418325 (-0.14455520433221114)
2134
+
2135
+ > BEST MODEL : xTTS/checkpoints/GPT_XTTS_FT-June-02-2025_12+22PM-0000000/best_model_11584.pth