Add trained xTTS model
Browse files- best_model.pth +3 -0
- best_model_11584.pth +3 -0
- config.json +193 -0
- events.out.tfevents.1748866975.eafda0218e5b.317.0 +3 -0
- train_gpt_xtts.py +232 -0
- trainer_0_log.txt +2135 -0
best_model.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d13484d2e09730daa3dc3ea46b28eee10d4bc45d4dbf42d190865b99e0b81ca1
|
| 3 |
+
size 5640358637
|
best_model_11584.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d13484d2e09730daa3dc3ea46b28eee10d4bc45d4dbf42d190865b99e0b81ca1
|
| 3 |
+
size 5640358637
|
config.json
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"output_path": "xTTS/checkpoints/",
|
| 3 |
+
"logger_uri": null,
|
| 4 |
+
"run_name": "GPT_XTTS_FT",
|
| 5 |
+
"project_name": "XTTS_trainer",
|
| 6 |
+
"run_description": [
|
| 7 |
+
"\n GPT XTTS training\n "
|
| 8 |
+
],
|
| 9 |
+
"print_step": 50,
|
| 10 |
+
"plot_step": 100,
|
| 11 |
+
"model_param_stats": false,
|
| 12 |
+
"wandb_entity": null,
|
| 13 |
+
"dashboard_logger": "tensorboard",
|
| 14 |
+
"save_on_interrupt": true,
|
| 15 |
+
"log_model_step": 100,
|
| 16 |
+
"save_step": 9999999999,
|
| 17 |
+
"save_n_checkpoints": 1,
|
| 18 |
+
"save_checkpoints": false,
|
| 19 |
+
"save_all_best": false,
|
| 20 |
+
"save_best_after": 10000,
|
| 21 |
+
"target_loss": null,
|
| 22 |
+
"print_eval": false,
|
| 23 |
+
"test_delay_epochs": 0,
|
| 24 |
+
"run_eval": true,
|
| 25 |
+
"run_eval_steps": null,
|
| 26 |
+
"distributed_backend": "nccl",
|
| 27 |
+
"distributed_url": "tcp://localhost:54321",
|
| 28 |
+
"mixed_precision": false,
|
| 29 |
+
"precision": "fp16",
|
| 30 |
+
"epochs": 2,
|
| 31 |
+
"batch_size": 4,
|
| 32 |
+
"eval_batch_size": 16,
|
| 33 |
+
"grad_clip": 0.0,
|
| 34 |
+
"scheduler_after_epoch": true,
|
| 35 |
+
"lr": 5e-06,
|
| 36 |
+
"optimizer": "AdamW",
|
| 37 |
+
"optimizer_params": {
|
| 38 |
+
"betas": [
|
| 39 |
+
0.9,
|
| 40 |
+
0.96
|
| 41 |
+
],
|
| 42 |
+
"eps": 1e-08,
|
| 43 |
+
"weight_decay": 0.01
|
| 44 |
+
},
|
| 45 |
+
"lr_scheduler": "MultiStepLR",
|
| 46 |
+
"lr_scheduler_params": {
|
| 47 |
+
"milestones": [
|
| 48 |
+
900000,
|
| 49 |
+
2700000,
|
| 50 |
+
5400000
|
| 51 |
+
],
|
| 52 |
+
"gamma": 0.5,
|
| 53 |
+
"last_epoch": -1
|
| 54 |
+
},
|
| 55 |
+
"use_grad_scaler": false,
|
| 56 |
+
"allow_tf32": false,
|
| 57 |
+
"cudnn_enable": true,
|
| 58 |
+
"cudnn_deterministic": false,
|
| 59 |
+
"cudnn_benchmark": false,
|
| 60 |
+
"training_seed": 54321,
|
| 61 |
+
"model": "xtts",
|
| 62 |
+
"num_loader_workers": 8,
|
| 63 |
+
"num_eval_loader_workers": 0,
|
| 64 |
+
"use_noise_augment": false,
|
| 65 |
+
"audio": {
|
| 66 |
+
"sample_rate": 22050,
|
| 67 |
+
"output_sample_rate": 24000,
|
| 68 |
+
"dvae_sample_rate": 22050
|
| 69 |
+
},
|
| 70 |
+
"use_phonemes": false,
|
| 71 |
+
"phonemizer": null,
|
| 72 |
+
"phoneme_language": null,
|
| 73 |
+
"compute_input_seq_cache": false,
|
| 74 |
+
"text_cleaner": null,
|
| 75 |
+
"enable_eos_bos_chars": false,
|
| 76 |
+
"test_sentences_file": "",
|
| 77 |
+
"phoneme_cache_path": null,
|
| 78 |
+
"characters": null,
|
| 79 |
+
"add_blank": false,
|
| 80 |
+
"batch_group_size": 0,
|
| 81 |
+
"loss_masking": null,
|
| 82 |
+
"min_audio_len": 1,
|
| 83 |
+
"max_audio_len": Infinity,
|
| 84 |
+
"min_text_len": 1,
|
| 85 |
+
"max_text_len": Infinity,
|
| 86 |
+
"compute_f0": false,
|
| 87 |
+
"compute_energy": false,
|
| 88 |
+
"compute_linear_spec": false,
|
| 89 |
+
"precompute_num_workers": 0,
|
| 90 |
+
"start_by_longest": false,
|
| 91 |
+
"shuffle": false,
|
| 92 |
+
"drop_last": false,
|
| 93 |
+
"datasets": [
|
| 94 |
+
{
|
| 95 |
+
"formatter": "",
|
| 96 |
+
"dataset_name": "",
|
| 97 |
+
"path": "",
|
| 98 |
+
"meta_file_train": "",
|
| 99 |
+
"ignored_speakers": null,
|
| 100 |
+
"language": "",
|
| 101 |
+
"phonemizer": "",
|
| 102 |
+
"meta_file_val": "",
|
| 103 |
+
"meta_file_attn_mask": ""
|
| 104 |
+
}
|
| 105 |
+
],
|
| 106 |
+
"test_sentences": [],
|
| 107 |
+
"eval_split_max_size": 256,
|
| 108 |
+
"eval_split_size": 0.01,
|
| 109 |
+
"use_speaker_weighted_sampler": false,
|
| 110 |
+
"speaker_weighted_sampler_alpha": 1.0,
|
| 111 |
+
"use_language_weighted_sampler": false,
|
| 112 |
+
"language_weighted_sampler_alpha": 1.0,
|
| 113 |
+
"use_length_weighted_sampler": false,
|
| 114 |
+
"length_weighted_sampler_alpha": 1.0,
|
| 115 |
+
"model_args": {
|
| 116 |
+
"gpt_batch_size": 1,
|
| 117 |
+
"enable_redaction": false,
|
| 118 |
+
"kv_cache": true,
|
| 119 |
+
"gpt_checkpoint": "",
|
| 120 |
+
"clvp_checkpoint": null,
|
| 121 |
+
"decoder_checkpoint": null,
|
| 122 |
+
"num_chars": 255,
|
| 123 |
+
"tokenizer_file": "xTTS/checkpoints/XTTS_v2.0_original_model_files/vocab.json",
|
| 124 |
+
"gpt_max_audio_tokens": 605,
|
| 125 |
+
"gpt_max_text_tokens": 402,
|
| 126 |
+
"gpt_max_prompt_tokens": 70,
|
| 127 |
+
"gpt_layers": 30,
|
| 128 |
+
"gpt_n_model_channels": 1024,
|
| 129 |
+
"gpt_n_heads": 16,
|
| 130 |
+
"gpt_number_text_tokens": 8000,
|
| 131 |
+
"gpt_start_text_token": 261,
|
| 132 |
+
"gpt_stop_text_token": 0,
|
| 133 |
+
"gpt_num_audio_tokens": 1026,
|
| 134 |
+
"gpt_start_audio_token": 1024,
|
| 135 |
+
"gpt_stop_audio_token": 1025,
|
| 136 |
+
"gpt_code_stride_len": 1024,
|
| 137 |
+
"gpt_use_masking_gt_prompt_approach": true,
|
| 138 |
+
"gpt_use_perceiver_resampler": true,
|
| 139 |
+
"input_sample_rate": 22050,
|
| 140 |
+
"output_sample_rate": 24000,
|
| 141 |
+
"output_hop_length": 256,
|
| 142 |
+
"decoder_input_dim": 1024,
|
| 143 |
+
"d_vector_dim": 512,
|
| 144 |
+
"cond_d_vector_in_each_upsampling_layer": true,
|
| 145 |
+
"duration_const": 102400,
|
| 146 |
+
"min_conditioning_length": 11025,
|
| 147 |
+
"max_conditioning_length": 132300,
|
| 148 |
+
"gpt_loss_text_ce_weight": 0.01,
|
| 149 |
+
"gpt_loss_mel_ce_weight": 1.0,
|
| 150 |
+
"debug_loading_failures": false,
|
| 151 |
+
"max_wav_length": 320000,
|
| 152 |
+
"max_text_length": 400,
|
| 153 |
+
"mel_norm_file": "xTTS/checkpoints/XTTS_v2.0_original_model_files/mel_stats.pth",
|
| 154 |
+
"dvae_checkpoint": "xTTS/checkpoints/XTTS_v2.0_original_model_files/dvae.pth",
|
| 155 |
+
"xtts_checkpoint": "xTTS/checkpoints/XTTS_v2.0_original_model_files/model.pth",
|
| 156 |
+
"vocoder": ""
|
| 157 |
+
},
|
| 158 |
+
"model_dir": null,
|
| 159 |
+
"languages": [
|
| 160 |
+
"en",
|
| 161 |
+
"es",
|
| 162 |
+
"fr",
|
| 163 |
+
"de",
|
| 164 |
+
"it",
|
| 165 |
+
"pt",
|
| 166 |
+
"pl",
|
| 167 |
+
"tr",
|
| 168 |
+
"ru",
|
| 169 |
+
"nl",
|
| 170 |
+
"cs",
|
| 171 |
+
"ar",
|
| 172 |
+
"zh-cn",
|
| 173 |
+
"hu",
|
| 174 |
+
"ko",
|
| 175 |
+
"ja",
|
| 176 |
+
"hi",
|
| 177 |
+
"wo"
|
| 178 |
+
],
|
| 179 |
+
"temperature": 0.75,
|
| 180 |
+
"length_penalty": 1.0,
|
| 181 |
+
"repetition_penalty": 5.0,
|
| 182 |
+
"top_k": 50,
|
| 183 |
+
"top_p": 0.85,
|
| 184 |
+
"num_gpt_outputs": 1,
|
| 185 |
+
"gpt_cond_len": 30,
|
| 186 |
+
"gpt_cond_chunk_len": 4,
|
| 187 |
+
"max_ref_len": 30,
|
| 188 |
+
"sound_norm_refs": false,
|
| 189 |
+
"optimizer_wd_only_on_weights": true,
|
| 190 |
+
"weighted_loss_attrs": null,
|
| 191 |
+
"weighted_loss_multipliers": null,
|
| 192 |
+
"github_branch": "inside_docker"
|
| 193 |
+
}
|
events.out.tfevents.1748866975.eafda0218e5b.317.0
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:92bbd9c8257e7635c9e148d4d60f774e3b306139eaa7dbc879c9709a077dd17f
|
| 3 |
+
size 60608
|
train_gpt_xtts.py
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import gc
|
| 3 |
+
|
| 4 |
+
from trainer import Trainer, TrainerArgs
|
| 5 |
+
|
| 6 |
+
from TTS.config.shared_configs import BaseDatasetConfig
|
| 7 |
+
from TTS.tts.datasets import load_tts_samples
|
| 8 |
+
from TTS.tts.layers.xtts.trainer.gpt_trainer import GPTArgs, GPTTrainer, GPTTrainerConfig, XttsAudioConfig
|
| 9 |
+
from TTS.utils.manage import ModelManager
|
| 10 |
+
|
| 11 |
+
from dataclasses import dataclass, field
|
| 12 |
+
from typing import Optional
|
| 13 |
+
from transformers import HfArgumentParser
|
| 14 |
+
|
| 15 |
+
import argparse
|
| 16 |
+
|
| 17 |
+
def create_xtts_trainer_parser():
|
| 18 |
+
parser = argparse.ArgumentParser(description="Arguments for XTTS Trainer")
|
| 19 |
+
|
| 20 |
+
parser.add_argument("--output_path", type=str, required=True,
|
| 21 |
+
help="Path to pretrained + checkpoint model")
|
| 22 |
+
parser.add_argument("--metadatas", nargs='+', type=str, required=True,
|
| 23 |
+
help="train_csv_path,eval_csv_path,language")
|
| 24 |
+
parser.add_argument("--num_epochs", type=int, default=1,
|
| 25 |
+
help="Number of epochs")
|
| 26 |
+
parser.add_argument("--batch_size", type=int, default=1,
|
| 27 |
+
help="Mini batch size")
|
| 28 |
+
parser.add_argument("--grad_acumm", type=int, default=1,
|
| 29 |
+
help="Grad accumulation steps")
|
| 30 |
+
parser.add_argument("--max_audio_length", type=int, default=255995,
|
| 31 |
+
help="Max audio length")
|
| 32 |
+
parser.add_argument("--max_text_length", type=int, default=200,
|
| 33 |
+
help="Max text length")
|
| 34 |
+
parser.add_argument("--weight_decay", type=float, default=1e-2,
|
| 35 |
+
help="Weight decay")
|
| 36 |
+
parser.add_argument("--lr", type=float, default=5e-6,
|
| 37 |
+
help="Learning rate")
|
| 38 |
+
parser.add_argument("--save_step", type=int, default=5000,
|
| 39 |
+
help="Save step")
|
| 40 |
+
|
| 41 |
+
return parser
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def train_gpt(metadatas, num_epochs, batch_size, grad_acumm, output_path, max_audio_length, max_text_length, lr, weight_decay, save_step):
|
| 46 |
+
# Logging parameters
|
| 47 |
+
RUN_NAME = "GPT_XTTS_FT"
|
| 48 |
+
PROJECT_NAME = "XTTS_trainer"
|
| 49 |
+
DASHBOARD_LOGGER = "tensorboard"
|
| 50 |
+
LOGGER_URI = None
|
| 51 |
+
|
| 52 |
+
# Set here the path that the checkpoints will be saved. Default: ./run/training/
|
| 53 |
+
# OUT_PATH = os.path.join(output_path, "run", "training")
|
| 54 |
+
OUT_PATH = output_path
|
| 55 |
+
|
| 56 |
+
# Training Parameters
|
| 57 |
+
OPTIMIZER_WD_ONLY_ON_WEIGHTS = True # for multi-gpu training please make it False
|
| 58 |
+
START_WITH_EVAL = False # if True it will star with evaluation
|
| 59 |
+
BATCH_SIZE = batch_size # set here the batch size
|
| 60 |
+
GRAD_ACUMM_STEPS = grad_acumm # set here the grad accumulation steps
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
# Define here the dataset that you want to use for the fine-tuning on.
|
| 64 |
+
DATASETS_CONFIG_LIST = []
|
| 65 |
+
for metadata in metadatas:
|
| 66 |
+
train_csv, eval_csv, language = metadata.split(",")
|
| 67 |
+
print(train_csv, eval_csv, language)
|
| 68 |
+
|
| 69 |
+
config_dataset = BaseDatasetConfig(
|
| 70 |
+
formatter="coqui",
|
| 71 |
+
dataset_name="ft_dataset",
|
| 72 |
+
path=os.path.dirname(train_csv),
|
| 73 |
+
meta_file_train=os.path.basename(train_csv),
|
| 74 |
+
meta_file_val=os.path.basename(eval_csv),
|
| 75 |
+
language=language,
|
| 76 |
+
)
|
| 77 |
+
|
| 78 |
+
DATASETS_CONFIG_LIST.append(config_dataset)
|
| 79 |
+
|
| 80 |
+
# Define the path where XTTS v2.0.1 files will be downloaded
|
| 81 |
+
CHECKPOINTS_OUT_PATH = os.path.join(OUT_PATH, "XTTS_v2.0_original_model_files/")
|
| 82 |
+
os.makedirs(CHECKPOINTS_OUT_PATH, exist_ok=True)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
# DVAE files
|
| 86 |
+
DVAE_CHECKPOINT_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/dvae.pth"
|
| 87 |
+
MEL_NORM_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/mel_stats.pth"
|
| 88 |
+
|
| 89 |
+
# Set the path to the downloaded files
|
| 90 |
+
DVAE_CHECKPOINT = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(DVAE_CHECKPOINT_LINK))
|
| 91 |
+
MEL_NORM_FILE = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(MEL_NORM_LINK))
|
| 92 |
+
|
| 93 |
+
# download DVAE files if needed
|
| 94 |
+
if not os.path.isfile(DVAE_CHECKPOINT) or not os.path.isfile(MEL_NORM_FILE):
|
| 95 |
+
print(" > Downloading DVAE files!")
|
| 96 |
+
ModelManager._download_model_files([MEL_NORM_LINK, DVAE_CHECKPOINT_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
# Download XTTS v2.0 checkpoint if needed
|
| 100 |
+
TOKENIZER_FILE_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/vocab.json"
|
| 101 |
+
XTTS_CHECKPOINT_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/model.pth"
|
| 102 |
+
XTTS_CONFIG_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/config.json"
|
| 103 |
+
|
| 104 |
+
# XTTS transfer learning parameters: You we need to provide the paths of XTTS model checkpoint that you want to do the fine tuning.
|
| 105 |
+
TOKENIZER_FILE = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(TOKENIZER_FILE_LINK)) # vocab.json file
|
| 106 |
+
XTTS_CHECKPOINT = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(XTTS_CHECKPOINT_LINK)) # model.pth file
|
| 107 |
+
XTTS_CONFIG_FILE = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(XTTS_CONFIG_LINK)) # config.json file
|
| 108 |
+
|
| 109 |
+
# download XTTS v2.0 files if needed
|
| 110 |
+
if not os.path.isfile(TOKENIZER_FILE):
|
| 111 |
+
print(" > Downloading XTTS v2.0 tokenizer!")
|
| 112 |
+
ModelManager._download_model_files(
|
| 113 |
+
[TOKENIZER_FILE_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True
|
| 114 |
+
)
|
| 115 |
+
if not os.path.isfile(XTTS_CHECKPOINT):
|
| 116 |
+
print(" > Downloading XTTS v2.0 checkpoint!")
|
| 117 |
+
ModelManager._download_model_files(
|
| 118 |
+
[XTTS_CHECKPOINT_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True
|
| 119 |
+
)
|
| 120 |
+
if not os.path.isfile(XTTS_CONFIG_FILE):
|
| 121 |
+
print(" > Downloading XTTS v2.0 config!")
|
| 122 |
+
ModelManager._download_model_files(
|
| 123 |
+
[XTTS_CONFIG_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
# init args and config
|
| 127 |
+
model_args = GPTArgs(
|
| 128 |
+
max_conditioning_length=132300, # 6 secs
|
| 129 |
+
min_conditioning_length=11025, # 0.5 secs
|
| 130 |
+
debug_loading_failures=False,
|
| 131 |
+
max_wav_length=max_audio_length, # ~11.6 seconds
|
| 132 |
+
max_text_length=max_text_length,
|
| 133 |
+
mel_norm_file=MEL_NORM_FILE,
|
| 134 |
+
dvae_checkpoint=DVAE_CHECKPOINT,
|
| 135 |
+
xtts_checkpoint=XTTS_CHECKPOINT, # checkpoint path of the model that you want to fine-tune
|
| 136 |
+
tokenizer_file=TOKENIZER_FILE,
|
| 137 |
+
gpt_num_audio_tokens=1026,
|
| 138 |
+
gpt_start_audio_token=1024,
|
| 139 |
+
gpt_stop_audio_token=1025,
|
| 140 |
+
gpt_use_masking_gt_prompt_approach=True,
|
| 141 |
+
gpt_use_perceiver_resampler=True,
|
| 142 |
+
)
|
| 143 |
+
# define audio config
|
| 144 |
+
audio_config = XttsAudioConfig(sample_rate=22050, dvae_sample_rate=22050, output_sample_rate=24000)
|
| 145 |
+
# training parameters config
|
| 146 |
+
|
| 147 |
+
config = GPTTrainerConfig()
|
| 148 |
+
|
| 149 |
+
config.load_json(XTTS_CONFIG_FILE)
|
| 150 |
+
|
| 151 |
+
config.epochs = num_epochs
|
| 152 |
+
config.output_path = OUT_PATH
|
| 153 |
+
config.model_args = model_args
|
| 154 |
+
config.run_name = RUN_NAME
|
| 155 |
+
config.project_name = PROJECT_NAME
|
| 156 |
+
config.run_description = """
|
| 157 |
+
GPT XTTS training
|
| 158 |
+
""",
|
| 159 |
+
config.dashboard_logger = DASHBOARD_LOGGER
|
| 160 |
+
config.logger_uri = LOGGER_URI
|
| 161 |
+
config.audio = audio_config
|
| 162 |
+
config.batch_size = BATCH_SIZE
|
| 163 |
+
config.num_loader_workers = 8
|
| 164 |
+
config.eval_split_max_size = 256
|
| 165 |
+
config.print_step = 50
|
| 166 |
+
config.plot_step = 100
|
| 167 |
+
config.log_model_step = 100
|
| 168 |
+
config.save_n_checkpoints = 1
|
| 169 |
+
config.save_checkpoints = False
|
| 170 |
+
config.print_eval = False
|
| 171 |
+
config.optimizer = "AdamW"
|
| 172 |
+
config.optimizer_wd_only_on_weights = OPTIMIZER_WD_ONLY_ON_WEIGHTS
|
| 173 |
+
config.optimizer_params = {"betas": [0.9, 0.96], "eps": 1e-8, "weight_decay": weight_decay}
|
| 174 |
+
config.lr = lr
|
| 175 |
+
config.lr_scheduler = "MultiStepLR"
|
| 176 |
+
config.lr_scheduler_params = {"milestones": [50000 * 18, 150000 * 18, 300000 * 18], "gamma": 0.5, "last_epoch": -1}
|
| 177 |
+
config.test_sentences = []
|
| 178 |
+
config.save_step = 9999999999
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
# init the model from config
|
| 182 |
+
model = GPTTrainer.init_from_config(config)
|
| 183 |
+
|
| 184 |
+
# load training samples
|
| 185 |
+
train_samples, eval_samples = load_tts_samples(
|
| 186 |
+
DATASETS_CONFIG_LIST,
|
| 187 |
+
eval_split=True,
|
| 188 |
+
eval_split_max_size=config.eval_split_max_size,
|
| 189 |
+
eval_split_size=config.eval_split_size,
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
# init the trainer and 🚀
|
| 193 |
+
trainer = Trainer(
|
| 194 |
+
TrainerArgs(
|
| 195 |
+
restore_path=None, # xtts checkpoint is restored via xtts_checkpoint key so no need of restore it using Trainer restore_path parameter
|
| 196 |
+
skip_train_epoch=False,
|
| 197 |
+
start_with_eval=START_WITH_EVAL,
|
| 198 |
+
grad_accum_steps=GRAD_ACUMM_STEPS
|
| 199 |
+
),
|
| 200 |
+
config,
|
| 201 |
+
output_path=os.path.join(output_path, "run", "training"),
|
| 202 |
+
model=model,
|
| 203 |
+
train_samples=train_samples,
|
| 204 |
+
eval_samples=eval_samples,
|
| 205 |
+
)
|
| 206 |
+
trainer.fit()
|
| 207 |
+
trainer_out_path = trainer.output_path
|
| 208 |
+
|
| 209 |
+
# deallocate VRAM and RAM
|
| 210 |
+
del model, trainer, train_samples, eval_samples
|
| 211 |
+
gc.collect()
|
| 212 |
+
|
| 213 |
+
return trainer_out_path
|
| 214 |
+
|
| 215 |
+
if __name__ == "__main__":
|
| 216 |
+
parser = create_xtts_trainer_parser()
|
| 217 |
+
args = parser.parse_args()
|
| 218 |
+
|
| 219 |
+
trainer_out_path = train_gpt(
|
| 220 |
+
metadatas=args.metadatas,
|
| 221 |
+
output_path=args.output_path,
|
| 222 |
+
num_epochs=args.num_epochs,
|
| 223 |
+
batch_size=args.batch_size,
|
| 224 |
+
grad_acumm=args.grad_acumm,
|
| 225 |
+
weight_decay=args.weight_decay,
|
| 226 |
+
lr=args.lr,
|
| 227 |
+
max_text_length=args.max_text_length,
|
| 228 |
+
max_audio_length=args.max_audio_length,
|
| 229 |
+
save_step=args.save_step
|
| 230 |
+
)
|
| 231 |
+
|
| 232 |
+
print(f"Checkpoint saved in dir: {trainer_out_path}")
|
trainer_0_log.txt
ADDED
|
@@ -0,0 +1,2135 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
> Training Environment:
|
| 2 |
+
| > Backend: Torch
|
| 3 |
+
| > Mixed precision: False
|
| 4 |
+
| > Precision: float32
|
| 5 |
+
| > Current device: 0
|
| 6 |
+
| > Num. of GPUs: 1
|
| 7 |
+
| > Num. of CPUs: 4
|
| 8 |
+
| > Num. of Torch Threads: 1
|
| 9 |
+
| > Torch seed: 54321
|
| 10 |
+
| > Torch CUDNN: True
|
| 11 |
+
| > Torch CUDNN deterministic: False
|
| 12 |
+
| > Torch CUDNN benchmark: False
|
| 13 |
+
| > Torch TF32 MatMul: False
|
| 14 |
+
> Start Tensorboard: tensorboard --logdir=xTTS/checkpoints/GPT_XTTS_FT-June-02-2025_12+22PM-0000000
|
| 15 |
+
|
| 16 |
+
> Model has 521144678 parameters
|
| 17 |
+
|
| 18 |
+
[4m[1m > EPOCH: 0/2[0m
|
| 19 |
+
--> xTTS/checkpoints/GPT_XTTS_FT-June-02-2025_12+22PM-0000000
|
| 20 |
+
|
| 21 |
+
[1m > TRAINING (2025-06-02 12:22:56) [0m
|
| 22 |
+
|
| 23 |
+
[1m --> TIME: 2025-06-02 12:23:01 -- STEP: 0/5792 -- GLOBAL_STEP: 0[0m
|
| 24 |
+
| > loss_text_ce: 0.23782861232757568 (0.23782861232757568)
|
| 25 |
+
| > loss_mel_ce: 4.941317081451416 (4.941317081451416)
|
| 26 |
+
| > loss: 1.2947864532470703 (1.2947864532470703)
|
| 27 |
+
| > current_lr: 5e-06
|
| 28 |
+
| > step_time: 1.3843 (1.3842837810516357)
|
| 29 |
+
| > loader_time: 4.3003 (4.3003199100494385)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
[1m --> TIME: 2025-06-02 12:23:30 -- STEP: 50/5792 -- GLOBAL_STEP: 50[0m
|
| 33 |
+
| > loss_text_ce: 0.23064814507961273 (0.24020873457193376)
|
| 34 |
+
| > loss_mel_ce: 4.853140830993652 (4.867191286087037)
|
| 35 |
+
| > loss: 1.2709472179412842 (1.2768500065803527)
|
| 36 |
+
| > current_lr: 5e-06
|
| 37 |
+
| > step_time: 0.2303 (0.3325099849700927)
|
| 38 |
+
| > loader_time: 0.0143 (0.016327280998229988)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
[1m --> TIME: 2025-06-02 12:23:59 -- STEP: 100/5792 -- GLOBAL_STEP: 100[0m
|
| 42 |
+
| > loss_text_ce: 0.2179468423128128 (0.2335291735827923)
|
| 43 |
+
| > loss_mel_ce: 4.29183292388916 (4.678085584640503)
|
| 44 |
+
| > loss: 1.1274449825286865 (1.2279036939144128)
|
| 45 |
+
| > current_lr: 5e-06
|
| 46 |
+
| > step_time: 0.2657 (0.3300197696685791)
|
| 47 |
+
| > loader_time: 0.0095 (0.015608358383178715)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
[1m --> TIME: 2025-06-02 12:24:28 -- STEP: 150/5792 -- GLOBAL_STEP: 150[0m
|
| 51 |
+
| > loss_text_ce: 0.21278369426727295 (0.22793374240398406)
|
| 52 |
+
| > loss_mel_ce: 4.24708890914917 (4.557877604166666)
|
| 53 |
+
| > loss: 1.114968180656433 (1.1964528389771778)
|
| 54 |
+
| > current_lr: 5e-06
|
| 55 |
+
| > step_time: 0.3034 (0.33196935812632256)
|
| 56 |
+
| > loader_time: 0.0147 (0.0150977897644043)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
[1m --> TIME: 2025-06-02 12:24:57 -- STEP: 200/5792 -- GLOBAL_STEP: 200[0m
|
| 60 |
+
| > loss_text_ce: 0.21909523010253906 (0.22294333018362522)
|
| 61 |
+
| > loss_mel_ce: 3.632870674133301 (4.465021697282792)
|
| 62 |
+
| > loss: 0.96299147605896 (1.1719912576675409)
|
| 63 |
+
| > current_lr: 5e-06
|
| 64 |
+
| > step_time: 0.137 (0.33124068975448623)
|
| 65 |
+
| > loader_time: 0.0134 (0.014897105693817142)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
[1m --> TIME: 2025-06-02 12:25:28 -- STEP: 250/5792 -- GLOBAL_STEP: 250[0m
|
| 69 |
+
| > loss_text_ce: 0.20100712776184082 (0.21740962487459184)
|
| 70 |
+
| > loss_mel_ce: 4.115545749664307 (4.400661633491516)
|
| 71 |
+
| > loss: 1.0791382789611816 (1.154517814159393)
|
| 72 |
+
| > current_lr: 5e-06
|
| 73 |
+
| > step_time: 0.4054 (0.33776610374450694)
|
| 74 |
+
| > loader_time: 0.0126 (0.014648478507995607)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
[1m --> TIME: 2025-06-02 12:25:57 -- STEP: 300/5792 -- GLOBAL_STEP: 300[0m
|
| 78 |
+
| > loss_text_ce: 0.1881532222032547 (0.21308185448249173)
|
| 79 |
+
| > loss_mel_ce: 4.062322616577148 (4.338319851557412)
|
| 80 |
+
| > loss: 1.0626189708709717 (1.1378504260381062)
|
| 81 |
+
| > current_lr: 5e-06
|
| 82 |
+
| > step_time: 0.3084 (0.33573377132415777)
|
| 83 |
+
| > loader_time: 0.0168 (0.014551755587259931)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
[1m --> TIME: 2025-06-02 12:26:25 -- STEP: 350/5792 -- GLOBAL_STEP: 350[0m
|
| 87 |
+
| > loss_text_ce: 0.18258750438690186 (0.20880227374179014)
|
| 88 |
+
| > loss_mel_ce: 3.5383872985839844 (4.286162930216108)
|
| 89 |
+
| > loss: 0.930243730545044 (1.1237413001060488)
|
| 90 |
+
| > current_lr: 5e-06
|
| 91 |
+
| > step_time: 0.2231 (0.3339259896959578)
|
| 92 |
+
| > loader_time: 0.0132 (0.014371032714843752)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
[1m --> TIME: 2025-06-02 12:26:54 -- STEP: 400/5792 -- GLOBAL_STEP: 400[0m
|
| 96 |
+
| > loss_text_ce: 0.17027834057807922 (0.20482054084539406)
|
| 97 |
+
| > loss_mel_ce: 3.622598648071289 (4.2505981731414835)
|
| 98 |
+
| > loss: 0.9482192397117615 (1.113854677826166)
|
| 99 |
+
| > current_lr: 5e-06
|
| 100 |
+
| > step_time: 0.3104 (0.3339052683115007)
|
| 101 |
+
| > loader_time: 0.0161 (0.014224189519882205)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
[1m --> TIME: 2025-06-02 12:27:23 -- STEP: 450/5792 -- GLOBAL_STEP: 450[0m
|
| 105 |
+
| > loss_text_ce: 0.1692354679107666 (0.20091567562686066)
|
| 106 |
+
| > loss_mel_ce: 3.857461452484131 (4.213876393636072)
|
| 107 |
+
| > loss: 1.0066742897033691 (1.1036980164051056)
|
| 108 |
+
| > current_lr: 5e-06
|
| 109 |
+
| > step_time: 0.3746 (0.33324571874406617)
|
| 110 |
+
| > loader_time: 0.0117 (0.014043218294779461)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
[1m --> TIME: 2025-06-02 12:27:52 -- STEP: 500/5792 -- GLOBAL_STEP: 500[0m
|
| 114 |
+
| > loss_text_ce: 0.15726038813591003 (0.19705737382173533)
|
| 115 |
+
| > loss_mel_ce: 3.882636547088623 (4.188746206283573)
|
| 116 |
+
| > loss: 1.0099742412567139 (1.0964508949518204)
|
| 117 |
+
| > current_lr: 5e-06
|
| 118 |
+
| > step_time: 0.2675 (0.3323551220893861)
|
| 119 |
+
| > loader_time: 0.0094 (0.01385477685928345)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
[1m --> TIME: 2025-06-02 12:28:21 -- STEP: 550/5792 -- GLOBAL_STEP: 550[0m
|
| 123 |
+
| > loss_text_ce: 0.14702577888965607 (0.1931881040605631)
|
| 124 |
+
| > loss_mel_ce: 3.913630723953247 (4.1642507709156416)
|
| 125 |
+
| > loss: 1.0151641368865967 (1.0893597186695445)
|
| 126 |
+
| > current_lr: 5e-06
|
| 127 |
+
| > step_time: 0.4815 (0.33244910023429186)
|
| 128 |
+
| > loader_time: 0.0126 (0.013701290217312902)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
[1m --> TIME: 2025-06-02 12:28:51 -- STEP: 600/5792 -- GLOBAL_STEP: 600[0m
|
| 132 |
+
| > loss_text_ce: 0.13721345365047455 (0.18944158929089705)
|
| 133 |
+
| > loss_mel_ce: 4.101404190063477 (4.144896006981534)
|
| 134 |
+
| > loss: 1.0596543550491333 (1.0835843993226688)
|
| 135 |
+
| > current_lr: 5e-06
|
| 136 |
+
| > step_time: 0.4056 (0.3338357198238373)
|
| 137 |
+
| > loader_time: 0.0101 (0.013622902234395351)
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
[1m --> TIME: 2025-06-02 12:29:20 -- STEP: 650/5792 -- GLOBAL_STEP: 650[0m
|
| 141 |
+
| > loss_text_ce: 0.142731174826622 (0.18570196391298224)
|
| 142 |
+
| > loss_mel_ce: 3.8508477210998535 (4.123029582683858)
|
| 143 |
+
| > loss: 0.9983947277069092 (1.0771828870589912)
|
| 144 |
+
| > current_lr: 5e-06
|
| 145 |
+
| > step_time: 0.2568 (0.33367859180156995)
|
| 146 |
+
| > loader_time: 0.0125 (0.01352635603684646)
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
[1m --> TIME: 2025-06-02 12:29:47 -- STEP: 700/5792 -- GLOBAL_STEP: 700[0m
|
| 150 |
+
| > loss_text_ce: 0.13017334043979645 (0.18195634870656907)
|
| 151 |
+
| > loss_mel_ce: 3.6967780590057373 (4.096020159721375)
|
| 152 |
+
| > loss: 0.9567378759384155 (1.069494128142085)
|
| 153 |
+
| > current_lr: 5e-06
|
| 154 |
+
| > step_time: 0.2121 (0.3316369649342126)
|
| 155 |
+
| > loader_time: 0.0094 (0.0133876953806196)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
[1m --> TIME: 2025-06-02 12:30:16 -- STEP: 750/5792 -- GLOBAL_STEP: 750[0m
|
| 159 |
+
| > loss_text_ce: 0.1326562911272049 (0.17836018381516147)
|
| 160 |
+
| > loss_mel_ce: 3.863213300704956 (4.077398623466496)
|
| 161 |
+
| > loss: 0.9989674091339111 (1.0639397028287254)
|
| 162 |
+
| > current_lr: 5e-06
|
| 163 |
+
| > step_time: 0.3401 (0.33136590449015263)
|
| 164 |
+
| > loader_time: 0.0089 (0.013331648508707687)
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
[1m --> TIME: 2025-06-02 12:30:45 -- STEP: 800/5792 -- GLOBAL_STEP: 800[0m
|
| 168 |
+
| > loss_text_ce: 0.11892132461071014 (0.17485223358497032)
|
| 169 |
+
| > loss_mel_ce: 3.5116255283355713 (4.059487291574481)
|
| 170 |
+
| > loss: 0.9076367020606995 (1.0585848820954558)
|
| 171 |
+
| > current_lr: 5e-06
|
| 172 |
+
| > step_time: 0.1597 (0.331714543998241)
|
| 173 |
+
| > loader_time: 0.0106 (0.013222561478614812)
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
[1m --> TIME: 2025-06-02 12:31:14 -- STEP: 850/5792 -- GLOBAL_STEP: 850[0m
|
| 177 |
+
| > loss_text_ce: 0.11038005352020264 (0.1714906403685318)
|
| 178 |
+
| > loss_mel_ce: 3.8610212802886963 (4.046719538183775)
|
| 179 |
+
| > loss: 0.9928503036499023 (1.0545525454072382)
|
| 180 |
+
| > current_lr: 5e-06
|
| 181 |
+
| > step_time: 0.3977 (0.3319235268761126)
|
| 182 |
+
| > loader_time: 0.012 (0.013164884623359236)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
[1m --> TIME: 2025-06-02 12:31:42 -- STEP: 900/5792 -- GLOBAL_STEP: 900[0m
|
| 186 |
+
| > loss_text_ce: 0.11592362821102142 (0.1682853105333117)
|
| 187 |
+
| > loss_mel_ce: 4.2247114181518555 (4.031507659223343)
|
| 188 |
+
| > loss: 1.0851587057113647 (1.0499482430352092)
|
| 189 |
+
| > current_lr: 5e-06
|
| 190 |
+
| > step_time: 0.3718 (0.3308940598699778)
|
| 191 |
+
| > loader_time: 0.0134 (0.01308101230197483)
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
[1m --> TIME: 2025-06-02 12:32:10 -- STEP: 950/5792 -- GLOBAL_STEP: 950[0m
|
| 195 |
+
| > loss_text_ce: 0.10771063715219498 (0.16517683673061828)
|
| 196 |
+
| > loss_mel_ce: 3.5032119750976562 (4.01872667036558)
|
| 197 |
+
| > loss: 0.9027306437492371 (1.045975877674001)
|
| 198 |
+
| > current_lr: 5e-06
|
| 199 |
+
| > step_time: 0.1779 (0.33019118836051503)
|
| 200 |
+
| > loader_time: 0.0148 (0.01302782359876131)
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
[1m --> TIME: 2025-06-02 12:32:38 -- STEP: 1000/5792 -- GLOBAL_STEP: 1000[0m
|
| 204 |
+
| > loss_text_ce: 0.12572963535785675 (0.16233218172937638)
|
| 205 |
+
| > loss_mel_ce: 3.0970044136047363 (4.003747127771376)
|
| 206 |
+
| > loss: 0.8056834936141968 (1.041519828259944)
|
| 207 |
+
| > current_lr: 5e-06
|
| 208 |
+
| > step_time: 0.1609 (0.32947225642204253)
|
| 209 |
+
| > loader_time: 0.0112 (0.01298054933547974)
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
[1m --> TIME: 2025-06-02 12:33:07 -- STEP: 1050/5792 -- GLOBAL_STEP: 1050[0m
|
| 213 |
+
| > loss_text_ce: 0.10119885206222534 (0.1596139285252208)
|
| 214 |
+
| > loss_mel_ce: 3.9164676666259766 (3.993313400404794)
|
| 215 |
+
| > loss: 1.004416584968567 (1.0382318332649403)
|
| 216 |
+
| > current_lr: 5e-06
|
| 217 |
+
| > step_time: 0.4415 (0.33021083627428294)
|
| 218 |
+
| > loader_time: 0.0149 (0.01293485868544806)
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
[1m --> TIME: 2025-06-02 12:33:36 -- STEP: 1100/5792 -- GLOBAL_STEP: 1100[0m
|
| 222 |
+
| > loss_text_ce: 0.09815080463886261 (0.1569338711080226)
|
| 223 |
+
| > loss_mel_ce: 3.761666774749756 (3.9810155831683764)
|
| 224 |
+
| > loss: 0.9649543762207031 (1.0344873643463295)
|
| 225 |
+
| > current_lr: 5e-06
|
| 226 |
+
| > step_time: 0.2093 (0.33017331686886836)
|
| 227 |
+
| > loader_time: 0.0095 (0.012836727445775815)
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
[1m --> TIME: 2025-06-02 12:34:03 -- STEP: 1150/5792 -- GLOBAL_STEP: 1150[0m
|
| 231 |
+
| > loss_text_ce: 0.09583915770053864 (0.15432746619634008)
|
| 232 |
+
| > loss_mel_ce: 3.7277543544769287 (3.967495829955391)
|
| 233 |
+
| > loss: 0.9558984041213989 (1.0304558247586937)
|
| 234 |
+
| > current_lr: 5e-06
|
| 235 |
+
| > step_time: 0.5857 (0.3293305108858188)
|
| 236 |
+
| > loader_time: 0.0121 (0.01275255389835524)
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
[1m --> TIME: 2025-06-02 12:34:32 -- STEP: 1200/5792 -- GLOBAL_STEP: 1200[0m
|
| 240 |
+
| > loss_text_ce: 0.09167931973934174 (0.15185882307589063)
|
| 241 |
+
| > loss_mel_ce: 3.893519163131714 (3.9543634712696076)
|
| 242 |
+
| > loss: 0.9962996244430542 (1.0265555742382992)
|
| 243 |
+
| > current_lr: 5e-06
|
| 244 |
+
| > step_time: 0.2053 (0.32926277975241314)
|
| 245 |
+
| > loader_time: 0.0085 (0.01267102936903636)
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
[1m --> TIME: 2025-06-02 12:35:03 -- STEP: 1250/5792 -- GLOBAL_STEP: 1250[0m
|
| 249 |
+
| > loss_text_ce: 0.09783565998077393 (0.14956970821619048)
|
| 250 |
+
| > loss_mel_ce: 3.702324867248535 (3.949050336265564)
|
| 251 |
+
| > loss: 0.9500401020050049 (1.024655011558532)
|
| 252 |
+
| > current_lr: 5e-06
|
| 253 |
+
| > step_time: 0.4952 (0.3306157491683954)
|
| 254 |
+
| > loader_time: 0.0104 (0.012619953727722172)
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
[1m --> TIME: 2025-06-02 12:35:31 -- STEP: 1300/5792 -- GLOBAL_STEP: 1300[0m
|
| 258 |
+
| > loss_text_ce: 0.08431262522935867 (0.14732415511057936)
|
| 259 |
+
| > loss_mel_ce: 3.927443027496338 (3.9411918537433333)
|
| 260 |
+
| > loss: 1.0029388666152954 (1.0221290025344256)
|
| 261 |
+
| > current_lr: 5e-06
|
| 262 |
+
| > step_time: 0.2377 (0.3301881036391619)
|
| 263 |
+
| > loader_time: 0.0087 (0.012530116668114296)
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
[1m --> TIME: 2025-06-02 12:36:01 -- STEP: 1350/5792 -- GLOBAL_STEP: 1350[0m
|
| 267 |
+
| > loss_text_ce: 0.08895734697580338 (0.14518685861870098)
|
| 268 |
+
| > loss_mel_ce: 3.606088638305664 (3.93310096723062)
|
| 269 |
+
| > loss: 0.9237614870071411 (1.0195719568376187)
|
| 270 |
+
| > current_lr: 5e-06
|
| 271 |
+
| > step_time: 0.5945 (0.33128830627158784)
|
| 272 |
+
| > loader_time: 0.0219 (0.012501915649131492)
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
[1m --> TIME: 2025-06-02 12:36:30 -- STEP: 1400/5792 -- GLOBAL_STEP: 1400[0m
|
| 276 |
+
| > loss_text_ce: 0.08065199851989746 (0.14312162278486157)
|
| 277 |
+
| > loss_mel_ce: 3.0727732181549072 (3.924280370473862)
|
| 278 |
+
| > loss: 0.7883563041687012 (1.0168504986166955)
|
| 279 |
+
| > current_lr: 5e-06
|
| 280 |
+
| > step_time: 0.1569 (0.3313496242250711)
|
| 281 |
+
| > loader_time: 0.0119 (0.012489057098116194)
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
[1m --> TIME: 2025-06-02 12:37:00 -- STEP: 1450/5792 -- GLOBAL_STEP: 1450[0m
|
| 285 |
+
| > loss_text_ce: 0.079571932554245 (0.14114773946075607)
|
| 286 |
+
| > loss_mel_ce: 3.952150821685791 (3.91605803686997)
|
| 287 |
+
| > loss: 1.0079306364059448 (1.0143014442509617)
|
| 288 |
+
| > current_lr: 5e-06
|
| 289 |
+
| > step_time: 0.3056 (0.33169992216702143)
|
| 290 |
+
| > loader_time: 0.0124 (0.012441920247571221)
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
[1m --> TIME: 2025-06-02 12:37:29 -- STEP: 1500/5792 -- GLOBAL_STEP: 1500[0m
|
| 294 |
+
| > loss_text_ce: 0.08127571642398834 (0.1392633452763161)
|
| 295 |
+
| > loss_mel_ce: 3.6380221843719482 (3.905871781508128)
|
| 296 |
+
| > loss: 0.9298244714736938 (1.0112837819655731)
|
| 297 |
+
| > current_lr: 5e-06
|
| 298 |
+
| > step_time: 0.2384 (0.33202595345179226)
|
| 299 |
+
| > loader_time: 0.0098 (0.012403525034586589)
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
[1m --> TIME: 2025-06-02 12:37:59 -- STEP: 1550/5792 -- GLOBAL_STEP: 1550[0m
|
| 303 |
+
| > loss_text_ce: 0.08357858657836914 (0.13743022495700477)
|
| 304 |
+
| > loss_mel_ce: 3.2771170139312744 (3.898509816354321)
|
| 305 |
+
| > loss: 0.8401739001274109 (1.0089850104624223)
|
| 306 |
+
| > current_lr: 5e-06
|
| 307 |
+
| > step_time: 0.1768 (0.33223780908892203)
|
| 308 |
+
| > loader_time: 0.0085 (0.012367438347108902)
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
[1m --> TIME: 2025-06-02 12:38:28 -- STEP: 1600/5792 -- GLOBAL_STEP: 1600[0m
|
| 312 |
+
| > loss_text_ce: 0.07955673336982727 (0.13564207764342429)
|
| 313 |
+
| > loss_mel_ce: 3.7596960067749023 (3.892578792124987)
|
| 314 |
+
| > loss: 0.9598131775856018 (1.007055217660963)
|
| 315 |
+
| > current_lr: 5e-06
|
| 316 |
+
| > step_time: 0.3524 (0.33262646302580806)
|
| 317 |
+
| > loader_time: 0.0101 (0.012325592041015625)
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
[1m --> TIME: 2025-06-02 12:38:57 -- STEP: 1650/5792 -- GLOBAL_STEP: 1650[0m
|
| 321 |
+
| > loss_text_ce: 0.08837619423866272 (0.13392942287253612)
|
| 322 |
+
| > loss_mel_ce: 3.777754068374634 (3.885421214681683)
|
| 323 |
+
| > loss: 0.9665325880050659 (1.0048376595973967)
|
| 324 |
+
| > current_lr: 5e-06
|
| 325 |
+
| > step_time: 0.3983 (0.3327004941304522)
|
| 326 |
+
| > loader_time: 0.0101 (0.012279371637286561)
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
[1m --> TIME: 2025-06-02 12:39:25 -- STEP: 1700/5792 -- GLOBAL_STEP: 1700[0m
|
| 330 |
+
| > loss_text_ce: 0.07985109090805054 (0.13228617378455745)
|
| 331 |
+
| > loss_mel_ce: 3.664367198944092 (3.8788135032092823)
|
| 332 |
+
| > loss: 0.9360545873641968 (1.0027749196922073)
|
| 333 |
+
| > current_lr: 5e-06
|
| 334 |
+
| > step_time: 0.1903 (0.33247794515946305)
|
| 335 |
+
| > loader_time: 0.0099 (0.012235631241517908)
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
[1m --> TIME: 2025-06-02 12:39:53 -- STEP: 1750/5792 -- GLOBAL_STEP: 1750[0m
|
| 339 |
+
| > loss_text_ce: 0.07385779917240143 (0.13067971873496242)
|
| 340 |
+
| > loss_mel_ce: 3.792656421661377 (3.8709569222586495)
|
| 341 |
+
| > loss: 0.9666285514831543 (1.0004091606821328)
|
| 342 |
+
| > current_lr: 5e-06
|
| 343 |
+
| > step_time: 0.2969 (0.3322374545506066)
|
| 344 |
+
| > loader_time: 0.0094 (0.012205468041556223)
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
[1m --> TIME: 2025-06-02 12:40:23 -- STEP: 1800/5792 -- GLOBAL_STEP: 1800[0m
|
| 348 |
+
| > loss_text_ce: 0.07535833865404129 (0.12911513355664064)
|
| 349 |
+
| > loss_mel_ce: 3.611762046813965 (3.8656086026297674)
|
| 350 |
+
| > loss: 0.9217801094055176 (0.9986809344093001)
|
| 351 |
+
| > current_lr: 5e-06
|
| 352 |
+
| > step_time: 0.4042 (0.3324654255972966)
|
| 353 |
+
| > loader_time: 0.014 (0.012192738983366225)
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
[1m --> TIME: 2025-06-02 12:40:51 -- STEP: 1850/5792 -- GLOBAL_STEP: 1850[0m
|
| 357 |
+
| > loss_text_ce: 0.0778559073805809 (0.12761117409814027)
|
| 358 |
+
| > loss_mel_ce: 3.0250251293182373 (3.857865214734464)
|
| 359 |
+
| > loss: 0.7757202386856079 (0.9963690974583493)
|
| 360 |
+
| > current_lr: 5e-06
|
| 361 |
+
| > step_time: 0.2565 (0.3320673637132384)
|
| 362 |
+
| > loader_time: 0.0119 (0.012154008891131426)
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
[1m --> TIME: 2025-06-02 12:41:21 -- STEP: 1900/5792 -- GLOBAL_STEP: 1900[0m
|
| 366 |
+
| > loss_text_ce: 0.07138916105031967 (0.12619867451096817)
|
| 367 |
+
| > loss_mel_ce: 3.775388717651367 (3.851701809858021)
|
| 368 |
+
| > loss: 0.9616944789886475 (0.9944751213726241)
|
| 369 |
+
| > current_lr: 5e-06
|
| 370 |
+
| > step_time: 0.2679 (0.3323917353780644)
|
| 371 |
+
| > loader_time: 0.0108 (0.012138623438383404)
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
[1m --> TIME: 2025-06-02 12:41:50 -- STEP: 1950/5792 -- GLOBAL_STEP: 1950[0m
|
| 375 |
+
| > loss_text_ce: 0.06446552276611328 (0.12474528638980308)
|
| 376 |
+
| > loss_mel_ce: 3.478724956512451 (3.8443781879620675)
|
| 377 |
+
| > loss: 0.8857976198196411 (0.9922808688114847)
|
| 378 |
+
| > current_lr: 5e-06
|
| 379 |
+
| > step_time: 0.267 (0.3323717740865852)
|
| 380 |
+
| > loader_time: 0.0102 (0.012119236114697578)
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
[1m --> TIME: 2025-06-02 12:42:16 -- STEP: 2000/5792 -- GLOBAL_STEP: 2000[0m
|
| 384 |
+
| > loss_text_ce: 0.07915454357862473 (0.12336141603440026)
|
| 385 |
+
| > loss_mel_ce: 3.8885855674743652 (3.8369900035858153)
|
| 386 |
+
| > loss: 0.9919350147247314 (0.9900878550410267)
|
| 387 |
+
| > current_lr: 5e-06
|
| 388 |
+
| > step_time: 0.3543 (0.3317557517290113)
|
| 389 |
+
| > loader_time: 0.0126 (0.012084858655929566)
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
[1m --> TIME: 2025-06-02 12:42:46 -- STEP: 2050/5792 -- GLOBAL_STEP: 2050[0m
|
| 393 |
+
| > loss_text_ce: 0.06675997376441956 (0.1220419639494359)
|
| 394 |
+
| > loss_mel_ce: 3.372281312942505 (3.8299568820581205)
|
| 395 |
+
| > loss: 0.8597603440284729 (0.9879997116181906)
|
| 396 |
+
| > current_lr: 5e-06
|
| 397 |
+
| > step_time: 0.3444 (0.3319971689363803)
|
| 398 |
+
| > loader_time: 0.0105 (0.01205671554658471)
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
[1m --> TIME: 2025-06-02 12:43:14 -- STEP: 2100/5792 -- GLOBAL_STEP: 2100[0m
|
| 402 |
+
| > loss_text_ce: 0.0644061341881752 (0.12074772182142432)
|
| 403 |
+
| > loss_mel_ce: 3.0573296546936035 (3.823917001769656)
|
| 404 |
+
| > loss: 0.7804339528083801 (0.9861661809682841)
|
| 405 |
+
| > current_lr: 5e-06
|
| 406 |
+
| > step_time: 0.1591 (0.33194411834080995)
|
| 407 |
+
| > loader_time: 0.0112 (0.012042945112500873)
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
[1m --> TIME: 2025-06-02 12:43:45 -- STEP: 2150/5792 -- GLOBAL_STEP: 2150[0m
|
| 411 |
+
| > loss_text_ce: 0.0625600665807724 (0.11952382788062077)
|
| 412 |
+
| > loss_mel_ce: 3.1828041076660156 (3.819017451530279)
|
| 413 |
+
| > loss: 0.8113410472869873 (0.984635319903839)
|
| 414 |
+
| > current_lr: 5e-06
|
| 415 |
+
| > step_time: 0.2213 (0.33232841746751635)
|
| 416 |
+
| > loader_time: 0.0153 (0.012043467122454977)
|
| 417 |
+
|
| 418 |
+
|
| 419 |
+
[1m --> TIME: 2025-06-02 12:44:12 -- STEP: 2200/5792 -- GLOBAL_STEP: 2200[0m
|
| 420 |
+
| > loss_text_ce: 0.07580503076314926 (0.11829185286367463)
|
| 421 |
+
| > loss_mel_ce: 3.471283435821533 (3.8122500800002705)
|
| 422 |
+
| > loss: 0.886772096157074 (0.9826354832540855)
|
| 423 |
+
| > current_lr: 5e-06
|
| 424 |
+
| > step_time: 0.2548 (0.33183346889235754)
|
| 425 |
+
| > loader_time: 0.0097 (0.012037343978881839)
|
| 426 |
+
|
| 427 |
+
|
| 428 |
+
[1m --> TIME: 2025-06-02 12:44:41 -- STEP: 2250/5792 -- GLOBAL_STEP: 2250[0m
|
| 429 |
+
| > loss_text_ce: 0.06772749871015549 (0.11713893160720648)
|
| 430 |
+
| > loss_mel_ce: 3.4121651649475098 (3.8072973964479235)
|
| 431 |
+
| > loss: 0.8699731826782227 (0.9811090819570749)
|
| 432 |
+
| > current_lr: 5e-06
|
| 433 |
+
| > step_time: 0.196 (0.33194303947024884)
|
| 434 |
+
| > loader_time: 0.0094 (0.012012425316704644)
|
| 435 |
+
|
| 436 |
+
|
| 437 |
+
[1m --> TIME: 2025-06-02 12:45:10 -- STEP: 2300/5792 -- GLOBAL_STEP: 2300[0m
|
| 438 |
+
| > loss_text_ce: 0.061648476868867874 (0.11599788975942377)
|
| 439 |
+
| > loss_mel_ce: 3.568927764892578 (3.802683669069539)
|
| 440 |
+
| > loss: 0.9076440334320068 (0.9796703896574348)
|
| 441 |
+
| > current_lr: 5e-06
|
| 442 |
+
| > step_time: 0.3098 (0.33178884682448023)
|
| 443 |
+
| > loader_time: 0.0146 (0.012008188081824262)
|
| 444 |
+
|
| 445 |
+
|
| 446 |
+
[1m --> TIME: 2025-06-02 12:45:40 -- STEP: 2350/5792 -- GLOBAL_STEP: 2350[0m
|
| 447 |
+
| > loss_text_ce: 0.06095200777053833 (0.11489018085947679)
|
| 448 |
+
| > loss_mel_ce: 3.651190996170044 (3.798658301576655)
|
| 449 |
+
| > loss: 0.9280357360839844 (0.9783871206324147)
|
| 450 |
+
| > current_lr: 5e-06
|
| 451 |
+
| > step_time: 0.34 (0.3323820060364746)
|
| 452 |
+
| > loader_time: 0.0123 (0.011982986064667397)
|
| 453 |
+
|
| 454 |
+
|
| 455 |
+
[1m --> TIME: 2025-06-02 12:46:08 -- STEP: 2400/5792 -- GLOBAL_STEP: 2400[0m
|
| 456 |
+
| > loss_text_ce: 0.06559105217456818 (0.11383588070515527)
|
| 457 |
+
| > loss_mel_ce: 3.4982616901397705 (3.793631868362427)
|
| 458 |
+
| > loss: 0.8909631967544556 (0.9768669372300303)
|
| 459 |
+
| > current_lr: 5e-06
|
| 460 |
+
| > step_time: 0.3963 (0.3320956921577454)
|
| 461 |
+
| > loader_time: 0.015 (0.011968173583348597)
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
[1m --> TIME: 2025-06-02 12:46:40 -- STEP: 2450/5792 -- GLOBAL_STEP: 2450[0m
|
| 465 |
+
| > loss_text_ce: 0.06471642851829529 (0.11277475817471118)
|
| 466 |
+
| > loss_mel_ce: 3.9599180221557617 (3.790205652275864)
|
| 467 |
+
| > loss: 1.0061585903167725 (0.9757451025807123)
|
| 468 |
+
| > current_lr: 5e-06
|
| 469 |
+
| > step_time: 0.43 (0.3328273964901357)
|
| 470 |
+
| > loader_time: 0.0128 (0.011948673481843914)
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
[1m --> TIME: 2025-06-02 12:47:08 -- STEP: 2500/5792 -- GLOBAL_STEP: 2500[0m
|
| 474 |
+
| > loss_text_ce: 0.058090440928936005 (0.11174730639308675)
|
| 475 |
+
| > loss_mel_ce: 3.680204153060913 (3.786430072784424)
|
| 476 |
+
| > loss: 0.9345736503601074 (0.9745443446636196)
|
| 477 |
+
| > current_lr: 5e-06
|
| 478 |
+
| > step_time: 0.4192 (0.3327032661437986)
|
| 479 |
+
| > loader_time: 0.014 (0.011920587730407725)
|
| 480 |
+
|
| 481 |
+
|
| 482 |
+
[1m --> TIME: 2025-06-02 12:47:37 -- STEP: 2550/5792 -- GLOBAL_STEP: 2550[0m
|
| 483 |
+
| > loss_text_ce: 0.06345102936029434 (0.11076159270224602)
|
| 484 |
+
| > loss_mel_ce: 3.3983774185180664 (3.7819082406922884)
|
| 485 |
+
| > loss: 0.8654571175575256 (0.9731674582584228)
|
| 486 |
+
| > current_lr: 5e-06
|
| 487 |
+
| > step_time: 0.3032 (0.3327015813191728)
|
| 488 |
+
| > loader_time: 0.0101 (0.01190789082471062)
|
| 489 |
+
|
| 490 |
+
|
| 491 |
+
[1m --> TIME: 2025-06-02 12:48:06 -- STEP: 2600/5792 -- GLOBAL_STEP: 2600[0m
|
| 492 |
+
| > loss_text_ce: 0.05978523939847946 (0.10978607040041899)
|
| 493 |
+
| > loss_mel_ce: 3.5986759662628174 (3.7784423384299646)
|
| 494 |
+
| > loss: 0.914615273475647 (0.9720571020704046)
|
| 495 |
+
| > current_lr: 5e-06
|
| 496 |
+
| > step_time: 0.2067 (0.33279164598538297)
|
| 497 |
+
| > loader_time: 0.0095 (0.01188810458550087)
|
| 498 |
+
|
| 499 |
+
|
| 500 |
+
[1m --> TIME: 2025-06-02 12:48:38 -- STEP: 2650/5792 -- GLOBAL_STEP: 2650[0m
|
| 501 |
+
| > loss_text_ce: 0.058306336402893066 (0.1088496894923582)
|
| 502 |
+
| > loss_mel_ce: 2.9056618213653564 (3.7759509926022226)
|
| 503 |
+
| > loss: 0.7409920692443848 (0.9712001704045058)
|
| 504 |
+
| > current_lr: 5e-06
|
| 505 |
+
| > step_time: 0.2274 (0.3335093621487883)
|
| 506 |
+
| > loader_time: 0.0143 (0.01186871078779113)
|
| 507 |
+
|
| 508 |
+
|
| 509 |
+
[1m --> TIME: 2025-06-02 12:49:06 -- STEP: 2700/5792 -- GLOBAL_STEP: 2700[0m
|
| 510 |
+
| > loss_text_ce: 0.06381174176931381 (0.1079658736467912)
|
| 511 |
+
| > loss_mel_ce: 3.7307090759277344 (3.7717624486817254)
|
| 512 |
+
| > loss: 0.9486302137374878 (0.9699320805293539)
|
| 513 |
+
| > current_lr: 5e-06
|
| 514 |
+
| > step_time: 0.3863 (0.33329126207916787)
|
| 515 |
+
| > loader_time: 0.0102 (0.01184851937823826)
|
| 516 |
+
|
| 517 |
+
|
| 518 |
+
[1m --> TIME: 2025-06-02 12:49:37 -- STEP: 2750/5792 -- GLOBAL_STEP: 2750[0m
|
| 519 |
+
| > loss_text_ce: 0.06515434384346008 (0.1070680166618389)
|
| 520 |
+
| > loss_mel_ce: 3.7349181175231934 (3.76892153228413)
|
| 521 |
+
| > loss: 0.9500181078910828 (0.9689973871707913)
|
| 522 |
+
| > current_lr: 5e-06
|
| 523 |
+
| > step_time: 0.1943 (0.33362000248648843)
|
| 524 |
+
| > loader_time: 0.0092 (0.011837096214294438)
|
| 525 |
+
|
| 526 |
+
|
| 527 |
+
[1m --> TIME: 2025-06-02 12:50:06 -- STEP: 2800/5792 -- GLOBAL_STEP: 2800[0m
|
| 528 |
+
| > loss_text_ce: 0.0626336857676506 (0.10621006238939493)
|
| 529 |
+
| > loss_mel_ce: 3.619314193725586 (3.7653382179566792)
|
| 530 |
+
| > loss: 0.9204869866371155 (0.9678870699873989)
|
| 531 |
+
| > current_lr: 5e-06
|
| 532 |
+
| > step_time: 0.3534 (0.3337795458521157)
|
| 533 |
+
| > loader_time: 0.0102 (0.011815384626388556)
|
| 534 |
+
|
| 535 |
+
|
| 536 |
+
[1m --> TIME: 2025-06-02 12:50:34 -- STEP: 2850/5792 -- GLOBAL_STEP: 2850[0m
|
| 537 |
+
| > loss_text_ce: 0.05888611823320389 (0.10537111821273949)
|
| 538 |
+
| > loss_mel_ce: 3.1223511695861816 (3.763010348939059)
|
| 539 |
+
| > loss: 0.79530930519104 (0.9670953666536428)
|
| 540 |
+
| > current_lr: 5e-06
|
| 541 |
+
| > step_time: 0.2558 (0.33350726955815335)
|
| 542 |
+
| > loader_time: 0.0118 (0.011799037080062068)
|
| 543 |
+
|
| 544 |
+
|
| 545 |
+
[1m --> TIME: 2025-06-02 12:51:03 -- STEP: 2900/5792 -- GLOBAL_STEP: 2900[0m
|
| 546 |
+
| > loss_text_ce: 0.0631546750664711 (0.1045509203009563)
|
| 547 |
+
| > loss_mel_ce: 3.269371271133423 (3.7583381495804624)
|
| 548 |
+
| > loss: 0.8331314921379089 (0.9657222673399692)
|
| 549 |
+
| > current_lr: 5e-06
|
| 550 |
+
| > step_time: 0.2418 (0.33377023384488846)
|
| 551 |
+
| > loader_time: 0.0097 (0.011781094896382304)
|
| 552 |
+
|
| 553 |
+
|
| 554 |
+
[1m --> TIME: 2025-06-02 12:51:33 -- STEP: 2950/5792 -- GLOBAL_STEP: 2950[0m
|
| 555 |
+
| > loss_text_ce: 0.054032497107982635 (0.10374996635999706)
|
| 556 |
+
| > loss_mel_ce: 3.7166588306427 (3.754722292625298)
|
| 557 |
+
| > loss: 0.942672848701477 (0.9646180645894193)
|
| 558 |
+
| > current_lr: 5e-06
|
| 559 |
+
| > step_time: 0.4931 (0.3338233034489517)
|
| 560 |
+
| > loader_time: 0.0146 (0.01176468857264115)
|
| 561 |
+
|
| 562 |
+
|
| 563 |
+
[1m --> TIME: 2025-06-02 12:52:03 -- STEP: 3000/5792 -- GLOBAL_STEP: 3000[0m
|
| 564 |
+
| > loss_text_ce: 0.04497094824910164 (0.10296552998696751)
|
| 565 |
+
| > loss_mel_ce: 3.8430159091949463 (3.7516442841688793)
|
| 566 |
+
| > loss: 0.9719967246055603 (0.9636524533629415)
|
| 567 |
+
| > current_lr: 5e-06
|
| 568 |
+
| > step_time: 0.2344 (0.3341527958710988)
|
| 569 |
+
| > loader_time: 0.0093 (0.011756853898366297)
|
| 570 |
+
|
| 571 |
+
|
| 572 |
+
[1m --> TIME: 2025-06-02 12:52:32 -- STEP: 3050/5792 -- GLOBAL_STEP: 3050[0m
|
| 573 |
+
| > loss_text_ce: 0.05044661462306976 (0.10219378712480172)
|
| 574 |
+
| > loss_mel_ce: 3.3637678623199463 (3.7479482764103373)
|
| 575 |
+
| > loss: 0.8535535931587219 (0.962535515730498)
|
| 576 |
+
| > current_lr: 5e-06
|
| 577 |
+
| > step_time: 0.3357 (0.33412718593097107)
|
| 578 |
+
| > loader_time: 0.0094 (0.011745000745429371)
|
| 579 |
+
|
| 580 |
+
|
| 581 |
+
[1m --> TIME: 2025-06-02 12:53:01 -- STEP: 3100/5792 -- GLOBAL_STEP: 3100[0m
|
| 582 |
+
| > loss_text_ce: 0.0588652640581131 (0.10145230594421574)
|
| 583 |
+
| > loss_mel_ce: 3.9438345432281494 (3.7447509037294697)
|
| 584 |
+
| > loss: 1.0006749629974365 (0.9615508023769622)
|
| 585 |
+
| > current_lr: 5e-06
|
| 586 |
+
| > step_time: 0.2943 (0.3342037938487145)
|
| 587 |
+
| > loader_time: 0.0097 (0.011726075987662043)
|
| 588 |
+
|
| 589 |
+
|
| 590 |
+
[1m --> TIME: 2025-06-02 12:53:30 -- STEP: 3150/5792 -- GLOBAL_STEP: 3150[0m
|
| 591 |
+
| > loss_text_ce: 0.051031917333602905 (0.10072415719902691)
|
| 592 |
+
| > loss_mel_ce: 3.401136636734009 (3.7408774338828192)
|
| 593 |
+
| > loss: 0.8630421161651611 (0.9604003977207908)
|
| 594 |
+
| > current_lr: 5e-06
|
| 595 |
+
| > step_time: 0.2545 (0.334141366822379)
|
| 596 |
+
| > loader_time: 0.0144 (0.011702652280292818)
|
| 597 |
+
|
| 598 |
+
|
| 599 |
+
[1m --> TIME: 2025-06-02 12:53:57 -- STEP: 3200/5792 -- GLOBAL_STEP: 3200[0m
|
| 600 |
+
| > loss_text_ce: 0.06074317917227745 (0.10001451563904981)
|
| 601 |
+
| > loss_mel_ce: 3.513090133666992 (3.7376667942106723)
|
| 602 |
+
| > loss: 0.8934583067893982 (0.9594203274324533)
|
| 603 |
+
| > current_lr: 5e-06
|
| 604 |
+
| > step_time: 0.2398 (0.3338389614969492)
|
| 605 |
+
| > loader_time: 0.012 (0.011692634075880054)
|
| 606 |
+
|
| 607 |
+
|
| 608 |
+
[1m --> TIME: 2025-06-02 12:54:24 -- STEP: 3250/5792 -- GLOBAL_STEP: 3250[0m
|
| 609 |
+
| > loss_text_ce: 0.05713744834065437 (0.0993218669753807)
|
| 610 |
+
| > loss_mel_ce: 3.3079988956451416 (3.7339409486330473)
|
| 611 |
+
| > loss: 0.8412840962409973 (0.9583157039055454)
|
| 612 |
+
| > current_lr: 5e-06
|
| 613 |
+
| > step_time: 0.2647 (0.3333787508010864)
|
| 614 |
+
| > loader_time: 0.01 (0.011673195472130414)
|
| 615 |
+
|
| 616 |
+
|
| 617 |
+
[1m --> TIME: 2025-06-02 12:54:53 -- STEP: 3300/5792 -- GLOBAL_STEP: 3300[0m
|
| 618 |
+
| > loss_text_ce: 0.049549344927072525 (0.09864641397401226)
|
| 619 |
+
| > loss_mel_ce: 3.5223379135131836 (3.731030137611158)
|
| 620 |
+
| > loss: 0.8929718136787415 (0.9574191379005256)
|
| 621 |
+
| > current_lr: 5e-06
|
| 622 |
+
| > step_time: 0.2054 (0.33343946623079707)
|
| 623 |
+
| > loader_time: 0.0094 (0.011662811438242599)
|
| 624 |
+
|
| 625 |
+
|
| 626 |
+
[1m --> TIME: 2025-06-02 12:55:21 -- STEP: 3350/5792 -- GLOBAL_STEP: 3350[0m
|
| 627 |
+
| > loss_text_ce: 0.053567271679639816 (0.09798200751974499)
|
| 628 |
+
| > loss_mel_ce: 3.276874542236328 (3.7281440085795388)
|
| 629 |
+
| > loss: 0.8326104283332825 (0.9565315039656053)
|
| 630 |
+
| > current_lr: 5e-06
|
| 631 |
+
| > step_time: 0.5258 (0.3332891092015736)
|
| 632 |
+
| > loader_time: 0.0106 (0.011652176344572612)
|
| 633 |
+
|
| 634 |
+
|
| 635 |
+
[1m --> TIME: 2025-06-02 12:55:52 -- STEP: 3400/5792 -- GLOBAL_STEP: 3400[0m
|
| 636 |
+
| > loss_text_ce: 0.05310482531785965 (0.09735800301765683)
|
| 637 |
+
| > loss_mel_ce: 3.462460517883301 (3.7253227573282577)
|
| 638 |
+
| > loss: 0.8788913488388062 (0.9556701900503212)
|
| 639 |
+
| > current_lr: 5e-06
|
| 640 |
+
| > step_time: 0.1824 (0.3334693743902094)
|
| 641 |
+
| > loader_time: 0.0087 (0.011638146919362689)
|
| 642 |
+
|
| 643 |
+
|
| 644 |
+
[1m --> TIME: 2025-06-02 12:56:21 -- STEP: 3450/5792 -- GLOBAL_STEP: 3450[0m
|
| 645 |
+
| > loss_text_ce: 0.05677446722984314 (0.0967331422789804)
|
| 646 |
+
| > loss_mel_ce: 3.994664430618286 (3.722395037084386)
|
| 647 |
+
| > loss: 1.0128597021102905 (0.954782044852989)
|
| 648 |
+
| > current_lr: 5e-06
|
| 649 |
+
| > step_time: 0.3058 (0.333492807927339)
|
| 650 |
+
| > loader_time: 0.0099 (0.011624124844868982)
|
| 651 |
+
|
| 652 |
+
|
| 653 |
+
[1m --> TIME: 2025-06-02 12:56:51 -- STEP: 3500/5792 -- GLOBAL_STEP: 3500[0m
|
| 654 |
+
| > loss_text_ce: 0.058832015842199326 (0.0961081453689506)
|
| 655 |
+
| > loss_mel_ce: 3.7743239402770996 (3.7200029888153074)
|
| 656 |
+
| > loss: 0.9582889676094055 (0.9540277835471287)
|
| 657 |
+
| > current_lr: 5e-06
|
| 658 |
+
| > step_time: 0.4594 (0.3338103825705392)
|
| 659 |
+
| > loader_time: 0.0135 (0.011612459046500073)
|
| 660 |
+
|
| 661 |
+
|
| 662 |
+
[1m --> TIME: 2025-06-02 12:57:20 -- STEP: 3550/5792 -- GLOBAL_STEP: 3550[0m
|
| 663 |
+
| > loss_text_ce: 0.050712406635284424 (0.09550555716832747)
|
| 664 |
+
| > loss_mel_ce: 3.5286216735839844 (3.7171594685567935)
|
| 665 |
+
| > loss: 0.894833505153656 (0.9531662564546287)
|
| 666 |
+
| > current_lr: 5e-06
|
| 667 |
+
| > step_time: 0.3831 (0.33380109598938845)
|
| 668 |
+
| > loader_time: 0.0088 (0.011606321603479524)
|
| 669 |
+
|
| 670 |
+
|
| 671 |
+
[1m --> TIME: 2025-06-02 12:57:48 -- STEP: 3600/5792 -- GLOBAL_STEP: 3600[0m
|
| 672 |
+
| > loss_text_ce: 0.0556088350713253 (0.09492813276644371)
|
| 673 |
+
| > loss_mel_ce: 3.395554780960083 (3.713761396010717)
|
| 674 |
+
| > loss: 0.8627908825874329 (0.9521723822421495)
|
| 675 |
+
| > current_lr: 5e-06
|
| 676 |
+
| > step_time: 0.2089 (0.3337154185109668)
|
| 677 |
+
| > loader_time: 0.0126 (0.01158974859449599)
|
| 678 |
+
|
| 679 |
+
|
| 680 |
+
[1m --> TIME: 2025-06-02 12:58:17 -- STEP: 3650/5792 -- GLOBAL_STEP: 3650[0m
|
| 681 |
+
| > loss_text_ce: 0.05382274463772774 (0.0943567333778697)
|
| 682 |
+
| > loss_mel_ce: 3.8336167335510254 (3.7102085386563655)
|
| 683 |
+
| > loss: 0.971859872341156 (0.9511413181317994)
|
| 684 |
+
| > current_lr: 5e-06
|
| 685 |
+
| > step_time: 0.3718 (0.33372791290283205)
|
| 686 |
+
| > loader_time: 0.0094 (0.011583965510538183)
|
| 687 |
+
|
| 688 |
+
|
| 689 |
+
[1m --> TIME: 2025-06-02 12:58:46 -- STEP: 3700/5792 -- GLOBAL_STEP: 3700[0m
|
| 690 |
+
| > loss_text_ce: 0.054553646594285965 (0.09378471502983882)
|
| 691 |
+
| > loss_mel_ce: 3.3460400104522705 (3.707569275224531)
|
| 692 |
+
| > loss: 0.8501484394073486 (0.9503384976612552)
|
| 693 |
+
| > current_lr: 5e-06
|
| 694 |
+
| > step_time: 0.239 (0.333674537491154)
|
| 695 |
+
| > loader_time: 0.0098 (0.01157456649316324)
|
| 696 |
+
|
| 697 |
+
|
| 698 |
+
[1m --> TIME: 2025-06-02 12:59:16 -- STEP: 3750/5792 -- GLOBAL_STEP: 3750[0m
|
| 699 |
+
| > loss_text_ce: 0.04969848692417145 (0.0932255767772594)
|
| 700 |
+
| > loss_mel_ce: 3.773103713989258 (3.705821061897278)
|
| 701 |
+
| > loss: 0.9557005763053894 (0.9497616597811379)
|
| 702 |
+
| > current_lr: 5e-06
|
| 703 |
+
| > step_time: 0.4401 (0.33399006907145184)
|
| 704 |
+
| > loader_time: 0.0105 (0.011564516321818038)
|
| 705 |
+
|
| 706 |
+
|
| 707 |
+
[1m --> TIME: 2025-06-02 12:59:45 -- STEP: 3800/5792 -- GLOBAL_STEP: 3800[0m
|
| 708 |
+
| > loss_text_ce: 0.05216987058520317 (0.09270269069232429)
|
| 709 |
+
| > loss_mel_ce: 4.144445419311523 (3.703308340373792)
|
| 710 |
+
| > loss: 1.0491538047790527 (0.9490027578567202)
|
| 711 |
+
| > current_lr: 5e-06
|
| 712 |
+
| > step_time: 0.3106 (0.3339822284171456)
|
| 713 |
+
| > loader_time: 0.0101 (0.011551208119643365)
|
| 714 |
+
|
| 715 |
+
|
| 716 |
+
[1m --> TIME: 2025-06-02 13:00:15 -- STEP: 3850/5792 -- GLOBAL_STEP: 3850[0m
|
| 717 |
+
| > loss_text_ce: 0.051087573170661926 (0.09218463780527755)
|
| 718 |
+
| > loss_mel_ce: 3.5129377841949463 (3.7008065200780895)
|
| 719 |
+
| > loss: 0.891006350517273 (0.9482477895625223)
|
| 720 |
+
| > current_lr: 5e-06
|
| 721 |
+
| > step_time: 0.4407 (0.33412671906607494)
|
| 722 |
+
| > loader_time: 0.0109 (0.011542596259674472)
|
| 723 |
+
|
| 724 |
+
|
| 725 |
+
[1m --> TIME: 2025-06-02 13:00:46 -- STEP: 3900/5792 -- GLOBAL_STEP: 3900[0m
|
| 726 |
+
| > loss_text_ce: 0.05810575187206268 (0.09166618832888501)
|
| 727 |
+
| > loss_mel_ce: 3.621946334838867 (3.698486832961058)
|
| 728 |
+
| > loss: 0.9200130105018616 (0.9475382554378262)
|
| 729 |
+
| > current_lr: 5e-06
|
| 730 |
+
| > step_time: 0.3832 (0.3345049955906012)
|
| 731 |
+
| > loader_time: 0.0108 (0.011529551591628641)
|
| 732 |
+
|
| 733 |
+
|
| 734 |
+
[1m --> TIME: 2025-06-02 13:01:15 -- STEP: 3950/5792 -- GLOBAL_STEP: 3950[0m
|
| 735 |
+
| > loss_text_ce: 0.05320081114768982 (0.09115826475186428)
|
| 736 |
+
| > loss_mel_ce: 3.2360873222351074 (3.695133606451976)
|
| 737 |
+
| > loss: 0.8223220109939575 (0.946572967918613)
|
| 738 |
+
| > current_lr: 5e-06
|
| 739 |
+
| > step_time: 0.2365 (0.33447549988951863)
|
| 740 |
+
| > loader_time: 0.0092 (0.011514630860920198)
|
| 741 |
+
|
| 742 |
+
|
| 743 |
+
[1m --> TIME: 2025-06-02 13:01:44 -- STEP: 4000/5792 -- GLOBAL_STEP: 4000[0m
|
| 744 |
+
| > loss_text_ce: 0.05098037049174309 (0.09065508700720956)
|
| 745 |
+
| > loss_mel_ce: 3.636732578277588 (3.6921278865337372)
|
| 746 |
+
| > loss: 0.9219282269477844 (0.9456957434713839)
|
| 747 |
+
| > current_lr: 5e-06
|
| 748 |
+
| > step_time: 0.3267 (0.33452891433238985)
|
| 749 |
+
| > loader_time: 0.0102 (0.01150596791505814)
|
| 750 |
+
|
| 751 |
+
|
| 752 |
+
[1m --> TIME: 2025-06-02 13:02:12 -- STEP: 4050/5792 -- GLOBAL_STEP: 4050[0m
|
| 753 |
+
| > loss_text_ce: 0.0510944202542305 (0.09017181752640514)
|
| 754 |
+
| > loss_mel_ce: 3.851416826248169 (3.6895573917436013)
|
| 755 |
+
| > loss: 0.9756278395652771 (0.9449323024131632)
|
| 756 |
+
| > current_lr: 5e-06
|
| 757 |
+
| > step_time: 0.3061 (0.33426993028617197)
|
| 758 |
+
| > loader_time: 0.0091 (0.01149895055794422)
|
| 759 |
+
|
| 760 |
+
|
| 761 |
+
[1m --> TIME: 2025-06-02 13:02:41 -- STEP: 4100/5792 -- GLOBAL_STEP: 4100[0m
|
| 762 |
+
| > loss_text_ce: 0.059732966125011444 (0.0897003380263723)
|
| 763 |
+
| > loss_mel_ce: 4.043025016784668 (3.6875316979245443)
|
| 764 |
+
| > loss: 1.0256894826889038 (0.9443080091040307)
|
| 765 |
+
| > current_lr: 5e-06
|
| 766 |
+
| > step_time: 0.3286 (0.33428408331987336)
|
| 767 |
+
| > loader_time: 0.0132 (0.011492944345241642)
|
| 768 |
+
|
| 769 |
+
|
| 770 |
+
[1m --> TIME: 2025-06-02 13:03:11 -- STEP: 4150/5792 -- GLOBAL_STEP: 4150[0m
|
| 771 |
+
| > loss_text_ce: 0.046467892825603485 (0.08922771856817126)
|
| 772 |
+
| > loss_mel_ce: 3.8112823963165283 (3.6847478388590984)
|
| 773 |
+
| > loss: 0.9644375443458557 (0.9434938894840605)
|
| 774 |
+
| > current_lr: 5e-06
|
| 775 |
+
| > step_time: 0.3452 (0.3344374258546944)
|
| 776 |
+
| > loader_time: 0.0125 (0.011488211528364437)
|
| 777 |
+
|
| 778 |
+
|
| 779 |
+
[1m --> TIME: 2025-06-02 13:03:41 -- STEP: 4200/5792 -- GLOBAL_STEP: 4200[0m
|
| 780 |
+
| > loss_text_ce: 0.05000675097107887 (0.08878737421174125)
|
| 781 |
+
| > loss_mel_ce: 3.3880319595336914 (3.6820667290119897)
|
| 782 |
+
| > loss: 0.8595097064971924 (0.9427135259338786)
|
| 783 |
+
| > current_lr: 5e-06
|
| 784 |
+
| > step_time: 0.2624 (0.33464160896482914)
|
| 785 |
+
| > loader_time: 0.0116 (0.011479487986791706)
|
| 786 |
+
|
| 787 |
+
|
| 788 |
+
[1m --> TIME: 2025-06-02 13:04:10 -- STEP: 4250/5792 -- GLOBAL_STEP: 4250[0m
|
| 789 |
+
| > loss_text_ce: 0.0450843945145607 (0.08833793204146267)
|
| 790 |
+
| > loss_mel_ce: 3.3565142154693604 (3.6793924145418058)
|
| 791 |
+
| > loss: 0.8503996729850769 (0.9419325868522416)
|
| 792 |
+
| > current_lr: 5e-06
|
| 793 |
+
| > step_time: 0.5804 (0.3347087397855871)
|
| 794 |
+
| > loader_time: 0.0107 (0.011472317415125237)
|
| 795 |
+
|
| 796 |
+
|
| 797 |
+
[1m --> TIME: 2025-06-02 13:04:41 -- STEP: 4300/5792 -- GLOBAL_STEP: 4300[0m
|
| 798 |
+
| > loss_text_ce: 0.04820011556148529 (0.08789586786960442)
|
| 799 |
+
| > loss_mel_ce: 3.48346209526062 (3.677155670066213)
|
| 800 |
+
| > loss: 0.8829155564308167 (0.9412628846806143)
|
| 801 |
+
| > current_lr: 5e-06
|
| 802 |
+
| > step_time: 0.2393 (0.33497945025909787)
|
| 803 |
+
| > loader_time: 0.0126 (0.011467096805572514)
|
| 804 |
+
|
| 805 |
+
|
| 806 |
+
[1m --> TIME: 2025-06-02 13:05:08 -- STEP: 4350/5792 -- GLOBAL_STEP: 4350[0m
|
| 807 |
+
| > loss_text_ce: 0.05529556795954704 (0.08746406216474098)
|
| 808 |
+
| > loss_mel_ce: 3.5293896198272705 (3.674542185136642)
|
| 809 |
+
| > loss: 0.8961712718009949 (0.9405015619869881)
|
| 810 |
+
| > current_lr: 5e-06
|
| 811 |
+
| > step_time: 0.3449 (0.3346871621033242)
|
| 812 |
+
| > loader_time: 0.0123 (0.011460946072107077)
|
| 813 |
+
|
| 814 |
+
|
| 815 |
+
[1m --> TIME: 2025-06-02 13:05:40 -- STEP: 4400/5792 -- GLOBAL_STEP: 4400[0m
|
| 816 |
+
| > loss_text_ce: 0.0525493361055851 (0.08705084911886261)
|
| 817 |
+
| > loss_mel_ce: 3.2084221839904785 (3.6720786437663175)
|
| 818 |
+
| > loss: 0.8152428865432739 (0.9397823734175065)
|
| 819 |
+
| > current_lr: 5e-06
|
| 820 |
+
| > step_time: 0.2394 (0.3350362862240186)
|
| 821 |
+
| > loader_time: 0.0123 (0.011455193866382943)
|
| 822 |
+
|
| 823 |
+
|
| 824 |
+
[1m --> TIME: 2025-06-02 13:06:08 -- STEP: 4450/5792 -- GLOBAL_STEP: 4450[0m
|
| 825 |
+
| > loss_text_ce: 0.048889096826314926 (0.0866304319134254)
|
| 826 |
+
| > loss_mel_ce: 3.5298917293548584 (3.670149624642362)
|
| 827 |
+
| > loss: 0.8946952223777771 (0.9391950143455109)
|
| 828 |
+
| > current_lr: 5e-06
|
| 829 |
+
| > step_time: 0.5265 (0.33503451711676124)
|
| 830 |
+
| > loader_time: 0.0111 (0.011446116265286221)
|
| 831 |
+
|
| 832 |
+
|
| 833 |
+
[1m --> TIME: 2025-06-02 13:06:37 -- STEP: 4500/5792 -- GLOBAL_STEP: 4500[0m
|
| 834 |
+
| > loss_text_ce: 0.05733109265565872 (0.08622805037432246)
|
| 835 |
+
| > loss_mel_ce: 3.3458251953125 (3.666845300144621)
|
| 836 |
+
| > loss: 0.8507890701293945 (0.9382683378458013)
|
| 837 |
+
| > current_lr: 5e-06
|
| 838 |
+
| > step_time: 0.2417 (0.33490972476535413)
|
| 839 |
+
| > loader_time: 0.0092 (0.011438722981346978)
|
| 840 |
+
|
| 841 |
+
|
| 842 |
+
[1m --> TIME: 2025-06-02 13:07:05 -- STEP: 4550/5792 -- GLOBAL_STEP: 4550[0m
|
| 843 |
+
| > loss_text_ce: 0.042130474001169205 (0.08583367971437321)
|
| 844 |
+
| > loss_mel_ce: 3.6195030212402344 (3.664244420135416)
|
| 845 |
+
| > loss: 0.9154083728790283 (0.9375195251716352)
|
| 846 |
+
| > current_lr: 5e-06
|
| 847 |
+
| > step_time: 0.492 (0.33481265240973224)
|
| 848 |
+
| > loader_time: 0.0104 (0.011426460035554658)
|
| 849 |
+
|
| 850 |
+
|
| 851 |
+
[1m --> TIME: 2025-06-02 13:07:33 -- STEP: 4600/5792 -- GLOBAL_STEP: 4600[0m
|
| 852 |
+
| > loss_text_ce: 0.0529852919280529 (0.08543671885748273)
|
| 853 |
+
| > loss_mel_ce: 3.0790507793426514 (3.6618555622515485)
|
| 854 |
+
| > loss: 0.7830089926719666 (0.9368230704898409)
|
| 855 |
+
| > current_lr: 5e-06
|
| 856 |
+
| > step_time: 0.2388 (0.3346719070102861)
|
| 857 |
+
| > loader_time: 0.0106 (0.011421545173810875)
|
| 858 |
+
|
| 859 |
+
|
| 860 |
+
[1m --> TIME: 2025-06-02 13:08:05 -- STEP: 4650/5792 -- GLOBAL_STEP: 4650[0m
|
| 861 |
+
| > loss_text_ce: 0.047616079449653625 (0.08505546001497138)
|
| 862 |
+
| > loss_mel_ce: 3.2773044109344482 (3.659559770348253)
|
| 863 |
+
| > loss: 0.831230103969574 (0.9361538077938931)
|
| 864 |
+
| > current_lr: 5e-06
|
| 865 |
+
| > step_time: 0.406 (0.3350888223032801)
|
| 866 |
+
| > loader_time: 0.0104 (0.011418454980337488)
|
| 867 |
+
|
| 868 |
+
|
| 869 |
+
[1m --> TIME: 2025-06-02 13:08:33 -- STEP: 4700/5792 -- GLOBAL_STEP: 4700[0m
|
| 870 |
+
| > loss_text_ce: 0.04613576456904411 (0.08467220809072895)
|
| 871 |
+
| > loss_mel_ce: 3.406162977218628 (3.657190910603139)
|
| 872 |
+
| > loss: 0.8630746603012085 (0.9354657798751862)
|
| 873 |
+
| > current_lr: 5e-06
|
| 874 |
+
| > step_time: 0.2167 (0.33492715972535186)
|
| 875 |
+
| > loader_time: 0.0128 (0.011416076599283414)
|
| 876 |
+
|
| 877 |
+
|
| 878 |
+
[1m --> TIME: 2025-06-02 13:09:03 -- STEP: 4750/5792 -- GLOBAL_STEP: 4750[0m
|
| 879 |
+
| > loss_text_ce: 0.052280884236097336 (0.08429848386896287)
|
| 880 |
+
| > loss_mel_ce: 3.40238356590271 (3.655417844571567)
|
| 881 |
+
| > loss: 0.8636661171913147 (0.9349290823058068)
|
| 882 |
+
| > current_lr: 5e-06
|
| 883 |
+
| > step_time: 0.2554 (0.3351128557104816)
|
| 884 |
+
| > loader_time: 0.0099 (0.011411911663256183)
|
| 885 |
+
|
| 886 |
+
|
| 887 |
+
[1m --> TIME: 2025-06-02 13:09:32 -- STEP: 4800/5792 -- GLOBAL_STEP: 4800[0m
|
| 888 |
+
| > loss_text_ce: 0.042021121829748154 (0.08392824066414815)
|
| 889 |
+
| > loss_mel_ce: 3.1501052379608154 (3.653419889857373)
|
| 890 |
+
| > loss: 0.7980315685272217 (0.9343370328471056)
|
| 891 |
+
| > current_lr: 5e-06
|
| 892 |
+
| > step_time: 0.2071 (0.33511566052834235)
|
| 893 |
+
| > loader_time: 0.0091 (0.011409775366385762)
|
| 894 |
+
|
| 895 |
+
|
| 896 |
+
[1m --> TIME: 2025-06-02 13:10:01 -- STEP: 4850/5792 -- GLOBAL_STEP: 4850[0m
|
| 897 |
+
| > loss_text_ce: 0.05289650708436966 (0.08356028458122745)
|
| 898 |
+
| > loss_mel_ce: 3.2315640449523926 (3.6508835728143922)
|
| 899 |
+
| > loss: 0.8211151361465454 (0.9336109645710768)
|
| 900 |
+
| > current_lr: 5e-06
|
| 901 |
+
| > step_time: 0.335 (0.3350185093437278)
|
| 902 |
+
| > loader_time: 0.0092 (0.011409686619473473)
|
| 903 |
+
|
| 904 |
+
|
| 905 |
+
[1m --> TIME: 2025-06-02 13:10:29 -- STEP: 4900/5792 -- GLOBAL_STEP: 4900[0m
|
| 906 |
+
| > loss_text_ce: 0.04954209551215172 (0.08320543044683895)
|
| 907 |
+
| > loss_mel_ce: 3.3401761054992676 (3.6483470648648804)
|
| 908 |
+
| > loss: 0.8474295735359192 (0.9328881240499254)
|
| 909 |
+
| > current_lr: 5e-06
|
| 910 |
+
| > step_time: 0.245 (0.3349105890916322)
|
| 911 |
+
| > loader_time: 0.01 (0.011410678065552989)
|
| 912 |
+
|
| 913 |
+
|
| 914 |
+
[1m --> TIME: 2025-06-02 13:11:00 -- STEP: 4950/5792 -- GLOBAL_STEP: 4950[0m
|
| 915 |
+
| > loss_text_ce: 0.04523472115397453 (0.08286285736118316)
|
| 916 |
+
| > loss_mel_ce: 3.421116352081299 (3.646339290888624)
|
| 917 |
+
| > loss: 0.86658775806427 (0.9323005372586868)
|
| 918 |
+
| > current_lr: 5e-06
|
| 919 |
+
| > step_time: 0.267 (0.33518396363113906)
|
| 920 |
+
| > loader_time: 0.0095 (0.011406980524159427)
|
| 921 |
+
|
| 922 |
+
|
| 923 |
+
[1m --> TIME: 2025-06-02 13:11:30 -- STEP: 5000/5792 -- GLOBAL_STEP: 5000[0m
|
| 924 |
+
| > loss_text_ce: 0.051856279373168945 (0.08252262841463104)
|
| 925 |
+
| > loss_mel_ce: 3.7696542739868164 (3.64465701274872)
|
| 926 |
+
| > loss: 0.9553776383399963 (0.931794910478591)
|
| 927 |
+
| > current_lr: 5e-06
|
| 928 |
+
| > step_time: 0.4771 (0.3353376725673679)
|
| 929 |
+
| > loader_time: 0.0107 (0.011403836870193468)
|
| 930 |
+
|
| 931 |
+
|
| 932 |
+
[1m --> TIME: 2025-06-02 13:12:00 -- STEP: 5050/5792 -- GLOBAL_STEP: 5050[0m
|
| 933 |
+
| > loss_text_ce: 0.048051297664642334 (0.08218267903026988)
|
| 934 |
+
| > loss_mel_ce: 3.2416117191314697 (3.6419559119007388)
|
| 935 |
+
| > loss: 0.8224157691001892 (0.9310346478990981)
|
| 936 |
+
| > current_lr: 5e-06
|
| 937 |
+
| > step_time: 0.2594 (0.33541135504694297)
|
| 938 |
+
| > loader_time: 0.0094 (0.011395886864992641)
|
| 939 |
+
|
| 940 |
+
|
| 941 |
+
[1m --> TIME: 2025-06-02 13:12:30 -- STEP: 5100/5792 -- GLOBAL_STEP: 5100[0m
|
| 942 |
+
| > loss_text_ce: 0.04729149118065834 (0.0818471597763256)
|
| 943 |
+
| > loss_mel_ce: 3.463068962097168 (3.6400194923083)
|
| 944 |
+
| > loss: 0.8775901198387146 (0.9304666631712624)
|
| 945 |
+
| > current_lr: 5e-06
|
| 946 |
+
| > step_time: 0.2577 (0.3356010669352963)
|
| 947 |
+
| > loader_time: 0.0093 (0.011388069274378751)
|
| 948 |
+
|
| 949 |
+
|
| 950 |
+
[1m --> TIME: 2025-06-02 13:12:59 -- STEP: 5150/5792 -- GLOBAL_STEP: 5150[0m
|
| 951 |
+
| > loss_text_ce: 0.049754828214645386 (0.08151657056822972)
|
| 952 |
+
| > loss_mel_ce: 3.3020455837249756 (3.638027641333424)
|
| 953 |
+
| > loss: 0.8379501104354858 (0.9298860531408801)
|
| 954 |
+
| > current_lr: 5e-06
|
| 955 |
+
| > step_time: 0.1726 (0.33566474298829063)
|
| 956 |
+
| > loader_time: 0.0116 (0.011384222206560154)
|
| 957 |
+
|
| 958 |
+
|
| 959 |
+
[1m --> TIME: 2025-06-02 13:13:28 -- STEP: 5200/5792 -- GLOBAL_STEP: 5200[0m
|
| 960 |
+
| > loss_text_ce: 0.05030031129717827 (0.08118891543852057)
|
| 961 |
+
| > loss_mel_ce: 3.18542742729187 (3.635307474182204)
|
| 962 |
+
| > loss: 0.8089319467544556 (0.9291240975948472)
|
| 963 |
+
| > current_lr: 5e-06
|
| 964 |
+
| > step_time: 0.3433 (0.3355758514771101)
|
| 965 |
+
| > loader_time: 0.0094 (0.011372857231360218)
|
| 966 |
+
|
| 967 |
+
|
| 968 |
+
[1m --> TIME: 2025-06-02 13:13:59 -- STEP: 5250/5792 -- GLOBAL_STEP: 5250[0m
|
| 969 |
+
| > loss_text_ce: 0.05008910596370697 (0.08087093743681945)
|
| 970 |
+
| > loss_mel_ce: 3.088693857192993 (3.6334633124669407)
|
| 971 |
+
| > loss: 0.7846957445144653 (0.9285835626579457)
|
| 972 |
+
| > current_lr: 5e-06
|
| 973 |
+
| > step_time: 0.3717 (0.33582283274332736)
|
| 974 |
+
| > loader_time: 0.0095 (0.011365627288818372)
|
| 975 |
+
|
| 976 |
+
|
| 977 |
+
[1m --> TIME: 2025-06-02 13:14:29 -- STEP: 5300/5792 -- GLOBAL_STEP: 5300[0m
|
| 978 |
+
| > loss_text_ce: 0.0471770241856575 (0.08056517972277057)
|
| 979 |
+
| > loss_mel_ce: 3.359286069869995 (3.6318132130155036)
|
| 980 |
+
| > loss: 0.8516157865524292 (0.9280945984035159)
|
| 981 |
+
| > current_lr: 5e-06
|
| 982 |
+
| > step_time: 0.3564 (0.33594532296342744)
|
| 983 |
+
| > loader_time: 0.0105 (0.011357401227051363)
|
| 984 |
+
|
| 985 |
+
|
| 986 |
+
[1m --> TIME: 2025-06-02 13:14:58 -- STEP: 5350/5792 -- GLOBAL_STEP: 5350[0m
|
| 987 |
+
| > loss_text_ce: 0.047488246113061905 (0.08026272471123812)
|
| 988 |
+
| > loss_mel_ce: 3.0119431018829346 (3.6297121964659658)
|
| 989 |
+
| > loss: 0.764857828617096 (0.9274937304937942)
|
| 990 |
+
| > current_lr: 5e-06
|
| 991 |
+
| > step_time: 0.2594 (0.33586654542762623)
|
| 992 |
+
| > loader_time: 0.0141 (0.011354769679987546)
|
| 993 |
+
|
| 994 |
+
|
| 995 |
+
[1m --> TIME: 2025-06-02 13:15:27 -- STEP: 5400/5792 -- GLOBAL_STEP: 5400[0m
|
| 996 |
+
| > loss_text_ce: 0.047491975128650665 (0.079949514229127)
|
| 997 |
+
| > loss_mel_ce: 3.455355405807495 (3.627370389611634)
|
| 998 |
+
| > loss: 0.8757118582725525 (0.9268299761966414)
|
| 999 |
+
| > current_lr: 5e-06
|
| 1000 |
+
| > step_time: 0.2422 (0.3358160152258706)
|
| 1001 |
+
| > loader_time: 0.0092 (0.011353673890784925)
|
| 1002 |
+
|
| 1003 |
+
|
| 1004 |
+
[1m --> TIME: 2025-06-02 13:15:54 -- STEP: 5450/5792 -- GLOBAL_STEP: 5450[0m
|
| 1005 |
+
| > loss_text_ce: 0.046924345195293427 (0.07964956542562501)
|
| 1006 |
+
| > loss_mel_ce: 3.266174554824829 (3.625191910944949)
|
| 1007 |
+
| > loss: 0.8282747268676758 (0.9262103693200899)
|
| 1008 |
+
| > current_lr: 5e-06
|
| 1009 |
+
| > step_time: 0.4257 (0.3356467504238875)
|
| 1010 |
+
| > loader_time: 0.0097 (0.011349368707849337)
|
| 1011 |
+
|
| 1012 |
+
|
| 1013 |
+
[1m --> TIME: 2025-06-02 13:16:25 -- STEP: 5500/5792 -- GLOBAL_STEP: 5500[0m
|
| 1014 |
+
| > loss_text_ce: 0.039876487106084824 (0.07935999841446274)
|
| 1015 |
+
| > loss_mel_ce: 3.372520685195923 (3.62339421679757)
|
| 1016 |
+
| > loss: 0.8530992865562439 (0.9256885540377001)
|
| 1017 |
+
| > current_lr: 5e-06
|
| 1018 |
+
| > step_time: 0.2624 (0.3357687810984538)
|
| 1019 |
+
| > loader_time: 0.0098 (0.011344045855782236)
|
| 1020 |
+
|
| 1021 |
+
|
| 1022 |
+
[1m --> TIME: 2025-06-02 13:16:53 -- STEP: 5550/5792 -- GLOBAL_STEP: 5550[0m
|
| 1023 |
+
| > loss_text_ce: 0.05153404176235199 (0.07907796377802799)
|
| 1024 |
+
| > loss_mel_ce: 3.4474453926086426 (3.6210933367411307)
|
| 1025 |
+
| > loss: 0.8747448325157166 (0.9250428253680728)
|
| 1026 |
+
| > current_lr: 5e-06
|
| 1027 |
+
| > step_time: 0.2201 (0.3357482298000451)
|
| 1028 |
+
| > loader_time: 0.0085 (0.011339900300309447)
|
| 1029 |
+
|
| 1030 |
+
|
| 1031 |
+
[1m --> TIME: 2025-06-02 13:17:23 -- STEP: 5600/5792 -- GLOBAL_STEP: 5600[0m
|
| 1032 |
+
| > loss_text_ce: 0.05078967288136482 (0.07880074134562218)
|
| 1033 |
+
| > loss_mel_ce: 3.4180071353912354 (3.6193902756060887)
|
| 1034 |
+
| > loss: 0.867199182510376 (0.9245477544835627)
|
| 1035 |
+
| > current_lr: 5e-06
|
| 1036 |
+
| > step_time: 0.2526 (0.33589044515575817)
|
| 1037 |
+
| > loader_time: 0.0093 (0.011337815778596043)
|
| 1038 |
+
|
| 1039 |
+
|
| 1040 |
+
[1m --> TIME: 2025-06-02 13:17:51 -- STEP: 5650/5792 -- GLOBAL_STEP: 5650[0m
|
| 1041 |
+
| > loss_text_ce: 0.053694888949394226 (0.07851750885227134)
|
| 1042 |
+
| > loss_mel_ce: 3.102844476699829 (3.6172207505724083)
|
| 1043 |
+
| > loss: 0.7891348600387573 (0.9239345651073785)
|
| 1044 |
+
| > current_lr: 5e-06
|
| 1045 |
+
| > step_time: 0.3004 (0.33572653774666705)
|
| 1046 |
+
| > loader_time: 0.0096 (0.011334677755305183)
|
| 1047 |
+
|
| 1048 |
+
|
| 1049 |
+
[1m --> TIME: 2025-06-02 13:18:20 -- STEP: 5700/5792 -- GLOBAL_STEP: 5700[0m
|
| 1050 |
+
| > loss_text_ce: 0.0459514856338501 (0.07823847596974794)
|
| 1051 |
+
| > loss_mel_ce: 3.5479061603546143 (3.615112853050233)
|
| 1052 |
+
| > loss: 0.8984644412994385 (0.9233378325533441)
|
| 1053 |
+
| > current_lr: 5e-06
|
| 1054 |
+
| > step_time: 0.3254 (0.3357153525687113)
|
| 1055 |
+
| > loader_time: 0.0106 (0.011330316192225392)
|
| 1056 |
+
|
| 1057 |
+
|
| 1058 |
+
[1m --> TIME: 2025-06-02 13:18:49 -- STEP: 5750/5792 -- GLOBAL_STEP: 5750[0m
|
| 1059 |
+
| > loss_text_ce: 0.04507908225059509 (0.07796999881967298)
|
| 1060 |
+
| > loss_mel_ce: 3.325566530227661 (3.613506336834121)
|
| 1061 |
+
| > loss: 0.8426613807678223 (0.9228690842441882)
|
| 1062 |
+
| > current_lr: 5e-06
|
| 1063 |
+
| > step_time: 0.2647 (0.3357090207804818)
|
| 1064 |
+
| > loader_time: 0.0132 (0.011328426651332674)
|
| 1065 |
+
|
| 1066 |
+
|
| 1067 |
+
[1m > EVALUATION [0m
|
| 1068 |
+
|
| 1069 |
+
|
| 1070 |
+
[1m--> EVAL PERFORMANCE[0m
|
| 1071 |
+
| > avg_loader_time: 0.2016423004495222 [0m(+0)
|
| 1072 |
+
| > avg_loss_text_ce: 0.04477991773036578 [0m(+0)
|
| 1073 |
+
| > avg_loss_mel_ce: 3.199991709768435 [0m(+0)
|
| 1074 |
+
| > avg_loss: 3.2447716307505363 [0m(+0)
|
| 1075 |
+
|
| 1076 |
+
|
| 1077 |
+
[4m[1m > EPOCH: 1/2[0m
|
| 1078 |
+
--> xTTS/checkpoints/GPT_XTTS_FT-June-02-2025_12+22PM-0000000
|
| 1079 |
+
|
| 1080 |
+
[1m > TRAINING (2025-06-02 13:26:46) [0m
|
| 1081 |
+
|
| 1082 |
+
[1m --> TIME: 2025-06-02 13:26:52 -- STEP: 8/5792 -- GLOBAL_STEP: 5800[0m
|
| 1083 |
+
| > loss_text_ce: 0.04962650686502457 (0.04589929943904281)
|
| 1084 |
+
| > loss_mel_ce: 3.476445436477661 (3.366139441728592)
|
| 1085 |
+
| > loss: 0.8815180063247681 (0.8530096858739853)
|
| 1086 |
+
| > current_lr: 5e-06
|
| 1087 |
+
| > step_time: 0.2652 (0.30354249477386475)
|
| 1088 |
+
| > loader_time: 0.0149 (0.011465787887573242)
|
| 1089 |
+
|
| 1090 |
+
|
| 1091 |
+
[1m --> TIME: 2025-06-02 13:27:22 -- STEP: 58/5792 -- GLOBAL_STEP: 5850[0m
|
| 1092 |
+
| > loss_text_ce: 0.043131984770298004 (0.046852813262877796)
|
| 1093 |
+
| > loss_mel_ce: 3.372774600982666 (3.4303042765321403)
|
| 1094 |
+
| > loss: 0.8539766669273376 (0.8692892707627395)
|
| 1095 |
+
| > current_lr: 5e-06
|
| 1096 |
+
| > step_time: 0.196 (0.3431942175174581)
|
| 1097 |
+
| > loader_time: 0.0127 (0.01153468263560328)
|
| 1098 |
+
|
| 1099 |
+
|
| 1100 |
+
[1m --> TIME: 2025-06-02 13:27:52 -- STEP: 108/5792 -- GLOBAL_STEP: 5900[0m
|
| 1101 |
+
| > loss_text_ce: 0.04623452201485634 (0.04628198635247019)
|
| 1102 |
+
| > loss_mel_ce: 2.833378553390503 (3.3930641324431807)
|
| 1103 |
+
| > loss: 0.719903290271759 (0.8598365292504981)
|
| 1104 |
+
| > current_lr: 5e-06
|
| 1105 |
+
| > step_time: 0.1939 (0.3511013057496811)
|
| 1106 |
+
| > loader_time: 0.0098 (0.011563482107939544)
|
| 1107 |
+
|
| 1108 |
+
|
| 1109 |
+
[1m --> TIME: 2025-06-02 13:28:22 -- STEP: 158/5792 -- GLOBAL_STEP: 5950[0m
|
| 1110 |
+
| > loss_text_ce: 0.04444996267557144 (0.04651183649143084)
|
| 1111 |
+
| > loss_mel_ce: 3.0688369274139404 (3.396478888354724)
|
| 1112 |
+
| > loss: 0.7783217430114746 (0.8607476810111275)
|
| 1113 |
+
| > current_lr: 5e-06
|
| 1114 |
+
| > step_time: 0.3297 (0.34766890278345425)
|
| 1115 |
+
| > loader_time: 0.0103 (0.01131993305834034)
|
| 1116 |
+
|
| 1117 |
+
|
| 1118 |
+
[1m --> TIME: 2025-06-02 13:28:52 -- STEP: 208/5792 -- GLOBAL_STEP: 6000[0m
|
| 1119 |
+
| > loss_text_ce: 0.05267365276813507 (0.04660090401123921)
|
| 1120 |
+
| > loss_mel_ce: 3.509939670562744 (3.395385509500137)
|
| 1121 |
+
| > loss: 0.8906533122062683 (0.8604966015196763)
|
| 1122 |
+
| > current_lr: 5e-06
|
| 1123 |
+
| > step_time: 0.2086 (0.34758124213952296)
|
| 1124 |
+
| > loader_time: 0.0099 (0.01136978314473079)
|
| 1125 |
+
|
| 1126 |
+
|
| 1127 |
+
[1m --> TIME: 2025-06-02 13:29:21 -- STEP: 258/5792 -- GLOBAL_STEP: 6050[0m
|
| 1128 |
+
| > loss_text_ce: 0.042573895305395126 (0.04650543012129243)
|
| 1129 |
+
| > loss_mel_ce: 3.198530673980713 (3.406408476275067)
|
| 1130 |
+
| > loss: 0.8102761507034302 (0.8632284749385922)
|
| 1131 |
+
| > current_lr: 5e-06
|
| 1132 |
+
| > step_time: 0.2678 (0.34636553313380997)
|
| 1133 |
+
| > loader_time: 0.0093 (0.011229625968045968)
|
| 1134 |
+
|
| 1135 |
+
|
| 1136 |
+
[1m --> TIME: 2025-06-02 13:29:52 -- STEP: 308/5792 -- GLOBAL_STEP: 6100[0m
|
| 1137 |
+
| > loss_text_ce: 0.04070745408535004 (0.04644479884391093)
|
| 1138 |
+
| > loss_mel_ce: 3.161195993423462 (3.409244562124277)
|
| 1139 |
+
| > loss: 0.8004758358001709 (0.8639223389811327)
|
| 1140 |
+
| > current_lr: 5e-06
|
| 1141 |
+
| > step_time: 0.2063 (0.34847222055707666)
|
| 1142 |
+
| > loader_time: 0.0096 (0.0111765776361738)
|
| 1143 |
+
|
| 1144 |
+
|
| 1145 |
+
[1m --> TIME: 2025-06-02 13:30:21 -- STEP: 358/5792 -- GLOBAL_STEP: 6150[0m
|
| 1146 |
+
| > loss_text_ce: 0.04625903442502022 (0.04643882370902503)
|
| 1147 |
+
| > loss_mel_ce: 3.097196578979492 (3.4112956583832896)
|
| 1148 |
+
| > loss: 0.7858638763427734 (0.8644336197296332)
|
| 1149 |
+
| > current_lr: 5e-06
|
| 1150 |
+
| > step_time: 0.2922 (0.3459169138743227)
|
| 1151 |
+
| > loader_time: 0.01 (0.011164890321273387)
|
| 1152 |
+
|
| 1153 |
+
|
| 1154 |
+
[1m --> TIME: 2025-06-02 13:30:50 -- STEP: 408/5792 -- GLOBAL_STEP: 6200[0m
|
| 1155 |
+
| > loss_text_ce: 0.048108890652656555 (0.04640357185374285)
|
| 1156 |
+
| > loss_mel_ce: 3.2346646785736084 (3.406046163217694)
|
| 1157 |
+
| > loss: 0.8206933736801147 (0.8631124335761161)
|
| 1158 |
+
| > current_lr: 5e-06
|
| 1159 |
+
| > step_time: 0.3294 (0.3447357386934993)
|
| 1160 |
+
| > loader_time: 0.0094 (0.01113292983933992)
|
| 1161 |
+
|
| 1162 |
+
|
| 1163 |
+
[1m --> TIME: 2025-06-02 13:31:20 -- STEP: 458/5792 -- GLOBAL_STEP: 6250[0m
|
| 1164 |
+
| > loss_text_ce: 0.048285942524671555 (0.046448883360585246)
|
| 1165 |
+
| > loss_mel_ce: 3.597545862197876 (3.4110943599559334)
|
| 1166 |
+
| > loss: 0.9114579558372498 (0.8643858104814085)
|
| 1167 |
+
| > current_lr: 5e-06
|
| 1168 |
+
| > step_time: 0.5765 (0.34515097307846554)
|
| 1169 |
+
| > loader_time: 0.0139 (0.011161539752410497)
|
| 1170 |
+
|
| 1171 |
+
|
| 1172 |
+
[1m --> TIME: 2025-06-02 13:31:50 -- STEP: 508/5792 -- GLOBAL_STEP: 6300[0m
|
| 1173 |
+
| > loss_text_ce: 0.05213044583797455 (0.04643937362372638)
|
| 1174 |
+
| > loss_mel_ce: 3.4266021251678467 (3.4056746645236577)
|
| 1175 |
+
| > loss: 0.8696831464767456 (0.8630285095276793)
|
| 1176 |
+
| > current_lr: 5e-06
|
| 1177 |
+
| > step_time: 0.3125 (0.3454259840522227)
|
| 1178 |
+
| > loader_time: 0.0139 (0.011176920781924038)
|
| 1179 |
+
|
| 1180 |
+
|
| 1181 |
+
[1m --> TIME: 2025-06-02 13:32:18 -- STEP: 558/5792 -- GLOBAL_STEP: 6350[0m
|
| 1182 |
+
| > loss_text_ce: 0.04106414318084717 (0.046370733649499944)
|
| 1183 |
+
| > loss_mel_ce: 3.4639627933502197 (3.397692242830885)
|
| 1184 |
+
| > loss: 0.8762567043304443 (0.8610157443203804)
|
| 1185 |
+
| > current_lr: 5e-06
|
| 1186 |
+
| > step_time: 0.1901 (0.3433396858981008)
|
| 1187 |
+
| > loader_time: 0.0088 (0.011147387138831576)
|
| 1188 |
+
|
| 1189 |
+
|
| 1190 |
+
[1m --> TIME: 2025-06-02 13:32:45 -- STEP: 608/5792 -- GLOBAL_STEP: 6400[0m
|
| 1191 |
+
| > loss_text_ce: 0.04276609793305397 (0.04636223264969883)
|
| 1192 |
+
| > loss_mel_ce: 3.2456772327423096 (3.394306463630576)
|
| 1193 |
+
| > loss: 0.8221108317375183 (0.8601671744530135)
|
| 1194 |
+
| > current_lr: 5e-06
|
| 1195 |
+
| > step_time: 0.2159 (0.3403207224450615)
|
| 1196 |
+
| > loader_time: 0.0096 (0.011096751611483725)
|
| 1197 |
+
|
| 1198 |
+
|
| 1199 |
+
[1m --> TIME: 2025-06-02 13:33:15 -- STEP: 658/5792 -- GLOBAL_STEP: 6450[0m
|
| 1200 |
+
| > loss_text_ce: 0.04221029207110405 (0.046298709672544465)
|
| 1201 |
+
| > loss_mel_ce: 3.376603364944458 (3.3917605297181384)
|
| 1202 |
+
| > loss: 0.854703426361084 (0.8595148100316702)
|
| 1203 |
+
| > current_lr: 5e-06
|
| 1204 |
+
| > step_time: 0.2959 (0.3403355002765601)
|
| 1205 |
+
| > loader_time: 0.0123 (0.01107824609634724)
|
| 1206 |
+
|
| 1207 |
+
|
| 1208 |
+
[1m --> TIME: 2025-06-02 13:33:45 -- STEP: 708/5792 -- GLOBAL_STEP: 6500[0m
|
| 1209 |
+
| > loss_text_ce: 0.0455298125743866 (0.04620274585161333)
|
| 1210 |
+
| > loss_mel_ce: 3.368074655532837 (3.3912849278099793)
|
| 1211 |
+
| > loss: 0.8534011244773865 (0.8593719188271266)
|
| 1212 |
+
| > current_lr: 5e-06
|
| 1213 |
+
| > step_time: 0.2983 (0.34120621048124494)
|
| 1214 |
+
| > loader_time: 0.0097 (0.011063059507790253)
|
| 1215 |
+
|
| 1216 |
+
|
| 1217 |
+
[1m --> TIME: 2025-06-02 13:34:16 -- STEP: 758/5792 -- GLOBAL_STEP: 6550[0m
|
| 1218 |
+
| > loss_text_ce: 0.04287204146385193 (0.046180617649078645)
|
| 1219 |
+
| > loss_mel_ce: 3.8676905632019043 (3.3931626326805056)
|
| 1220 |
+
| > loss: 0.9776406288146973 (0.8598358130234839)
|
| 1221 |
+
| > current_lr: 5e-06
|
| 1222 |
+
| > step_time: 0.4669 (0.3418154071692112)
|
| 1223 |
+
| > loader_time: 0.0107 (0.011052280743078057)
|
| 1224 |
+
|
| 1225 |
+
|
| 1226 |
+
[1m --> TIME: 2025-06-02 13:34:44 -- STEP: 808/5792 -- GLOBAL_STEP: 6600[0m
|
| 1227 |
+
| > loss_text_ce: 0.05415653437376022 (0.046192656469411446)
|
| 1228 |
+
| > loss_mel_ce: 3.0190370082855225 (3.3884078290792976)
|
| 1229 |
+
| > loss: 0.7682983875274658 (0.8586501218747382)
|
| 1230 |
+
| > current_lr: 5e-06
|
| 1231 |
+
| > step_time: 0.2155 (0.3408614199350376)
|
| 1232 |
+
| > loader_time: 0.0131 (0.011081166196577612)
|
| 1233 |
+
|
| 1234 |
+
|
| 1235 |
+
[1m --> TIME: 2025-06-02 13:35:13 -- STEP: 858/5792 -- GLOBAL_STEP: 6650[0m
|
| 1236 |
+
| > loss_text_ce: 0.0525716096162796 (0.04613834280425154)
|
| 1237 |
+
| > loss_mel_ce: 3.157501459121704 (3.3880747826227218)
|
| 1238 |
+
| > loss: 0.8025182485580444 (0.8585532816834658)
|
| 1239 |
+
| > current_lr: 5e-06
|
| 1240 |
+
| > step_time: 0.3441 (0.34018004690850534)
|
| 1241 |
+
| > loader_time: 0.0116 (0.01113530639168265)
|
| 1242 |
+
|
| 1243 |
+
|
| 1244 |
+
[1m --> TIME: 2025-06-02 13:35:43 -- STEP: 908/5792 -- GLOBAL_STEP: 6700[0m
|
| 1245 |
+
| > loss_text_ce: 0.04388509690761566 (0.04611085562060055)
|
| 1246 |
+
| > loss_mel_ce: 3.5000290870666504 (3.382610232031818)
|
| 1247 |
+
| > loss: 0.8859785199165344 (0.8571802720361866)
|
| 1248 |
+
| > current_lr: 5e-06
|
| 1249 |
+
| > step_time: 0.2732 (0.3402811882254311)
|
| 1250 |
+
| > loader_time: 0.0098 (0.01113482561405534)
|
| 1251 |
+
|
| 1252 |
+
|
| 1253 |
+
[1m --> TIME: 2025-06-02 13:36:12 -- STEP: 958/5792 -- GLOBAL_STEP: 6750[0m
|
| 1254 |
+
| > loss_text_ce: 0.04200979322195053 (0.046016689794359696)
|
| 1255 |
+
| > loss_mel_ce: 3.2025482654571533 (3.386244950812147)
|
| 1256 |
+
| > loss: 0.8111395239830017 (0.858065410110796)
|
| 1257 |
+
| > current_lr: 5e-06
|
| 1258 |
+
| > step_time: 0.3037 (0.3404792111701408)
|
| 1259 |
+
| > loader_time: 0.013 (0.011161978409036457)
|
| 1260 |
+
|
| 1261 |
+
|
| 1262 |
+
[1m --> TIME: 2025-06-02 13:36:43 -- STEP: 1008/5792 -- GLOBAL_STEP: 6800[0m
|
| 1263 |
+
| > loss_text_ce: 0.04656248539686203 (0.045978479368227784)
|
| 1264 |
+
| > loss_mel_ce: 3.424415349960327 (3.3894127905368805)
|
| 1265 |
+
| > loss: 0.8677444458007812 (0.8588478174947555)
|
| 1266 |
+
| > current_lr: 5e-06
|
| 1267 |
+
| > step_time: 0.2899 (0.34167367078009103)
|
| 1268 |
+
| > loader_time: 0.0134 (0.011164178214375927)
|
| 1269 |
+
|
| 1270 |
+
|
| 1271 |
+
[1m --> TIME: 2025-06-02 13:37:12 -- STEP: 1058/5792 -- GLOBAL_STEP: 6850[0m
|
| 1272 |
+
| > loss_text_ce: 0.044847793877124786 (0.04588977389727736)
|
| 1273 |
+
| > loss_mel_ce: 3.753209352493286 (3.3859440919807593)
|
| 1274 |
+
| > loss: 0.9495142698287964 (0.8579584664342983)
|
| 1275 |
+
| > current_lr: 5e-06
|
| 1276 |
+
| > step_time: 0.4274 (0.3409366860056194)
|
| 1277 |
+
| > loader_time: 0.0106 (0.01115854092942293)
|
| 1278 |
+
|
| 1279 |
+
|
| 1280 |
+
[1m --> TIME: 2025-06-02 13:37:40 -- STEP: 1108/5792 -- GLOBAL_STEP: 6900[0m
|
| 1281 |
+
| > loss_text_ce: 0.04726967215538025 (0.04582478893443348)
|
| 1282 |
+
| > loss_mel_ce: 3.5899248123168945 (3.3846601099313816)
|
| 1283 |
+
| > loss: 0.9092985987663269 (0.8576212246172693)
|
| 1284 |
+
| > current_lr: 5e-06
|
| 1285 |
+
| > step_time: 0.2437 (0.34045616211873964)
|
| 1286 |
+
| > loader_time: 0.0129 (0.01116837792448188)
|
| 1287 |
+
|
| 1288 |
+
|
| 1289 |
+
[1m --> TIME: 2025-06-02 13:38:07 -- STEP: 1158/5792 -- GLOBAL_STEP: 6950[0m
|
| 1290 |
+
| > loss_text_ce: 0.04457278549671173 (0.04580180395864244)
|
| 1291 |
+
| > loss_mel_ce: 3.8512332439422607 (3.3820696817984452)
|
| 1292 |
+
| > loss: 0.973951518535614 (0.8569678713620632)
|
| 1293 |
+
| > current_lr: 5e-06
|
| 1294 |
+
| > step_time: 0.4823 (0.33921163193302456)
|
| 1295 |
+
| > loader_time: 0.0142 (0.011172776601063361)
|
| 1296 |
+
|
| 1297 |
+
|
| 1298 |
+
[1m --> TIME: 2025-06-02 13:38:35 -- STEP: 1208/5792 -- GLOBAL_STEP: 7000[0m
|
| 1299 |
+
| > loss_text_ce: 0.04959622770547867 (0.04575508566178527)
|
| 1300 |
+
| > loss_mel_ce: 3.364575147628784 (3.3837944628387104)
|
| 1301 |
+
| > loss: 0.8535428643226624 (0.8573873870518821)
|
| 1302 |
+
| > current_lr: 5e-06
|
| 1303 |
+
| > step_time: 0.2847 (0.3381390390017176)
|
| 1304 |
+
| > loader_time: 0.0111 (0.011142225849707395)
|
| 1305 |
+
|
| 1306 |
+
|
| 1307 |
+
[1m --> TIME: 2025-06-02 13:39:03 -- STEP: 1258/5792 -- GLOBAL_STEP: 7050[0m
|
| 1308 |
+
| > loss_text_ce: 0.04021007567644119 (0.04573283367309072)
|
| 1309 |
+
| > loss_mel_ce: 3.0035552978515625 (3.3824539863058805)
|
| 1310 |
+
| > loss: 0.7609413266181946 (0.857046704982157)
|
| 1311 |
+
| > current_lr: 5e-06
|
| 1312 |
+
| > step_time: 0.2549 (0.33753872296768467)
|
| 1313 |
+
| > loader_time: 0.0098 (0.011134225537553312)
|
| 1314 |
+
|
| 1315 |
+
|
| 1316 |
+
[1m --> TIME: 2025-06-02 13:39:33 -- STEP: 1308/5792 -- GLOBAL_STEP: 7100[0m
|
| 1317 |
+
| > loss_text_ce: 0.04839656129479408 (0.04570206533263035)
|
| 1318 |
+
| > loss_mel_ce: 3.5808684825897217 (3.3811058704641623)
|
| 1319 |
+
| > loss: 0.907316267490387 (0.8567019838623313)
|
| 1320 |
+
| > current_lr: 5e-06
|
| 1321 |
+
| > step_time: 0.2648 (0.33785488281045833)
|
| 1322 |
+
| > loader_time: 0.0126 (0.01116103261982629)
|
| 1323 |
+
|
| 1324 |
+
|
| 1325 |
+
[1m --> TIME: 2025-06-02 13:40:03 -- STEP: 1358/5792 -- GLOBAL_STEP: 7150[0m
|
| 1326 |
+
| > loss_text_ce: 0.04666044935584068 (0.04564322057743835)
|
| 1327 |
+
| > loss_mel_ce: 3.6217894554138184 (3.380934479310341)
|
| 1328 |
+
| > loss: 0.9171124696731567 (0.8566444249321679)
|
| 1329 |
+
| > current_lr: 5e-06
|
| 1330 |
+
| > step_time: 0.3762 (0.33808904807241025)
|
| 1331 |
+
| > loader_time: 0.0109 (0.011158927726464696)
|
| 1332 |
+
|
| 1333 |
+
|
| 1334 |
+
[1m --> TIME: 2025-06-02 13:40:32 -- STEP: 1408/5792 -- GLOBAL_STEP: 7200[0m
|
| 1335 |
+
| > loss_text_ce: 0.04733112081885338 (0.04561400014790704)
|
| 1336 |
+
| > loss_mel_ce: 3.2079885005950928 (3.3785352163355467)
|
| 1337 |
+
| > loss: 0.8138298988342285 (0.8560373041440138)
|
| 1338 |
+
| > current_lr: 5e-06
|
| 1339 |
+
| > step_time: 0.3122 (0.33810166117142526)
|
| 1340 |
+
| > loader_time: 0.0126 (0.01115033944899386)
|
| 1341 |
+
|
| 1342 |
+
|
| 1343 |
+
[1m --> TIME: 2025-06-02 13:41:01 -- STEP: 1458/5792 -- GLOBAL_STEP: 7250[0m
|
| 1344 |
+
| > loss_text_ce: 0.04578738287091255 (0.04558497288140411)
|
| 1345 |
+
| > loss_mel_ce: 3.535698652267456 (3.3770085259229563)
|
| 1346 |
+
| > loss: 0.8953714966773987 (0.8556483748109578)
|
| 1347 |
+
| > current_lr: 5e-06
|
| 1348 |
+
| > step_time: 0.4838 (0.3378712419619776)
|
| 1349 |
+
| > loader_time: 0.0114 (0.011147373661255806)
|
| 1350 |
+
|
| 1351 |
+
|
| 1352 |
+
[1m --> TIME: 2025-06-02 13:41:31 -- STEP: 1508/5792 -- GLOBAL_STEP: 7300[0m
|
| 1353 |
+
| > loss_text_ce: 0.04656374827027321 (0.0455558022918965)
|
| 1354 |
+
| > loss_mel_ce: 3.0295422077178955 (3.3743492266859874)
|
| 1355 |
+
| > loss: 0.769026517868042 (0.8549762572944953)
|
| 1356 |
+
| > current_lr: 5e-06
|
| 1357 |
+
| > step_time: 0.1911 (0.3378510261720308)
|
| 1358 |
+
| > loader_time: 0.0125 (0.011148763430213425)
|
| 1359 |
+
|
| 1360 |
+
|
| 1361 |
+
[1m --> TIME: 2025-06-02 13:42:00 -- STEP: 1558/5792 -- GLOBAL_STEP: 7350[0m
|
| 1362 |
+
| > loss_text_ce: 0.04521656408905983 (0.04553002290743304)
|
| 1363 |
+
| > loss_mel_ce: 3.1920225620269775 (3.3758582573034954)
|
| 1364 |
+
| > loss: 0.8093097805976868 (0.8553470702165203)
|
| 1365 |
+
| > current_lr: 5e-06
|
| 1366 |
+
| > step_time: 0.2611 (0.33788349662114764)
|
| 1367 |
+
| > loader_time: 0.0099 (0.011138344300726721)
|
| 1368 |
+
|
| 1369 |
+
|
| 1370 |
+
[1m --> TIME: 2025-06-02 13:42:28 -- STEP: 1608/5792 -- GLOBAL_STEP: 7400[0m
|
| 1371 |
+
| > loss_text_ce: 0.04438360407948494 (0.04550991758741251)
|
| 1372 |
+
| > loss_mel_ce: 3.5675711631774902 (3.3738133661189487)
|
| 1373 |
+
| > loss: 0.9029886722564697 (0.8548308210215757)
|
| 1374 |
+
| > current_lr: 5e-06
|
| 1375 |
+
| > step_time: 0.2961 (0.33742392522778664)
|
| 1376 |
+
| > loader_time: 0.0104 (0.011117274488382674)
|
| 1377 |
+
|
| 1378 |
+
|
| 1379 |
+
[1m --> TIME: 2025-06-02 13:42:59 -- STEP: 1658/5792 -- GLOBAL_STEP: 7450[0m
|
| 1380 |
+
| > loss_text_ce: 0.04301420599222183 (0.045442521491226136)
|
| 1381 |
+
| > loss_mel_ce: 3.6960628032684326 (3.37488229901713)
|
| 1382 |
+
| > loss: 0.9347692728042603 (0.8550812052371849)
|
| 1383 |
+
| > current_lr: 5e-06
|
| 1384 |
+
| > step_time: 0.3433 (0.3380332888970184)
|
| 1385 |
+
| > loader_time: 0.0102 (0.011103294151683483)
|
| 1386 |
+
|
| 1387 |
+
|
| 1388 |
+
[1m --> TIME: 2025-06-02 13:43:27 -- STEP: 1708/5792 -- GLOBAL_STEP: 7500[0m
|
| 1389 |
+
| > loss_text_ce: 0.043877944350242615 (0.0453859308027865)
|
| 1390 |
+
| > loss_mel_ce: 2.9800140857696533 (3.373501475298433)
|
| 1391 |
+
| > loss: 0.7559729814529419 (0.854721851594554)
|
| 1392 |
+
| > current_lr: 5e-06
|
| 1393 |
+
| > step_time: 0.2149 (0.3374889004425923)
|
| 1394 |
+
| > loader_time: 0.0099 (0.011108080732180309)
|
| 1395 |
+
|
| 1396 |
+
|
| 1397 |
+
[1m --> TIME: 2025-06-02 13:43:55 -- STEP: 1758/5792 -- GLOBAL_STEP: 7550[0m
|
| 1398 |
+
| > loss_text_ce: 0.03403538465499878 (0.04534231755878262)
|
| 1399 |
+
| > loss_mel_ce: 2.907336711883545 (3.3728728446266083)
|
| 1400 |
+
| > loss: 0.7353430390357971 (0.8545537905956154)
|
| 1401 |
+
| > current_lr: 5e-06
|
| 1402 |
+
| > step_time: 0.2952 (0.33705969068377484)
|
| 1403 |
+
| > loader_time: 0.0126 (0.011120306347010488)
|
| 1404 |
+
|
| 1405 |
+
|
| 1406 |
+
[1m --> TIME: 2025-06-02 13:44:24 -- STEP: 1808/5792 -- GLOBAL_STEP: 7600[0m
|
| 1407 |
+
| > loss_text_ce: 0.036792825907468796 (0.04528982559552794)
|
| 1408 |
+
| > loss_mel_ce: 3.696239948272705 (3.3725063265424917)
|
| 1409 |
+
| > loss: 0.9332581758499146 (0.8544490379569804)
|
| 1410 |
+
| > current_lr: 5e-06
|
| 1411 |
+
| > step_time: 0.2613 (0.33688090253720226)
|
| 1412 |
+
| > loader_time: 0.0099 (0.011115989721981826)
|
| 1413 |
+
|
| 1414 |
+
|
| 1415 |
+
[1m --> TIME: 2025-06-02 13:44:52 -- STEP: 1858/5792 -- GLOBAL_STEP: 7650[0m
|
| 1416 |
+
| > loss_text_ce: 0.0398641973733902 (0.04524051058893085)
|
| 1417 |
+
| > loss_mel_ce: 3.4899098873138428 (3.3701283141280918)
|
| 1418 |
+
| > loss: 0.8824435472488403 (0.8538422060025648)
|
| 1419 |
+
| > current_lr: 5e-06
|
| 1420 |
+
| > step_time: 0.2554 (0.3364653570659706)
|
| 1421 |
+
| > loader_time: 0.0106 (0.011119012812110403)
|
| 1422 |
+
|
| 1423 |
+
|
| 1424 |
+
[1m --> TIME: 2025-06-02 13:45:21 -- STEP: 1908/5792 -- GLOBAL_STEP: 7700[0m
|
| 1425 |
+
| > loss_text_ce: 0.041133489459753036 (0.04522936787179024)
|
| 1426 |
+
| > loss_mel_ce: 3.302356481552124 (3.369759842034906)
|
| 1427 |
+
| > loss: 0.83587247133255 (0.8537473023187688)
|
| 1428 |
+
| > current_lr: 5e-06
|
| 1429 |
+
| > step_time: 0.2272 (0.33635687903038347)
|
| 1430 |
+
| > loader_time: 0.0118 (0.011123857283242345)
|
| 1431 |
+
|
| 1432 |
+
|
| 1433 |
+
[1m --> TIME: 2025-06-02 13:45:49 -- STEP: 1958/5792 -- GLOBAL_STEP: 7750[0m
|
| 1434 |
+
| > loss_text_ce: 0.04508072882890701 (0.045202246298242925)
|
| 1435 |
+
| > loss_mel_ce: 4.015277862548828 (3.3691428928988953)
|
| 1436 |
+
| > loss: 1.0150896310806274 (0.8535862845712101)
|
| 1437 |
+
| > current_lr: 5e-06
|
| 1438 |
+
| > step_time: 0.2578 (0.33611000692272075)
|
| 1439 |
+
| > loader_time: 0.01 (0.01110899363158307)
|
| 1440 |
+
|
| 1441 |
+
|
| 1442 |
+
[1m --> TIME: 2025-06-02 13:46:17 -- STEP: 2008/5792 -- GLOBAL_STEP: 7800[0m
|
| 1443 |
+
| > loss_text_ce: 0.040393561124801636 (0.04518387101839232)
|
| 1444 |
+
| > loss_mel_ce: 3.0478756427764893 (3.367659538628096)
|
| 1445 |
+
| > loss: 0.7720673084259033 (0.8532108522625083)
|
| 1446 |
+
| > current_lr: 5e-06
|
| 1447 |
+
| > step_time: 0.2072 (0.33570773442427937)
|
| 1448 |
+
| > loader_time: 0.0096 (0.011115385597920515)
|
| 1449 |
+
|
| 1450 |
+
|
| 1451 |
+
[1m --> TIME: 2025-06-02 13:46:46 -- STEP: 2058/5792 -- GLOBAL_STEP: 7850[0m
|
| 1452 |
+
| > loss_text_ce: 0.04632216691970825 (0.04517081452024949)
|
| 1453 |
+
| > loss_mel_ce: 3.593982458114624 (3.366837971527908)
|
| 1454 |
+
| > loss: 0.9100761413574219 (0.8530021963534248)
|
| 1455 |
+
| > current_lr: 5e-06
|
| 1456 |
+
| > step_time: 0.6081 (0.3358027338402379)
|
| 1457 |
+
| > loader_time: 0.0109 (0.011119001460839988)
|
| 1458 |
+
|
| 1459 |
+
|
| 1460 |
+
[1m --> TIME: 2025-06-02 13:47:17 -- STEP: 2108/5792 -- GLOBAL_STEP: 7900[0m
|
| 1461 |
+
| > loss_text_ce: 0.046658266335725784 (0.04515279076756657)
|
| 1462 |
+
| > loss_mel_ce: 3.4903433322906494 (3.3665078355872438)
|
| 1463 |
+
| > loss: 0.8842504024505615 (0.8529151564983534)
|
| 1464 |
+
| > current_lr: 5e-06
|
| 1465 |
+
| > step_time: 0.4083 (0.3364675910242141)
|
| 1466 |
+
| > loader_time: 0.0108 (0.011129756353837944)
|
| 1467 |
+
|
| 1468 |
+
|
| 1469 |
+
[1m --> TIME: 2025-06-02 13:47:46 -- STEP: 2158/5792 -- GLOBAL_STEP: 7950[0m
|
| 1470 |
+
| > loss_text_ce: 0.04326501116156578 (0.04511675107844374)
|
| 1471 |
+
| > loss_mel_ce: 3.7370195388793945 (3.365736954072983)
|
| 1472 |
+
| > loss: 0.9450711607933044 (0.8527134262211792)
|
| 1473 |
+
| > current_lr: 5e-06
|
| 1474 |
+
| > step_time: 0.3271 (0.3362241120554982)
|
| 1475 |
+
| > loader_time: 0.0134 (0.011123432065735718)
|
| 1476 |
+
|
| 1477 |
+
|
| 1478 |
+
[1m --> TIME: 2025-06-02 13:48:15 -- STEP: 2208/5792 -- GLOBAL_STEP: 8000[0m
|
| 1479 |
+
| > loss_text_ce: 0.04933100938796997 (0.04507167210238918)
|
| 1480 |
+
| > loss_mel_ce: 3.107172966003418 (3.364644525707631)
|
| 1481 |
+
| > loss: 0.7891259789466858 (0.8524290493594995)
|
| 1482 |
+
| > current_lr: 5e-06
|
| 1483 |
+
| > step_time: 0.2153 (0.33627322402553284)
|
| 1484 |
+
| > loader_time: 0.0104 (0.011131968619166944)
|
| 1485 |
+
|
| 1486 |
+
|
| 1487 |
+
[1m --> TIME: 2025-06-02 13:48:43 -- STEP: 2258/5792 -- GLOBAL_STEP: 8050[0m
|
| 1488 |
+
| > loss_text_ce: 0.0509863905608654 (0.04505748897576433)
|
| 1489 |
+
| > loss_mel_ce: 3.511996269226074 (3.3630264050354457)
|
| 1490 |
+
| > loss: 0.8907456398010254 (0.8520209733578247)
|
| 1491 |
+
| > current_lr: 5e-06
|
| 1492 |
+
| > step_time: 0.4846 (0.3359105811908882)
|
| 1493 |
+
| > loader_time: 0.0105 (0.011143911930182215)
|
| 1494 |
+
|
| 1495 |
+
|
| 1496 |
+
[1m --> TIME: 2025-06-02 13:49:15 -- STEP: 2308/5792 -- GLOBAL_STEP: 8100[0m
|
| 1497 |
+
| > loss_text_ce: 0.04393137991428375 (0.045034569963341514)
|
| 1498 |
+
| > loss_mel_ce: 2.9540281295776367 (3.362665162962478)
|
| 1499 |
+
| > loss: 0.7494899034500122 (0.8519249330411955)
|
| 1500 |
+
| > current_lr: 5e-06
|
| 1501 |
+
| > step_time: 0.2141 (0.33653108003441157)
|
| 1502 |
+
| > loader_time: 0.0099 (0.011160700325329545)
|
| 1503 |
+
|
| 1504 |
+
|
| 1505 |
+
[1m --> TIME: 2025-06-02 13:49:45 -- STEP: 2358/5792 -- GLOBAL_STEP: 8150[0m
|
| 1506 |
+
| > loss_text_ce: 0.04599863290786743 (0.04497464137914904)
|
| 1507 |
+
| > loss_mel_ce: 3.3983516693115234 (3.363921059098053)
|
| 1508 |
+
| > loss: 0.8610875606536865 (0.8522239249070076)
|
| 1509 |
+
| > current_lr: 5e-06
|
| 1510 |
+
| > step_time: 0.3547 (0.3368334209039317)
|
| 1511 |
+
| > loader_time: 0.0101 (0.011168763332188992)
|
| 1512 |
+
|
| 1513 |
+
|
| 1514 |
+
[1m --> TIME: 2025-06-02 13:50:13 -- STEP: 2408/5792 -- GLOBAL_STEP: 8200[0m
|
| 1515 |
+
| > loss_text_ce: 0.0410335548222065 (0.04494161312808699)
|
| 1516 |
+
| > loss_mel_ce: 2.8338351249694824 (3.362981556558925)
|
| 1517 |
+
| > loss: 0.7187171578407288 (0.8519807922285653)
|
| 1518 |
+
| > current_lr: 5e-06
|
| 1519 |
+
| > step_time: 0.1891 (0.33666275852542393)
|
| 1520 |
+
| > loader_time: 0.0106 (0.011154194111839877)
|
| 1521 |
+
|
| 1522 |
+
|
| 1523 |
+
[1m --> TIME: 2025-06-02 13:50:44 -- STEP: 2458/5792 -- GLOBAL_STEP: 8250[0m
|
| 1524 |
+
| > loss_text_ce: 0.04467436298727989 (0.04491210582857818)
|
| 1525 |
+
| > loss_mel_ce: 3.4023947715759277 (3.362881237942576)
|
| 1526 |
+
| > loss: 0.8617672920227051 (0.8519483357546673)
|
| 1527 |
+
| > current_lr: 5e-06
|
| 1528 |
+
| > step_time: 0.2659 (0.3370825775858832)
|
| 1529 |
+
| > loader_time: 0.0133 (0.011155853725430246)
|
| 1530 |
+
|
| 1531 |
+
|
| 1532 |
+
[1m --> TIME: 2025-06-02 13:51:14 -- STEP: 2508/5792 -- GLOBAL_STEP: 8300[0m
|
| 1533 |
+
| > loss_text_ce: 0.0431218296289444 (0.04488258450210355)
|
| 1534 |
+
| > loss_mel_ce: 3.392765998840332 (3.3614071411759463)
|
| 1535 |
+
| > loss: 0.8589719533920288 (0.8515724312793311)
|
| 1536 |
+
| > current_lr: 5e-06
|
| 1537 |
+
| > step_time: 0.3281 (0.33738990387087664)
|
| 1538 |
+
| > loader_time: 0.0105 (0.011148617028049297)
|
| 1539 |
+
|
| 1540 |
+
|
| 1541 |
+
[1m --> TIME: 2025-06-02 13:51:43 -- STEP: 2558/5792 -- GLOBAL_STEP: 8350[0m
|
| 1542 |
+
| > loss_text_ce: 0.042658887803554535 (0.0448320767120622)
|
| 1543 |
+
| > loss_mel_ce: 3.3791749477386475 (3.3615849678920893)
|
| 1544 |
+
| > loss: 0.8554584383964539 (0.8516042609993973)
|
| 1545 |
+
| > current_lr: 5e-06
|
| 1546 |
+
| > step_time: 0.2925 (0.3374677144334692)
|
| 1547 |
+
| > loader_time: 0.0098 (0.011138459217557677)
|
| 1548 |
+
|
| 1549 |
+
|
| 1550 |
+
[1m --> TIME: 2025-06-02 13:52:12 -- STEP: 2608/5792 -- GLOBAL_STEP: 8400[0m
|
| 1551 |
+
| > loss_text_ce: 0.03552236780524254 (0.04479173863623748)
|
| 1552 |
+
| > loss_mel_ce: 2.71285343170166 (3.360946973682912)
|
| 1553 |
+
| > loss: 0.68709397315979 (0.851434677976406)
|
| 1554 |
+
| > current_lr: 5e-06
|
| 1555 |
+
| > step_time: 0.1824 (0.33746330266349933)
|
| 1556 |
+
| > loader_time: 0.012 (0.01112092354911966)
|
| 1557 |
+
|
| 1558 |
+
|
| 1559 |
+
[1m --> TIME: 2025-06-02 13:52:40 -- STEP: 2658/5792 -- GLOBAL_STEP: 8450[0m
|
| 1560 |
+
| > loss_text_ce: 0.045843303203582764 (0.0447824644737725)
|
| 1561 |
+
| > loss_mel_ce: 3.9877352714538574 (3.3594930995398657)
|
| 1562 |
+
| > loss: 1.0083945989608765 (0.8510688909289527)
|
| 1563 |
+
| > current_lr: 5e-06
|
| 1564 |
+
| > step_time: 0.2632 (0.337073947003769)
|
| 1565 |
+
| > loader_time: 0.0093 (0.011125095810900938)
|
| 1566 |
+
|
| 1567 |
+
|
| 1568 |
+
[1m --> TIME: 2025-06-02 13:53:08 -- STEP: 2708/5792 -- GLOBAL_STEP: 8500[0m
|
| 1569 |
+
| > loss_text_ce: 0.04365628957748413 (0.04472194665195517)
|
| 1570 |
+
| > loss_mel_ce: 3.502896308898926 (3.358532962189819)
|
| 1571 |
+
| > loss: 0.8866381645202637 (0.8508137271710651)
|
| 1572 |
+
| > current_lr: 5e-06
|
| 1573 |
+
| > step_time: 0.3144 (0.3368678473299331)
|
| 1574 |
+
| > loader_time: 0.014 (0.011116211136123433)
|
| 1575 |
+
|
| 1576 |
+
|
| 1577 |
+
[1m --> TIME: 2025-06-02 13:53:36 -- STEP: 2758/5792 -- GLOBAL_STEP: 8550[0m
|
| 1578 |
+
| > loss_text_ce: 0.044692471623420715 (0.044700241800292285)
|
| 1579 |
+
| > loss_mel_ce: 3.10170578956604 (3.356952504717499)
|
| 1580 |
+
| > loss: 0.7865995764732361 (0.8504131865769383)
|
| 1581 |
+
| > current_lr: 5e-06
|
| 1582 |
+
| > step_time: 0.1751 (0.33657982109075396)
|
| 1583 |
+
| > loader_time: 0.0096 (0.011113227234965547)
|
| 1584 |
+
|
| 1585 |
+
|
| 1586 |
+
[1m --> TIME: 2025-06-02 13:54:05 -- STEP: 2808/5792 -- GLOBAL_STEP: 8600[0m
|
| 1587 |
+
| > loss_text_ce: 0.04175104200839996 (0.044659487993995044)
|
| 1588 |
+
| > loss_mel_ce: 3.7599270343780518 (3.3555039678543714)
|
| 1589 |
+
| > loss: 0.950419545173645 (0.8500408639340655)
|
| 1590 |
+
| > current_lr: 5e-06
|
| 1591 |
+
| > step_time: 0.3366 (0.3363867896234893)
|
| 1592 |
+
| > loader_time: 0.0102 (0.011102559050263819)
|
| 1593 |
+
|
| 1594 |
+
|
| 1595 |
+
[1m --> TIME: 2025-06-02 13:54:33 -- STEP: 2858/5792 -- GLOBAL_STEP: 8650[0m
|
| 1596 |
+
| > loss_text_ce: 0.04298544302582741 (0.04463345923137111)
|
| 1597 |
+
| > loss_mel_ce: 3.1992127895355225 (3.3554093194558448)
|
| 1598 |
+
| > loss: 0.8105495572090149 (0.8500106946582804)
|
| 1599 |
+
| > current_lr: 5e-06
|
| 1600 |
+
| > step_time: 0.293 (0.3361897283371222)
|
| 1601 |
+
| > loader_time: 0.0101 (0.011095891055566914)
|
| 1602 |
+
|
| 1603 |
+
|
| 1604 |
+
[1m --> TIME: 2025-06-02 13:55:02 -- STEP: 2908/5792 -- GLOBAL_STEP: 8700[0m
|
| 1605 |
+
| > loss_text_ce: 0.04670070484280586 (0.044607090496649)
|
| 1606 |
+
| > loss_mel_ce: 3.638002872467041 (3.354981057827541)
|
| 1607 |
+
| > loss: 0.9211758971214294 (0.8498970369771223)
|
| 1608 |
+
| > current_lr: 5e-06
|
| 1609 |
+
| > step_time: 0.2085 (0.33621489690127415)
|
| 1610 |
+
| > loader_time: 0.0093 (0.011086543947663218)
|
| 1611 |
+
|
| 1612 |
+
|
| 1613 |
+
[1m --> TIME: 2025-06-02 13:55:31 -- STEP: 2958/5792 -- GLOBAL_STEP: 8750[0m
|
| 1614 |
+
| > loss_text_ce: 0.04170145094394684 (0.044566551267476086)
|
| 1615 |
+
| > loss_mel_ce: 3.448326826095581 (3.35389118762013)
|
| 1616 |
+
| > loss: 0.8725070953369141 (0.849614434604148)
|
| 1617 |
+
| > current_lr: 5e-06
|
| 1618 |
+
| > step_time: 0.4418 (0.3360969547970708)
|
| 1619 |
+
| > loader_time: 0.0106 (0.01107192184572044)
|
| 1620 |
+
|
| 1621 |
+
|
| 1622 |
+
[1m --> TIME: 2025-06-02 13:55:59 -- STEP: 3008/5792 -- GLOBAL_STEP: 8800[0m
|
| 1623 |
+
| > loss_text_ce: 0.04650944098830223 (0.04452610574786532)
|
| 1624 |
+
| > loss_mel_ce: 3.232806921005249 (3.3524444404434646)
|
| 1625 |
+
| > loss: 0.8198291063308716 (0.8492426364781689)
|
| 1626 |
+
| > current_lr: 5e-06
|
| 1627 |
+
| > step_time: 0.4633 (0.33591527674109456)
|
| 1628 |
+
| > loader_time: 0.0122 (0.011064917958797276)
|
| 1629 |
+
|
| 1630 |
+
|
| 1631 |
+
[1m --> TIME: 2025-06-02 13:56:28 -- STEP: 3058/5792 -- GLOBAL_STEP: 8850[0m
|
| 1632 |
+
| > loss_text_ce: 0.04007389768958092 (0.04448877311780065)
|
| 1633 |
+
| > loss_mel_ce: 3.012786626815796 (3.3516998684741526)
|
| 1634 |
+
| > loss: 0.7632151246070862 (0.8490471603669237)
|
| 1635 |
+
| > current_lr: 5e-06
|
| 1636 |
+
| > step_time: 0.2232 (0.33596821498683804)
|
| 1637 |
+
| > loader_time: 0.0119 (0.01105607171866376)
|
| 1638 |
+
|
| 1639 |
+
|
| 1640 |
+
[1m --> TIME: 2025-06-02 13:56:56 -- STEP: 3108/5792 -- GLOBAL_STEP: 8900[0m
|
| 1641 |
+
| > loss_text_ce: 0.03358534350991249 (0.04445769784173796)
|
| 1642 |
+
| > loss_mel_ce: 3.2545511722564697 (3.3518611339221738)
|
| 1643 |
+
| > loss: 0.8220341205596924 (0.8490797078555766)
|
| 1644 |
+
| > current_lr: 5e-06
|
| 1645 |
+
| > step_time: 0.2089 (0.33573530938481194)
|
| 1646 |
+
| > loader_time: 0.0094 (0.011041443857293623)
|
| 1647 |
+
|
| 1648 |
+
|
| 1649 |
+
[1m --> TIME: 2025-06-02 13:57:26 -- STEP: 3158/5792 -- GLOBAL_STEP: 8950[0m
|
| 1650 |
+
| > loss_text_ce: 0.037105049937963486 (0.04442861938803087)
|
| 1651 |
+
| > loss_mel_ce: 3.5840868949890137 (3.351841525762725)
|
| 1652 |
+
| > loss: 0.9052979946136475 (0.8490675361832911)
|
| 1653 |
+
| > current_lr: 5e-06
|
| 1654 |
+
| > step_time: 0.4385 (0.33593913925985125)
|
| 1655 |
+
| > loader_time: 0.0127 (0.011041003590222006)
|
| 1656 |
+
|
| 1657 |
+
|
| 1658 |
+
[1m --> TIME: 2025-06-02 13:57:54 -- STEP: 3208/5792 -- GLOBAL_STEP: 9000[0m
|
| 1659 |
+
| > loss_text_ce: 0.04580637440085411 (0.04439754502504062)
|
| 1660 |
+
| > loss_mel_ce: 3.0522005558013916 (3.351788088567833)
|
| 1661 |
+
| > loss: 0.7745017409324646 (0.8490464082754161)
|
| 1662 |
+
| > current_lr: 5e-06
|
| 1663 |
+
| > step_time: 0.1902 (0.33578265449055483)
|
| 1664 |
+
| > loader_time: 0.0086 (0.011037279245562075)
|
| 1665 |
+
|
| 1666 |
+
|
| 1667 |
+
[1m --> TIME: 2025-06-02 13:58:22 -- STEP: 3258/5792 -- GLOBAL_STEP: 9050[0m
|
| 1668 |
+
| > loss_text_ce: 0.03635700047016144 (0.044366618606934825)
|
| 1669 |
+
| > loss_mel_ce: 3.045945167541504 (3.3514977041529024)
|
| 1670 |
+
| > loss: 0.7705755233764648 (0.8489660805467447)
|
| 1671 |
+
| > current_lr: 5e-06
|
| 1672 |
+
| > step_time: 0.2663 (0.3356093359256983)
|
| 1673 |
+
| > loader_time: 0.0145 (0.011036152924548766)
|
| 1674 |
+
|
| 1675 |
+
|
| 1676 |
+
[1m --> TIME: 2025-06-02 13:58:52 -- STEP: 3308/5792 -- GLOBAL_STEP: 9100[0m
|
| 1677 |
+
| > loss_text_ce: 0.0383525975048542 (0.04434114113729335)
|
| 1678 |
+
| > loss_mel_ce: 3.439059257507324 (3.3502290691401098)
|
| 1679 |
+
| > loss: 0.8693529367446899 (0.848642552417884)
|
| 1680 |
+
| > current_lr: 5e-06
|
| 1681 |
+
| > step_time: 0.2088 (0.33574586695886793)
|
| 1682 |
+
| > loader_time: 0.0124 (0.011047434129334404)
|
| 1683 |
+
|
| 1684 |
+
|
| 1685 |
+
[1m --> TIME: 2025-06-02 13:59:20 -- STEP: 3358/5792 -- GLOBAL_STEP: 9150[0m
|
| 1686 |
+
| > loss_text_ce: 0.0411563478410244 (0.04431965041788048)
|
| 1687 |
+
| > loss_mel_ce: 3.5818145275115967 (3.3501473820017798)
|
| 1688 |
+
| > loss: 0.9057427048683167 (0.8486167579587544)
|
| 1689 |
+
| > current_lr: 5e-06
|
| 1690 |
+
| > step_time: 0.333 (0.3356304534352067)
|
| 1691 |
+
| > loader_time: 0.0102 (0.01104049429856572)
|
| 1692 |
+
|
| 1693 |
+
|
| 1694 |
+
[1m --> TIME: 2025-06-02 13:59:49 -- STEP: 3408/5792 -- GLOBAL_STEP: 9200[0m
|
| 1695 |
+
| > loss_text_ce: 0.04104316234588623 (0.044287355214427344)
|
| 1696 |
+
| > loss_mel_ce: 3.3351967334747314 (3.349034378436249)
|
| 1697 |
+
| > loss: 0.844059944152832 (0.8483304332224694)
|
| 1698 |
+
| > current_lr: 5e-06
|
| 1699 |
+
| > step_time: 0.3537 (0.3354787281561352)
|
| 1700 |
+
| > loader_time: 0.0143 (0.01104401031010586)
|
| 1701 |
+
|
| 1702 |
+
|
| 1703 |
+
[1m --> TIME: 2025-06-02 14:00:17 -- STEP: 3458/5792 -- GLOBAL_STEP: 9250[0m
|
| 1704 |
+
| > loss_text_ce: 0.04712622985243797 (0.04426270988896412)
|
| 1705 |
+
| > loss_mel_ce: 3.5113916397094727 (3.3474128444874607)
|
| 1706 |
+
| > loss: 0.8896294832229614 (0.8479188883352856)
|
| 1707 |
+
| > current_lr: 5e-06
|
| 1708 |
+
| > step_time: 0.4075 (0.3352865505383985)
|
| 1709 |
+
| > loader_time: 0.0109 (0.011042501417593558)
|
| 1710 |
+
|
| 1711 |
+
|
| 1712 |
+
[1m --> TIME: 2025-06-02 14:00:47 -- STEP: 3508/5792 -- GLOBAL_STEP: 9300[0m
|
| 1713 |
+
| > loss_text_ce: 0.0433344691991806 (0.04423293619052409)
|
| 1714 |
+
| > loss_mel_ce: 3.186034679412842 (3.3471053151611314)
|
| 1715 |
+
| > loss: 0.8073422908782959 (0.8478345625960566)
|
| 1716 |
+
| > current_lr: 5e-06
|
| 1717 |
+
| > step_time: 0.3399 (0.33543082470225155)
|
| 1718 |
+
| > loader_time: 0.0136 (0.011043719921851497)
|
| 1719 |
+
|
| 1720 |
+
|
| 1721 |
+
[1m --> TIME: 2025-06-02 14:01:15 -- STEP: 3558/5792 -- GLOBAL_STEP: 9350[0m
|
| 1722 |
+
| > loss_text_ce: 0.0385572724044323 (0.04420625756611444)
|
| 1723 |
+
| > loss_mel_ce: 3.43131160736084 (3.3458053263604737)
|
| 1724 |
+
| > loss: 0.8674672245979309 (0.8475028957591555)
|
| 1725 |
+
| > current_lr: 5e-06
|
| 1726 |
+
| > step_time: 0.2685 (0.3353362489510554)
|
| 1727 |
+
| > loader_time: 0.0128 (0.011042438242259877)
|
| 1728 |
+
|
| 1729 |
+
|
| 1730 |
+
[1m --> TIME: 2025-06-02 14:01:45 -- STEP: 3608/5792 -- GLOBAL_STEP: 9400[0m
|
| 1731 |
+
| > loss_text_ce: 0.04384002462029457 (0.04419183030212402)
|
| 1732 |
+
| > loss_mel_ce: 3.5863473415374756 (3.3455215218722683)
|
| 1733 |
+
| > loss: 0.9075468182563782 (0.8474283378463888)
|
| 1734 |
+
| > current_lr: 5e-06
|
| 1735 |
+
| > step_time: 0.3275 (0.33544651070349557)
|
| 1736 |
+
| > loader_time: 0.01 (0.011052065572294573)
|
| 1737 |
+
|
| 1738 |
+
|
| 1739 |
+
[1m --> TIME: 2025-06-02 14:02:12 -- STEP: 3658/5792 -- GLOBAL_STEP: 9450[0m
|
| 1740 |
+
| > loss_text_ce: 0.04530433937907219 (0.04417096679045439)
|
| 1741 |
+
| > loss_mel_ce: 3.4153072834014893 (3.3448606153991842)
|
| 1742 |
+
| > loss: 0.865152895450592 (0.8472578953595157)
|
| 1743 |
+
| > current_lr: 5e-06
|
| 1744 |
+
| > step_time: 0.314 (0.3351731634061808)
|
| 1745 |
+
| > loader_time: 0.0125 (0.011047935929045499)
|
| 1746 |
+
|
| 1747 |
+
|
| 1748 |
+
[1m --> TIME: 2025-06-02 14:02:42 -- STEP: 3708/5792 -- GLOBAL_STEP: 9500[0m
|
| 1749 |
+
| > loss_text_ce: 0.0346110537648201 (0.04413479716469813)
|
| 1750 |
+
| > loss_mel_ce: 3.222661018371582 (3.3444809126622466)
|
| 1751 |
+
| > loss: 0.8143180012702942 (0.8471539272588174)
|
| 1752 |
+
| > current_lr: 5e-06
|
| 1753 |
+
| > step_time: 0.3003 (0.33524002194276065)
|
| 1754 |
+
| > loader_time: 0.011 (0.011049625685814244)
|
| 1755 |
+
|
| 1756 |
+
|
| 1757 |
+
[1m --> TIME: 2025-06-02 14:03:09 -- STEP: 3758/5792 -- GLOBAL_STEP: 9550[0m
|
| 1758 |
+
| > loss_text_ce: 0.04709426686167717 (0.04409706911509297)
|
| 1759 |
+
| > loss_mel_ce: 3.4125912189483643 (3.343986088002597)
|
| 1760 |
+
| > loss: 0.8649213910102844 (0.8470207890213136)
|
| 1761 |
+
| > current_lr: 5e-06
|
| 1762 |
+
| > step_time: 0.4286 (0.33501876009595527)
|
| 1763 |
+
| > loader_time: 0.0134 (0.011049652048855007)
|
| 1764 |
+
|
| 1765 |
+
|
| 1766 |
+
[1m --> TIME: 2025-06-02 14:03:38 -- STEP: 3808/5792 -- GLOBAL_STEP: 9600[0m
|
| 1767 |
+
| > loss_text_ce: 0.041285887360572815 (0.044069222885305306)
|
| 1768 |
+
| > loss_mel_ce: 3.439079523086548 (3.3440659681037688)
|
| 1769 |
+
| > loss: 0.8700913786888123 (0.8470337975676318)
|
| 1770 |
+
| > current_lr: 5e-06
|
| 1771 |
+
| > step_time: 0.2723 (0.334857460267905)
|
| 1772 |
+
| > loader_time: 0.0094 (0.011047247202456485)
|
| 1773 |
+
|
| 1774 |
+
|
| 1775 |
+
[1m --> TIME: 2025-06-02 14:04:07 -- STEP: 3858/5792 -- GLOBAL_STEP: 9650[0m
|
| 1776 |
+
| > loss_text_ce: 0.0423085018992424 (0.04404395498178935)
|
| 1777 |
+
| > loss_mel_ce: 3.5037617683410645 (3.343483012102007)
|
| 1778 |
+
| > loss: 0.8865175843238831 (0.8468817415912264)
|
| 1779 |
+
| > current_lr: 5e-06
|
| 1780 |
+
| > step_time: 0.41 (0.33492562775787443)
|
| 1781 |
+
| > loader_time: 0.0131 (0.011050497329933387)
|
| 1782 |
+
|
| 1783 |
+
|
| 1784 |
+
[1m --> TIME: 2025-06-02 14:04:37 -- STEP: 3908/5792 -- GLOBAL_STEP: 9700[0m
|
| 1785 |
+
| > loss_text_ce: 0.04542999714612961 (0.04403033828412392)
|
| 1786 |
+
| > loss_mel_ce: 2.891977071762085 (3.342988579214656)
|
| 1787 |
+
| > loss: 0.7343517541885376 (0.8467547291987015)
|
| 1788 |
+
| > current_lr: 5e-06
|
| 1789 |
+
| > step_time: 0.1907 (0.335096147006708)
|
| 1790 |
+
| > loader_time: 0.0098 (0.011049572435591811)
|
| 1791 |
+
|
| 1792 |
+
|
| 1793 |
+
[1m --> TIME: 2025-06-02 14:05:07 -- STEP: 3958/5792 -- GLOBAL_STEP: 9750[0m
|
| 1794 |
+
| > loss_text_ce: 0.0466097928583622 (0.044010729583553475)
|
| 1795 |
+
| > loss_mel_ce: 3.508066177368164 (3.3432548332238445)
|
| 1796 |
+
| > loss: 0.8886690139770508 (0.8468163905419621)
|
| 1797 |
+
| > current_lr: 5e-06
|
| 1798 |
+
| > step_time: 0.3011 (0.3353043986788423)
|
| 1799 |
+
| > loader_time: 0.013 (0.011049168347228097)
|
| 1800 |
+
|
| 1801 |
+
|
| 1802 |
+
[1m --> TIME: 2025-06-02 14:05:38 -- STEP: 4008/5792 -- GLOBAL_STEP: 9800[0m
|
| 1803 |
+
| > loss_text_ce: 0.04837203770875931 (0.04398115760806629)
|
| 1804 |
+
| > loss_mel_ce: 3.3902106285095215 (3.3430067203834857)
|
| 1805 |
+
| > loss: 0.859645664691925 (0.8467469693955307)
|
| 1806 |
+
| > current_lr: 5e-06
|
| 1807 |
+
| > step_time: 0.313 (0.33554493548151554)
|
| 1808 |
+
| > loader_time: 0.01 (0.011050231026080303)
|
| 1809 |
+
|
| 1810 |
+
|
| 1811 |
+
[1m --> TIME: 2025-06-02 14:06:08 -- STEP: 4058/5792 -- GLOBAL_STEP: 9850[0m
|
| 1812 |
+
| > loss_text_ce: 0.04397377371788025 (0.043955045121553474)
|
| 1813 |
+
| > loss_mel_ce: 3.5290706157684326 (3.342233718645989)
|
| 1814 |
+
| > loss: 0.8932610750198364 (0.8465471908251834)
|
| 1815 |
+
| > current_lr: 5e-06
|
| 1816 |
+
| > step_time: 0.4423 (0.3356970511615546)
|
| 1817 |
+
| > loader_time: 0.0139 (0.01104889202728947)
|
| 1818 |
+
|
| 1819 |
+
|
| 1820 |
+
[1m --> TIME: 2025-06-02 14:06:36 -- STEP: 4108/5792 -- GLOBAL_STEP: 9900[0m
|
| 1821 |
+
| > loss_text_ce: 0.034252800047397614 (0.04393632656793966)
|
| 1822 |
+
| > loss_mel_ce: 3.536194086074829 (3.3410506406916842)
|
| 1823 |
+
| > loss: 0.8926117420196533 (0.846246741690329)
|
| 1824 |
+
| > current_lr: 5e-06
|
| 1825 |
+
| > step_time: 0.2459 (0.33544389101175215)
|
| 1826 |
+
| > loader_time: 0.0095 (0.011051703973545407)
|
| 1827 |
+
|
| 1828 |
+
|
| 1829 |
+
[1m --> TIME: 2025-06-02 14:07:05 -- STEP: 4158/5792 -- GLOBAL_STEP: 9950[0m
|
| 1830 |
+
| > loss_text_ce: 0.03575450927019119 (0.04390224255218264)
|
| 1831 |
+
| > loss_mel_ce: 2.652836799621582 (3.3396065647983955)
|
| 1832 |
+
| > loss: 0.672147810459137 (0.8458772017268847)
|
| 1833 |
+
| > current_lr: 5e-06
|
| 1834 |
+
| > step_time: 0.3049 (0.33547147149475504)
|
| 1835 |
+
| > loader_time: 0.0129 (0.01104814040655123)
|
| 1836 |
+
|
| 1837 |
+
|
| 1838 |
+
[1m --> TIME: 2025-06-02 14:07:34 -- STEP: 4208/5792 -- GLOBAL_STEP: 10000[0m
|
| 1839 |
+
| > loss_text_ce: 0.044553492218256 (0.04387786551847873)
|
| 1840 |
+
| > loss_mel_ce: 3.6881346702575684 (3.3393874958780767)
|
| 1841 |
+
| > loss: 0.9331720471382141 (0.8458163402549665)
|
| 1842 |
+
| > current_lr: 5e-06
|
| 1843 |
+
| > step_time: 0.24 (0.33554055975190633)
|
| 1844 |
+
| > loader_time: 0.0136 (0.011045128107070909)
|
| 1845 |
+
|
| 1846 |
+
|
| 1847 |
+
[1m --> TIME: 2025-06-02 14:08:05 -- STEP: 4258/5792 -- GLOBAL_STEP: 10050[0m
|
| 1848 |
+
| > loss_text_ce: 0.04713189974427223 (0.043857144453016465)
|
| 1849 |
+
| > loss_mel_ce: 3.250263214111328 (3.3402409040631116)
|
| 1850 |
+
| > loss: 0.8243488073348999 (0.8460245119847841)
|
| 1851 |
+
| > current_lr: 5e-06
|
| 1852 |
+
| > step_time: 0.4278 (0.33582793624818846)
|
| 1853 |
+
| > loader_time: 0.0109 (0.011045684160453815)
|
| 1854 |
+
|
| 1855 |
+
|
| 1856 |
+
[1m --> TIME: 2025-06-02 14:08:36 -- STEP: 4308/5792 -- GLOBAL_STEP: 10100[0m
|
| 1857 |
+
| > loss_text_ce: 0.04142797365784645 (0.04383900594753162)
|
| 1858 |
+
| > loss_mel_ce: 3.029611110687256 (3.339805153861793)
|
| 1859 |
+
| > loss: 0.7677597999572754 (0.8459110398229447)
|
| 1860 |
+
| > current_lr: 5e-06
|
| 1861 |
+
| > step_time: 0.2426 (0.3360530724653853)
|
| 1862 |
+
| > loader_time: 0.0125 (0.01104891161843372)
|
| 1863 |
+
|
| 1864 |
+
|
| 1865 |
+
[1m --> TIME: 2025-06-02 14:09:06 -- STEP: 4358/5792 -- GLOBAL_STEP: 10150[0m
|
| 1866 |
+
| > loss_text_ce: 0.04575802758336067 (0.04381198164147425)
|
| 1867 |
+
| > loss_mel_ce: 3.206411838531494 (3.3394206824681243)
|
| 1868 |
+
| > loss: 0.8130424618721008 (0.8458081659180897)
|
| 1869 |
+
| > current_lr: 5e-06
|
| 1870 |
+
| > step_time: 0.4958 (0.33611179339114383)
|
| 1871 |
+
| > loader_time: 0.0112 (0.011051389699465848)
|
| 1872 |
+
|
| 1873 |
+
|
| 1874 |
+
[1m --> TIME: 2025-06-02 14:09:35 -- STEP: 4408/5792 -- GLOBAL_STEP: 10200[0m
|
| 1875 |
+
| > loss_text_ce: 0.04181518405675888 (0.04377960568140123)
|
| 1876 |
+
| > loss_mel_ce: 2.947584629058838 (3.3390845118763206)
|
| 1877 |
+
| > loss: 0.7473499774932861 (0.8457160292703737)
|
| 1878 |
+
| > current_lr: 5e-06
|
| 1879 |
+
| > step_time: 0.2769 (0.33606701202271466)
|
| 1880 |
+
| > loader_time: 0.0094 (0.011053307681680795)
|
| 1881 |
+
|
| 1882 |
+
|
| 1883 |
+
[1m --> TIME: 2025-06-02 14:10:06 -- STEP: 4458/5792 -- GLOBAL_STEP: 10250[0m
|
| 1884 |
+
| > loss_text_ce: 0.04814762994647026 (0.0437453800015162)
|
| 1885 |
+
| > loss_mel_ce: 2.8046014308929443 (3.3391340181077727)
|
| 1886 |
+
| > loss: 0.7131872773170471 (0.8457198493841137)
|
| 1887 |
+
| > current_lr: 5e-06
|
| 1888 |
+
| > step_time: 0.2371 (0.3363838041240513)
|
| 1889 |
+
| > loader_time: 0.0094 (0.011047967242895073)
|
| 1890 |
+
|
| 1891 |
+
|
| 1892 |
+
[1m --> TIME: 2025-06-02 14:10:36 -- STEP: 4508/5792 -- GLOBAL_STEP: 10300[0m
|
| 1893 |
+
| > loss_text_ce: 0.045141834765672684 (0.043721168837076246)
|
| 1894 |
+
| > loss_mel_ce: 3.2513136863708496 (3.3381951364036566)
|
| 1895 |
+
| > loss: 0.8241139054298401 (0.8454790762043034)
|
| 1896 |
+
| > current_lr: 5e-06
|
| 1897 |
+
| > step_time: 0.2119 (0.3364703716063989)
|
| 1898 |
+
| > loader_time: 0.0097 (0.011043780920456853)
|
| 1899 |
+
|
| 1900 |
+
|
| 1901 |
+
[1m --> TIME: 2025-06-02 14:11:04 -- STEP: 4558/5792 -- GLOBAL_STEP: 10350[0m
|
| 1902 |
+
| > loss_text_ce: 0.046194713562726974 (0.04369950713651088)
|
| 1903 |
+
| > loss_mel_ce: 3.618640184402466 (3.3370097836364305)
|
| 1904 |
+
| > loss: 0.9162087440490723 (0.8451773226261132)
|
| 1905 |
+
| > current_lr: 5e-06
|
| 1906 |
+
| > step_time: 0.4156 (0.3364097547405593)
|
| 1907 |
+
| > loader_time: 0.013 (0.011043213553049927)
|
| 1908 |
+
|
| 1909 |
+
|
| 1910 |
+
[1m --> TIME: 2025-06-02 14:11:34 -- STEP: 4608/5792 -- GLOBAL_STEP: 10400[0m
|
| 1911 |
+
| > loss_text_ce: 0.044357966631650925 (0.0436816014767424)
|
| 1912 |
+
| > loss_mel_ce: 3.656270742416382 (3.336337390355765)
|
| 1913 |
+
| > loss: 0.9251571893692017 (0.8450047479062854)
|
| 1914 |
+
| > current_lr: 5e-06
|
| 1915 |
+
| > step_time: 0.2739 (0.3365074931126504)
|
| 1916 |
+
| > loader_time: 0.0125 (0.011044854091273402)
|
| 1917 |
+
|
| 1918 |
+
|
| 1919 |
+
[1m --> TIME: 2025-06-02 14:12:03 -- STEP: 4658/5792 -- GLOBAL_STEP: 10450[0m
|
| 1920 |
+
| > loss_text_ce: 0.03670710697770119 (0.04365839876879141)
|
| 1921 |
+
| > loss_mel_ce: 2.583122968673706 (3.335039994193535)
|
| 1922 |
+
| > loss: 0.6549575328826904 (0.8446745981777001)
|
| 1923 |
+
| > current_lr: 5e-06
|
| 1924 |
+
| > step_time: 0.1691 (0.336374844487208)
|
| 1925 |
+
| > loader_time: 0.0109 (0.011051791632518373)
|
| 1926 |
+
|
| 1927 |
+
|
| 1928 |
+
[1m --> TIME: 2025-06-02 14:12:31 -- STEP: 4708/5792 -- GLOBAL_STEP: 10500[0m
|
| 1929 |
+
| > loss_text_ce: 0.04842893406748772 (0.04363689536684546)
|
| 1930 |
+
| > loss_mel_ce: 3.4084410667419434 (3.33495435723837)
|
| 1931 |
+
| > loss: 0.8642175197601318 (0.8446478130788037)
|
| 1932 |
+
| > current_lr: 5e-06
|
| 1933 |
+
| > step_time: 0.1822 (0.33627098611379563)
|
| 1934 |
+
| > loader_time: 0.0117 (0.011046174631953332)
|
| 1935 |
+
|
| 1936 |
+
|
| 1937 |
+
[1m --> TIME: 2025-06-02 14:13:01 -- STEP: 4758/5792 -- GLOBAL_STEP: 10550[0m
|
| 1938 |
+
| > loss_text_ce: 0.042808979749679565 (0.043614106523299764)
|
| 1939 |
+
| > loss_mel_ce: 3.381239652633667 (3.3350225664477926)
|
| 1940 |
+
| > loss: 0.8560121655464172 (0.8446591681745581)
|
| 1941 |
+
| > current_lr: 5e-06
|
| 1942 |
+
| > step_time: 0.293 (0.3363845983740162)
|
| 1943 |
+
| > loader_time: 0.0131 (0.011052533276394754)
|
| 1944 |
+
|
| 1945 |
+
|
| 1946 |
+
[1m --> TIME: 2025-06-02 14:13:31 -- STEP: 4808/5792 -- GLOBAL_STEP: 10600[0m
|
| 1947 |
+
| > loss_text_ce: 0.04915136843919754 (0.04358621223631336)
|
| 1948 |
+
| > loss_mel_ce: 2.8472352027893066 (3.3347837737515045)
|
| 1949 |
+
| > loss: 0.7240966558456421 (0.8445924964180201)
|
| 1950 |
+
| > current_lr: 5e-06
|
| 1951 |
+
| > step_time: 0.1576 (0.3364169633329015)
|
| 1952 |
+
| > loader_time: 0.0093 (0.011056910884162139)
|
| 1953 |
+
|
| 1954 |
+
|
| 1955 |
+
[1m --> TIME: 2025-06-02 14:14:02 -- STEP: 4858/5792 -- GLOBAL_STEP: 10650[0m
|
| 1956 |
+
| > loss_text_ce: 0.038126781582832336 (0.0435614236114802)
|
| 1957 |
+
| > loss_mel_ce: 3.1234774589538574 (3.3353497294939896)
|
| 1958 |
+
| > loss: 0.790401041507721 (0.8447277882109941)
|
| 1959 |
+
| > current_lr: 5e-06
|
| 1960 |
+
| > step_time: 0.2953 (0.3366490865548782)
|
| 1961 |
+
| > loader_time: 0.0091 (0.01105543447159208)
|
| 1962 |
+
|
| 1963 |
+
|
| 1964 |
+
[1m --> TIME: 2025-06-02 14:14:32 -- STEP: 4908/5792 -- GLOBAL_STEP: 10700[0m
|
| 1965 |
+
| > loss_text_ce: 0.04095133766531944 (0.043551660765238684)
|
| 1966 |
+
| > loss_mel_ce: 3.398160457611084 (3.335550274903345)
|
| 1967 |
+
| > loss: 0.8597779273986816 (0.8447754838623056)
|
| 1968 |
+
| > current_lr: 5e-06
|
| 1969 |
+
| > step_time: 0.2428 (0.33676366579658956)
|
| 1970 |
+
| > loader_time: 0.0128 (0.011054953036506476)
|
| 1971 |
+
|
| 1972 |
+
|
| 1973 |
+
[1m --> TIME: 2025-06-02 14:15:02 -- STEP: 4958/5792 -- GLOBAL_STEP: 10750[0m
|
| 1974 |
+
| > loss_text_ce: 0.04319563880562782 (0.0435207473597263)
|
| 1975 |
+
| > loss_mel_ce: 2.871285915374756 (3.335179869231316)
|
| 1976 |
+
| > loss: 0.7286204099655151 (0.844675154106998)
|
| 1977 |
+
| > current_lr: 5e-06
|
| 1978 |
+
| > step_time: 0.1926 (0.33679866516671897)
|
| 1979 |
+
| > loader_time: 0.0087 (0.011052699087126775)
|
| 1980 |
+
|
| 1981 |
+
|
| 1982 |
+
[1m --> TIME: 2025-06-02 14:15:30 -- STEP: 5008/5792 -- GLOBAL_STEP: 10800[0m
|
| 1983 |
+
| > loss_text_ce: 0.04007870703935623 (0.043486832795789604)
|
| 1984 |
+
| > loss_mel_ce: 3.0385284423828125 (3.3343036392340646)
|
| 1985 |
+
| > loss: 0.7696517705917358 (0.8444476179516727)
|
| 1986 |
+
| > current_lr: 5e-06
|
| 1987 |
+
| > step_time: 0.3957 (0.3367528500267494)
|
| 1988 |
+
| > loader_time: 0.011 (0.011056139684332816)
|
| 1989 |
+
|
| 1990 |
+
|
| 1991 |
+
[1m --> TIME: 2025-06-02 14:16:00 -- STEP: 5058/5792 -- GLOBAL_STEP: 10850[0m
|
| 1992 |
+
| > loss_text_ce: 0.04264426231384277 (0.04346949416229308)
|
| 1993 |
+
| > loss_mel_ce: 3.440608263015747 (3.333787598596726)
|
| 1994 |
+
| > loss: 0.8708131313323975 (0.8443142731536648)
|
| 1995 |
+
| > current_lr: 5e-06
|
| 1996 |
+
| > step_time: 0.3941 (0.3367869499032324)
|
| 1997 |
+
| > loader_time: 0.0124 (0.01105723639352842)
|
| 1998 |
+
|
| 1999 |
+
|
| 2000 |
+
[1m --> TIME: 2025-06-02 14:16:28 -- STEP: 5108/5792 -- GLOBAL_STEP: 10900[0m
|
| 2001 |
+
| > loss_text_ce: 0.034066058695316315 (0.043442419822748615)
|
| 2002 |
+
| > loss_mel_ce: 3.139413595199585 (3.3330060827050625)
|
| 2003 |
+
| > loss: 0.7933698892593384 (0.8441121256388804)
|
| 2004 |
+
| > current_lr: 5e-06
|
| 2005 |
+
| > step_time: 0.1858 (0.33673182608284047)
|
| 2006 |
+
| > loader_time: 0.0091 (0.011057687393836698)
|
| 2007 |
+
|
| 2008 |
+
|
| 2009 |
+
[1m --> TIME: 2025-06-02 14:16:56 -- STEP: 5158/5792 -- GLOBAL_STEP: 10950[0m
|
| 2010 |
+
| > loss_text_ce: 0.04518938437104225 (0.04340843048297699)
|
| 2011 |
+
| > loss_mel_ce: 3.1620471477508545 (3.3323658000780383)
|
| 2012 |
+
| > loss: 0.8018091320991516 (0.843943557682865)
|
| 2013 |
+
| > current_lr: 5e-06
|
| 2014 |
+
| > step_time: 0.2567 (0.3365875314307241)
|
| 2015 |
+
| > loader_time: 0.0094 (0.011058349256230763)
|
| 2016 |
+
|
| 2017 |
+
|
| 2018 |
+
[1m --> TIME: 2025-06-02 14:17:24 -- STEP: 5208/5792 -- GLOBAL_STEP: 11000[0m
|
| 2019 |
+
| > loss_text_ce: 0.04442836344242096 (0.04338480853733139)
|
| 2020 |
+
| > loss_mel_ce: 3.0665812492370605 (3.3311940293799163)
|
| 2021 |
+
| > loss: 0.7777523994445801 (0.8436447095280412)
|
| 2022 |
+
| > current_lr: 5e-06
|
| 2023 |
+
| > step_time: 0.2419 (0.3364083989882436)
|
| 2024 |
+
| > loader_time: 0.0102 (0.011055767169928952)
|
| 2025 |
+
|
| 2026 |
+
|
| 2027 |
+
[1m --> TIME: 2025-06-02 14:17:53 -- STEP: 5258/5792 -- GLOBAL_STEP: 11050[0m
|
| 2028 |
+
| > loss_text_ce: 0.04343422129750252 (0.04335219707671583)
|
| 2029 |
+
| > loss_mel_ce: 2.8784594535827637 (3.3301943961365135)
|
| 2030 |
+
| > loss: 0.7304733991622925 (0.8433866483510418)
|
| 2031 |
+
| > current_lr: 5e-06
|
| 2032 |
+
| > step_time: 0.3317 (0.3363045960517387)
|
| 2033 |
+
| > loader_time: 0.0101 (0.011054212260763195)
|
| 2034 |
+
|
| 2035 |
+
|
| 2036 |
+
[1m --> TIME: 2025-06-02 14:18:21 -- STEP: 5308/5792 -- GLOBAL_STEP: 11100[0m
|
| 2037 |
+
| > loss_text_ce: 0.04893643409013748 (0.04332292118490856)
|
| 2038 |
+
| > loss_mel_ce: 3.3955016136169434 (3.3295369724986603)
|
| 2039 |
+
| > loss: 0.8611094951629639 (0.8432149734929163)
|
| 2040 |
+
| > current_lr: 5e-06
|
| 2041 |
+
| > step_time: 0.326 (0.33622330490053426)
|
| 2042 |
+
| > loader_time: 0.0105 (0.011051318837721364)
|
| 2043 |
+
|
| 2044 |
+
|
| 2045 |
+
[1m --> TIME: 2025-06-02 14:18:52 -- STEP: 5358/5792 -- GLOBAL_STEP: 11150[0m
|
| 2046 |
+
| > loss_text_ce: 0.044002778828144073 (0.04330665982714582)
|
| 2047 |
+
| > loss_mel_ce: 4.1681904792785645 (3.3292624402642472)
|
| 2048 |
+
| > loss: 1.0530483722686768 (0.8431422751134073)
|
| 2049 |
+
| > current_lr: 5e-06
|
| 2050 |
+
| > step_time: 0.5552 (0.3363442467237067)
|
| 2051 |
+
| > loader_time: 0.0151 (0.011057644159146863)
|
| 2052 |
+
|
| 2053 |
+
|
| 2054 |
+
[1m --> TIME: 2025-06-02 14:19:20 -- STEP: 5408/5792 -- GLOBAL_STEP: 11200[0m
|
| 2055 |
+
| > loss_text_ce: 0.038751423358917236 (0.043278468308141)
|
| 2056 |
+
| > loss_mel_ce: 3.3097198009490967 (3.3285519173864784)
|
| 2057 |
+
| > loss: 0.8371177911758423 (0.8429575964764371)
|
| 2058 |
+
| > current_lr: 5e-06
|
| 2059 |
+
| > step_time: 0.2749 (0.3362092402647945)
|
| 2060 |
+
| > loader_time: 0.0135 (0.011058031013731425)
|
| 2061 |
+
|
| 2062 |
+
|
| 2063 |
+
[1m --> TIME: 2025-06-02 14:19:48 -- STEP: 5458/5792 -- GLOBAL_STEP: 11250[0m
|
| 2064 |
+
| > loss_text_ce: 0.041902702301740646 (0.04325453944254458)
|
| 2065 |
+
| > loss_mel_ce: 2.8589980602264404 (3.3282593212620424)
|
| 2066 |
+
| > loss: 0.7252252101898193 (0.8428784652632549)
|
| 2067 |
+
| > current_lr: 5e-06
|
| 2068 |
+
| > step_time: 0.2551 (0.33609242695248986)
|
| 2069 |
+
| > loader_time: 0.0099 (0.011059924594296262)
|
| 2070 |
+
|
| 2071 |
+
|
| 2072 |
+
[1m --> TIME: 2025-06-02 14:20:16 -- STEP: 5508/5792 -- GLOBAL_STEP: 11300[0m
|
| 2073 |
+
| > loss_text_ce: 0.036019548773765564 (0.0432351567869034)
|
| 2074 |
+
| > loss_mel_ce: 2.981379270553589 (3.3276002054294125)
|
| 2075 |
+
| > loss: 0.7543497085571289 (0.842708840619937)
|
| 2076 |
+
| > current_lr: 5e-06
|
| 2077 |
+
| > step_time: 0.1859 (0.3359816792483949)
|
| 2078 |
+
| > loader_time: 0.0119 (0.011056781118125613)
|
| 2079 |
+
|
| 2080 |
+
|
| 2081 |
+
[1m --> TIME: 2025-06-02 14:20:47 -- STEP: 5558/5792 -- GLOBAL_STEP: 11350[0m
|
| 2082 |
+
| > loss_text_ce: 0.04409126192331314 (0.04321853997813861)
|
| 2083 |
+
| > loss_mel_ce: 3.3099846839904785 (3.3272555071314964)
|
| 2084 |
+
| > loss: 0.8385189771652222 (0.842618511869736)
|
| 2085 |
+
| > current_lr: 5e-06
|
| 2086 |
+
| > step_time: 0.3187 (0.33623436845091076)
|
| 2087 |
+
| > loader_time: 0.0121 (0.011060114171102605)
|
| 2088 |
+
|
| 2089 |
+
|
| 2090 |
+
[1m --> TIME: 2025-06-02 14:21:16 -- STEP: 5608/5792 -- GLOBAL_STEP: 11400[0m
|
| 2091 |
+
| > loss_text_ce: 0.03766907751560211 (0.043189963759633916)
|
| 2092 |
+
| > loss_mel_ce: 3.2820019721984863 (3.325929513628915)
|
| 2093 |
+
| > loss: 0.8299177885055542 (0.8422798694412984)
|
| 2094 |
+
| > current_lr: 5e-06
|
| 2095 |
+
| > step_time: 0.3434 (0.3362132219037052)
|
| 2096 |
+
| > loader_time: 0.0099 (0.011059219843990964)
|
| 2097 |
+
|
| 2098 |
+
|
| 2099 |
+
[1m --> TIME: 2025-06-02 14:21:45 -- STEP: 5658/5792 -- GLOBAL_STEP: 11450[0m
|
| 2100 |
+
| > loss_text_ce: 0.0405101515352726 (0.04316613374270707)
|
| 2101 |
+
| > loss_mel_ce: 3.357116937637329 (3.325798077269804)
|
| 2102 |
+
| > loss: 0.8494067788124084 (0.8422410528686782)
|
| 2103 |
+
| > current_lr: 5e-06
|
| 2104 |
+
| > step_time: 0.2667 (0.3361453908035056)
|
| 2105 |
+
| > loader_time: 0.0096 (0.011055887103628554)
|
| 2106 |
+
|
| 2107 |
+
|
| 2108 |
+
[1m --> TIME: 2025-06-02 14:22:11 -- STEP: 5708/5792 -- GLOBAL_STEP: 11500[0m
|
| 2109 |
+
| > loss_text_ce: 0.033331647515296936 (0.04313662911477462)
|
| 2110 |
+
| > loss_mel_ce: 2.8956968784332275 (3.3249534104951377)
|
| 2111 |
+
| > loss: 0.7322571277618408 (0.8420225100117953)
|
| 2112 |
+
| > current_lr: 5e-06
|
| 2113 |
+
| > step_time: 0.1596 (0.3358227129406929)
|
| 2114 |
+
| > loader_time: 0.0095 (0.011055266556409003)
|
| 2115 |
+
|
| 2116 |
+
|
| 2117 |
+
[1m --> TIME: 2025-06-02 14:22:39 -- STEP: 5758/5792 -- GLOBAL_STEP: 11550[0m
|
| 2118 |
+
| > loss_text_ce: 0.042040519416332245 (0.04311639427965342)
|
| 2119 |
+
| > loss_mel_ce: 2.8961079120635986 (3.3243683234152175)
|
| 2120 |
+
| > loss: 0.7345371246337891 (0.8418711795600674)
|
| 2121 |
+
| > current_lr: 5e-06
|
| 2122 |
+
| > step_time: 0.2629 (0.33571524755041415)
|
| 2123 |
+
| > loader_time: 0.0125 (0.011058961322382958)
|
| 2124 |
+
|
| 2125 |
+
|
| 2126 |
+
[1m > EVALUATION [0m
|
| 2127 |
+
|
| 2128 |
+
|
| 2129 |
+
[1m--> EVAL PERFORMANCE[0m
|
| 2130 |
+
| > avg_loader_time:[92m 0.20065663089859956 [0m(-0.0009856695509226354)
|
| 2131 |
+
| > avg_loss_text_ce:[92m 0.0388087550504396 [0m(-0.00597116267992618)
|
| 2132 |
+
| > avg_loss_mel_ce:[92m 3.061407669115872 [0m(-0.1385840406525629)
|
| 2133 |
+
| > avg_loss:[92m 3.100216426418325 [0m(-0.14455520433221114)
|
| 2134 |
+
|
| 2135 |
+
> BEST MODEL : xTTS/checkpoints/GPT_XTTS_FT-June-02-2025_12+22PM-0000000/best_model_11584.pth
|