train.py

import logging
import multiprocessing
import time

logging.getLogger('matplotlib').setLevel(logging.WARNING)
import os
import paddle
#paddle.device.set_device("cpu") #开启可用CPU进行炼丹
trainer:str = "admin" 
from paddle.nn import functional as F
from paddle.io import DataLoader
from visualdl import LogWriter
from paddle.amp import auto_cast, GradScaler

import modules.commons as commons
import utils
from data_utils import TextAudioSpeakerLoader, TextAudioCollate
from models import (
    SynthesizerTrn,
    MultiPeriodDiscriminator,
)
from modules.losses import (
    kl_loss,
    generator_loss, discriminator_loss, feature_loss
)

from modules.mel_processing import mel_spectrogram_torch, spec_to_mel_torch

paddle.set_flags({'FLAGS_cudnn_exhaustive_search': True}) # 使用穷举搜索方法来选择卷积算法
global_step = 0
trainers:list[str] = []
start_time = time.time()


def main():
    """Assume Single Node Multi GPUs Training Only"""
    #assert torch.cuda.is_available(), "CPU training is not allowed."
    hps = utils.get_hparams()

    n_gpus = paddle.device.cuda.device_count()
    os.environ['MASTER_ADDR'] = 'localhost'
    os.environ['MASTER_PORT'] = hps.train.port

    run(n_gpus, hps, )


def run(n_gpus, hps):
    global global_step,trainers,trainer

    trainer = hps.trainer
    rank = 0
    if rank == 0:
        logger = utils.get_logger(hps.model_dir)
        logger.info(hps)
        utils.check_git_hash(hps.model_dir)
        writer = LogWriter(logdir=hps.model_dir)
        writer_eval = LogWriter(logdir=os.path.join(hps.model_dir, "eval"))

    paddle.seed(hps.train.seed)
    paddle.device.set_device('cpu' if paddle.device.get_device() == 'cpu' else 'gpu:' + str(rank))
    collate_fn = TextAudioCollate()
    train_dataset = TextAudioSpeakerLoader(hps.data.training_files, hps)
    num_workers = 5 if multiprocessing.cpu_count() > 4 else multiprocessing.cpu_count()
    train_loader = DataLoader(dataset = train_dataset,
                             num_workers=num_workers,
                             shuffle=False,
                             batch_size=hps.train.batch_size,
                             collate_fn=collate_fn)
    if rank == 0:
        eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps)
        eval_loader = DataLoader(dataset = eval_dataset,
                                num_workers = 1,
                                shuffle = False,
                                batch_size = 1,
                                drop_last = False,
                                collate_fn = collate_fn)

    net_g = SynthesizerTrn(
        hps.data.filter_length // 2 + 1,
        hps.train.segment_size // hps.data.hop_length,
        **hps.model)
    net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm)
    optim_g = paddle.optimizer.AdamW(
        parameters = net_g.parameters(),
        learning_rate = hps.train.learning_rate,
        beta1 = hps.train.betas[0],
        beta2 = hps.train.betas[1],
        epsilon = hps.train.eps)
    optim_d = paddle.optimizer.AdamW(
        parameters = net_d.parameters(),
        learning_rate = hps.train.learning_rate,
        beta1 = hps.train.betas[0],
        beta2 = hps.train.betas[1],
        epsilon = hps.train.eps)

    skip_optimizer = False
    try:
        _, _, _, epoch_str, trainers = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "G_*.pdparams"), net_g,
                                                   optim_g, skip_optimizer)
        _, _, _, epoch_str, trainers = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "D_*.pdparams"), net_d,
                                                   optim_d, skip_optimizer)
        if trainer not in trainers:
            trainers.append(trainer)
        epoch_str = max(epoch_str, 1)
        global_step = (epoch_str - 1) * len(train_loader)
    except Exception as e:
        print(e)
        logger.info("加载旧检查点失败……")
        epoch_str = 1
        global_step = 0
    if skip_optimizer:
        epoch_str = 1
        global_step = 0

    scheduler_g = paddle.optimizer.lr.ExponentialDecay(hps.train.learning_rate, gamma = hps.train.lr_decay, last_epoch = epoch_str - 2)
    scheduler_d = paddle.optimizer.lr.ExponentialDecay(hps.train.learning_rate, gamma = hps.train.lr_decay, last_epoch = epoch_str - 2)

    optim_g = paddle.optimizer.AdamW(
        parameters = net_g.parameters(),
        learning_rate = scheduler_g,
        beta1 = hps.train.betas[0],
        beta2 = hps.train.betas[1],
        epsilon = hps.train.eps)
    optim_d = paddle.optimizer.AdamW(
        parameters = net_d.parameters(),
        learning_rate = scheduler_d,
        beta1 = hps.train.betas[0],
        beta2 = hps.train.betas[1],
        epsilon = hps.train.eps)

    scaler = GradScaler(enable = hps.train.fp16_run)

    for epoch in range(epoch_str, hps.train.epochs + 1):
        if rank == 0:
            train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler,
                               [train_loader, eval_loader], logger, [writer, writer_eval])
        else:
            train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler,
                               [train_loader, None], None, None)
        scheduler_g.step()
        scheduler_d.step()


def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler:GradScaler, loaders, logger:logging.Logger, writers:list or None):
    net_g, net_d = nets
    optim_g, optim_d = optims
    scheduler_g, scheduler_d = schedulers
    train_loader, eval_loader = loaders
    if writers is not None:
        writer, writer_eval = writers

    # train_loader.batch_sampler.set_epoch(epoch)
    global global_step

    net_g.train()
    net_d.train()
    for batch_idx, items in enumerate(train_loader):
        c, f0, spec, y, spk, lengths, uv = items
        g = spk
        mel = spec_to_mel_torch(
            spec,
            hps.data.filter_length,
            hps.data.n_mel_channels,
            hps.data.sampling_rate,
            hps.data.mel_fmin,
            hps.data.mel_fmax)

        with auto_cast(enable=hps.train.fp16_run):

            y_hat, ids_slice, z_mask, \
            (z, z_p, m_p, logs_p, m_q, logs_q), pred_lf0, norm_lf0, lf0 = net_g(c, f0, uv, spec, g=g, c_lengths=lengths,
                                                                                spec_lengths=lengths)

            y_mel = commons.slice_segments(mel, ids_slice, hps.train.segment_size // hps.data.hop_length)
            y_hat_mel = mel_spectrogram_torch(
                y_hat.squeeze(1),
                hps.data.filter_length,
                hps.data.n_mel_channels,
                hps.data.sampling_rate,
                hps.data.hop_length,
                hps.data.win_length,
                hps.data.mel_fmin,
                hps.data.mel_fmax
            )
            y = commons.slice_segments(y, ids_slice * hps.data.hop_length, hps.train.segment_size)  # slice

            # Discriminator
            y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach())

            with auto_cast(enable=False):
                loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(y_d_hat_r, y_d_hat_g)
                loss_disc_all = loss_disc

        optim_d.clear_grad()
        scaler.scale(loss_disc_all).backward(retain_graph = True) # 将 Tensor 乘上缩放因子，返回缩放后的输出，返回loss然后反向传播
        scaler.unscale_(optim_d) # 将参数的梯度除去缩放比例。
        grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None)
        scaler.step(optim_d)
        with auto_cast(enable=hps.train.fp16_run):
            # Generator
            y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(y, y_hat)
            with auto_cast(enable=False):
                loss_mel = F.l1_loss(y_mel, y_hat_mel) * hps.train.c_mel
                loss_kl = kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * hps.train.c_kl
                loss_fm = feature_loss(fmap_r, fmap_g)
                loss_gen, losses_gen = generator_loss(y_d_hat_g)
                loss_lf0 = F.mse_loss(pred_lf0, lf0)
                loss_gen_all = loss_gen + loss_fm + loss_mel + loss_kl + loss_lf0
        optim_g.clear_grad()
        scaler.scale(loss_gen_all).backward(retain_graph = True)
        scaler.unscale_(optim_g)
        grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None)
        scaler.step(optim_g)
        scaler.update()

        #lr = optim_g.state_dict()['LR_Scheduler']['last_lr'] # paddle优化器特有的字典
        #losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_kl]
        #logger.info(f"损失：{[x.item() for x in losses]}，步数：{global_step}，学习率：{lr}") # 梅花自己看的~

        if rank == 0:
            if global_step % hps.train.log_interval == 0:
                lr = optim_g.state_dict()['LR_Scheduler']['last_lr'] # paddle优化器特有的字典
                losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_kl]
                logger.info('训练回合：{} [{:.0f}%]'.format(
                    epoch,
                    100. * batch_idx / len(train_loader)))
                logger.info(f"损失：{[x.item() for x in losses]}，步数：{global_step}，学习率：{lr}")

                scalar_dict = {"损失/生成器/总损失": loss_gen_all, "损失/判别器/总损失": loss_disc_all, "学习率": lr,
                               "归一化判别器梯度": grad_norm_d, "归一化生成器梯度": grad_norm_g}
                scalar_dict.update({"损失/生成器/特征匹配损失": loss_fm, "损失/生成器/梅尔频谱损失": loss_mel, "损失/生成器/KL散度": loss_kl,
                                    "损失/生成器/基音损失": loss_lf0})

                image_dict = {
                    "切片/原始梅尔频谱图": utils.plot_spectrogram_to_numpy(y_mel[0].detach().numpy()),
                    "切片/生成梅尔频谱图": utils.plot_spectrogram_to_numpy(y_hat_mel[0].detach().numpy()),
                    "全部/梅尔频谱图": utils.plot_spectrogram_to_numpy(mel[0].detach().numpy()),
                    "全部/基音损失": utils.plot_data_to_numpy(lf0[0, 0, :].numpy(),
                                                          pred_lf0[0, 0, :].detach().numpy()),
                    "全部/归一化基音损失": utils.plot_data_to_numpy(lf0[0, 0, :].numpy(),
                                                               norm_lf0[0, 0, :].detach().numpy())
                }

                utils.summarize(
                    writer=writer,
                    global_step=global_step,
                    images=image_dict,
                    scalars=scalar_dict
                )
            if global_step % hps.train.eval_interval == 0:
                if hps.clean_logs:
                    os.system('clear')
                evaluate(hps, net_g, eval_loader, writer_eval)
                utils.save_checkpoint(net_g, optim_g, hps.train.learning_rate, epoch,
                                      os.path.join(hps.model_dir, "G_{}.pdparams".format(global_step)), trainers)
                utils.save_checkpoint(net_d, optim_d, hps.train.learning_rate, epoch,
                                      os.path.join(hps.model_dir, "D_{}.pdparams".format(global_step)), trainers)
                keep_ckpts = getattr(hps.train, 'keep_ckpts', 0)
                if keep_ckpts > 0:
                    utils.clean_checkpoints(path_to_models=hps.model_dir, n_ckpts_to_keep=keep_ckpts, sort_by_time=True)

        global_step += 1

    if rank == 0:
        global start_time
        now = time.time()
        durtaion = format(now - start_time, '.2f')
        logger.info(f'====> 回合：{epoch}, 消耗 {durtaion} 秒')
        start_time = now


def evaluate(hps, generator, eval_loader, writer_eval):
    generator.eval()
    image_dict = {}
    audio_dict = {}
    with paddle.no_grad():
        for batch_idx, items in enumerate(eval_loader):
            c, f0, spec, y, spk, _, uv = items
            g = spk[:1]
            spec, y = spec[:1], y[:1]
            c = c[:1]
            f0 = f0[:1]
            uv= uv[:1]
            mel = spec_to_mel_torch(
                spec,
                hps.data.filter_length,
                hps.data.n_mel_channels,
                hps.data.sampling_rate,
                hps.data.mel_fmin,
                hps.data.mel_fmax)
            y_hat = generator.infer(c, f0, uv, g=g)
            y_hat_mel = mel_spectrogram_torch(
                y_hat.squeeze(1).cast('float32'),
                hps.data.filter_length,
                hps.data.n_mel_channels,
                hps.data.sampling_rate,
                hps.data.hop_length,
                hps.data.win_length,
                hps.data.mel_fmin,
                hps.data.mel_fmax
            )

            audio_dict.update({
                f"生成器测试数据/音频_{batch_idx}": y_hat[0],
                f"地标真实数据/音频_{batch_idx}": y[0]
            })
        image_dict.update({
            "生成器测试数据/梅尔频谱图": utils.plot_spectrogram_to_numpy(y_hat_mel[0].numpy()),
            "地标真实数据/梅尔频谱图": utils.plot_spectrogram_to_numpy(mel[0].numpy())
        })
    utils.summarize(
        writer=writer_eval,
        global_step=global_step,
        images=image_dict,
        audios=audio_dict,
        audio_sampling_rate=hps.data.sampling_rate
    )
    generator.train()


if __name__ == "__main__":
    main()