Trainer

    def update(self, x):
        """Update the running statistics with a new value x."""
        self.count += 1
        delta = x - self.mean
        self.mean += delta / self.count
        delta2 = x - self.mean
        self.M2 += delta * delta2

    @property
    def variance(self):
        """Return the sample variance. Returns NaN if fewer than two samples."""
        return self.M2 / (self.count - 1) if self.count > 1 else float('nan')

    @property
    def std(self):
        """Return the sample standard deviation."""
        return math.sqrt(self.variance)

class Trainer:
    def __init__(
        self,
        model: UniModel,
        epochs,
        learning_rate,
        num_warmup_updates=20000,
        save_per_updates=1000,
        checkpoint_path=None,
        batch_size=32,
        batch_size_type: str = "sample",
        max_samples=32,
        grad_accumulation_steps=1,
        max_grad_norm=1.0,
        noise_scheduler: str | None = None,
        duration_predictor: torch.nn.Module | None = None,
        wandb_project="test_e2-tts",
        wandb_run_name="test_run",
        wandb_resume_id: str = None,
        last_per_steps=None,
        log_step=1000,
        accelerate_kwargs: dict = dict(),
        bnb_optimizer: bool = False,
        scale: float = 1.0,

        # training parameters for DMDSpeech
        num_student_step: int = 1,
        gen_update_ratio: int = 5,
        lambda_discriminator_loss: float = 1.0,
        lambda_generator_loss: float = 1.0,
        lambda_ctc_loss: float = 1.0,
        lambda_sim_loss: float = 1.0,

        num_GAN: int = 5000,
        num_D: int = 500,
        num_ctc: int = 5000,
        num_sim: int = 10000,
        num_simu: int = 1000,
    ):
        ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=True)

        logger = "wandb" if wandb.api.api_key else None
        print(f"Using logger: {logger}")

        self.accelerator = Accelerator(
            log_with=logger,
            kwargs_handlers=[ddp_kwargs],
            gradient_accumulation_steps=grad_accumulation_steps,
            **accelerate_kwargs,
        )

        if logger == "wandb":
            if exists(wandb_resume_id):
                init_kwargs = {"wandb": {"resume": "allow", "name": wandb_run_name, "id": wandb_resume_id}}
            else:
                init_kwargs = {"wandb": {"resume": "allow", "name": wandb_run_name}}
            self.accelerator.init_trackers(
                project_name=wandb_project,
                init_kwargs=init_kwargs,
                config={
                    "epochs": epochs,
                    "learning_rate": learning_rate,
                    "num_warmup_updates": num_warmup_updates,
                    "batch_size": batch_size,
                    "batch_size_type": batch_size_type,
                    "max_samples": max_samples,
                    "grad_accumulation_steps": grad_accumulation_steps,
                    "max_grad_norm": max_grad_norm,
                    "gpus": self.accelerator.num_processes,
                    "noise_scheduler": noise_scheduler,
                },
            )

        self.model = model

        self.scale = scale

        self.epochs = epochs
        self.num_warmup_updates = num_warmup_updates
        self.save_per_updates = save_per_updates
        self.last_per_steps = default(last_per_steps, save_per_updates * grad_accumulation_steps)
        self.checkpoint_path = default(checkpoint_path, "ckpts/test_e2-tts")

        self.batch_size = batch_size
        self.batch_size_type = batch_size_type
        self.max_samples = max_samples
        self.grad_accumulation_steps = grad_accumulation_steps
        self.max_grad_norm = max_grad_norm

        self.noise_scheduler = noise_scheduler

        self.duration_predictor = duration_predictor

        self.log_step = log_step

        self.gen_update_ratio = gen_update_ratio # number of generator updates per guidance (fake score function and discriminator) update
        self.lambda_discriminator_loss = lambda_discriminator_loss # weight for discriminator loss (L_adv)
        self.lambda_generator_loss = lambda_generator_loss # weight for generator loss (L_adv)
        self.lambda_ctc_loss = lambda_ctc_loss # weight for ctc loss
        self.lambda_sim_loss = lambda_sim_loss # weight for similarity loss

        # create distillation schedule for student model
        self.student_steps = (
                torch.linspace(0.0, 1.0, num_student_step + 1)[:-1])

        self.GAN = model.guidance_model.gen_cls_loss # whether to use GAN training
        self.num_GAN = num_GAN # number of steps before adversarial training
        self.num_D = num_D # number of steps to train the discriminator before adversarial training
        self.num_ctc = num_ctc # number of steps before CTC training
        self.num_sim = num_sim # number of steps before similarity training
        self.num_simu = num_simu # number of steps before using simulated data

        # Assuming `self.model.fake_unet.parameters()` and `self.model.guidance_model.parameters()` are accessible
        if bnb_optimizer:
            import bitsandbytes as bnb
            self.optimizer_generator = bnb.optim.AdamW8bit(self.model.feedforward_model.parameters(), lr=learning_rate)
            self.optimizer_guidance = bnb.optim.AdamW8bit(self.model.guidance_model.parameters(), lr=learning_rate)
        else:
            self.optimizer_generator = AdamW(self.model.feedforward_model.parameters(), lr=learning_rate, eps=1e-7)
            self.optimizer_guidance = AdamW(self.model.guidance_model.parameters(), lr=learning_rate, eps=1e-7)

        self.model, self.optimizer_generator, self.optimizer_guidance = self.accelerator.prepare(self.model, self.optimizer_generator, self.optimizer_guidance)

        self.generator_norm = RunningStats()
        self.guidance_norm = RunningStats()

    @property
    def is_main(self):
        return self.accelerator.is_main_process

    def save_checkpoint(self, step, last=False):
        self.accelerator.wait_for_everyone()
        if self.is_main:
            checkpoint = dict(
                model_state_dict=self.accelerator.unwrap_model(self.model).state_dict(),
                optimizer_generator_state_dict=self.accelerator.unwrap_model(self.optimizer_generator).state_dict(),
                optimizer_guidance_state_dict=self.accelerator.unwrap_model(self.optimizer_guidance).state_dict(),
                scheduler_generator_state_dict=self.scheduler_generator.state_dict(),
                scheduler_guidance_state_dict=self.scheduler_guidance.state_dict(),
                step=step,
            )

            if not os.path.exists(self.checkpoint_path):
                os.makedirs(self.checkpoint_path)
            if last:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_last.pt")
                print(f"Saved last checkpoint at step {step}")
            else:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_{step}.pt")

    def load_checkpoint(self):
        if (
            not exists(self.checkpoint_path)
            or not os.path.exists(self.checkpoint_path)
            or not os.listdir(self.checkpoint_path)
        ):
            return 0

        self.accelerator.wait_for_everyone()
        if "model_last.pt" in os.listdir(self.checkpoint_path):
            latest_checkpoint = "model_last.pt"
        else:
            latest_checkpoint = sorted(
                [f for f in os.listdir(self.checkpoint_path) if f.endswith(".pt")],
                key=lambda x: int("".join(filter(str.isdigit, x))),
            )[-1]
        # checkpoint = torch.load(f"{self.checkpoint_path}/{latest_checkpoint}", map_location=self.accelerator.device)  # rather use accelerator.load_state ಥ_ಥ
        checkpoint = torch.load(f"{self.checkpoint_path}/{latest_checkpoint}", weights_only=True, map_location="cpu")

        self.accelerator.unwrap_model(self.model).load_state_dict(checkpoint["model_state_dict"], strict=False)
        # self.accelerator.unwrap_model(self.optimizer_generator).load_state_dict(checkpoint["optimizer_generator_state_dict"])
        # self.accelerator.unwrap_model(self.optimizer_guidance).load_state_dict(checkpoint["optimizer_guidance_state_dict"])
        # if self.scheduler_guidance:
        #     self.scheduler_guidance.load_state_dict(checkpoint["scheduler_guidance_state_dict"])
        # if self.scheduler_generator:
        #     self.scheduler_generator.load_state_dict(checkpoint["scheduler_generator_state_dict"])
        step = checkpoint["step"]

        del checkpoint
        gc.collect()
        return step

    def train(self, train_dataset: Dataset, num_workers=64, resumable_with_seed: int = None, vocoder: nn.Module = None):
        if exists(resumable_with_seed):
            generator = torch.Generator()
            generator.manual_seed(resumable_with_seed)
        else:
            generator = None

        if self.batch_size_type == "sample":
            train_dataloader = DataLoader(
                train_dataset,
                collate_fn=collate_fn,
                num_workers=num_workers,
                pin_memory=True,
                persistent_workers=True,
                batch_size=self.batch_size,
                shuffle=True,
                generator=generator,
            )
        elif self.batch_size_type == "frame":
            self.accelerator.even_batches = False
            sampler = SequentialSampler(train_dataset)
            batch_sampler = DynamicBatchSampler(
                sampler, self.batch_size, max_samples=self.max_samples, random_seed=resumable_with_seed, drop_last=False
            )
            train_dataloader = DataLoader(
                train_dataset,
                collate_fn=collate_fn,
                num_workers=num_workers,
                pin_memory=True,
                persistent_workers=True,
                batch_sampler=batch_sampler,
            )
        else:
            raise ValueError(f"batch_size_type must be either 'sample' or 'frame', but received {self.batch_size_type}")

        #  accelerator.prepare() dispatches batches to devices;
        #  which means the length of dataloader calculated before, should consider the number of devices
        warmup_steps = (
            self.num_warmup_updates * self.accelerator.num_processes
        )

        # consider a fixed warmup steps while using accelerate multi-gpu ddp
        # otherwise by default with split_batches=False, warmup steps change with num_processes
        total_steps = len(train_dataloader) * self.epochs / self.grad_accumulation_steps
        decay_steps = total_steps - warmup_steps

        warmup_scheduler_generator = LinearLR(self.optimizer_generator, start_factor=1e-8, end_factor=1.0, total_iters=warmup_steps // (self.gen_update_ratio * self.grad_accumulation_steps))
        decay_scheduler_generator = LinearLR(self.optimizer_generator, start_factor=1.0, end_factor=1e-8, total_iters=decay_steps // (self.gen_update_ratio * self.grad_accumulation_steps))
        self.scheduler_generator = SequentialLR(self.optimizer_generator, schedulers=[warmup_scheduler_generator, decay_scheduler_generator], milestones=[warmup_steps // (self.gen_update_ratio * self.grad_accumulation_steps)])

        warmup_scheduler_guidance = LinearLR(self.optimizer_guidance, start_factor=1e-8, end_factor=1.0, total_iters=warmup_steps)
        decay_scheduler_guidance = LinearLR(self.optimizer_guidance, start_factor=1.0, end_factor=1e-8, total_iters=decay_steps)
        self.scheduler_guidance = SequentialLR(self.optimizer_guidance, schedulers=[warmup_scheduler_guidance, decay_scheduler_guidance], milestones=[warmup_steps])

        train_dataloader, self.scheduler_generator, self.scheduler_guidance = self.accelerator.prepare(
            train_dataloader, self.scheduler_generator, self.scheduler_guidance
        )  # actual steps = 1 gpu steps / gpus
        start_step = self.load_checkpoint()
        global_step = start_step

        if exists(resumable_with_seed):
            orig_epoch_step = len(train_dataloader)
            skipped_epoch = int(start_step // orig_epoch_step)
            skipped_batch = start_step % orig_epoch_step
            skipped_dataloader = self.accelerator.skip_first_batches(train_dataloader, num_batches=skipped_batch)
        else:
            skipped_epoch = 0

        for epoch in range(skipped_epoch, self.epochs):
            self.model.train()
            if exists(resumable_with_seed) and epoch == skipped_epoch:
                progress_bar = tqdm(
                    skipped_dataloader,
                    desc=f"Epoch {epoch+1}/{self.epochs}",
                    unit="step",
                    disable=not self.accelerator.is_local_main_process,
                    initial=skipped_batch,
                    total=orig_epoch_step,
                )
            else:
                progress_bar = tqdm(
                    train_dataloader,
                    desc=f"Epoch {epoch+1}/{self.epochs}",
                    unit="step",
                    disable=not self.accelerator.is_local_main_process,
                )

            for batch in progress_bar:
                update_generator = global_step % self.gen_update_ratio == 0

                with self.accelerator.accumulate(self.model):
                    metrics = {}
                    text_inputs = batch["text"]
                    mel_spec = batch["mel"].permute(0, 2, 1)
                    mel_lengths = batch["mel_lengths"]

                    mel_spec = mel_spec / self.scale

                    guidance_loss_dict, guidance_log_dict = self.model(inp=mel_spec,
                                                                text=text_inputs,
                                                                lens=mel_lengths,
                                                                student_steps=self.student_steps,
                                                                update_generator=False,
                                                                use_simulated=global_step >= self.num_simu,
                                                                )

                    # if self.GAN and update_generator:
                    #     # only add discriminator loss if GAN is enabled and generator is being updated
                    #     guidance_cls_loss = guidance_loss_dict["guidance_cls_loss"] * (self.lambda_discriminator_loss if global_step >= self.num_GAN and update_generator else 0)
                    #     metrics['loss/discriminator_loss'] = guidance_loss_dict["guidance_cls_loss"]
                    #     self.accelerator.backward(guidance_cls_loss, retain_graph=True)

                    #     if self.max_grad_norm > 0 and self.accelerator.sync_gradients:
                    #         metrics['grad_norm_guidance'] = self.accelerator.clip_grad_norm_(self.model.parameters(), self.max_grad_norm)

                    guidance_loss = 0
                    guidance_loss += guidance_loss_dict["loss_fake_mean"]
                    metrics['loss/fake_score'] = guidance_loss_dict["loss_fake_mean"]
                    metrics["loss/guidance_loss"] = guidance_loss

                    if self.GAN and update_generator:
                        # only add discriminator loss if GAN is enabled and generator is being updated
                        guidance_cls_loss = guidance_loss_dict["guidance_cls_loss"] * (self.lambda_discriminator_loss if global_step >= self.num_GAN and update_generator else 0)
                        metrics['loss/discriminator_loss'] = guidance_loss_dict["guidance_cls_loss"]

                        guidance_loss += guidance_cls_loss

                    self.accelerator.backward(guidance_loss)

                    if self.max_grad_norm > 0 and self.accelerator.sync_gradients:
                        metrics['grad_norm_guidance'] = self.accelerator.clip_grad_norm_(self.model.parameters(), self.max_grad_norm)

                        # if self.guidance_norm.count < 100:
                        #     self.guidance_norm.update(metrics['grad_norm_guidance'])

                        # if metrics['grad_norm_guidance'] > self.guidance_norm.mean + 5 * self.guidance_norm.std:
                        #     self.optimizer_generator.zero_grad()
                        #     self.optimizer_guidance.zero_grad()
                        #     print("Gradient explosion detected. Skipping batch.")
                        # elif self.guidance_norm.count >= 100:
                        #     self.guidance_norm.update(metrics['grad_norm_guidance'])

                    self.optimizer_guidance.step()
                    self.scheduler_guidance.step()
                    self.optimizer_guidance.zero_grad()
                    self.optimizer_generator.zero_grad()  # zero out the generator's gradient as well

                    if update_generator:
                        generator_loss_dict, generator_log_dict = self.model(inp=mel_spec,
                                                                        text=text_inputs,
                                                                        lens=mel_lengths,
                                                                        student_steps=self.student_steps,
                                                                        update_generator=True,
                                                                        use_simulated=global_step >= self.num_ctc,
                                                                        )
                        # if self.GAN:
                        #     gen_cls_loss = generator_loss_dict["gen_cls_loss"] * (self.lambda_generator_loss if global_step >= (self.num_GAN + self.num_D) and update_generator else 0)
                        #     metrics["loss/gen_cls_loss"] = generator_loss_dict["gen_cls_loss"]

                        #     self.accelerator.backward(gen_cls_loss, retain_graph=True)

                        #     if self.max_grad_norm > 0 and self.accelerator.sync_gradients:
                        #         metrics['grad_norm_generator'] = self.accelerator.clip_grad_norm_(self.model.parameters(), self.max_grad_norm)

                        generator_loss = 0
                        generator_loss += generator_loss_dict["loss_dm"]
                        if "loss_mse" in generator_loss_dict:
                            generator_loss += generator_loss_dict["loss_mse"]
                        generator_loss += generator_loss_dict["loss_ctc"] * (self.lambda_ctc_loss if global_step >= self.num_ctc else 0)
                        generator_loss += generator_loss_dict["loss_sim"] * (self.lambda_sim_loss if global_step >= self.num_sim else 0)
                        generator_loss += generator_loss_dict["loss_kl"] * (self.lambda_ctc_loss if global_step >= self.num_ctc else 0)
                        if self.GAN:
                            gen_cls_loss = generator_loss_dict["gen_cls_loss"] * (self.lambda_generator_loss if global_step >= (self.num_GAN + self.num_D) and update_generator else 0)
                            metrics["loss/gen_cls_loss"] = generator_loss_dict["gen_cls_loss"]
                            generator_loss += gen_cls_loss

                        metrics['loss/dm_loss'] = generator_loss_dict["loss_dm"]
                        metrics['loss/ctc_loss'] = generator_loss_dict["loss_ctc"]

                        metrics['loss/similarity_loss'] = generator_loss_dict["loss_sim"]
                        metrics['loss/generator_loss'] = generator_loss

                        if "loss_mse" in generator_loss_dict and ge
# ... truncated ...


        """Update the running statistics with a new value x."""
        self.count += 1
        delta = x - self.mean
        self.mean += delta / self.count
        delta2 = x - self.mean
        self.M2 += delta * delta2

    @property
    def variance(self):
        """Return the sample variance. Returns NaN if fewer than two samples."""
        return self.M2 / (self.count - 1) if self.count > 1 else float('nan')

    @property
    def std(self):
        """Return the sample standard deviation."""
        return math.sqrt(self.variance)

class Trainer:
    def __init__(
        self,
        model: UniModel,
        epochs,
        learning_rate,
        num_warmup_updates=20000,
        save_per_updates=1000,
        checkpoint_path=None,
        batch_size=32,
        batch_size_type: str = "sample",
        max_samples=32,
        grad_accumulation_steps=1,
        max_grad_norm=1.0,
        noise_scheduler: str | None = None,
        duration_predictor: torch.nn.Module | None = None,
        wandb_project="test_e2-tts",
        wandb_run_name="test_run",
        wandb_resume_id: str = None,
        last_per_steps=None,
        log_step=1000,
        accelerate_kwargs: dict = dict(),
        bnb_optimizer: bool = False,
        scale: float = 1.0,

        # training parameters for DMDSpeech
        num_student_step: int = 1,
        gen_update_ratio: int = 5,
        lambda_discriminator_loss: float = 1.0,
        lambda_generator_loss: float = 1.0,
        lambda_ctc_loss: float = 1.0,
        lambda_sim_loss: float = 1.0,

        num_GAN: int = 5000,
        num_D: int = 500,
        num_ctc: int = 5000,
        num_sim: int = 10000,
        num_simu: int = 1000,
    ):
        ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=True)

        logger = "wandb" if wandb.api.api_key else None
        print(f"Using logger: {logger}")

        self.accelerator = Accelerator(
            log_with=logger,
            kwargs_handlers=[ddp_kwargs],
            gradient_accumulation_steps=grad_accumulation_steps,
            **accelerate_kwargs,
        )

        if logger == "wandb":
            if exists(wandb_resume_id):
                init_kwargs = {"wandb": {"resume": "allow", "name": wandb_run_name, "id": wandb_resume_id}}
            else:
                init_kwargs = {"wandb": {"resume": "allow", "name": wandb_run_name}}
            self.accelerator.init_trackers(
                project_name=wandb_project,
                init_kwargs=init_kwargs,
                config={
                    "epochs": epochs,
                    "learning_rate": learning_rate,
                    "num_warmup_updates": num_warmup_updates,
                    "batch_size": batch_size,
                    "batch_size_type": batch_size_type,
                    "max_samples": max_samples,
                    "grad_accumulation_steps": grad_accumulation_steps,
                    "max_grad_norm": max_grad_norm,
                    "gpus": self.accelerator.num_processes,
                    "noise_scheduler": noise_scheduler,
                },
            )

        self.model = model

        self.scale = scale

        self.epochs = epochs
        self.num_warmup_updates = num_warmup_updates
        self.save_per_updates = save_per_updates
        self.last_per_steps = default(last_per_steps, save_per_updates * grad_accumulation_steps)
        self.checkpoint_path = default(checkpoint_path, "ckpts/test_e2-tts")

        self.batch_size = batch_size
        self.batch_size_type = batch_size_type
        self.max_samples = max_samples
        self.grad_accumulation_steps = grad_accumulation_steps
        self.max_grad_norm = max_grad_norm

        self.noise_scheduler = noise_scheduler

        self.duration_predictor = duration_predictor

        self.log_step = log_step

        self.gen_update_ratio = gen_update_ratio # number of generator updates per guidance (fake score function and discriminator) update
        self.lambda_discriminator_loss = lambda_discriminator_loss # weight for discriminator loss (L_adv)
        self.lambda_generator_loss = lambda_generator_loss # weight for generator loss (L_adv)
        self.lambda_ctc_loss = lambda_ctc_loss # weight for ctc loss
        self.lambda_sim_loss = lambda_sim_loss # weight for similarity loss

        # create distillation schedule for student model
        self.student_steps = (
                torch.linspace(0.0, 1.0, num_student_step + 1)[:-1])

        self.GAN = model.guidance_model.gen_cls_loss # whether to use GAN training
        self.num_GAN = num_GAN # number of steps before adversarial training
        self.num_D = num_D # number of steps to train the discriminator before adversarial training
        self.num_ctc = num_ctc # number of steps before CTC training
        self.num_sim = num_sim # number of steps before similarity training
        self.num_simu = num_simu # number of steps before using simulated data

        # Assuming `self.model.fake_unet.parameters()` and `self.model.guidance_model.parameters()` are accessible
        if bnb_optimizer:
            import bitsandbytes as bnb
            self.optimizer_generator = bnb.optim.AdamW8bit(self.model.feedforward_model.parameters(), lr=learning_rate)
            self.optimizer_guidance = bnb.optim.AdamW8bit(self.model.guidance_model.parameters(), lr=learning_rate)
        else:
            self.optimizer_generator = AdamW(self.model.feedforward_model.parameters(), lr=learning_rate, eps=1e-7)
            self.optimizer_guidance = AdamW(self.model.guidance_model.parameters(), lr=learning_rate, eps=1e-7)

        self.model, self.optimizer_generator, self.optimizer_guidance = self.accelerator.prepare(self.model, self.optimizer_generator, self.optimizer_guidance)

        self.generator_norm = RunningStats()
        self.guidance_norm = RunningStats()

    @property
    def is_main(self):
        return self.accelerator.is_main_process

    def save_checkpoint(self, step, last=False):
        self.accelerator.wait_for_everyone()
        if self.is_main:
            checkpoint = dict(
                model_state_dict=self.accelerator.unwrap_model(self.model).state_dict(),
                optimizer_generator_state_dict=self.accelerator.unwrap_model(self.optimizer_generator).state_dict(),
                optimizer_guidance_state_dict=self.accelerator.unwrap_model(self.optimizer_guidance).state_dict(),
                scheduler_generator_state_dict=self.scheduler_generator.state_dict(),
                scheduler_guidance_state_dict=self.scheduler_guidance.state_dict(),
                step=step,
            )

            if not os.path.exists(self.checkpoint_path):
                os.makedirs(self.checkpoint_path)


update

from accelerate import Accelerator
from accelerate.utils import DistributedDataParallelKwargs

from unimodel import UniModel
from f5_tts.model import CFM
from f5_tts.model.utils import exists, default
from f5_tts.model.dataset import DynamicBatchSampler, collate_fn

# trainer

import math

class RunningStats:
    def __init__(self):
        self.count = 0
        self.mean = 0.0
        self.M2 = 0.0  # Sum of squared differences from the current mean

    def update(self, x):
        """Update the running statistics with a new value x."""
        self.count += 1
        delta = x - self.mean
        self.mean += delta / self.count
        delta2 = x - self.mean
        self.M2 += delta * delta2

    @property
    def variance(self):
        """Return the sample variance. Returns NaN if fewer than two samples."""
        return self.M2 / (self.count - 1) if self.count > 1 else float('nan')

    @property
    def std(self):
        """Return the sample standard deviation."""
        return math.sqrt(self.variance)

class Trainer:
    def __init__(
        self,
        model: UniModel,
        epochs,
        learning_rate,
        num_warmup_updates=20000,


variance

# trainer

import math

class RunningStats:
    def __init__(self):
        self.count = 0
        self.mean = 0.0
        self.M2 = 0.0  # Sum of squared differences from the current mean

    def update(self, x):
        """Update the running statistics with a new value x."""
        self.count += 1
        delta = x - self.mean
        self.mean += delta / self.count
        delta2 = x - self.mean
        self.M2 += delta * delta2

    @property
    def variance(self):
        """Return the sample variance. Returns NaN if fewer than two samples."""
        return self.M2 / (self.count - 1) if self.count > 1 else float('nan')

    @property
    def std(self):
        """Return the sample standard deviation."""
        return math.sqrt(self.variance)

class Trainer:
    def __init__(
        self,
        model: UniModel,
        epochs,
        learning_rate,
        num_warmup_updates=20000,
        save_per_updates=1000,
        checkpoint_path=None,
        batch_size=32,
        batch_size_type: str = "sample",
        max_samples=32,


std

class RunningStats:
    def __init__(self):
        self.count = 0
        self.mean = 0.0
        self.M2 = 0.0  # Sum of squared differences from the current mean

    def update(self, x):
        """Update the running statistics with a new value x."""
        self.count += 1
        delta = x - self.mean
        self.mean += delta / self.count
        delta2 = x - self.mean
        self.M2 += delta * delta2

    @property
    def variance(self):
        """Return the sample variance. Returns NaN if fewer than two samples."""
        return self.M2 / (self.count - 1) if self.count > 1 else float('nan')

    @property
    def std(self):
        """Return the sample standard deviation."""
        return math.sqrt(self.variance)

class Trainer:
    def __init__(
        self,
        model: UniModel,
        epochs,
        learning_rate,
        num_warmup_updates=20000,
        save_per_updates=1000,
        checkpoint_path=None,
        batch_size=32,
        batch_size_type: str = "sample",
        max_samples=32,
        grad_accumulation_steps=1,
        max_grad_norm=1.0,
        noise_scheduler: str | None = None,
        duration_predictor: torch.nn.Module | None = None,
        wandb_project="test_e2-tts",


        self.num_ctc = num_ctc # number of steps before CTC training
        self.num_sim = num_sim # number of steps before similarity training
        self.num_simu = num_simu # number of steps before using simulated data

        # Assuming `self.model.fake_unet.parameters()` and `self.model.guidance_model.parameters()` are accessible
        if bnb_optimizer:
            import bitsandbytes as bnb
            self.optimizer_generator = bnb.optim.AdamW8bit(self.model.feedforward_model.parameters(), lr=learning_rate)
            self.optimizer_guidance = bnb.optim.AdamW8bit(self.model.guidance_model.parameters(), lr=learning_rate)
        else:
            self.optimizer_generator = AdamW(self.model.feedforward_model.parameters(), lr=learning_rate, eps=1e-7)
            self.optimizer_guidance = AdamW(self.model.guidance_model.parameters(), lr=learning_rate, eps=1e-7)

        self.model, self.optimizer_generator, self.optimizer_guidance = self.accelerator.prepare(self.model, self.optimizer_generator, self.optimizer_guidance)

        self.generator_norm = RunningStats()
        self.guidance_norm = RunningStats()

    @property
    def is_main(self):
        return self.accelerator.is_main_process

    def save_checkpoint(self, step, last=False):
        self.accelerator.wait_for_everyone()
        if self.is_main:
            checkpoint = dict(
                model_state_dict=self.accelerator.unwrap_model(self.model).state_dict(),
                optimizer_generator_state_dict=self.accelerator.unwrap_model(self.optimizer_generator).state_dict(),
                optimizer_guidance_state_dict=self.accelerator.unwrap_model(self.optimizer_guidance).state_dict(),
                scheduler_generator_state_dict=self.scheduler_generator.state_dict(),
                scheduler_guidance_state_dict=self.scheduler_guidance.state_dict(),
                step=step,
            )

            if not os.path.exists(self.checkpoint_path):
                os.makedirs(self.checkpoint_path)
            if last:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_last.pt")
                print(f"Saved last checkpoint at step {step}")
            else:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_{step}.pt")


        # Assuming `self.model.fake_unet.parameters()` and `self.model.guidance_model.parameters()` are accessible
        if bnb_optimizer:
            import bitsandbytes as bnb
            self.optimizer_generator = bnb.optim.AdamW8bit(self.model.feedforward_model.parameters(), lr=learning_rate)
            self.optimizer_guidance = bnb.optim.AdamW8bit(self.model.guidance_model.parameters(), lr=learning_rate)
        else:
            self.optimizer_generator = AdamW(self.model.feedforward_model.parameters(), lr=learning_rate, eps=1e-7)
            self.optimizer_guidance = AdamW(self.model.guidance_model.parameters(), lr=learning_rate, eps=1e-7)

        self.model, self.optimizer_generator, self.optimizer_guidance = self.accelerator.prepare(self.model, self.optimizer_generator, self.optimizer_guidance)

        self.generator_norm = RunningStats()
        self.guidance_norm = RunningStats()

    @property
    def is_main(self):
        return self.accelerator.is_main_process

    def save_checkpoint(self, step, last=False):
        self.accelerator.wait_for_everyone()
        if self.is_main:
            checkpoint = dict(
                model_state_dict=self.accelerator.unwrap_model(self.model).state_dict(),
                optimizer_generator_state_dict=self.accelerator.unwrap_model(self.optimizer_generator).state_dict(),
                optimizer_guidance_state_dict=self.accelerator.unwrap_model(self.optimizer_guidance).state_dict(),
                scheduler_generator_state_dict=self.scheduler_generator.state_dict(),
                scheduler_guidance_state_dict=self.scheduler_guidance.state_dict(),
                step=step,
            )

            if not os.path.exists(self.checkpoint_path):
                os.makedirs(self.checkpoint_path)
            if last:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_last.pt")
                print(f"Saved last checkpoint at step {step}")
            else:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_{step}.pt")

    def load_checkpoint(self):
        if (
            not exists(self.checkpoint_path)
            or not os.path.exists(self.checkpoint_path)
            or not os.listdir(self.checkpoint_path)
        ):
            return 0

        self.accelerator.wait_for_everyone()
        if "model_last.pt" in os.listdir(self.checkpoint_path):
            latest_checkpoint = "model_last.pt"
        else:
            latest_checkpoint = sorted(
                [f for f in os.listdir(self.checkpoint_path) if f.endswith(".pt")],
                key=lambda x: int("".join(filter(str.isdigit, x))),
            )[-1]
        # checkpoint = torch.load(f"{self.checkpoint_path}/{latest_checkpoint}", map_location=self.accelerator.device)  # rather use accelerator.load_state ಥ_ಥ
        checkpoint = torch.load(f"{self.checkpoint_path}/{latest_checkpoint}", weights_only=True, map_location="cpu")


    def save_checkpoint(self, step, last=False):
        self.accelerator.wait_for_everyone()
        if self.is_main:
            checkpoint = dict(
                model_state_dict=self.accelerator.unwrap_model(self.model).state_dict(),
                optimizer_generator_state_dict=self.accelerator.unwrap_model(self.optimizer_generator).state_dict(),
                optimizer_guidance_state_dict=self.accelerator.unwrap_model(self.optimizer_guidance).state_dict(),
                scheduler_generator_state_dict=self.scheduler_generator.state_dict(),
                scheduler_guidance_state_dict=self.scheduler_guidance.state_dict(),
                step=step,
            )

            if not os.path.exists(self.checkpoint_path):
                os.makedirs(self.checkpoint_path)
            if last:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_last.pt")
                print(f"Saved last checkpoint at step {step}")
            else:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_{step}.pt")

    def load_checkpoint(self):
        if (
            not exists(self.checkpoint_path)
            or not os.path.exists(self.checkpoint_path)
            or not os.listdir(self.checkpoint_path)
        ):
            return 0

        self.accelerator.wait_for_everyone()
        if "model_last.pt" in os.listdir(self.checkpoint_path):
            latest_checkpoint = "model_last.pt"
        else:
            latest_checkpoint = sorted(
                [f for f in os.listdir(self.checkpoint_path) if f.endswith(".pt")],
                key=lambda x: int("".join(filter(str.isdigit, x))),
            )[-1]
        # checkpoint = torch.load(f"{self.checkpoint_path}/{latest_checkpoint}", map_location=self.accelerator.device)  # rather use accelerator.load_state ಥ_ಥ
        checkpoint = torch.load(f"{self.checkpoint_path}/{latest_checkpoint}", weights_only=True, map_location="cpu")

        self.accelerator.unwrap_model(self.model).load_state_dict(checkpoint["model_state_dict"], strict=False)
        # self.accelerator.unwrap_model(self.optimizer_generator).load_state_dict(checkpoint["optimizer_generator_state_dict"])
        # self.accelerator.unwrap_model(self.optimizer_guidance).load_state_dict(checkpoint["optimizer_guidance_state_dict"])
        # if self.scheduler_guidance:
        #     self.scheduler_guidance.load_state_dict(checkpoint["scheduler_guidance_state_dict"])
        # if self.scheduler_generator:
        #     self.scheduler_generator.load_state_dict(checkpoint["scheduler_generator_state_dict"])
        step = checkpoint["step"]

        del checkpoint
        gc.collect()
        return step

    def train(self, train_dataset: Dataset, num_workers=64, resumable_with_seed: int = None, vocoder: nn.Module = None):
        if exists(resumable_with_seed):
            generator = torch.Generator()
            generator.manual_seed(resumable_with_seed)
        else:
            generator = None

        if self.batch_size_type == "sample":
            train_dataloader = DataLoader(
                train_dataset,
                collate_fn=collate_fn,
                num_workers=num_workers,
                pin_memory=True,
                persistent_workers=True,
                batch_size=self.batch_size,
                shuffle=True,
                generator=generator,
            )


train

                [f for f in os.listdir(self.checkpoint_path) if f.endswith(".pt")],
                key=lambda x: int("".join(filter(str.isdigit, x))),
            )[-1]
        # checkpoint = torch.load(f"{self.checkpoint_path}/{latest_checkpoint}", map_location=self.accelerator.device)  # rather use accelerator.load_state ಥ_ಥ
        checkpoint = torch.load(f"{self.checkpoint_path}/{latest_checkpoint}", weights_only=True, map_location="cpu")

        self.accelerator.unwrap_model(self.model).load_state_dict(checkpoint["model_state_dict"], strict=False)
        # self.accelerator.unwrap_model(self.optimizer_generator).load_state_dict(checkpoint["optimizer_generator_state_dict"])
        # self.accelerator.unwrap_model(self.optimizer_guidance).load_state_dict(checkpoint["optimizer_guidance_state_dict"])
        # if self.scheduler_guidance:
        #     self.scheduler_guidance.load_state_dict(checkpoint["scheduler_guidance_state_dict"])
        # if self.scheduler_generator:
        #     self.scheduler_generator.load_state_dict(checkpoint["scheduler_generator_state_dict"])
        step = checkpoint["step"]

        del checkpoint
        gc.collect()
        return step

    def train(self, train_dataset: Dataset, num_workers=64, resumable_with_seed: int = None, vocoder: nn.Module = None):
        if exists(resumable_with_seed):
            generator = torch.Generator()
            generator.manual_seed(resumable_with_seed)
        else:
            generator = None

        if self.batch_size_type == "sample":
            train_dataloader = DataLoader(
                train_dataset,
                collate_fn=collate_fn,
                num_workers=num_workers,
                pin_memory=True,
                persistent_workers=True,
                batch_size=self.batch_size,
                shuffle=True,
                generator=generator,
            )
        elif self.batch_size_type == "frame":
            self.accelerator.even_batches = False
            sampler = SequentialSampler(train_dataset)
            batch_sampler = DynamicBatchSampler(
                sampler, self.batch_size, max_samples=self.max_samples, random_seed=resumable_with_seed, drop_last=False
            )
            train_dataloader = DataLoader(
                train_dataset,
                collate_fn=collate_fn,
                num_workers=num_workers,
                pin_memory=True,
                persistent_workers=True,
                batch_sampler=batch_sampler,
            )
        else:
            raise ValueError(f"batch_size_type must be either 'sample' or 'frame', but received {self.batch_size_type}")

        #  accelerator.prepare() dispatches batches to devices;
        #  which means the length of dataloader calculated before, should consider the number of devices
        warmup_steps = (
            self.num_warmup_updates * self.accelerator.num_processes
        )

        # consider a fixed warmup steps while using accelerate multi-gpu ddp
        # otherwise by default with split_batches=False, warmup steps change with num_processes
        total_steps = len(train_dataloader) * self.epochs / self.grad_accumulation_steps
        decay_steps = total_steps - warmup_steps

        warmup_scheduler_generator = LinearLR(self.optimizer_generator, start_factor=1e-8, end_factor=1.0, total_iters=warmup_steps // (self.gen_update_ratio * self.grad_accumulation_steps))
        decay_scheduler_generator = LinearLR(self.optimizer_generator, start_factor=1.0, end_factor=1e-8, total_iters=decay_steps // (self.gen_update_ratio * self.grad_accumulation_steps))
        self.scheduler_generator = SequentialLR(self.optimizer_generator, schedulers=[warmup_scheduler_generator, decay_scheduler_generator], milestones=[warmup_steps // (self.gen_update_ratio * self.grad_accumulation_steps)])

        warmup_scheduler_guidance = LinearLR(self.optimizer_guidance, start_factor=1e-8, end_factor=1.0, total_iters=warmup_steps)
        decay_scheduler_guidance = LinearLR(self.optimizer_guidance, start_factor=1.0, end_factor=1e-8, total_iters=decay_steps)
        self.scheduler_guidance = SequentialLR(self.optimizer_guidance, schedulers=[warmup_scheduler_guidance, decay_scheduler_guidance], milestones=[warmup_steps])

        train_dataloader, self.scheduler_generator, self.scheduler_guidance = self.accelerator.prepare(
            train_dataloader, self.scheduler_generator, self.scheduler_guidance
        )  # actual steps = 1 gpu steps / gpus
        start_step = self.load_checkpoint()
        global_step = start_step

        if exists(resumable_with_seed):
            orig_epoch_step = len(train_dataloader)
            skipped_epoch = int(start_step // orig_epoch_step)
            skipped_batch = start_step % orig_epoch_step
            skipped_dataloader = self.accelerator.skip_first_batches(train_dataloader, num_batches=skipped_batch)
        else:
            skipped_epoch = 0

        for epoch in range(skipped_epoch, self.epochs):
            self.model.train()
            if exists(resumable_with_seed) and epoch == skipped_epoch:
                progress_bar = tqdm(
                    skipped_dataloader,
                    desc=f"Epoch {epoch+1}/{self.epochs}",
                    unit="step",
                    disable=not self.accelerator.is_local_main_process,
                    initial=skipped_batch,
                    total=orig_epoch_step,
                )
            else:
                progress_bar = tqdm(
                    train_dataloader,
                    desc=f"Epoch {epoch+1}/{self.epochs}",
                    unit="step",
                    disable=not self.accelerator.is_local_main_process,
                )

            for batch in progress_bar:
                update_generator = global_step % self.gen_update_ratio == 0

                with self.accelerator.accumulate(self.model):
                    metrics = {}
                    text_inputs = batch["text"]
                    mel_spec = batch["mel"].permute(0, 2, 1)
                    mel_lengths = batch["mel_lengths"]

                    mel_spec = mel_spec / self.scale

                    guidance_loss_dict, guidance_log_dict = self.model(inp=mel_spec,
                                                                text=text_inputs,
                                                                lens=mel_lengths,
                                                                student_steps=self.student_steps,
                                                                update_generator=False,
                                                                use_simulated=global_step >= self.num_simu,
                                                                )

                    # if self.GAN and update_generator:
                    #     # only add discriminator loss if GAN is enabled and generator is being updated
                    #     guidance_cls_loss = guidance_loss_dict["guidance_cls_loss"] * (self.lambda_discriminator_loss if global_step >= self.num_GAN and update_generator else 0)
                    #     metrics['loss/discriminator_loss'] = guidance_loss_dict["guidance_cls_loss"]
                    #     self.accelerator.backward(guidance_cls_loss, retain_graph=True)

                    #     if self.max_grad_norm > 0 and self.accelerator.sync_gradients:
                    #         metrics['grad_norm_guidance'] = self.accelerator.clip_grad_norm_(self.model.parameters(), self.max_grad_norm)

                    guidance_loss = 0
                    guidance_loss += guidance_loss_dict["loss_fake_mean"]
                    metrics['loss/fake_score'] = guidance_loss_dict["loss_fake_mean"]
                    metrics["loss/guidance_loss"] = guidance_loss

                    if self.GAN and update_generator:
                        # only add discriminator loss if GAN is enabled and generator is being updated
                        guidance_cls_loss = guidance_loss_dict["guidance_cls_loss"] * (self.lambda_discriminator_loss if global_step >= self.num_GAN and update_generator else 0)
                        metrics['loss/discriminator_loss'] = guidance_loss_dict["guidance_cls_loss"]

                        guidance_loss += guidance_cls_loss

                    self.accelerator.backward(guidance_loss)

                    if self.max_grad_norm > 0 and self.accelerator.sync_gradients:
                        metrics['grad_norm_guidance'] = self.accelerator.clip_grad_norm_(self.model.parameters(), self.max_grad_norm)

                        # if self.guidance_norm.count < 100:
                        #     self.guidance_norm.update(metrics['grad_norm_guidance'])

                        # if metrics['grad_norm_guidance'] > self.guidance_norm.mean + 5 * self.guidance_norm.std:
                        #     self.optimizer_generator.zero_grad()
                        #     self.optimizer_guidance.zero_grad()
                        #     print("Gradient explosion detected. Skipping batch.")
                        # elif self.guidance_norm.count >= 100:
                        #     self.guidance_norm.update(metrics['grad_norm_guidance'])

                    self.optimizer_guidance.step()
                    self.scheduler_guidance.step()
                    self.optimizer_guidance.zero_grad()
                    self.optimizer_generator.zero_grad()  # zero out the generator's gradient as well

                    if update_generator:
                        generator_loss_dict, generator_log_dict = self.model(inp=mel_spec,
                                                                        text=text_inputs,
                                                                        lens=mel_lengths,
                                                                        student_steps=self.student_steps,
                                                                        update_generator=True,
                                                                        use_simulated=global_step >= self.num_ctc,
                                                                        )
                        # if self.GAN:
                        #     gen_cls_loss = generator_loss_dict["gen_cls_loss"] * (self.lambda_generator_loss if global_step >= (self.num_GAN + self.num_D) and update_generator else 0)
                        #     metrics["loss/gen_cls_loss"] = generator_loss_dict["gen_cls_loss"]

                        #     self.accelerator.backward(gen_cls_loss, retain_graph=True)

                        #     if self.max_grad_norm > 0 and self.accelerator.sync_gradients:
                        #         metrics['grad_norm_generator'] = self.accelerator.clip_grad_norm_(self.model.parameters(), self.max_grad_norm)

                        generator_loss = 0
                        generator_loss += generator_loss_dict["loss_dm"]
                        if "loss_mse" in generator_loss_dict:
                            generator_loss += generator_loss_dict["loss_mse"]
                        generator_loss += generator_loss_dict["loss_ctc"] * (self.lambda_ctc_loss if global_step >= self.num_ctc else 0)
                        generator_loss += generator_loss_dict["loss_sim"] * (self.lambda_sim_loss if global_step >= self.num_sim else 0)
                        generator_loss += generator_loss_dict["loss_kl"] * (self.lambda_ctc_loss if global_step >= self.num_ctc else 0)
                        if self.GAN:
                            gen_cls_loss = generator_loss_dict["gen_cls_loss"] * (self.lambda_generator_loss if global_step >= (self.num_GAN + self.num_D) and update_generator else 0)
                            metrics["loss/gen_cls_loss"] = generator_loss_dict["gen_cls_loss"]
                            generator_loss += gen_cls_loss

                        metrics['loss/dm_loss'] = generator_loss_dict["loss_dm"]
                        metrics['loss/ctc_loss'] = generator_loss_dict["loss_ctc"]

                        metrics['loss/similarity_loss'] = generator_loss_dict["loss_sim"]
                        metrics['loss/generator_loss'] = generator_loss

                        if "loss_mse" in generator_loss_dict and generator_loss_dict["loss_mse"] != 0:
                            metrics['loss/mse_loss'] = generator_loss_dict["loss_mse"]
                        if "loss_kl" in generator_loss_dict and generator_loss_dict["loss_kl"] != 0:
                            metrics['loss/kl_loss'] = generator_loss_dict["loss_kl"]

                        self.accelerator.backward(generator_loss)

                        if self.max_grad_norm > 0 and self.accelerator.sync_gradients:
                            metrics['grad_norm_generator'] = self.accelerator.clip_grad_norm_(self.model.parameters(), self.max_grad_norm)
                            # self.generator_norm.update(metrics['grad_norm_generator'])

                            # if metrics['grad_norm_generator'] > self.generator_norm.mean + 15 * self.generator_norm.std:
                            #     self.optimizer_generator.zero_grad()
                            #     self.optimizer_guidance.zero_grad()
                            #     update_generator = False
                            #     print("Gradient explosion detected. Skipping batch.")

                        if update_generator:
                            self.optimizer_generator.step()
                            self.scheduler_generator.step()
                            self.optimizer_generator.zero_grad()
                            self.optimizer_guidance.zero_grad()  # zero out the guidance's gradient as well

                global_step += 1

                if self.accelerator.is_local_main_process:
                    self.accelerator.log({**metrics,
                                          "lr_generator": self.scheduler_generator.get_last_lr()[0],
                                          "lr_guidance": self.scheduler_guidance.get_last_lr()[0],
                                          }
                                         , step=global_step)

                if global_step % self.log_step == 0 and self.accelerator.is_local_main_process and vocoder is not None:
                    # log the first batch of the epoch
                    with torch.no_grad():
                        generator_input = generator_log_dict['generator_input'][0].unsqueeze(0).permute(0, 2, 1) * self.scale
                        generator_input = vocoder.decode(generator_input.float().cpu())
                        generator_input = wandb.Audio(
                            generator_input.float().numpy().squeeze(),
                            sample_rate=24000,
                            caption="time: " + str(generator_log_dict['time'][0].float().cpu().numpy())
                        )

                        generator_output = generator_log_dict['generator_output'][0].unsqueeze(0).permute(0, 2, 1) * self.scale
                        generator_output = vocoder.decode(generator_output.float().cpu())
                        generator_output = wandb.Audio(
                            generator_output.float().numpy().squeeze(),
                            sample_rate=24000,
                            caption="time: " + str(generator_log_dict['time'][0].float().cpu().numpy())
                        )

                        generator_cond = generator_log_dict['generator_cond'][0].unsqueeze(0).permute(0, 2, 1) * self.scale
                        generator_cond = vocoder.decode(generator_cond.float().cpu())
                        generator_cond = wandb.Audio(
                            generator_cond.float().numpy().squeeze(),
                            sample_rate=24000,
                            caption="time: " + str(generator_log_dict['time'][0].float().cpu().numpy())
                        )

                        ground_truth = generator_log_dict['ground_truth'][0].unsqueeze(0).permute(0, 2, 1) * self.scale
                        ground_truth = vocoder.decode(ground_truth.float().cpu())
                        ground_truth = wandb.Audio(
                            ground_truth.float().numpy().squeeze(),
                            sample_rate=24000,
                            caption="time: " + str(generator_log_dict['time'][0].float().cpu().numpy())
                        )

                        dmtrain_noisy_inp = generator_log_dict['dmtrain_noisy_inp'][0].unsqueeze(0).permute(0, 2, 1) * self.scale
                        dmtrain_noisy_inp = vocoder.decode(dmtrain_noisy_inp.float().cpu())
                        dmtrain_noisy_inp = wandb.Audio(
                            dmtrain_noisy_inp.float().numpy().squeeze(),
                            sample_rate=24000,
                            caption="dmtrain_time: " + str(generator_log_dict['dmtrain_time'][0].float().cpu().numpy())
                        )

                        dmtrain_pred_real_image = generator_log_dict['dmtrain_pred_real_image'][0].unsqueeze(0).permute(0, 2, 1) * self.scale
                        dmtrain_pred_real_image = vocoder.decode(dmtrain_pred_real_image.float().cpu())
                        dmtrain_pred_real_image = wandb.Audio(
                            dmtrain_pred_real_image.float().numpy().squeeze(),
                            sample_rate=24000,
                            caption="dmtrain_time: " + str(generator_log_dict['dmtrain_time'][0].float().cpu().numpy())
                        )

                        dmtrain_pred_fake_image = generator_log_dict['dmtrain_pred_fake_image'][0].unsqueeze(0).permute(0, 2, 1) * self.scale
                        dmtrain_pred_fake_image = vocoder.decode(dmtrain_pred_fake_image.float().cpu())
                        dmtrain_pred_fake_image = wandb.Audio(
                            dmtrain_pred_fake_image.float().numpy().squeeze(),
                            sample_rate=24000,
                            caption="dmtrain_time: " + str(generator_log_dict['dmtrain_time'][0].float().cpu().numpy())
                        )

                        self.accelerator.log({"noisy_input": generator_input,
                                              "output": generator_output,
                                                "cond": generator_cond,
                                                "ground_truth": ground_truth,
                                                "dmtrain_noisy_inp": dmtrain_noisy_inp,
                                                "dmtrain_pred_real_image": dmtrain_pred_real_image,
                                                "dmtrain_pred_fake_image": dmtrain_pred_fake_image,

                                             }, step=global_step)

                progress_bar.set_postfix(step=str(global_step), metrics=metrics)

                if global_step % (self.save_per_updates * self.grad_accumulation_steps) == 0:
                    self.save_checkpoint(global_step)

                if global_step % self.last_per_steps == 0:
                    self.save_checkpoint(global_step, last=True)

        self.save_checkpoint(global_step, last=True)

        self.accelerator.end_training()


"""
ein notation:
b - batch
n - sequence
nt - text sequence
nw - raw wave length
d - dimension
"""

from __future__ import annotations
from typing import Callable
from random import random
import numpy as np

import torch
from torch import nn
import torch.nn.functional as F

from f5_tts.model import DiT

from f5_tts.model.utils import (
    default,
    exists,
    list_str_to_idx,
    list_str_to_tensor,
    lens_to_mask,
    mask_from_frac_lengths,
)

from discriminator_conformer import ConformerDiscirminator
from ctcmodel import ConformerCTC
from ecapa_tdnn import ECAPA_TDNN

class NoOpContext:
    def __enter__(self):
        pass

    def __exit__(self, *args):
        pass

def predict_flow(transformer, # flow model
                  x, # noisy input
                  cond, # mask (prompt mask + length mask)
                  text, # text input
                  time, # time step
                  second_time=None,
                  cfg_strength=1.0
):
    pred = transformer(
        x=x,
        cond=cond,
        text=text, time=time,
        second_time=second_time,
        drop_audio_cond=False,
        drop_text=False
    )

    if cfg_strength < 1e-5:
        return pred

    null_pred = transformer(
            x=x,
            cond=cond,
            text=text, time=time,
            second_time=second_time,
            drop_audio_cond=True,
            drop_text=True
    )

    return pred + (pred - null_pred) * cfg_strength

def _kl_dist_func(x, y):
    log_probs = F.log_softmax(x, dim=2)
    target_probs  = F.log_softmax(y, dim=2)
    return torch.nn.functional.kl_div(log_probs, target_probs, reduction="batchmean", log_target=True)

class Guidance(nn.Module):
    def __init__(self,
                real_unet: DiT, # teacher flow model
                fake_unet: DiT, # student flow model

                use_fp16: bool = True,
                real_guidance_scale: float = 0.0,
                fake_guidance_scale: float = 0.0,
                gen_cls_loss: bool = False,

                sv_path_en: str = "",
                sv_path_zh: str = "",
                ctc_path: str = "",
                sway_coeff: float = 0.0,
                scale: float = 1.0,
                ):
        super().__init__()
        self.vocab_size = real_unet.vocab_size

        if ctc_path != "":
            model = ConformerCTC(vocab_size=real_unet.vocab_size, mel_dim=real_unet.mel_dim, num_heads=8, d_hid=512, nlayers=6)
            self.ctc_model = model.eval()
            self.ctc_model.requires_grad_(False)
            self.ctc_model.load_state_dict(torch.load(ctc_path, weights_only=True, map_location='cpu')['model_state_dict'])

        if sv_path_en != "":
            model = ECAPA_TDNN()
            self.sv_model_en = model.eval()
            self.sv_model_en.requires_grad_(False)
            self.sv_model_en.load_state_dict(torch.load(sv_path, weights_only=True, map_location='cpu')['model_state_dict'])

        if sv_path_zh != "":
            model = ECAPA_TDNN()
            self.sv_model_zh = model.eval()
            self.sv_model_zh.requires_grad_(False)
            self.sv_model_zh.load_state_dict(torch.load(sv_path_zh, weights_only=True, map_location='cpu')['model_state_dict'])

        self.scale = scale

        self.real_unet = real_unet
        self.real_unet.requires_grad_(False) # no update on the teacher model

        self.fake_unet = fake_unet
        self.fake_unet.requires_grad_(True) # update the student model

        self.real_guidance_scale = real_guidance_scale
        self.fake_guidance_scale = fake_guidance_scale

        assert self.fake_guidance_scale == 0, "no guidance for fake"

        self.use_fp16 = use_fp16

        self.gen_cls_loss = gen_cls_loss

        self.sway_coeff = sway_coeff

        if self.gen_cls_loss:
            self.cls_pred_branch = ConformerDiscirminator(
                input_dim=(self.fake_unet.depth + 1) * self.fake_unet.dim + 3 * 512, # 3 is the number of layers from the CTC model
                num_layers=3,
                channels=self.fake_unet.dim // 2,
            )

            self.cls_pred_branch.requires_grad_(True)

        self.network_context_manager = torch.autocast(device_type="cuda", dtype=torch.float16) if self.use_fp16 else NoOpContext()

        from f5_tts.model.utils import get_tokenizer
        from torch.utils.data import DataLoader, Dataset, SequentialSampler
        from f5_tts.model.dataset import load_dataset
        from f5_tts.model.dataset import DynamicBatchSampler, collate_fn

        bsz = 16

        tokenizer = "pinyin"  # 'pinyin', 'char', or 'custom'
        tokenizer_path = None  # if tokenizer = 'custom', define the path to the tokenizer you want to use (should be vocab.txt)
        dataset_name = "Emilia_ZH_EN"
        if tokenizer == "custom":
            tokenizer_path = tokenizer_path
        else:
            tokenizer_path = dataset_name
        vocab_char_map, vocab_size = get_tokenizer(tokenizer_path, tokenizer)

        self.vocab_char_map = vocab_char_map

    def compute_distribution_matching_loss(
        self,

        inp: float["b n d"] | float["b nw"],  # mel or raw wave, ground truth latent
        text: int["b nt"] | list[str],  # text input
        *,
        second_time: torch.Tensor | None = None,  # second time step for flow prediction
        rand_span_mask: bool["b n d"] | bool["b nw"] | None = None,  # combined mask (prompt mask + padding mask)
    ):
        """
        Compute DMD loss (L_DMD) between the student distribution and teacher distribution.
        Following the DMDSpeech logic:
        - Sample time t
        - Construct noisy input phi = (1 - t)*x0 + t*x1, where x0 is noise and x1 is inp
        - Predict flows with teacher (f_phi) and student (G_theta)
        - Compute gradient that aligns student distribution with teacher distribution

        The code is adapted from F5-TTS but conceptualized per DMD:
        L_DMD encourages p_theta to match p_data via the difference between teacher and student predictions.
        """

        original_inp = inp

        with torch.no_grad():
            batch, seq_len, dtype, device = *inp.shape[:2], inp.dtype, inp.device

            # mel is x1
            x1 = inp

            # x0 is gaussian noise
            x0 = torch.randn_like(x1)

            # time step
            time = torch.rand((batch,), dtype=dtype, device=device)

            # get flow
            t = time.unsqueeze(-1).unsqueeze(-1)
            # t = t + self.sway_coeff * (torch.cos(torch.pi / 2 * t) - 1 + t)
            sigma_t, alpha_t = (1 - t), t

            phi = (1 - t) * x0 + t * x1 # noisy x
            flow = x1 - x0 # flow target

            # only predict what is within the random mask span for infilling
            cond = torch.where(rand_span_mask[..., None], torch.zeros_like(x1), x1)

            # run at full precision as autocast and no_grad doesn't work well together
            with self.network_context_manager:
                pred_fake = predict_flow(
                    self.fake_unet,
                    phi,
                    cond, # mask (prompt mask + length mask)
                    text, # text input
                    time, # time step
                    second_time=second_time,
                    cfg_strength=self.fake_guidance_scale
                )

            # pred = (x1 - x0), thus phi + (1-t) * pred = (1 - t) * x0 + t * x1 + (1 - t) * (x1 - x0) = (1 - t) * x1 + t * x1 = x1
            pred_fake_image = phi + (1 - t) * pred_fake
            pred_fake_image[~rand_span_mask] = inp[~rand_span_mask]

            with self.network_context_manager:
                pred_real = predict_flow(
                    self.real_unet, phi, cond, text, time, cfg_strength=self.real_guidance_scale
                )

            pred_real_image = phi + (1 - t) * pred_real
            pred_real_image[~rand_span_mask] = inp[~rand_span_mask]

            p_real = (inp - pred_real_image)
            p_fake = (inp - pred_fake_image)

            grad = (p_real - p_fake) / torch.abs(p_real).mean(dim=[1, 2], keepdim=True)
            grad = torch.nan_to_num(grad)

            # grad  = grad / sigma_t # pred_fake - pred_real
            # grad = grad * (1 + sigma_t / alpha_t)

            # grad = grad / (1 + sigma_t / alpha_t) # noise
            # grad = grad / sigma_t # score difference
            # grad = grad * alpha_t
            # grad = grad * (sigma_t ** 2 / alpha_t)

            # grad = grad * (alpha_t + sigma_t ** 2 / alpha_t)

        # The DMD loss: MSE to move student distribution closer to teacher distribution
        # Only optimize over the masked region
        loss = 0.5 * F.mse_loss(original_inp.float(), (original_inp-grad).detach().float(), reduction="none") * rand_span_mask.unsqueeze(
                -1
            )
        loss = loss.sum() / (rand_span_mask.sum() * grad.size(-1))

        loss_dict = {
            "loss_dm": loss
        }

        dm_log_dict = {
            "dmtrain_time": time.detach().float(),
            "dmtrain_noisy_inp": phi.detach().float(),
            "dmtrain_pred_real_image": pred_real_image.detach().float(),
            "dmtrain_pred_fake_image": pred_fake_image.detach().float(),
            "dmtrain_grad": grad.detach().float(),
            "dmtrain_gradient_norm": torch.norm(grad).item()
        }

        return loss_dict, dm_log_dict

    def compute_ctc_sv_loss(
        self,
        real_inp: torch.Tensor,   # real data latent
        fake_inp: torch.Tensor,   # student-generated data latent
        text: torch.Tensor,
        text_lens: torch.Tensor,
        rand_span_mask: torch.Tensor,
        second_time: torch.Tensor | None = None,
    ):
        """
        Compute CTC + SV loss for direct metric optimization, as described in DMDSpeech.
        - CTC loss reduces WER
        - SV loss improves speaker similarity

        Both CTC and SV models operate on latents.
        """

        # compute CTC loss
        out, layer, ctc_loss = self.ctc_model(fake_inp * self.scale, text, text_lens)  # lengths from rand_span_mask or known

        with torch.no_grad():
            real_out, real_layers, ctc_loss_test = self.ctc_model(real_inp * self.scale, text, text_lens)
            real_logits = real_out.log_softmax(dim=2)
            # emb_real = self.sv_model(real_inp * self.scale) # snippet from prompt region

        fake_logits = out.log_softmax(dim=2)
        kl_loss = F.kl_div(fake_logits, real_logits, reduction="mean", log_target=True)

        # For SV:
        # Extract speaker embeddings from real (prompt) and fake:
        # emb_fake = self.sv_model(fake_inp * self.scale)
        # sv_loss = 1 - F.cosine_similarity(emb_real, emb_fake, dim=-1).mean()

        input_lengths = rand_span_mask.sum(axis=-1).cpu().numpy()
        prompt_lengths = real_inp.size(1) - rand_span_mask.sum(axis=-1).cpu().numpy()

        chunks_real = []
        chunks_fake = []
        mel_len = min([int(input_lengths.min().item() - 1), 300])

        for bib in range(len(input_lengths)):
            prompt_length = int(prompt_lengths[bib].item())
            mel_length = int(input_lengths[bib].item())
            mask = rand_span_mask[bib]
            mask = torch.where(mask)[0]

            prompt_start = mask[0].cpu().numpy()
            prompt_end = mask[-1].cpu().numpy()

            if prompt_end - mel_len <= prompt_start:
                random_start = np.random.randint(0, mel_length - mel_len)
            else:
                random_start = np.random.randint(prompt_start, prompt_end - mel_len)

            chunks_fake.append(fake_inp[bib, random_start:random_start + mel_len, :])
            chunks_real.append(real_inp[bib, :mel_len, :])

        chunks_real = torch.stack(chunks_real, dim=0)
        chunks_fake = torch.stack(chunks_fake, dim=0)

        with torch.no_grad():
            emb_real_en = self.sv_model_en(chunks_real * self.scale)
        emb_fake_en = self.sv_model_en(chunks_fake * self.scale)

        sv_loss_en = 1 - F.cosine_similarity(emb_real_en, emb_fake_en, dim=-1).mean()

        with torch.no_grad():
            emb_real_zh = self.sv_model_zh(chunks_real * self.scale)
        emb_fake_zh = self.sv_model_zh(chunks_fake * self.scale)

        sv_loss_zh = 1 - F.cosine_similarity(emb_real_zh, emb_fake_zh, dim=-1).mean()

        sv_loss = (sv_loss_en + sv_loss_zh) / 2

        return {
            "loss_ctc": ctc_loss,
            'loss_kl': kl_loss,
            "loss_sim": sv_loss
        }, layer, real_layers

    def compute_loss_fake(
        self,
        inp: torch.Tensor, # student generator output
        text: torch.Tensor | list[str],
        rand_span_mask: torch.Tensor,
        second_time: torch.Tensor | None = None,
    ):
        """
        Compute flow loss for the fake flow model, which is trained to estimate the flow (score) of the student distribution.

        This is the same as L_diff in the paper.
        """

        # Similar to distribution matching, but only train fake to predict flow directly
        batch, seq_len, dtype, device = *inp.shape[:2], inp.dtype, inp.device

        if isinstance(text, list):
            if exists(self.vocab_char_map):
                text = list_str_to_idx(text, self.vocab_char_map).to(device)
            else:
                text = list_str_to_tensor(text).to(device)
            assert text.shape[0] == batch

        # Sample a time
        time = torch.rand((batch,), dtype=dtype, device=device)

        x1 = inp
        x0 = torch.randn_like(x1)
        t = time.unsqueeze(-1).unsqueeze(-1)

        phi = (1 - t) * x0 + t * x1
        flow = x1 - x0
        cond = torch.where(rand_span_mask[..., None], torch.zeros_like(x1), x1)

        with self.network_context_manager:
            pred = self.fake_unet(
                x=phi,
                cond=cond,
                text=text,
                time=time,
                second_time=second_time,
                drop_audio_cond=False,
                drop_text=False # make sure the cfg=1
            )

        # Compute MSE between predicted flow and actual flow, masked by rand_span_mask
        loss = F.mse_loss(pred, flow, reduction="none")
        loss = loss[rand_span_mask].mean()

        loss_dict = {
            "loss_fake_mean": loss
        }
        log_dict = {
            "faketrain_noisy_inp": phi.detach().float(),
            "faketrain_x1": x1.detach().float(),
            "faketrain_pred_flow": pred.detach().float(),
        }

        return loss_dict, log_dict

    def compute_cls_logits(
        self,
        inp: torch.Tensor, # student generator output
        layer: torch.Tensor,
        text: torch.Tensor,
        rand_span_mask: torch.Tensor,
        second_time: torch.Tensor | None = None,
        guidance: bool = False,
    ):
        '''
        Compute adversarial loss logits for the generator.

        This is used to compute L_adv in the paper.

        '''
        context_no_grad = torch.no_grad if guidance else NoOpContext

        with context_no_grad():
            # If we are not doing generator classification loss, return zeros
            if not self.gen_cls_loss:
                return torch.zeros_like(inp[..., 0])  # shape (b, n)

            # For classification, we need some representation:
            # We'll mimic the logic from compute_loss_fake

            batch, seq_len, dtype, device = *inp.shape[:2], inp.dtype, inp.device
            if isinstance(text, list):
                if exists(self.vocab_char_map):
                    text = list_str_to_idx(text, self.vocab_char_map).to(device)
                else:
                    text = list_str_to_tensor(text).to(device)
                assert text.shape[0] == batch

            # Sample a time
            time = torch.rand((batch,), dtype=dtype, device=device)

            x1 = inp
            x0 = torch.randn_like(x1)
            t = time.unsqueeze(-1).unsqueeze(-1)

            phi = (1 - t) * x0 + t * x1
            cond = torch.where(rand_span_mask[..., None], torch.zeros_like(x1), x1)

            with self.network_context_manager:
                layers = self.fake_unet(
                    x=phi,
                    cond=cond,
                    text=text,
                    time=time,
                    second_time=second_time,
                    drop_audio_cond=False,
                    drop_text=False, # make sure the cfg=1
                    classify_mode=True
                )
                # layers = torch.stack(layers, dim=0)

        if guidance:
            layers = [layer.detach() for layer in layers]
        layer = layer[-3:] # only use the last 3 layers
        layer = [l.transpose(-1, -2) for l in layer]
        # layer = [F.interpolate(l, mode='nearest', scale_factor=4).transpose(-1, -2) for l in layer]
        if layer[0].size(1) < layers[0].size(1):
            layer = [F.pad(l, (0, 0, 0, layers[0].size(1) - l.size(1))) for l in layer]

        layers = layer + layers
        # logits: (b, 1)
        logits = self.cls_pred_branch(layers)

        return logits, layers

    def compute_generator_cls_loss(
        self,
        inp: torch.Tensor, # student generator output
        layer: torch.Tensor,
        real_layers: torch.Tensor,
        text: torch.Tensor,
        rand_span_mask: torch.Tensor,
        second_time: torch.Tensor | None = None,
        mse_loss: bool = False,
        mse_inp: torch.Tensor | None = None,
    ):
        '''
        Compute the adversarial loss for the generator.
        '''

        # Compute classification loss for generator:
        if not self.gen_cls_loss:
            return {"gen_cls_loss": 0}

        logits, fake_layers = self.compute_cls_logits(inp, layer, text, rand_span_mask, second_time, guidance=False)

        loss = ((1 - logits) ** 2).mean()

        return {"gen_cls_loss": loss, "loss_mse": 0}

    def compute_guidance_cls_loss(
        self,
        fake_inp: torch.Tensor,
        text: torch.Tensor,
        rand_span_mask: torch.Tensor,
        real_data: dict,
        second_time: torch.Tensor | None = None,
    ):
        '''
        This function computes the adversarial loss for the discirminator.

        The discriminator is trained to classify the generator output as real or fake.
        '''

        with torch.no_grad():
            # get layers from CTC model
            _, layer = self.ctc_model(fake_inp * self.scale)

        logits_fake, _ = self.compute_cls_logits(fake_inp.detach(), layer, text, rand_span_mask, second_time, guidance=True)
        loss_fake = (logits_fake**2).mean()

        real_inp = real_data["inp"]

        with torch.no_grad():
            # get layers from CTC model
            _, layer = self.ctc_model(real_inp * self.scale)

        logits_real, _ = self.compute_cls_logits(real_inp.detach(), layer, text, rand_span_mask, second_time, guidance=True)
        loss_real = ((1 - logits_real)**2).mean()

        classification_loss = loss_real + loss_fake

        loss_dict = {
            "guidance_cls_loss": classification_loss
        }
        log_dict = {
            "pred_realism_on_real": loss_real.detach().item(),
            "pred_realism_on_fake": loss_fake.detach().item()
        }

        return loss_dict, log_dict

    def generator_forward(
        self,
        inp: torch.Tensor,
        text: torch.Tensor,
        text_lens: torch.Tensor,
        text_normalized: to
# ... truncated ...


NoOpContext

import torch
from torch import nn
import torch.nn.functional as F

from f5_tts.model import DiT

from f5_tts.model.utils import (
    default,
    exists,
    list_str_to_idx,
    list_str_to_tensor,
    lens_to_mask,
    mask_from_frac_lengths,
)

from discriminator_conformer import ConformerDiscirminator
from ctcmodel import ConformerCTC
from ecapa_tdnn import ECAPA_TDNN

class NoOpContext:
    def __enter__(self):
        pass

    def __exit__(self, *args):
        pass

def predict_flow(transformer, # flow model
                  x, # noisy input
                  cond, # mask (prompt mask + length mask)
                  text, # text input
                  time, # time step
                  second_time=None,
                  cfg_strength=1.0
):
    pred = transformer(
        x=x,
        cond=cond,
        text=text, time=time,
        second_time=second_time,
        drop_audio_cond=False,
        drop_text=False
    )

    if cfg_strength < 1e-5:
        return pred


from f5_tts.model.utils import (
    default,
    exists,
    list_str_to_idx,
    list_str_to_tensor,
    lens_to_mask,
    mask_from_frac_lengths,
)

from discriminator_conformer import ConformerDiscirminator
from ctcmodel import ConformerCTC
from ecapa_tdnn import ECAPA_TDNN

class NoOpContext:
    def __enter__(self):
        pass

    def __exit__(self, *args):
        pass

def predict_flow(transformer, # flow model
                  x, # noisy input
                  cond, # mask (prompt mask + length mask)
                  text, # text input
                  time, # time step
                  second_time=None,
                  cfg_strength=1.0
):
    pred = transformer(
        x=x,
        cond=cond,
        text=text, time=time,
        second_time=second_time,
        drop_audio_cond=False,
        drop_text=False
    )

    if cfg_strength < 1e-5:
        return pred

    null_pred = transformer(
            x=x,
            cond=cond,
            text=text, time=time,
            second_time=second_time,
            drop_audio_cond=True,
            drop_text=True
    )

    return pred + (pred - null_pred) * cfg_strength

def _kl_dist_func(x, y):
    log_probs = F.log_softmax(x, dim=2)
    target_probs  = F.log_softmax(y, dim=2)
    return torch.nn.functional.kl_div(log_probs, target_probs, reduction="batchmean", log_target=True)

class Guidance(nn.Module):
    def __init__(self,
                real_unet: DiT, # teacher flow model
                fake_unet: DiT, # student flow model

                use_fp16: bool = True,
                real_guidance_scale: float = 0.0,
                fake_guidance_scale: float = 0.0,
                gen_cls_loss: bool = False,

                sv_path_en: str = "",
                sv_path_zh: str = "",
                ctc_path: str = "",


        text=text, time=time,
        second_time=second_time,
        drop_audio_cond=False,
        drop_text=False
    )

    if cfg_strength < 1e-5:
        return pred

    null_pred = transformer(
            x=x,
            cond=cond,
            text=text, time=time,
            second_time=second_time,
            drop_audio_cond=True,
            drop_text=True
    )

    return pred + (pred - null_pred) * cfg_strength

def _kl_dist_func(x, y):
    log_probs = F.log_softmax(x, dim=2)
    target_probs  = F.log_softmax(y, dim=2)
    return torch.nn.functional.kl_div(log_probs, target_probs, reduction="batchmean", log_target=True)

class Guidance(nn.Module):
    def __init__(self,
                real_unet: DiT, # teacher flow model
                fake_unet: DiT, # student flow model

                use_fp16: bool = True,
                real_guidance_scale: float = 0.0,
                fake_guidance_scale: float = 0.0,
                gen_cls_loss: bool = False,

                sv_path_en: str = "",
                sv_path_zh: str = "",
                ctc_path: str = "",
                sway_coeff: float = 0.0,
                scale: float = 1.0,
                ):
        super().__init__()
        self.vocab_size = real_unet.vocab_size