UniModel

from torch import nn
import torch
import copy
import os

from f5_tts.model import DiT, UNetT
from pathlib import Path
from guidance_model import Guidance
from f5_tts.model.utils import (
    default,
    exists,
    list_str_to_idx,
    list_str_to_tensor,
    lens_to_mask,
    mask_from_frac_lengths,
    sample_consecutive_steps,
    sample_from_list,
)

class UniModel(nn.Module):
    def __init__(self,
                 model: DiT, # teacher model (dit model)
                 checkpoint_path: str = "",
                 second_time: bool = True,
                 use_fp16: bool = True,
                 real_guidance_scale: float = 2.0,
                 fake_guidance_scale: float = 0.0,
                 gen_cls_loss: bool = False,
                 sway_coeff: float = -1.0,
                 vocab_char_map: dict[str, int] | None = None,
                 frac_lengths_mask: tuple[float, float] = (0.7, 1.0)):

        super().__init__()

        if checkpoint_path != "":
            if "model_last.pt" in os.listdir(checkpoint_path):
                latest_checkpoint = "model_last.pt"
            else:
                latest_checkpoint = sorted(
                    [f for f in os.listdir(checkpoint_path) if f.endswith(".pt")],
                    key=lambda x: int("".join(filter(str.isdigit, x))),
                )[-1]
            checkpoint = torch.load(f"{checkpoint_path}/{latest_checkpoint}", weights_only=True, map_location="cpu")

            if "scale" in checkpoint:
                self.scale = checkpoint["scale"]
            else:
                self.scale = 1.0
            print(f"Loaded teacher model with scale: {self.scale}")

            if "step" in checkpoint:
                state = checkpoint["model_state_dict"]
            else:
                checkpoint["model_state_dict"] = {
                    k.replace("ema_model.", ""): v
                    for k, v in checkpoint["ema_model_state_dict"].items()
                    if k not in ["initted", "step"]
                }
                state = checkpoint["model_state_dict"]

            # only load the DiT module
            filtered_state_dict = {
                k.replace("transformer.", ""): v
                for k, v in state.items()
                if k.startswith("transformer.")
            }

            model.load_state_dict(filtered_state_dict, strict=False)
        else:
            self.scale = 1.0

        real_unet = copy.deepcopy(model)
        real_unet.time_embed2 = None

        fake_unet = copy.deepcopy(model)

        # Instantiate Guidance, which internally uses real_unet and fake_unet initialized from the teacher
        self.guidance_model = Guidance(
            real_unet=real_unet,
            fake_unet=fake_unet,
            use_fp16=use_fp16,
            real_guidance_scale=real_guidance_scale,
            fake_guidance_scale=fake_guidance_scale,
            gen_cls_loss=gen_cls_loss,
            sway_coeff=sway_coeff,
        )

        self.feedforward_model = copy.deepcopy(model) # initialize the student model
        self.feedforward_model.requires_grad_(True)
        self.feedforward_model.time_embed2 = None

        self.vocab_char_map = vocab_char_map
        self.frac_lengths_mask = frac_lengths_mask

        self.second_time = second_time # fake_unet.time_embed2 is not None

    def forward(self,
                inp: float["b n d"],  # mel
                text: int["b nt"] | list[str],
                *,
                lens: int["b"] | None = None,
                student_steps: list[int] = [0, 0.25, 0.5, 0.75],
                update_generator: bool = False,
    ):
        """
        Forward pass that routes to either generator_forward or guidance_forward
        in the Guidance class, depending on the arguments.

        Parameters:
        -----------
        generator_turn: bool
            If True, run the generator forward pass (distribution matching loss, etc.)
        guidance_turn: bool
            If True, run the guidance forward pass (fake loss, cls loss, etc.)
        data_dict: dict
            Input dictionary containing the necessary keys for the forward passes.
            Expected keys may include:
                "inp": Tensor (B, N, D) - input mel or latent
                "text": Tensor or list[str] - text input
                "rand_span_mask": Tensor (B, N) - boolean mask
                "real_data": dict with keys like:
                     "inp", "text", "rand_span_mask"

        Returns:
        --------
        loss_dict: dict[str, Tensor]
            Dictionary of losses.
        log_dict: dict[str, Tensor or float]
            Dictionary of logging tensors or values.
        """

        batch, seq_len, dtype, device = *inp.shape[:2], inp.dtype, inp.device

        # handle text as string
        if isinstance(text, list):
            if exists(self.vocab_char_map):
                text = list_str_to_idx(text, self.vocab_char_map).to(device)
            else:
                text = list_str_to_tensor(text).to(device)
            assert text.shape[0] == batch

        # lens and mask
        if not exists(lens):
            lens = torch.full((batch,), seq_len, device=device)

        mask = lens_to_mask(lens, length=seq_len)  # useless here, as collate_fn will pad to max length in batch

        # sample from the list of student steps
        time = sample_from_list(student_steps, batch).to(device)
        c_time, p_time = sample_consecutive_steps(student_steps)
        time = torch.ones_like(time) * c_time
        p_time = torch.ones_like(time) * p_time

        frac_lengths = torch.zeros((batch,), device=device).float().uniform_(*self.frac_lengths_mask)
        rand_span_mask = mask_from_frac_lengths(lens, frac_lengths)

        if exists(mask):
            rand_span_mask &= mask

        # # use generated output from previous step as input
        with torch.no_grad():
            x1 = inp
            x0 = torch.randn_like(x1)
            t = p_time.unsqueeze(-1).unsqueeze(-1)
            phi = (1 - t) * x0 + t * x1
            cond = torch.where(rand_span_mask[..., None], torch.zeros_like(x1), x1)

            pred = self.feedforward_model(
                x=phi,
                cond=cond,
                text=text,
                time=p_time,
                drop_audio_cond=False,
                drop_text=False # make sure the cfg=1
            ) # flow prediction

            # predicted mel spectrogram
            output = phi + (1 - t) * pred
            output[~rand_span_mask] = inp[~rand_span_mask]

        # forward diffusion
        x1 = output
        x0 = torch.randn_like(x1)
        t = time.unsqueeze(-1).unsqueeze(-1)
        phi = (1 - t) * x0 + t * x1
        cond = torch.where(rand_span_mask[..., None], torch.zeros_like(x1), x1)

        with torch.no_grad() if not update_generator else contextlib.nullcontext():
            pred = self.feedforward_model(
                x=phi,
                cond=cond,
                text=text,
                time=time,
                drop_audio_cond=False,
                drop_text=False # make sure no cfg is used
            )

            # predicted mel spectrogram
            output = phi + (1 - t) * pred
            output[~rand_span_mask] = inp[~rand_span_mask]

        if update_generator:
            generator_data_dict = {
                "inp": output,
                "text": text,
                "rand_span_mask": rand_span_mask,
                "second_time": time if self.second_time else None,
                "mse_loss": time.mean() == student_steps[-1].mean(),
                "real_data": {
                    "inp": inp,
                    "text": text,
                    "rand_span_mask": rand_span_mask
                }
            }

            # avoid any side effects of gradient accumulation
            # self.guidance_model.requires_grad_(False)
            # self.feedforward_model.requires_grad_(True)
            generator_loss_dict, generator_log_dict = self.guidance_model(
                generator_turn=True,
                guidance_turn=False,
                generator_data_dict=generator_data_dict,
                guidance_data_dict=None
            )

            generator_log_dict['ground_truth'] = x1
            generator_log_dict['generator_input'] = phi
            generator_log_dict['generator_output'] = output
            generator_log_dict['generator_cond'] = cond
            generator_log_dict['time'] = time

            return generator_loss_dict, generator_log_dict
        else:
            guidance_data_dict = {
                "inp": output.detach(),
                "text": text,
                "rand_span_mask": rand_span_mask,
                "second_time": time if self.second_time else None,
                "real_data": {
                    "inp": inp,
                    "text": text,
                    "rand_span_mask": rand_span_mask
                }
            }

            # avoid any side effects of gradient accumulation
            # self.feedforward_model.requires_grad_(False)
            # self.guidance_model.requires_grad_(True)
            guidance_loss_dict, guidance_log_dict = self.guidance_model(
                generator_turn=False,
                guidance_turn=True,
                generator_data_dict=None,
                guidance_data_dict=guidance_data_dict
            )
            # self.feedforward_model.requires_grad_(True)

            return guidance_loss_dict, guidance_log_dict

        # return guidance_loss_dict, guidance_log_dict, generator_loss_dict, generator_log_dict

if __name__ == "__main__":

    from f5_tts.model.utils import get_tokenizer
    from torch.utils.data import DataLoader, Dataset, SequentialSampler
    from f5_tts.model.dataset import load_dataset
    from f5_tts.model.dataset import DynamicBatchSampler, collate_fn

    bsz = 16

    tokenizer = "pinyin"  # 'pinyin', 'char', or 'custom'
    tokenizer_path = None  # if tokenizer = 'custom', define the path to the tokenizer you want to use (should be vocab.txt)
    dataset_name = "Emilia_ZH_EN"
    if tokenizer == "custom":
        tokenizer_path = tokenizer_path
    else:
        tokenizer_path = dataset_name


from torch import nn
import torch
import copy
import os

from f5_tts.model import DiT, UNetT
from pathlib import Path
from guidance_model import Guidance
from f5_tts.model.utils import (
    default,
    exists,
    list_str_to_idx,
    list_str_to_tensor,
    lens_to_mask,
    mask_from_frac_lengths,
    sample_consecutive_steps,
    sample_from_list,
)

class UniModel(nn.Module):
    def __init__(self,
                 model: DiT, # teacher model (dit model)
                 checkpoint_path: str = "",
                 second_time: bool = True,
                 use_fp16: bool = True,
                 real_guidance_scale: float = 2.0,
                 fake_guidance_scale: float = 0.0,
                 gen_cls_loss: bool = False,
                 sway_coeff: float = -1.0,
                 vocab_char_map: dict[str, int] | None = None,
                 frac_lengths_mask: tuple[float, float] = (0.7, 1.0)):

        super().__init__()

        if checkpoint_path != "":
            if "model_last.pt" in os.listdir(checkpoint_path):
                latest_checkpoint = "model_last.pt"
            else:
                latest_checkpoint = sorted(
                    [f for f in os.listdir(checkpoint_path) if f.endswith(".pt")],
                    key=lambda x: int("".join(filter(str.isdigit, x))),
                )[-1]
            checkpoint = torch.load(f"{checkpoint_path}/{latest_checkpoint}", weights_only=True, map_location="cpu")

            if "scale" in checkpoint:
                self.scale = checkpoint["scale"]
            else:
                self.scale = 1.0
            print(f"Loaded teacher model with scale: {self.scale}")

            if "step" in checkpoint:
                state = checkpoint["model_state_dict"]
            else:
                checkpoint["model_state_dict"] = {
                    k.replace("ema_model.", ""): v
                    for k, v in checkpoint["ema_model_state_dict"].items()
                    if k not in ["initted", "step"]
                }
                state = checkpoint["model_state_dict"]

            # only load the DiT module
            filtered_state_dict = {
                k.replace("transformer.", ""): v
                for k, v in state.items()
                if k.startswith("transformer.")
            }

            model.load_state_dict(filtered_state_dict, strict=False)
        else:
            self.scale = 1.0

        real_unet = copy.deepcopy(model)
        real_unet.time_embed2 = None

        fake_unet = copy.deepcopy(model)

        # Instantiate Guidance, which internally uses real_unet and fake_unet initialized from the teacher
        self.guidance_model = Guidance(
            real_unet=real_unet,
            fake_unet=fake_unet,
            use_fp16=use_fp16,
            real_guidance_scale=real_guidance_scale,
            fake_guidance_scale=fake_guidance_scale,
            gen_cls_loss=gen_cls_loss,
            sway_coeff=sway_coeff,
        )

        self.feedforward_model = copy.deepcopy(model) # initialize the student model
        self.feedforward_model.requires_grad_(True)
        self.feedforward_model.time_embed2 = None

        self.vocab_char_map = vocab_char_map
        self.frac_lengths_mask = frac_lengths_mask

        self.second_time = second_time # fake_unet.time_embed2 is not None

    def forward(self,
                inp: float["b n d"],  # mel
                text: int["b nt"] | list[str],
                *,
                lens: int["b"] | None = None,
                student_steps: list[int] = [0, 0.25, 0.5, 0.75],
                update_generator: bool = False,
    ):
        """
        Forward pass that routes to either generator_forward or guidance_forward
        in the Guidance class, depending on the arguments.

        Parameters:
        -----------
        generator_turn: bool
            If True, run the generator forward pass (distribution matching loss, etc.)
        guidance_turn: bool
            If True, run the guidance forward pass (fake loss, cls loss, etc.)
        data_dict: dict


forward

        # Instantiate Guidance, which internally uses real_unet and fake_unet initialized from the teacher
        self.guidance_model = Guidance(
            real_unet=real_unet,
            fake_unet=fake_unet,
            use_fp16=use_fp16,
            real_guidance_scale=real_guidance_scale,
            fake_guidance_scale=fake_guidance_scale,
            gen_cls_loss=gen_cls_loss,
            sway_coeff=sway_coeff,
        )

        self.feedforward_model = copy.deepcopy(model) # initialize the student model
        self.feedforward_model.requires_grad_(True)
        self.feedforward_model.time_embed2 = None

        self.vocab_char_map = vocab_char_map
        self.frac_lengths_mask = frac_lengths_mask

        self.second_time = second_time # fake_unet.time_embed2 is not None

    def forward(self,
                inp: float["b n d"],  # mel
                text: int["b nt"] | list[str],
                *,
                lens: int["b"] | None = None,
                student_steps: list[int] = [0, 0.25, 0.5, 0.75],
                update_generator: bool = False,
    ):
        """
        Forward pass that routes to either generator_forward or guidance_forward
        in the Guidance class, depending on the arguments.

        Parameters:
        -----------
        generator_turn: bool
            If True, run the generator forward pass (distribution matching loss, etc.)
        guidance_turn: bool
            If True, run the guidance forward pass (fake loss, cls loss, etc.)
        data_dict: dict
            Input dictionary containing the necessary keys for the forward passes.
            Expected keys may include:
                "inp": Tensor (B, N, D) - input mel or latent
                "text": Tensor or list[str] - text input
                "rand_span_mask": Tensor (B, N) - boolean mask
                "real_data": dict with keys like:
                     "inp", "text", "rand_span_mask"

        Returns:
        --------
        loss_dict: dict[str, Tensor]
            Dictionary of losses.
        log_dict: dict[str, Tensor or float]
            Dictionary of logging tensors or values.
        """

        batch, seq_len, dtype, device = *inp.shape[:2], inp.dtype, inp.device

        # handle text as string
        if isinstance(text, list):
            if exists(self.vocab_char_map):
                text = list_str_to_idx(text, self.vocab_char_map).to(device)
            else:
                text = list_str_to_tensor(text).to(device)
            assert text.shape[0] == batch

        # lens and mask
        if not exists(lens):
            lens = torch.full((batch,), seq_len, device=device)

        mask = lens_to_mask(lens, length=seq_len)  # useless here, as collate_fn will pad to max length in batch

        # sample from the list of student steps
        time = sample_from_list(student_steps, batch).to(device)
        c_time, p_time = sample_consecutive_steps(student_steps)
        time = torch.ones_like(time) * c_time
        p_time = torch.ones_like(time) * p_time

        frac_lengths = torch.zeros((batch,), device=device).float().uniform_(*self.frac_lengths_mask)
        rand_span_mask = mask_from_frac_lengths(lens, frac_lengths)

        if exists(mask):
            rand_span_mask &= mask

        # # use generated output from previous step as input
        with torch.no_grad():
            x1 = inp
            x0 = torch.randn_like(x1)
            t = p_time.unsqueeze(-1).unsqueeze(-1)
            phi = (1 - t) * x0 + t * x1
            cond = torch.where(rand_span_mask[..., None], torch.zeros_like(x1), x1)

            pred = self.feedforward_model(
                x=phi,
                cond=cond,
                text=text,
                time=p_time,
                drop_audio_cond=False,
                drop_text=False # make sure the cfg=1
            ) # flow prediction

            # predicted mel spectrogram
            output = phi + (1 - t) * pred
            output[~rand_span_mask] = inp[~rand_span_mask]

        # forward diffusion
        x1 = output
        x0 = torch.randn_like(x1)
        t = time.unsqueeze(-1).unsqueeze(-1)
        phi = (1 - t) * x0 + t * x1
        cond = torch.where(rand_span_mask[..., None], torch.zeros_like(x1), x1)

        with torch.no_grad() if not update_generator else contextlib.nullcontext():
            pred = self.feedforward_model(
                x=phi,
                cond=cond,
                text=text,
                time=time,
                drop_audio_cond=False,
                drop_text=False # make sure no cfg is used
            )

            # predicted mel spectrogram
            output = phi + (1 - t) * pred
            output[~rand_span_mask] = inp[~rand_span_mask]

        if update_generator:
            generator_data_dict = {
                "inp": output,
                "text": text,
                "rand_span_mask": rand_span_mask,
                "second_time": time if self.second_time else None,
                "mse_loss": time.mean() == student_steps[-1].mean(),
                "real_data": {
                    "inp": inp,
                    "text": text,
                    "rand_span_mask": rand_span_mask
                }
            }

            # avoid any side effects of gradient accumulation
            # self.guidance_model.requires_grad_(False)
            # self.feedforward_model.requires_grad_(True)
            generator_loss_dict, generator_log_dict = self.guidance_model(
                generator_turn=True,
                guidance_turn=False,
                generator_data_dict=generator_data_dict,
                guidance_data_dict=None
            )

            generator_log_dict['ground_truth'] = x1
            generator_log_dict['generator_input'] = phi
            generator_log_dict['generator_output'] = output
            generator_log_dict['generator_cond'] = cond
            generator_log_dict['time'] = time

            return generator_loss_dict, generator_log_dict
        else:
            guidance_data_dict = {
                "inp": output.detach(),
                "text": text,
                "rand_span_mask": rand_span_mask,
                "second_time": time if self.second_time else None,
                "real_data": {
                    "inp": inp,
                    "text": text,
                    "rand_span_mask": rand_span_mask
                }
            }

            # avoid any side effects of gradient accumulation
            # self.feedforward_model.requires_grad_(False)
            # self.guidance_model.requires_grad_(True)
            guidance_loss_dict, guidance_log_dict = self.guidance_model(
                generator_turn=False,
                guidance_turn=True,
                generator_data_dict=None,
                guidance_data_dict=guidance_data_dict
            )
            # self.feedforward_model.requires_grad_(True)

            return guidance_loss_dict, guidance_log_dict

        # return guidance_loss_dict, guidance_log_dict, generator_loss_dict, generator_log_dict

if __name__ == "__main__":

    from f5_tts.model.utils import get_tokenizer
    from torch.utils.data import DataLoader, Dataset, SequentialSampler
    from f5_tts.model.dataset import load_dataset
    from f5_tts.model.dataset import DynamicBatchSampler, collate_fn

    bsz = 16

    tokenizer = "pinyin"  # 'pinyin', 'char', or 'custom'
    tokenizer_path = None  # if tokenizer = 'custom', define the path to the tokenizer you want to use (should be vocab.txt)
    dataset_name = "Emilia_ZH_EN"
    if tokenizer == "custom":
        tokenizer_path = tokenizer_path
    else:
        tokenizer_path = dataset_name


import torch
import torch.nn as nn

# from tts_encode import tts_encode

def calculate_remaining_lengths(mel_lengths):
    B = mel_lengths.shape[0]
    max_L = mel_lengths.max().item()  # Get the maximum length in the batch

    # Create a range tensor: shape (max_L,), [0, 1, 2, ..., max_L-1]
    range_tensor = torch.arange(max_L, device=mel_lengths.device).expand(B, max_L)

    # Compute targets using broadcasting: (L-1) - range_tensor
    remain_lengths = (mel_lengths[:, None] - 1 - range_tensor).clamp(min=0)

    return remain_lengths

class PositionalEncoding(nn.Module):
    def __init__(self, hidden_dim, max_len=4096):
        super().__init__()
        pe = torch.zeros(max_len, hidden_dim)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, hidden_dim, 2).float() * (-torch.log(torch.tensor(10000.0)) / hidden_dim))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        self.pe = pe.unsqueeze(0)  # Shape: (1, max_len, hidden_dim)

    def forward(self, x):
        x = x + self.pe[:, :x.size(1)].to(x.device)
        return x

class SpeechLengthPredictor(nn.Module):

    def __init__(self,
        vocab_size=2545, n_mel=100, hidden_dim=256,
        n_text_layer=4, n_cross_layer=4, n_head=8,
        output_dim=1,
    ):
        super().__init__()

        # Text Encoder: Embedding + Transformer Layers
        self.text_embedder = nn.Embedding(vocab_size+1, hidden_dim, padding_idx=vocab_size)
        self.text_pe = PositionalEncoding(hidden_dim)
        encoder_layer = nn.TransformerEncoderLayer(
            d_model=hidden_dim, nhead=n_head, dim_feedforward=hidden_dim*2, batch_first=True
        )
        self.text_encoder = nn.TransformerEncoder(encoder_layer, num_layers=n_text_layer)

        # Mel Spectrogram Embedder
        self.mel_embedder = nn.Linear(n_mel, hidden_dim)
        self.mel_pe = PositionalEncoding(hidden_dim)

        # Transformer Decoder Layers with Cross-Attention in Every Layer
        decoder_layer = nn.TransformerDecoderLayer(
            d_model=hidden_dim, nhead=n_head, dim_feedforward=hidden_dim*2, batch_first=True
        )
        self.decoder = nn.TransformerDecoder(decoder_layer, num_layers=n_cross_layer)

        # Final Classification Layer
        self.predictor = nn.Linear(hidden_dim, output_dim)

    def forward(self, text_ids, mel):
        # Encode text
        text_embedded = self.text_pe(self.text_embedder(text_ids))
        text_features = self.text_encoder(text_embedded) # (B, L_text, D)

        # Encode Mel spectrogram
        mel_features = self.mel_pe(self.mel_embedder(mel))  # (B, L_mel, D)

        # Causal Masking for Decoder
        seq_len = mel_features.size(1)
        causal_mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1).bool().to(mel.device)
        # causal_mask = torch.triu(
        #     torch.full((seq_len, seq_len), float('-inf'), device=mel.device), diagonal=1
        # )

        # Transformer Decoder with Cross-Attention in Each Layer
        decoder_out = self.decoder(mel_features, text_features, tgt_mask=causal_mask)

        # Length Prediction
        length_logits = self.predictor(decoder_out).squeeze(-1)
        return length_logits


import torch
import torch.nn as nn

# from tts_encode import tts_encode

def calculate_remaining_lengths(mel_lengths):
    B = mel_lengths.shape[0]
    max_L = mel_lengths.max().item()  # Get the maximum length in the batch

    # Create a range tensor: shape (max_L,), [0, 1, 2, ..., max_L-1]
    range_tensor = torch.arange(max_L, device=mel_lengths.device).expand(B, max_L)

    # Compute targets using broadcasting: (L-1) - range_tensor
    remain_lengths = (mel_lengths[:, None] - 1 - range_tensor).clamp(min=0)

    return remain_lengths

class PositionalEncoding(nn.Module):
    def __init__(self, hidden_dim, max_len=4096):
        super().__init__()
        pe = torch.zeros(max_len, hidden_dim)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, hidden_dim, 2).float() * (-torch.log(torch.tensor(10000.0)) / hidden_dim))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        self.pe = pe.unsqueeze(0)  # Shape: (1, max_len, hidden_dim)

    def forward(self, x):
        x = x + self.pe[:, :x.size(1)].to(x.device)
        return x

class SpeechLengthPredictor(nn.Module):

    def __init__(self,


PositionalEncoding

import torch
import torch.nn as nn

# from tts_encode import tts_encode

def calculate_remaining_lengths(mel_lengths):
    B = mel_lengths.shape[0]
    max_L = mel_lengths.max().item()  # Get the maximum length in the batch

    # Create a range tensor: shape (max_L,), [0, 1, 2, ..., max_L-1]
    range_tensor = torch.arange(max_L, device=mel_lengths.device).expand(B, max_L)

    # Compute targets using broadcasting: (L-1) - range_tensor
    remain_lengths = (mel_lengths[:, None] - 1 - range_tensor).clamp(min=0)

    return remain_lengths

class PositionalEncoding(nn.Module):
    def __init__(self, hidden_dim, max_len=4096):
        super().__init__()
        pe = torch.zeros(max_len, hidden_dim)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, hidden_dim, 2).float() * (-torch.log(torch.tensor(10000.0)) / hidden_dim))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        self.pe = pe.unsqueeze(0)  # Shape: (1, max_len, hidden_dim)

    def forward(self, x):
        x = x + self.pe[:, :x.size(1)].to(x.device)
        return x

class SpeechLengthPredictor(nn.Module):

    def __init__(self,
        vocab_size=2545, n_mel=100, hidden_dim=256,
        n_text_layer=4, n_cross_layer=4, n_head=8,
        output_dim=1,
    ):
        super().__init__()

        # Text Encoder: Embedding + Transformer Layers
        self.text_embedder = nn.Embedding(vocab_size+1, hidden_dim, padding_idx=vocab_size)
        self.text_pe = PositionalEncoding(hidden_dim)
        encoder_layer = nn.TransformerEncoderLayer(
            d_model=hidden_dim, nhead=n_head, dim_feedforward=hidden_dim*2, batch_first=True
        )
        self.text_encoder = nn.TransformerEncoder(encoder_layer, num_layers=n_text_layer)

        # Mel Spectrogram Embedder


SpeechLengthPredictor

    remain_lengths = (mel_lengths[:, None] - 1 - range_tensor).clamp(min=0)

    return remain_lengths

class PositionalEncoding(nn.Module):
    def __init__(self, hidden_dim, max_len=4096):
        super().__init__()
        pe = torch.zeros(max_len, hidden_dim)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, hidden_dim, 2).float() * (-torch.log(torch.tensor(10000.0)) / hidden_dim))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        self.pe = pe.unsqueeze(0)  # Shape: (1, max_len, hidden_dim)

    def forward(self, x):
        x = x + self.pe[:, :x.size(1)].to(x.device)
        return x

class SpeechLengthPredictor(nn.Module):

    def __init__(self,
        vocab_size=2545, n_mel=100, hidden_dim=256,
        n_text_layer=4, n_cross_layer=4, n_head=8,
        output_dim=1,
    ):
        super().__init__()

        # Text Encoder: Embedding + Transformer Layers
        self.text_embedder = nn.Embedding(vocab_size+1, hidden_dim, padding_idx=vocab_size)
        self.text_pe = PositionalEncoding(hidden_dim)
        encoder_layer = nn.TransformerEncoderLayer(
            d_model=hidden_dim, nhead=n_head, dim_feedforward=hidden_dim*2, batch_first=True
        )
        self.text_encoder = nn.TransformerEncoder(encoder_layer, num_layers=n_text_layer)

        # Mel Spectrogram Embedder
        self.mel_embedder = nn.Linear(n_mel, hidden_dim)
        self.mel_pe = PositionalEncoding(hidden_dim)

        # Transformer Decoder Layers with Cross-Attention in Every Layer
        decoder_layer = nn.TransformerDecoderLayer(
            d_model=hidden_dim, nhead=n_head, dim_feedforward=hidden_dim*2, batch_first=True
        )
        self.decoder = nn.TransformerDecoder(decoder_layer, num_layers=n_cross_layer)

        # Final Classification Layer
        self.predictor = nn.Linear(hidden_dim, output_dim)

    def forward(self, text_ids, mel):
        # Encode text
        text_embedded = self.text_pe(self.text_embedder(text_ids))
        text_features = self.text_encoder(text_embedded) # (B, L_text, D)

        # Encode Mel spectrogram
        mel_features = self.mel_pe(self.mel_embedder(mel))  # (B, L_mel, D)

        # Causal Masking for Decoder
        seq_len = mel_features.size(1)
        causal_mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1).bool().to(mel.device)
        # causal_mask = torch.triu(
        #     torch.full((seq_len, seq_len), float('-inf'), device=mel.device), diagonal=1
        # )

        # Transformer Decoder with Cross-Attention in Each Layer
        decoder_out = self.decoder(mel_features, text_features, tgt_mask=causal_mask)

        # Length Prediction
        length_logits = self.predictor(decoder_out).squeeze(-1)
        return length_logits


    return remain_lengths

class PositionalEncoding(nn.Module):
    def __init__(self, hidden_dim, max_len=4096):
        super().__init__()
        pe = torch.zeros(max_len, hidden_dim)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, hidden_dim, 2).float() * (-torch.log(torch.tensor(10000.0)) / hidden_dim))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        self.pe = pe.unsqueeze(0)  # Shape: (1, max_len, hidden_dim)

    def forward(self, x):
        x = x + self.pe[:, :x.size(1)].to(x.device)
        return x

class SpeechLengthPredictor(nn.Module):

    def __init__(self,
        vocab_size=2545, n_mel=100, hidden_dim=256,
        n_text_layer=4, n_cross_layer=4, n_head=8,
        output_dim=1,
    ):
        super().__init__()

        # Text Encoder: Embedding + Transformer Layers
        self.text_embedder = nn.Embedding(vocab_size+1, hidden_dim, padding_idx=vocab_size)
        self.text_pe = PositionalEncoding(hidden_dim)
        encoder_layer = nn.TransformerEncoderLayer(
            d_model=hidden_dim, nhead=n_head, dim_feedforward=hidden_dim*2, batch_first=True
        )
        self.text_encoder = nn.TransformerEncoder(encoder_layer, num_layers=n_text_layer)

        # Mel Spectrogram Embedder
        self.mel_embedder = nn.Linear(n_mel, hidden_dim)
        self.mel_pe = PositionalEncoding(hidden_dim)

        # Transformer Decoder Layers with Cross-Attention in Every Layer
        decoder_layer = nn.TransformerDecoderLayer(
            d_model=hidden_dim, nhead=n_head, dim_feedforward=hidden_dim*2, batch_first=True
        )
        self.decoder = nn.TransformerDecoder(decoder_layer, num_layers=n_cross_layer)

        # Final Classification Layer
        self.predictor = nn.Linear(hidden_dim, output_dim)

    def forward(self, text_ids, mel):
        # Encode text
        text_embedded = self.text_pe(self.text_embedder(text_ids))
        text_features = self.text_encoder(text_embedded) # (B, L_text, D)

        # Encode Mel spectrogram
        mel_features = self.mel_pe(self.mel_embedder(mel))  # (B, L_mel, D)

        # Causal Masking for Decoder
        seq_len = mel_features.size(1)
        causal_mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1).bool().to(mel.device)
        # causal_mask = torch.triu(
        #     torch.full((seq_len, seq_len), float('-inf'), device=mel.device), diagonal=1
        # )

        # Transformer Decoder with Cross-Attention in Each Layer
        decoder_out = self.decoder(mel_features, text_features, tgt_mask=causal_mask)

        # Length Prediction


forward

        self.text_embedder = nn.Embedding(vocab_size+1, hidden_dim, padding_idx=vocab_size)
        self.text_pe = PositionalEncoding(hidden_dim)
        encoder_layer = nn.TransformerEncoderLayer(
            d_model=hidden_dim, nhead=n_head, dim_feedforward=hidden_dim*2, batch_first=True
        )
        self.text_encoder = nn.TransformerEncoder(encoder_layer, num_layers=n_text_layer)

        # Mel Spectrogram Embedder
        self.mel_embedder = nn.Linear(n_mel, hidden_dim)
        self.mel_pe = PositionalEncoding(hidden_dim)

        # Transformer Decoder Layers with Cross-Attention in Every Layer
        decoder_layer = nn.TransformerDecoderLayer(
            d_model=hidden_dim, nhead=n_head, dim_feedforward=hidden_dim*2, batch_first=True
        )
        self.decoder = nn.TransformerDecoder(decoder_layer, num_layers=n_cross_layer)

        # Final Classification Layer
        self.predictor = nn.Linear(hidden_dim, output_dim)

    def forward(self, text_ids, mel):
        # Encode text
        text_embedded = self.text_pe(self.text_embedder(text_ids))
        text_features = self.text_encoder(text_embedded) # (B, L_text, D)

        # Encode Mel spectrogram
        mel_features = self.mel_pe(self.mel_embedder(mel))  # (B, L_mel, D)

        # Causal Masking for Decoder
        seq_len = mel_features.size(1)
        causal_mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1).bool().to(mel.device)
        # causal_mask = torch.triu(
        #     torch.full((seq_len, seq_len), float('-inf'), device=mel.device), diagonal=1
        # )

        # Transformer Decoder with Cross-Attention in Each Layer
        decoder_out = self.decoder(mel_features, text_features, tgt_mask=causal_mask)

        # Length Prediction
        length_logits = self.predictor(decoder_out).squeeze(-1)
        return length_logits


import os
import gc
import json
import random
import time
import io
import copy
from typing import List, Dict, Any, Optional, Callable, Tuple, Union

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import AdamW
from torch.optim.lr_scheduler import LinearLR, SequentialLR
from torch.utils.data import DataLoader, Dataset, SequentialSampler, Subset
from tqdm import tqdm

from accelerate import Accelerator
from accelerate.utils import DistributedDataParallelKwargs
import wandb

from f5_tts.model.dataset import collate_fn, DynamicBatchSampler
from f5_tts.model.utils import list_str_to_idx

# torch.autograd.set_detect_anomaly(True)
# os.environ['HYDRA_FULL_ERROR'] = 'True'

def safe_sample(logits, temperature=1.0):
    """
    logits: Tensor of shape (B, n_class)
    temperature: Sampling temperature (higher => more random)
    """
    # Apply temperature scaling
    scaled_logits = logits / temperature

    # Compute categorical distribution
    probs = F.softmax(scaled_logits, dim=-1)

    # Sample from the distribution once per batch element
    samples = torch.multinomial(probs, num_samples=1)  # (B, 1)

    # Convert to one-hot encoding
    one_hot_samples = torch.zeros_like(probs).scatter_(1, samples, 1)

    return one_hot_samples

class GRPODurationTrainer:
    """
    Trainer class that implements GRPO (Generative Reinforcement Learning from Preference Optimization)
    for a duration predictor in text-to-speech synthesis.
    """
    def __init__(
        self,
        model,                      # Duration predictor model
        inference_fn,               # Function to generate speech
        reward_fn,                  # Function to compute rewards from generated speech

        vocab_size: int,            # Size of the vocabulary
        vocab_char_map: dict,       # Mapping from characters to token IDs

        # Duration model parameters
        n_class: int = 301,         # Number of duration classes
        n_frame_per_class: int = 10, # Number of frames per class
        gumbel_tau: int = 0.7,

        # GRPO parameters
        beta: float = 0.04,         # KL regularization weight
        clip_param: float = 0.2,    # PPO clip parameter
        num_pre_samples: int = 8,   # Number of samples per prompt
        compute_gen_logps: bool = True, # Whether to compute generation log probabilities

        # Training parameters
        learning_rate: float = 5e-6,
        num_warmup_updates: int = 10000,
        save_per_updates: int = 10000,
        checkpoint_path: Optional[str] = None,
        all_steps: int = 100000,     # Total training steps

        # Batch parameters
        batch_size: int = 8,
        batch_size_type: str = "sample",
        max_samples: int = 16,
        grad_accumulation_steps: int = 2,
        max_grad_norm: float = 1.0,

        # Logging parameters
        logger: Optional[str] = "wandb",
        wandb_project: str = "tts-duration-grpo",
        wandb_run_name: str = "grpo_run",
        wandb_resume_id: Optional[str] = None,

        accelerate_kwargs: dict = dict(),
    ):
        # Initialize accelerator for distributed training
        ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=False)

        if logger == "wandb" and not wandb.api.api_key:
            logger = None
        print(f"Using logger: {logger}")

        self.accelerator = Accelerator(
            log_with=logger if logger == "wandb" else None,
            kwargs_handlers=[ddp_kwargs],
            gradient_accumulation_steps=grad_accumulation_steps,
            **accelerate_kwargs,
        )

        self.logger = logger
        if self.logger == "wandb":
            if wandb_resume_id:
                init_kwargs = {"wandb": {"resume": "allow", "name": wandb_run_name, "id": wandb_resume_id}}
            else:
                init_kwargs = {"wandb": {"resume": "allow", "name": wandb_run_name}}

            self.accelerator.init_trackers(
                project_name=wandb_project,
                init_kwargs=init_kwargs,
                config={
                    "learning_rate": learning_rate,
                    "num_warmup_updates": num_warmup_updates,
                    "batch_size": batch_size,
                    "beta": beta,
                    "clip_param": clip_param,
                    "num_pre_samples": num_pre_samples,
                    "n_class": n_class,
                    "n_frame_per_class": n_frame_per_class,
                    "all_steps": all_steps,
                    "grad_accumulation_steps": grad_accumulation_steps,
                    "max_grad_norm": max_grad_norm,
                    "gpus": self.accelerator.num_processes,
                },
            )
        elif self.logger == "tensorboard":
            from torch.utils.tensorboard import SummaryWriter
            self.writer = SummaryWriter(log_dir=f"runs/{wandb_run_name}")

        # Store model, inference function, and reward function
        self.model = model

        # Create reference model (frozen clone of the initial model)
        self.ref_model = copy.deepcopy(model)
        for param in self.ref_model.parameters():
            param.requires_grad = False
        self.ref_model.eval()

        # prepare inference_fn
        self.inference_fn = inference_fn
        self.inference_fn.scale = self.inference_fn.scale.to(self.accelerator.device)
        self.inference_fn.tts_model = self.inference_fn.tts_model.to(self.accelerator.device)
        # prepare reward_fn
        self.reward_fn = reward_fn

        # Store vocabulary and mapping
        self.vocab_size = vocab_size
        self.vocab_char_map = vocab_char_map

        # Store duration model parameters
        self.n_class = n_class
        self.n_frame_per_class = n_frame_per_class
        self.gumbel_tau = gumbel_tau

        # Store GRPO parameters
        self.beta = beta
        self.clip_param = clip_param
        self.num_pre_samples = num_pre_samples
        self.compute_gen_logps = compute_gen_logps

        # Store training parameters
        self.learning_rate = learning_rate
        self.num_warmup_updates: int = num_warmup_updates
        self.save_per_updates = save_per_updates
        self.checkpoint_path = checkpoint_path or f"ckpts/{wandb_run_name}"
        self.all_steps = all_steps

        # Store batch parameters
        self.batch_size = batch_size
        self.batch_size_type = batch_size_type
        self.max_samples = max_samples
        self.grad_accumulation_steps = grad_accumulation_steps
        self.max_grad_norm = max_grad_norm

        # Initialize optimizer
        self.optimizer = AdamW(model.parameters(), lr=learning_rate)

        # Prepare model and optimizer with accelerator
        self.model, self.optimizer = self.accelerator.prepare(self.model, self.optimizer)
        self.ref_model = self.accelerator.prepare(self.ref_model)
        self.reward_fn, self.inference_fn = self.accelerator.prepare(self.reward_fn, self.inference_fn)

        # GRPO batch queue
        self.batch_queue = []

        # Store distributed rank
        self.rank = torch.distributed.get_rank() if torch.distributed.is_initialized() else 0

        self.device = f'cuda:{self.rank}'

    @property
    def is_main(self):
        return self.accelerator.is_main_process

    def save_checkpoint(self, step, last=False):
        """Save model and optimizer state"""
        self.accelerator.wait_for_everyone()
        if self.is_main:
            checkpoint = dict(
                model_state_dict=self.accelerator.unwrap_model(self.model).state_dict(),
                optimizer_state_dict=self.accelerator.unwrap_model(self.optimizer).state_dict(),
                scheduler_state_dict=self.scheduler.state_dict() if hasattr(self, 'scheduler') else None,
                step=step,
            )
            if not os.path.exists(self.checkpoint_path):
                os.makedirs(self.checkpoint_path)
            if last:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_last.pt")
            else:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_{step}.pt")

    def load_checkpoint(self):
        """Load latest checkpoint if available"""
        if (
            not self.checkpoint_path
            or not os.path.exists(self.checkpoint_path)
            or not any(filename.endswith(".pt") for filename in os.listdir(self.checkpoint_path))
        ):
            return 0

        self.accelerator.wait_for_everyone()
        if "model_last.pt" in os.listdir(self.checkpoint_path):
            latest_checkpoint = "model_last.pt"
        else:
            latest_checkpoint = sorted(
                [f for f in os.listdir(self.checkpoint_path) if f.endswith(".pt")],
                key=lambda x: int("".join(filter(str.isdigit, x))),
            )[-1]

        print(f'Loading checkpoint: {latest_checkpoint}')
        checkpoint = torch.load(
            f"{self.checkpoint_path}/{latest_checkpoint}",
            weights_only=True,
            map_location="cpu"
        )

        if "step" in checkpoint:
            self.accelerator.unwrap_model(self.model).load_state_dict(checkpoint["model_state_dict"])
            self.accelerator.unwrap_model(self.optimizer).load_state_dict(checkpoint["optimizer_state_dict"])
            if hasattr(self, 'scheduler') and checkpoint["scheduler_state_dict"]:
                self.scheduler.load_state_dict(checkpoint["scheduler_state_dict"])
            step = checkpoint["step"]
        else:
            self.accelerator.unwrap_model(self.model).load_state_dict(checkpoint["model_state_dict"])
            step = 0

        del checkpoint
        gc.collect()

        print(f'Successfully loaded checkpoint at step {step}')
        return step

    @torch.no_grad()
    def get_ref_logps(self, text_ids, mel, sampled_classes):
        """
        Get log probabilities from the reference model for the sampled classes
        """
        B = text_ids.shape[0]
        K = self.num_pre_samples
        with torch.no_grad():
            ref_logits = self.ref_model(text_ids=text_ids, mel=mel)[:, -1, :]
            ref_logits = ref_logits.unsqueeze(1).repeat(1, K, 1).view(B*K, -1)
            ref_log_probs = F.log_softmax(ref_logits, dim=-1)
            ref_logps = torch.gather(
                ref_log_probs,
                dim=-1,
                index=sampled_classes.unsqueeze(-1)
            ).squeeze(-1)
        return ref_logps

    @torch.no_grad()
    def generate_duration_samples(self, batch_inputs):
        """
        Generate multiple duration predictions from the model for each input
        and evaluate them using the inference function and reward model

        Args:
            batch_inputs: Dictionary with text, prompt audio, etc.

        Returns:
            Dictionary with duration samples, rewards, and reference logits
        """

        if self.rank == 0:
            print("Generating duration samples...")

        # all_logits = []
        all_text_ids = []
        all_mels = []
        all_sampled_classes = []
        all_durations = []
        all_rewards = []
        all_gen_logps = []

        all_ctc_loss = []
        all_sv_loss = []

        # Fetch batch inputs
        # prompt_mel = batch_inputs['mel'].permute(0, 2, 1).to(self.device)
        prompt_mel = batch_inputs['mel'].permute(0, 2, 1) # (B, T, 100)
        prompt_text = batch_inputs['text']

        batch_size = prompt_mel.shape[0]

        # Shift text to unpair 'mel' and 'text'; The shifted text will be synthesized
        target_text = batch_inputs['target_text']
        target_text_lengths = torch.LongTensor([len(t) for t in target_text]).to(prompt_mel.device)
        try:
            full_text = [prompt+[' ']+target for prompt, target in zip(prompt_text, target_text)]
        except:
            target_text = [batch_inputs['text'][-1]] + batch_inputs['text'][:-1]
            target_text_lengths = batch_inputs['text_lengths'].clone().roll(1, 0)
            full_text = [prompt+[' ']+target for prompt, target in zip(prompt_text, target_text)]

        # Goes to reward model
        target_text_ids = list_str_to_idx(target_text, self.vocab_char_map).to(self.accelerator.device)    # to device, the dataloader only gives list

        # Goes to duration model and TTS
        full_text_ids = list_str_to_idx(full_text, self.vocab_char_map).to(self.accelerator.device)

        # Deepcopy to separate text_ids for SLP and TTS
        slp_text_ids = full_text_ids.detach().clone()
        slp_text_ids = slp_text_ids.masked_fill(slp_text_ids==-1, self.vocab_size) # (B, L)

        # Pre-compute duration logits
        K = self.num_pre_samples
        B, T, _ = prompt_mel.shape
        _, L = slp_text_ids.shape
        # prompt_mel_k_repeats = prompt_mel.unsqueeze(1).repeat(1, K, 1, 1)  # (B, K, T, 100)
        # slp_text_ids_k_repeats = slp_text_ids.unsqueeze(1).repeat(1, K, 1)  # (B, K, L)

        # Run model once for B inputs
        old_logits = self.model(
            text_ids=slp_text_ids, # (B, L)
            mel=prompt_mel         # (B, T, 100)
        )[:, -1, :]  # (B, n_class)

        # Repeat each result K times along batch dimension
        old_logits = old_logits.unsqueeze(1).repeat(1, K, 1) # (B, K, n_class)
        # logits_nograd = logits_grad.detach().clone().view(B, K, -1) # (B, K, n_class)

        for _full_text_ids, _target_text_ids, _target_text_lengths, \
            _prompt_mel, _old_logits in zip(
                full_text_ids, target_text_ids, target_text_lengths,
                prompt_mel, old_logits
            ):

            duration_sample = F.gumbel_softmax(_old_logits, tau=self.gumbel_tau, hard=True, dim=-1)
            duration2frames = torch.arange(self.n_class).float().to(self.accelerator.device) * self.n_frame_per_class
            est_frames = (duration_sample * duration2frames).sum(-1) # (K, )

            # Compute log probabilities of the samples
            sampled_classes = duration_sample.argmax(dim=-1)
            log_probs = F.log_softmax(_old_logits, dim=-1)
            gen_logps = torch.gather(
                log_probs,
                dim=-1,
                index=sampled_classes.unsqueeze(-1)
            ).squeeze(-1)  # Shape: [K, n_class]

            # Generate speech using the sampled durations
            sampled_rewards = []

            for i in range(K):
                cur_duration = est_frames[i]
                if cur_duration == 0:
                    cur_duration = cur_duration + 50 # prevent 0 duration
                infer_full_text_ids = _full_text_ids.unsqueeze(0)
                infer_prompt_mel = _prompt_mel.unsqueeze(0)
                cur_duration = cur_duration.unsqueeze(0)
                infer_target_text_ids = _target_text_ids.unsqueeze(0)
                infer_target_text_lengths = _target_text_lengths.unsqueeze(0)
                with torch.inference_mode():
                    try:
                        _est_mel = self.inference_fn(
                            full_text_ids=infer_full_text_ids,
                            prompt_mel=infer_prompt_mel,
                            target_duration=cur_duration,
                            teacher_steps=0
                        )
                        _est_mel = _est_mel.permute(0, 2, 1) # (1, T, 100)

                        loss_dict = self.reward_fn(
                            prompt_mel=infer_prompt_mel,
                            est_mel=_est_mel,
                            target_text_id=infer_target_text_ids,
                            target_text_length=infer_target_text_lengths
                        )
                        # #TODO reweight the loss for reward
                        reward_sim = loss_dict['loss_sim'] # 0 to 1
                        reward_ctc = loss_dict['loss_ctc']
                        reward = -(reward_ctc + reward_sim * 3)
                        all_ctc_loss.append(reward_ctc)
                        all_sv_loss.append(reward_sim)
                    except Exception as e:
                        if self.rank == 0:
                            print(f"Error in speech synthesis: {e}")
                        reward = torch.tensor(-1.0).to(cur_duration.device)

                    sampled_rewards.append(reward)
                        # list with length of K
            sampled_rewards = torch.stack(sampled_rewards)  # (K, )
            # Normalize rewards
            if (sampled_rewards.max() - sampled_rewards.min()).item() > 1e-6:
                sampled_rewards = (sampled_rewards - sampled_rewards.mean()) / (sampled_rewards.std() + 1e-8)

            # Store all data
            # all_logits.append(duration_logits)
            # all_text_ids.append(duration_input_expanded["text_ids"])
            # all_mels.append(duration_input_expanded["mel"])
            all_sampled_classes.append(sampled_classes)
            all_durations.append(est_frames)
            all_gen_logps.append(gen_logps)
            all_rewards.extend(sampled_rewards)  # list with length of B*K

        # Concatenate all data
        # logits = torch.cat(all_logits, dim=0)
        # text_ids = torch.cat(all_text_ids, dim=0)
        # mels = torch.cat(all_mels, dim=0)
        sampled_classes = torch.cat(all_sampled_classes, dim=0)
        durations = torch.cat(all_durations, dim=0)
        rewards = torch.stack(all_rewards)    # use stack to keep the same device of elements
        gen_logps = torch.cat(all_gen_logps, dim=0)

        ctc_losses = torch.stack(all_ctc_loss)
        sv_losses = torch.stack(all_sv_loss)

        if self.is_main:
            self.accelerator.log({
                "ctc_loss": ctc_losses.mean().item(),
                "sv_loss": sv_losses.mean().item(),
                "reward": rewards.mean().item(),
                "reward_min": rewards.min().item(),
                "reward_max": rewards.max().item(),
            }, step=self.global_step)

        # # Normalize rewards
        # if (rewards.max() - rewards.min()).item() > 1e-6:
        #     rewards = (rewards - rewards.mean()) / (rewards.std() + 1e-8)

        ref_logps = self.get_ref_logps(slp_text_ids, prompt_mel, sampled_classes)

        # Create batch dict similar to Qwen2.5 implementation
        batch_outputs = {
            # "logits": logits_grad,
            "text_ids": slp_text_ids,
            "prompt_mel": prompt_mel,
            "rewards": rewards,
            "refs": ref_logps,
            "sampled_classes": sampled_classes,
            "durations": durations,
        }

        if self.compute_gen_logps:
            batch_outputs["gen_logps"] = gen_logps

        if self.rank == 0:
            print(f"Generated {len(rewards)} samples with reward min/mean/max: {rewards.min().item():.4f}/{rewards.mean().item():.4f}/{rewards.max().item():.4f}")

        return batch_outputs

    def GRPO_step(self, batch):
        """
        Perform a GRPO update step

        Args:
            batch: Dictionary with inputs, rewards, reference logits, etc.

        Returns:
            Loss value
        """
        # Extract batch data
        # NOTE: why .unsqueeze(1) ???
        rewards = batch['rewards'] #.unsqueeze(1)
        ref_logps = batch['refs']  # (B)
        sampled_classes = batch['sampled_classes'] # (B)
        prompt_mel = batch['prompt_mel']
        text_ids = batch['text_ids']

        # Forward pass to get current model logits
        K = self.num_pre_samples
        B, T, _ = prompt_mel.shape
        _, L = text_ids.shape
        cur_logits = self.model(
# ... truncated ...


import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import AdamW
from torch.optim.lr_scheduler import LinearLR, SequentialLR
from torch.utils.data import DataLoader, Dataset, SequentialSampler, Subset
from tqdm import tqdm

from accelerate import Accelerator
from accelerate.utils import DistributedDataParallelKwargs
import wandb

from f5_tts.model.dataset import collate_fn, DynamicBatchSampler
from f5_tts.model.utils import list_str_to_idx

# torch.autograd.set_detect_anomaly(True)
# os.environ['HYDRA_FULL_ERROR'] = 'True'

def safe_sample(logits, temperature=1.0):
    """
    logits: Tensor of shape (B, n_class)
    temperature: Sampling temperature (higher => more random)
    """
    # Apply temperature scaling
    scaled_logits = logits / temperature

    # Compute categorical distribution
    probs = F.softmax(scaled_logits, dim=-1)

    # Sample from the distribution once per batch element
    samples = torch.multinomial(probs, num_samples=1)  # (B, 1)

    # Convert to one-hot encoding
    one_hot_samples = torch.zeros_like(probs).scatter_(1, samples, 1)

    return one_hot_samples

class GRPODurationTrainer:
    """
    Trainer class that implements GRPO (Generative Reinforcement Learning from Preference Optimization)
    for a duration predictor in text-to-speech synthesis.
    """
    def __init__(
        self,
        model,                      # Duration predictor model
        inference_fn,               # Function to generate speech
        reward_fn,                  # Function to compute rewards from generated speech

        vocab_size: int,            # Size of the vocabulary
        vocab_char_map: dict,       # Mapping from characters to token IDs

        # Duration model parameters
        n_class: int = 301,         # Number of duration classes
        n_frame_per_class: int = 10, # Number of frames per class
        gumbel_tau: int = 0.7,


GRPODurationTrainer

def safe_sample(logits, temperature=1.0):
    """
    logits: Tensor of shape (B, n_class)
    temperature: Sampling temperature (higher => more random)
    """
    # Apply temperature scaling
    scaled_logits = logits / temperature

    # Compute categorical distribution
    probs = F.softmax(scaled_logits, dim=-1)

    # Sample from the distribution once per batch element
    samples = torch.multinomial(probs, num_samples=1)  # (B, 1)

    # Convert to one-hot encoding
    one_hot_samples = torch.zeros_like(probs).scatter_(1, samples, 1)

    return one_hot_samples

class GRPODurationTrainer:
    """
    Trainer class that implements GRPO (Generative Reinforcement Learning from Preference Optimization)
    for a duration predictor in text-to-speech synthesis.
    """
    def __init__(
        self,
        model,                      # Duration predictor model
        inference_fn,               # Function to generate speech
        reward_fn,                  # Function to compute rewards from generated speech

        vocab_size: int,            # Size of the vocabulary
        vocab_char_map: dict,       # Mapping from characters to token IDs

        # Duration model parameters
        n_class: int = 301,         # Number of duration classes
        n_frame_per_class: int = 10, # Number of frames per class
        gumbel_tau: int = 0.7,

        # GRPO parameters
        beta: float = 0.04,         # KL regularization weight
        clip_param: float = 0.2,    # PPO clip parameter
        num_pre_samples: int = 8,   # Number of samples per prompt
        compute_gen_logps: bool = True, # Whether to compute generation log probabilities

        # Training parameters
        learning_rate: float = 5e-6,
        num_warmup_updates: int = 10000,
        save_per_updates: int = 10000,
        checkpoint_path: Optional[str] = None,
        all_steps: int = 100000,     # Total training steps

        # Batch parameters
        batch_size: int = 8,
        batch_size_type: str = "sample",
        max_samples: int = 16,
        grad_accumulation_steps: int = 2,
        max_grad_norm: float = 1.0,

        # Logging parameters
        logger: Optional[str] = "wandb",
        wandb_project: str = "tts-duration-grpo",
        wandb_run_name: str = "grpo_run",
        wandb_resume_id: Optional[str] = None,

        accelerate_kwargs: dict = dict(),
    ):
        # Initialize accelerator for distributed training
        ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=False)

        if logger == "wandb" and not wandb.api.api_key:
            logger = None
        print(f"Using logger: {logger}")

        self.accelerator = Accelerator(
            log_with=logger if logger == "wandb" else None,
            kwargs_handlers=[ddp_kwargs],
            gradient_accumulation_steps=grad_accumulation_steps,
            **accelerate_kwargs,
        )

        self.logger = logger
        if self.logger == "wandb":
            if wandb_resume_id:
                init_kwargs = {"wandb": {"resume": "allow", "name": wandb_run_name, "id": wandb_resume_id}}
            else:
                init_kwargs = {"wandb": {"resume": "allow", "name": wandb_run_name}}

            self.accelerator.init_trackers(
                project_name=wandb_project,
                init_kwargs=init_kwargs,
                config={
                    "learning_rate": learning_rate,
                    "num_warmup_updates": num_warmup_updates,
                    "batch_size": batch_size,
                    "beta": beta,
                    "clip_param": clip_param,
                    "num_pre_samples": num_pre_samples,
                    "n_class": n_class,
                    "n_frame_per_class": n_frame_per_class,
                    "all_steps": all_steps,
                    "grad_accumulation_steps": grad_accumulation_steps,
                    "max_grad_norm": max_grad_norm,
                    "gpus": self.accelerator.num_processes,
                },
            )
        elif self.logger == "tensorboard":
            from torch.utils.tensorboard import SummaryWriter
            self.writer = SummaryWriter(log_dir=f"runs/{wandb_run_name}")

        # Store model, inference function, and reward function
        self.model = model

        # Create reference model (frozen clone of the initial model)
        self.ref_model = copy.deepcopy(model)
        for param in self.ref_model.parameters():
            param.requires_grad = False
        self.ref_model.eval()

        # prepare inference_fn
        self.inference_fn = inference_fn
        self.inference_fn.scale = self.inference_fn.scale.to(self.accelerator.device)
        self.inference_fn.tts_model = self.inference_fn.tts_model.to(self.accelerator.device)
        # prepare reward_fn
        self.reward_fn = reward_fn

        # Store vocabulary and mapping
        self.vocab_size = vocab_size
        self.vocab_char_map = vocab_char_map

        # Store duration model parameters
        self.n_class = n_class
        self.n_frame_per_class = n_frame_per_class
        self.gumbel_tau = gumbel_tau

        # Store GRPO parameters
        self.beta = beta
        self.clip_param = clip_param
        self.num_pre_samples = num_pre_samples
        self.compute_gen_logps = compute_gen_logps

        # Store training parameters
        self.learning_rate = learning_rate
        self.num_warmup_updates: int = num_warmup_updates
        self.save_per_updates = save_per_updates
        self.checkpoint_path = checkpoint_path or f"ckpts/{wandb_run_name}"
        self.all_steps = all_steps

        # Store batch parameters
        self.batch_size = batch_size
        self.batch_size_type = batch_size_type
        self.max_samples = max_samples
        self.grad_accumulation_steps = grad_accumulation_steps
        self.max_grad_norm = max_grad_norm

        # Initialize optimizer
        self.optimizer = AdamW(model.parameters(), lr=learning_rate)

        # Prepare model and optimizer with accelerator
        self.model, self.optimizer = self.accelerator.prepare(self.model, self.optimizer)
        self.ref_model = self.accelerator.prepare(self.ref_model)
        self.reward_fn, self.inference_fn = self.accelerator.prepare(self.reward_fn, self.inference_fn)

        # GRPO batch queue
        self.batch_queue = []

        # Store distributed rank
        self.rank = torch.distributed.get_rank() if torch.distributed.is_initialized() else 0

        self.device = f'cuda:{self.rank}'

    @property
    def is_main(self):
        return self.accelerator.is_main_process

    def save_checkpoint(self, step, last=False):
        """Save model and optimizer state"""
        self.accelerator.wait_for_everyone()
        if self.is_main:
            checkpoint = dict(
                model_state_dict=self.accelerator.unwrap_model(self.model).state_dict(),
                optimizer_state_dict=self.accelerator.unwrap_model(self.optimizer).state_dict(),
                scheduler_state_dict=self.scheduler.state_dict() if hasattr(self, 'scheduler') else None,
                step=step,
            )
            if not os.path.exists(self.checkpoint_path):
                os.makedirs(self.checkpoint_path)
            if last:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_last.pt")
            else:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_{step}.pt")

    def load_checkpoint(self):
        """Load latest checkpoint if available"""
        if (
            not self.checkpoint_path
            or not os.path.exists(self.checkpoint_path)
            or not any(filename.endswith(".pt") for filename in os.listdir(self.checkpoint_path))
        ):
            return 0

        self.accelerator.wait_for_everyone()
        if "model_last.pt" in os.listdir(self.checkpoint_path):
            latest_checkpoint = "model_last.pt"
        else:
            latest_checkpoint = sorted(
                [f for f in os.listdir(self.checkpoint_path) if f.endswith(".pt")],
                key=lambda x: int("".join(filter(str.isdigit, x))),
            )[-1]

        print(f'Loading checkpoint: {latest_checkpoint}')
        checkpoint = torch.load(
            f"{self.checkpoint_path}/{latest_checkpoint}",
            weights_only=True,
            map_location="cpu"
        )

        if "step" in checkpoint:
            self.accelerator.unwrap_model(self.model).load_state_dict(checkpoint["model_state_dict"])
            self.accelerator.unwrap_model(self.optimizer).load_state_dict(checkpoint["optimizer_state_dict"])
            if hasattr(self, 'scheduler') and checkpoint["scheduler_state_dict"]:
                self.scheduler.load_state_dict(checkpoint["scheduler_state_dict"])
            step = checkpoint["step"]
        else:
            self.accelerator.unwrap_model(self.model).load_state_dict(checkpoint["model_state_dict"])
            step = 0

        del checkpoint
        gc.collect()

        print(f'Successfully loaded checkpoint at step {step}')
        return step

    @torch.no_grad()
    def get_ref_logps(self, text_ids, mel, sampled_classes):
        """
        Get log probabilities from the reference model for the sampled classes
        """
        B = text_ids.shape[0]
        K = self.num_pre_samples
        with torch.no_grad():
            ref_logits = self.ref_model(text_ids=text_ids, mel=mel)[:, -1, :]
            ref_logits = ref_logits.unsqueeze(1).repeat(1, K, 1).view(B*K, -1)
            ref_log_probs = F.log_softmax(ref_logits, dim=-1)
            ref_logps = torch.gather(
                ref_log_probs,
                dim=-1,
                index=sampled_classes.unsqueeze(-1)
            ).squeeze(-1)
        return ref_logps

    @torch.no_grad()
    def generate_duration_samples(self, batch_inputs):
        """
        Generate multiple duration predictions from the model for each input
        and evaluate them using the inference function and reward model

        Args:
            batch_inputs: Dictionary with text, prompt audio, etc.

        Returns:
            Dictionary with duration samples, rewards, and reference logits
        """

        if self.rank == 0:
            print("Generating duration samples...")

        # all_logits = []
        all_text_ids = []
        all_mels = []
        all_sampled_classes = []
        all_durations = []
        all_rewards = []
        all_gen_logps = []

        all_ctc_loss = []
        all_sv_loss = []

        # Fetch batch inputs
        # prompt_mel = batch_inputs['mel'].permute(0, 2, 1).to(self.device)
        prompt_mel = batch_inputs['mel'].permute(0, 2, 1) # (B, T, 100)
        prompt_text = batch_inputs['text']

        batch_size = prompt_mel.shape[0]

        # Shift text to unpair 'mel' and 'text'; The shifted text will be synthesized
        target_text = batch_inputs['target_text']
        target_text_lengths = torch.LongTensor([len(t) for t in target_text]).to(prompt_mel.device)
        try:
            full_text = [prompt+[' ']+target for prompt, target in zip(prompt_text, target_text)]
        except:
            target_text = [batch_inputs['text'][-1]] + batch_inputs['text'][:-1]
            target_text_lengths = batch_inputs['text_lengths'].clone().roll(1, 0)
            full_text = [prompt+[' ']+target for prompt, target in zip(prompt_text, target_text)]

        # Goes to reward model
        target_text_ids = list_str_to_idx(target_text, self.vocab_char_map).to(self.accelerator.device)    # to device, the dataloader only gives list

        # Goes to duration model and TTS
        full_text_ids = list_str_to_idx(full_text, self.vocab_char_map).to(self.accelerator.device)

        # Deepcopy to separate text_ids for SLP and TTS
        slp_text_ids = full_text_ids.detach().clone()
        slp_text_ids = slp_text_ids.masked_fill(slp_text_ids==-1, self.vocab_size) # (B, L)

        # Pre-compute duration logits
        K = self.num_pre_samples
        B, T, _ = prompt_mel.shape
        _, L = slp_text_ids.shape
        # prompt_mel_k_repeats = prompt_mel.unsqueeze(1).repeat(1, K, 1, 1)  # (B, K, T, 100)
        # slp_text_ids_k_repeats = slp_text_ids.unsqueeze(1).repeat(1, K, 1)  # (B, K, L)

        # Run model once for B inputs
        old_logits = self.model(
            text_ids=slp_text_ids, # (B, L)
            mel=prompt_mel         # (B, T, 100)
        )[:, -1, :]  # (B, n_class)

        # Repeat each result K times along batch dimension
        old_logits = old_logits.unsqueeze(1).repeat(1, K, 1) # (B, K, n_class)
        # logits_nograd = logits_grad.detach().clone().view(B, K, -1) # (B, K, n_class)

        for _full_text_ids, _target_text_ids, _target_text_lengths, \
            _prompt_mel, _old_logits in zip(
                full_text_ids, target_text_ids, target_text_lengths,
                prompt_mel, old_logits
            ):

            duration_sample = F.gumbel_softmax(_old_logits, tau=self.gumbel_tau, hard=True, dim=-1)
            duration2frames = torch.arange(self.n_class).float().to(self.accelerator.device) * self.n_frame_per_class
            est_frames = (duration_sample * duration2frames).sum(-1) # (K, )

            # Compute log probabilities of the samples
            sampled_classes = duration_sample.argmax(dim=-1)
            log_probs = F.log_softmax(_old_logits, dim=-1)
            gen_logps = torch.gather(
                log_probs,
                dim=-1,
                index=sampled_classes.unsqueeze(-1)
            ).squeeze(-1)  # Shape: [K, n_class]

            # Generate speech using the sampled durations
            sampled_rewards = []

            for i in range(K):
                cur_duration = est_frames[i]
                if cur_duration == 0:
                    cur_duration = cur_duration + 50 # prevent 0 duration
                infer_full_text_ids = _full_text_ids.unsqueeze(0)
                infer_prompt_mel = _prompt_mel.unsqueeze(0)
                cur_duration = cur_duration.unsqueeze(0)
                infer_target_text_ids = _target_text_ids.unsqueeze(0)
                infer_target_text_lengths = _target_text_lengths.unsqueeze(0)
                with torch.inference_mode():
                    try:
                        _est_mel = self.inference_fn(
                            full_text_ids=infer_full_text_ids,
                            prompt_mel=infer_prompt_mel,
                            target_duration=cur_duration,
                            teacher_steps=0
                        )
                        _est_mel = _est_mel.permute(0, 2, 1) # (1, T, 100)

                        loss_dict = self.reward_fn(
                            prompt_mel=infer_prompt_mel,
                            est_mel=_est_mel,
                            target_text_id=infer_target_text_ids,
                            target_text_length=infer_target_text_lengths
                        )
                        # #TODO reweight the loss for reward
                        reward_sim = loss_dict['loss_sim'] # 0 to 1
                        reward_ctc = loss_dict['loss_ctc']
                        reward = -(reward_ctc + reward_sim * 3)
                        all_ctc_loss.append(reward_ctc)
                        all_sv_loss.append(reward_sim)
                    except Exception as e:
                        if self.rank == 0:
                            print(f"Error in speech synthesis: {e}")
                        reward = torch.tensor(-1.0).to(cur_duration.device)

                    sampled_rewards.append(reward)
                        # list with length of K
            sampled_rewards = torch.stack(sampled_rewards)  # (K, )
            # Normalize rewards
            if (sampled_rewards.max() - sampled_rewards.min()).item() > 1e-6:
                sampled_rewards = (sampled_rewards - sampled_rewards.mean()) / (sampled_rewards.std() + 1e-8)

            # Store all data
            # all_logits.append(duration_logits)
            # all_text_ids.append(duration_input_expanded["text_ids"])
            # all_mels.append(duration_input_expanded["mel"])
            all_sampled_classes.append(sampled_classes)
            all_durations.append(est_frames)
            all_gen_logps.append(gen_logps)
            all_rewards.extend(sampled_rewards)  # list with length of B*K

        # Concatenate all data
        # logits = torch.cat(all_logits, dim=0)
        # text_ids = torch.cat(all_text_ids, dim=0)
        # mels = torch.cat(all_mels, dim=0)
        sampled_classes = torch.cat(all_sampled_classes, dim=0)
        durations = torch.cat(all_durations, dim=0)
        rewards = torch.stack(all_rewards)    # use stack to keep the same device of elements
        gen_logps = torch.cat(all_gen_logps, dim=0)

        ctc_losses = torch.stack(all_ctc_loss)
        sv_losses = torch.stack(all_sv_loss)

        if self.is_main:
            self.accelerator.log({
                "ctc_loss": ctc_losses.mean().item(),
                "sv_loss": sv_losses.mean().item(),
                "reward": rewards.mean().item(),
                "reward_min": rewards.min().item(),
                "reward_max": rewards.max().item(),
            }, step=self.global_step)

        # # Normalize rewards
        # if (rewards.max() - rewards.min()).item() > 1e-6:
        #     rewards = (rewards - rewards.mean()) / (rewards.std() + 1e-8)

        ref_logps = self.get_ref_logps(slp_text_ids, prompt_mel, sampled_classes)

        # Create batch dict similar to Qwen2.5 implementation
        batch_outputs = {
            # "logits": logits_grad,
            "text_ids": slp_text_ids,
            "prompt_mel": prompt_mel,
            "rewards": rewards,
            "refs": ref_logps,
            "sampled_classes": sampled_classes,
            "durations": durations,
        }

        if self.compute_gen_logps:
            batch_outputs["gen_logps"] = gen_logps

        if self.rank == 0:
            print(f"Generated {len(rewards)} samples with reward min/mean/max: {rewards.min().item():.4f}/{rewards.mean().item():.4f}/{rewards.max().item():.4f}")

        return batch_outputs

    def GRPO_step(self, batch):
        """
        Perform a GRPO update step

        Args:
            batch: Dictionary with inputs, rewards, reference logits, etc.

        Returns:
            Loss value
        """
        # Extract batch data
        # NOTE: why .unsqueeze(1) ???
        rewards = batch['rewards'] #.unsqueeze(1)
        ref_logps = batch['refs']  # (B)
        sampled_classes = batch['sampled_classes'] # (B)
        prompt_mel = batch['prompt_mel']
        text_ids = batch['text_ids']

        # Forward pass to get current model logits
        K = self.num_pre_samples
        B, T, _ = prompt_mel.shape
        _, L = text_ids.shape
        cur_logits = self.model(
            text_ids=text_ids, # (B, L)
            mel=prompt_mel         # (B, T, 100)
        )[:, -1, :]
        cur_logits = cur_logits.unsqueeze(1).repeat(1, K, 1).view(B*K, -1)

        # Compute current log probabilities for sampled actions
        log_probs = F.log_softmax(cur_logits, dim=-1)
        cur_logps = torch.gather(
            log_probs,
            dim=-1,
            index=sampled_classes.unsqueeze(-1)
        ).squeeze(-1)  # (B)

        # KL divergence computation (same as in Qwen2.5 code)
        # KL = exp(ref - cur) - (ref - cur) - 1
        kl_div = torch.exp(ref_logps - cur_logps) - (ref_logps - cur_logps) - 1 # (B)

        # Compute probability ratio for PPO
# ... truncated ...


    # Apply temperature scaling
    scaled_logits = logits / temperature

    # Compute categorical distribution
    probs = F.softmax(scaled_logits, dim=-1)

    # Sample from the distribution once per batch element
    samples = torch.multinomial(probs, num_samples=1)  # (B, 1)

    # Convert to one-hot encoding
    one_hot_samples = torch.zeros_like(probs).scatter_(1, samples, 1)

    return one_hot_samples

class GRPODurationTrainer:
    """
    Trainer class that implements GRPO (Generative Reinforcement Learning from Preference Optimization)
    for a duration predictor in text-to-speech synthesis.
    """
    def __init__(
        self,
        model,                      # Duration predictor model
        inference_fn,               # Function to generate speech
        reward_fn,                  # Function to compute rewards from generated speech

        vocab_size: int,            # Size of the vocabulary
        vocab_char_map: dict,       # Mapping from characters to token IDs

        # Duration model parameters
        n_class: int = 301,         # Number of duration classes
        n_frame_per_class: int = 10, # Number of frames per class
        gumbel_tau: int = 0.7,

        # GRPO parameters
        beta: float = 0.04,         # KL regularization weight
        clip_param: float = 0.2,    # PPO clip parameter
        num_pre_samples: int = 8,   # Number of samples per prompt
        compute_gen_logps: bool = True, # Whether to compute generation log probabilities

        # Training parameters
        learning_rate: float = 5e-6,
        num_warmup_updates: int = 10000,
        save_per_updates: int = 10000,
        checkpoint_path: Optional[str] = None,
        all_steps: int = 100000,     # Total training steps

        # Batch parameters
        batch_size: int = 8,
        batch_size_type: str = "sample",
        max_samples: int = 16,
        grad_accumulation_steps: int = 2,
        max_grad_norm: float = 1.0,

        # Logging parameters
        logger: Optional[str] = "wandb",
        wandb_project: str = "tts-duration-grpo",
        wandb_run_name: str = "grpo_run",
        wandb_resume_id: Optional[str] = None,

        accelerate_kwargs: dict = dict(),
    ):
        # Initialize accelerator for distributed training
        ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=False)

        if logger == "wandb" and not wandb.api.api_key:
            logger = None
        print(f"Using logger: {logger}")

        self.accelerator = Accelerator(
            log_with=logger if logger == "wandb" else None,
            kwargs_handlers=[ddp_kwargs],
            gradient_accumulation_steps=grad_accumulation_steps,
            **accelerate_kwargs,
        )

        self.logger = logger
        if self.logger == "wandb":
            if wandb_resume_id:
                init_kwargs = {"wandb": {"resume": "allow", "name": wandb_run_name, "id": wandb_resume_id}}
            else:
                init_kwargs = {"wandb": {"resume": "allow", "name": wandb_run_name}}

            self.accelerator.init_trackers(
                project_name=wandb_project,
                init_kwargs=init_kwargs,
                config={
                    "learning_rate": learning_rate,
                    "num_warmup_updates": num_warmup_updates,
                    "batch_size": batch_size,
                    "beta": beta,
                    "clip_param": clip_param,
                    "num_pre_samples": num_pre_samples,
                    "n_class": n_class,
                    "n_frame_per_class": n_frame_per_class,
                    "all_steps": all_steps,
                    "grad_accumulation_steps": grad_accumulation_steps,
                    "max_grad_norm": max_grad_norm,
                    "gpus": self.accelerator.num_processes,
                },
            )
        elif self.logger == "tensorboard":
            from torch.utils.tensorboard import SummaryWriter
            self.writer = SummaryWriter(log_dir=f"runs/{wandb_run_name}")

        # Store model, inference function, and reward function
        self.model = model

        # Create reference model (frozen clone of the initial model)
        self.ref_model = copy.deepcopy(model)
        for param in self.ref_model.parameters():
            param.requires_grad = False
        self.ref_model.eval()

        # prepare inference_fn
        self.inference_fn = inference_fn
        self.inference_fn.scale = self.inference_fn.scale.to(self.accelerator.device)
        self.inference_fn.tts_model = self.inference_fn.tts_model.to(self.accelerator.device)
        # prepare reward_fn
        self.reward_fn = reward_fn

        # Store vocabulary and mapping
        self.vocab_size = vocab_size
        self.vocab_char_map = vocab_char_map

        # Store duration model parameters
        self.n_class = n_class
        self.n_frame_per_class = n_frame_per_class
        self.gumbel_tau = gumbel_tau

        # Store GRPO parameters
        self.beta = beta
        self.clip_param = clip_param
        self.num_pre_samples = num_pre_samples
        self.compute_gen_logps = compute_gen_logps

        # Store training parameters
        self.learning_rate = learning_rate
        self.num_warmup_updates: int = num_warmup_updates
        self.save_per_updates = save_per_updates
        self.checkpoint_path = checkpoint_path or f"ckpts/{wandb_run_name}"
        self.all_steps = all_steps

        # Store batch parameters
        self.batch_size = batch_size
        self.batch_size_type = batch_size_type
        self.max_samples = max_samples
        self.grad_accumulation_steps = grad_accumulation_steps
        self.max_grad_norm = max_grad_norm

        # Initialize optimizer
        self.optimizer = AdamW(model.parameters(), lr=learning_rate)

        # Prepare model and optimizer with accelerator
        self.model, self.optimizer = self.accelerator.prepare(self.model, self.optimizer)
        self.ref_model = self.accelerator.prepare(self.ref_model)
        self.reward_fn, self.inference_fn = self.accelerator.prepare(self.reward_fn, self.inference_fn)

        # GRPO batch queue
        self.batch_queue = []

        # Store distributed rank
        self.rank = torch.distributed.get_rank() if torch.distributed.is_initialized() else 0

        self.device = f'cuda:{self.rank}'

    @property
    def is_main(self):
        return self.accelerator.is_main_process

    def save_checkpoint(self, step, last=False):
        """Save model and optimizer state"""
        self.accelerator.wait_for_everyone()
        if self.is_main:
            checkpoint = dict(
                model_state_dict=self.accelerator.unwrap_model(self.model).state_dict(),
                optimizer_state_dict=self.accelerator.unwrap_model(self.optimizer).state_dict(),
                scheduler_state_dict=self.scheduler.state_dict() if hasattr(self, 'scheduler') else None,
                step=step,
            )
            if not os.path.exists(self.checkpoint_path):
                os.makedirs(self.checkpoint_path)
            if last:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_last.pt")
            else:


        self.grad_accumulation_steps = grad_accumulation_steps
        self.max_grad_norm = max_grad_norm

        # Initialize optimizer
        self.optimizer = AdamW(model.parameters(), lr=learning_rate)

        # Prepare model and optimizer with accelerator
        self.model, self.optimizer = self.accelerator.prepare(self.model, self.optimizer)
        self.ref_model = self.accelerator.prepare(self.ref_model)
        self.reward_fn, self.inference_fn = self.accelerator.prepare(self.reward_fn, self.inference_fn)

        # GRPO batch queue
        self.batch_queue = []

        # Store distributed rank
        self.rank = torch.distributed.get_rank() if torch.distributed.is_initialized() else 0

        self.device = f'cuda:{self.rank}'

    @property
    def is_main(self):
        return self.accelerator.is_main_process

    def save_checkpoint(self, step, last=False):
        """Save model and optimizer state"""
        self.accelerator.wait_for_everyone()
        if self.is_main:
            checkpoint = dict(
                model_state_dict=self.accelerator.unwrap_model(self.model).state_dict(),
                optimizer_state_dict=self.accelerator.unwrap_model(self.optimizer).state_dict(),
                scheduler_state_dict=self.scheduler.state_dict() if hasattr(self, 'scheduler') else None,
                step=step,
            )
            if not os.path.exists(self.checkpoint_path):
                os.makedirs(self.checkpoint_path)
            if last:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_last.pt")
            else:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_{step}.pt")

    def load_checkpoint(self):
        """Load latest checkpoint if available"""


        # Initialize optimizer
        self.optimizer = AdamW(model.parameters(), lr=learning_rate)

        # Prepare model and optimizer with accelerator
        self.model, self.optimizer = self.accelerator.prepare(self.model, self.optimizer)
        self.ref_model = self.accelerator.prepare(self.ref_model)
        self.reward_fn, self.inference_fn = self.accelerator.prepare(self.reward_fn, self.inference_fn)

        # GRPO batch queue
        self.batch_queue = []

        # Store distributed rank
        self.rank = torch.distributed.get_rank() if torch.distributed.is_initialized() else 0

        self.device = f'cuda:{self.rank}'

    @property
    def is_main(self):
        return self.accelerator.is_main_process

    def save_checkpoint(self, step, last=False):
        """Save model and optimizer state"""
        self.accelerator.wait_for_everyone()
        if self.is_main:
            checkpoint = dict(
                model_state_dict=self.accelerator.unwrap_model(self.model).state_dict(),
                optimizer_state_dict=self.accelerator.unwrap_model(self.optimizer).state_dict(),
                scheduler_state_dict=self.scheduler.state_dict() if hasattr(self, 'scheduler') else None,
                step=step,
            )
            if not os.path.exists(self.checkpoint_path):
                os.makedirs(self.checkpoint_path)
            if last:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_last.pt")
            else:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_{step}.pt")

    def load_checkpoint(self):
        """Load latest checkpoint if available"""
        if (
            not self.checkpoint_path
            or not os.path.exists(self.checkpoint_path)
            or not any(filename.endswith(".pt") for filename in os.listdir(self.checkpoint_path))
        ):
            return 0

        self.accelerator.wait_for_everyone()
        if "model_last.pt" in os.listdir(self.checkpoint_path):
            latest_checkpoint = "model_last.pt"
        else:
            latest_checkpoint = sorted(
                [f for f in os.listdir(self.checkpoint_path) if f.endswith(".pt")],
                key=lambda x: int("".join(filter(str.isdigit, x))),
            )[-1]

        print(f'Loading checkpoint: {latest_checkpoint}')


    def is_main(self):
        return self.accelerator.is_main_process

    def save_checkpoint(self, step, last=False):
        """Save model and optimizer state"""
        self.accelerator.wait_for_everyone()
        if self.is_main:
            checkpoint = dict(
                model_state_dict=self.accelerator.unwrap_model(self.model).state_dict(),
                optimizer_state_dict=self.accelerator.unwrap_model(self.optimizer).state_dict(),
                scheduler_state_dict=self.scheduler.state_dict() if hasattr(self, 'scheduler') else None,
                step=step,
            )
            if not os.path.exists(self.checkpoint_path):
                os.makedirs(self.checkpoint_path)
            if last:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_last.pt")
            else:
                self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_{step}.pt")

    def load_checkpoint(self):
        """Load latest checkpoint if available"""
        if (
            not self.checkpoint_path
            or not os.path.exists(self.checkpoint_path)
            or not any(filename.endswith(".pt") for filename in os.listdir(self.checkpoint_path))
        ):
            return 0

        self.accelerator.wait_for_everyone()
        if "model_last.pt" in os.listdir(self.checkpoint_path):
            latest_checkpoint = "model_last.pt"
        else:
            latest_checkpoint = sorted(
                [f for f in os.listdir(self.checkpoint_path) if f.endswith(".pt")],
                key=lambda x: int("".join(filter(str.isdigit, x))),
            )[-1]

        print(f'Loading checkpoint: {latest_checkpoint}')
        checkpoint = torch.load(
            f"{self.checkpoint_path}/{latest_checkpoint}",
            weights_only=True,
            map_location="cpu"
        )

        if "step" in checkpoint:
            self.accelerator.unwrap_model(self.model).load_state_dict(checkpoint["model_state_dict"])
            self.accelerator.unwrap_model(self.optimizer).load_state_dict(checkpoint["optimizer_state_dict"])
            if hasattr(self, 'scheduler') and checkpoint["scheduler_state_dict"]:
                self.scheduler.load_state_dict(checkpoint["scheduler_state_dict"])
            step = checkpoint["step"]
        else:
            self.accelerator.unwrap_model(self.model).load_state_dict(checkpoint["model_state_dict"])
            step = 0

        del checkpoint
        gc.collect()

        print(f'Successfully loaded checkpoint at step {step}')
        return step

    @torch.no_grad()
    def get_ref_logps(self, text_ids, mel, sampled_classes):
        """
        Get log probabilities from the reference model for the sampled classes
        """
        B = text_ids.shape[0]
        K = self.num_pre_samples
        with torch.no_grad():
            ref_logits = self.ref_model(text_ids=text_ids, mel=mel)[:, -1, :]
            ref_logits = ref_logits.unsqueeze(1).repeat(1, K, 1).view(B*K, -1)
            ref_log_probs = F.log_softmax(ref_logits, dim=-1)
            ref_logps = torch.gather(
                ref_log_probs,
                dim=-1,
                index=sampled_classes.unsqueeze(-1)
            ).squeeze(-1)
        return ref_logps

    @torch.no_grad()


            map_location="cpu"
        )

        if "step" in checkpoint:
            self.accelerator.unwrap_model(self.model).load_state_dict(checkpoint["model_state_dict"])
            self.accelerator.unwrap_model(self.optimizer).load_state_dict(checkpoint["optimizer_state_dict"])
            if hasattr(self, 'scheduler') and checkpoint["scheduler_state_dict"]:
                self.scheduler.load_state_dict(checkpoint["scheduler_state_dict"])
            step = checkpoint["step"]
        else:
            self.accelerator.unwrap_model(self.model).load_state_dict(checkpoint["model_state_dict"])
            step = 0

        del checkpoint
        gc.collect()

        print(f'Successfully loaded checkpoint at step {step}')
        return step

    @torch.no_grad()
    def get_ref_logps(self, text_ids, mel, sampled_classes):
        """
        Get log probabilities from the reference model for the sampled classes
        """
        B = text_ids.shape[0]
        K = self.num_pre_samples
        with torch.no_grad():
            ref_logits = self.ref_model(text_ids=text_ids, mel=mel)[:, -1, :]
            ref_logits = ref_logits.unsqueeze(1).repeat(1, K, 1).view(B*K, -1)
            ref_log_probs = F.log_softmax(ref_logits, dim=-1)
            ref_logps = torch.gather(
                ref_log_probs,
                dim=-1,
                index=sampled_classes.unsqueeze(-1)
            ).squeeze(-1)
        return ref_logps

    @torch.no_grad()
    def generate_duration_samples(self, batch_inputs):
        """
        Generate multiple duration predictions from the model for each input
        and evaluate them using the inference function and reward model

        Args:
            batch_inputs: Dictionary with text, prompt audio, etc.

        Returns:
            Dictionary with duration samples, rewards, and reference logits
        """

        if self.rank == 0:
            print("Generating duration samples...")

        # all_logits = []
        all_text_ids = []
        all_mels = []