test

from __future__ import annotations
from typing import List, Tuple, Any, Optional

import torch
import torch.nn as nn
import torch.optim as optim
from torch import Tensor
from loguru import logger
from copy import deepcopy

# Define the end-of-sequence token ID (you should set this according to your tokenizer)
eos_token_id = 2  # Example EOS token ID

class Node:
    """
    A class representing a node in the tree of thoughts.

    Attributes:
        sequence (List[int]): The sequence of tokens from the root to this node.
        children (List[Node]): The list of child nodes.
    """

    def __init__(self, sequence: List[int]):
        self.sequence: List[int] = sequence
        self.children: List[Node] = []

    def add_child(self, child_node: Node):
        """Adds a child node to the current node."""
        self.children.append(child_node)

class PolicyModel(nn.Module):
    """
    Policy model π_θ to generate sequences.

    Args:
        vocab_size (int): Vocabulary size.
        hidden_size (int): Hidden layer size.
        num_layers (int): Number of transformer layers.
    """

    def __init__(self, vocab_size: int, hidden_size: int, num_layers: int):
        super(PolicyModel, self).__init__()
        self.embedding = nn.Embedding(vocab_size, hidden_size)
        self.transformer = nn.Transformer(
            d_model=hidden_size,
            nhead=8,
            num_encoder_layers=num_layers,
            num_decoder_layers=num_layers,
        )
        self.fc_out = nn.Linear(hidden_size, vocab_size)

    def forward(self, src: Tensor, tgt: Tensor) -> Tensor:
        """
        Forward pass of the policy model.

        Args:
            src (Tensor): Source sequence tensor (prompt), shape (S, N).
            tgt (Tensor): Target sequence tensor (continuation), shape (T, N).

        Returns:
            Tensor: Logits over the vocabulary, shape (T, N, V).
        """
        src_emb = self.embedding(src)  # (S, N, E)
        tgt_emb = self.embedding(tgt)  # (T, N, E)
        memory = self.transformer.encoder(src_emb)
        output = self.transformer.decoder(tgt_emb, memory)
        logits = self.fc_out(output)  # (T, N, V)
        return logits

class RewardModel(nn.Module):
    """
    Reward model R(s) to compute rewards for sequences.

    Args:
        vocab_size (int): Vocabulary size.
        hidden_size (int): Hidden layer size.
        num_layers (int): Number of transformer layers.
    """

    def __init__(self, vocab_size: int, hidden_size: int, num_layers: int):
        super(RewardModel, self).__init__()
        self.embedding = nn.Embedding(vocab_size, hidden_size)
        encoder_layer = nn.TransformerEncoderLayer(
            d_model=hidden_size, nhead=8, dim_feedforward=hidden_size * 4
        )
        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
        self.fc_out = nn.Linear(hidden_size, 1)

    def forward(self, sequence: Tensor) -> Tensor:
        """
        Forward pass of the reward model.

        Args:
            sequence (Tensor): Sequence tensor, shape (S, N).

        Returns:
            Tensor: Scalar reward value, shape (N).
        """
        emb = self.embedding(sequence)  # (S, N, E)
        output = self.transformer(emb)  # (S, N, E)
        # Take the mean over the sequence length
        pooled_output = output.mean(dim=0)  # (N, E)
        reward = self.fc_out(pooled_output)  # (N, 1)
        return reward.squeeze(-1)  # (N)

def sample_sequence(
    model: PolicyModel, context: List[int], max_length: int, eos_token_id: int
) -> List[int]:
    """
    Samples a continuation from the model given the context.

    Args:
        model (PolicyModel): The policy model.
        context (List[int]): The context sequence (list of token ids).
        max_length (int): Maximum length of the continuation.
        eos_token_id (int): End-of-sequence token ID.

    Returns:
        List[int]: Sampled continuation tokens.
    """
    model.eval()
    generated = context.copy()
    with torch.no_grad():
        for _ in range(max_length):
            src = torch.tensor(context).unsqueeze(1)  # (S, 1)
            tgt_input = torch.tensor(generated).unsqueeze(1)  # (T, 1)
            logits = model(src, tgt_input)
            next_token_logits = logits[-1, 0, :]  # (V)
            probabilities = torch.softmax(next_token_logits, dim=-1)
            next_token_id = torch.multinomial(probabilities, num_samples=1).item()
            generated.append(next_token_id)
            if next_token_id == eos_token_id:
                break
    continuation = generated[len(context) :]
    return continuation

def compute_log_probability(
    model: PolicyModel, sequence: List[int], requires_grad: bool = False
) -> Tensor:
    """
    Computes the total log probability of the sequence under the model.

    Args:
        model (PolicyModel): The model (policy or reference).
        sequence (List[int]): The sequence of token ids.
        requires_grad (bool): Whether to compute gradients.

    Returns:
        Tensor: Total log probability of the sequence.
    """
    sequence_tensor = torch.tensor(sequence).unsqueeze(1)  # (S, 1)
    src = sequence_tensor[:-1, :]  # (S-1, 1)
    tgt = sequence_tensor[:-1, :]  # (S-1, 1)
    target_ids = sequence_tensor[1:, 0]  # (S-1)
    if not requires_grad:
        model.eval()
        with torch.no_grad():
            logits = model(src, tgt)  # (T, N, V)
            logits = logits.squeeze(1)  # (S-1, V)
            log_probs = torch.log_softmax(logits, dim=-1)  # (S-1, V)
            token_logprobs = log_probs[range(len(target_ids)), target_ids]  # (S-1)
            total_logprob = token_logprobs.sum()  # Scalar
    else:
        model.train()
        logits = model(src, tgt)
        logits = logits.squeeze(1)
        log_probs = torch.log_softmax(logits, dim=-1)
        token_logprobs = log_probs[range(len(target_ids)), target_ids]
        total_logprob = token_logprobs.sum()
    return total_logprob

def compute_reward(reward_model: RewardModel, sequence: List[int]) -> float:
    """
    Computes the reward of a sequence using the reward model.

    Args:
        reward_model (RewardModel): The reward model.
        sequence (List[int]): The sequence of token ids.

    Returns:
        float: Reward value.
    """
    reward_model.eval()
    with torch.no_grad():
        input_ids = torch.tensor(sequence).unsqueeze(1)  # (S, 1)
        reward = reward_model(input_ids)  # (1)
        return reward.item()

def train_policy_model(
    policy_model: PolicyModel,
    reward_model: RewardModel,
    prompts: List[List[int]],
    vocab_size: int,
    eos_token_id: int,
    beta: float = 0.1,
    D: int = 3,
    B: int = 2,
    max_length: int = 10,
    learning_rate: float = 1e-4,
    T_max: int = 1000,
    update_reference_model_every: Optional[int] = None,
) -> None:
    """
    Trains the policy model using RLHF with DPO and Monte Carlo Tree of Thoughts.

    Args:
        policy_model (PolicyModel): The policy model to train.
        reward_model (RewardModel): The reward model.
        prompts (List[List[int]]): The training dataset of prompts.
        vocab_size (int): Vocabulary size.
        eos_token_id (int): End-of-sequence token id.
        beta (float, optional): Beta parameter for DPO loss. Defaults to 0.1.
        D (int, optional): Maximum depth of the tree. Defaults to 3.
        B (int, optional): Number of branches per node. Defaults to 2.
        max_length (int, optional): Maximum length of continuations. Defaults to 10.
        learning_rate (float, optional): Learning rate for optimizer. Defaults to 1e-4.
        T_max (int, optional): Maximum number of training iterations. Defaults to 1000.
        update_reference_model_every (Optional[int], optional): Number of iterations after which to update the reference model. If None, keep fixed.

    """
    optimizer = optim.Adam(policy_model.parameters(), lr=learning_rate)
    reference_model = deepcopy(policy_model)
    reference_model.eval()
    for param in reference_model.parameters():
        param.requires_grad = False

    for t in range(1, T_max + 1):
        logger.info(f"Starting iteration {t}/{T_max}")
        for prompt_idx, prompt in enumerate(prompts):
            logger.info(f"Processing prompt {prompt_idx + 1}/{len(prompts)}")
            # Initialize tree with root node
            root_node = Node(sequence=prompt)
            frontier = [root_node]
            # Build the tree up to depth D
            for d in range(1, D + 1):
                logger.debug(f"Depth {d}/{D}")
                new_frontier = []
                for node in frontier:
                    for b in range(B):
                        continuation = sample_sequence(
                            policy_model, node.sequence, max_length, eos_token_id
                        )
                        child_sequence = node.sequence + continuation
                        child_node = Node(sequence=child_sequence)
                        node.add_child(child_node)
                        new_frontier.append(child_node)
                frontier = new_frontier
                if not frontier:
                    logger.debug("Frontier is empty. Breaking out of depth loop.")
                    break
            # Collect leaf nodes
            leaf_nodes = frontier
            if not leaf_nodes:
                logger.warning("No leaf nodes generated for this prompt.")
                continue
            # Evaluate leaf nodes
            rewards = []
            for node in leaf_nodes:
                sequence = node.sequence
                reward = compute_reward(reward_model, sequence)
                rewards.append((node, reward))
            # Rank sequences based on rewards
            rewards.sort(key=lambda x: x[1], reverse=True)
            ranked_nodes = [node for node, reward in rewards]
            # Create preference pairs
            preference_pairs = []
            M = len(ranked_nodes)
            for i in range(M - 1):
                for j in range(i + 1, M):
                    preferred_node = ranked_nodes[i]
                    unpreferred_node = ranked_nodes[j]
                    preference_pairs.append((preferred_node, unpreferred_node))
            # Compute DPO loss
            losses = []
            for preferred_node, unpreferred_node in preference_pairs:
                s_i = preferred_node.sequence
                s_j = unpreferred_node.sequence
                # Compute log probabilities
                policy_logprob_s_i = compute_log_probability(
                    policy_model, s_i, requires_grad=True
                )
                policy_logprob_s_j = compute_log_probability(
                    policy_model, s_j, requires_grad=True
                )
                ref_logprob_s_i = compute_log_probability(
                    reference_model, s_i, requires_grad=False
                )
                ref_logprob_s_j = compute_log_probability(
                    reference_model, s_j, requires_grad=False
                )
                # Compute log ratios
                policy_log_ratio = policy_logprob_s_i - policy_logprob_s_j  # tensor
                ref_log_ratio = ref_logprob_s_i - ref_logprob_s_j  # tensor
                # Compute loss
                loss = -torch.log(
                    torch.sigmoid(beta * (policy_log_ratio - ref_log_ratio))
                )
                losses.append(loss)
            if losses:
                total_loss = torch.stack(losses).mean()
                optimizer.zero_grad()
                total_loss.backward()
                optimizer.step()
                logger.info(f"Loss: {total_loss.item():.4f}")
            else:
                logger.info("No preference pairs generated.")
        # Optionally update the reference model
        if update_reference_model_every is not None and t % update_reference_model_every == 0:
            logger.info("Updating reference model.")
            reference_model = deepcopy(policy_model)
            reference_model.eval()
            for param in reference_model.parameters():
                param.requires_grad = False

# Example usage (you need to define your own data and models)
if __name__ == "__main__":
    # Initialize models with example hyperparameters
    vocab_size = 5000
    hidden_size = 256
    num_layers = 2

    policy_model = PolicyModel(vocab_size, hidden_size, num_layers)
    reward_model = RewardModel(vocab_size, hidden_size, num_layers)

    # Example prompts (list of token IDs)
    prompts = [
        [1, 5, 20],
        [1, 15, 30],
        [1, 25, 40],
    ]

    # Training parameters
    beta = 0.1
    D = 3
    B = 2
    max_length = 10
    learning_rate = 1e-4
    T_max = 10
    update_reference_model_every = 5

    # Train the policy model
    train_policy_model(
        policy_model,
        reward_model,
        prompts,
        vocab_size,
        eos_token_id,
        beta=beta,
        D=D,
        B=B,
        max_length=max_length,
        learning_rate=learning_rate,
        T_max=T_max,
        update_reference_model_every=update_reference_model_every,
    )


Node

from __future__ import annotations
from typing import List, Tuple, Any, Optional

import torch
import torch.nn as nn
import torch.optim as optim
from torch import Tensor
from loguru import logger
from copy import deepcopy

# Define the end-of-sequence token ID (you should set this according to your tokenizer)
eos_token_id = 2  # Example EOS token ID

class Node:
    """
    A class representing a node in the tree of thoughts.

    Attributes:
        sequence (List[int]): The sequence of tokens from the root to this node.
        children (List[Node]): The list of child nodes.
    """

    def __init__(self, sequence: List[int]):
        self.sequence: List[int] = sequence
        self.children: List[Node] = []

    def add_child(self, child_node: Node):
        """Adds a child node to the current node."""
        self.children.append(child_node)

class PolicyModel(nn.Module):
    """
    Policy model π_θ to generate sequences.

    Args:
        vocab_size (int): Vocabulary size.
        hidden_size (int): Hidden layer size.
        num_layers (int): Number of transformer layers.
    """

    def __init__(self, vocab_size: int, hidden_size: int, num_layers: int):
        super(PolicyModel, self).__init__()
        self.embedding = nn.Embedding(vocab_size, hidden_size)
        self.transformer = nn.Transformer(
            d_model=hidden_size,
            nhead=8,
            num_encoder_layers=num_layers,
            num_decoder_layers=num_layers,


PolicyModel

class Node:
    """
    A class representing a node in the tree of thoughts.

    Attributes:
        sequence (List[int]): The sequence of tokens from the root to this node.
        children (List[Node]): The list of child nodes.
    """

    def __init__(self, sequence: List[int]):
        self.sequence: List[int] = sequence
        self.children: List[Node] = []

    def add_child(self, child_node: Node):
        """Adds a child node to the current node."""
        self.children.append(child_node)

class PolicyModel(nn.Module):
    """
    Policy model π_θ to generate sequences.

    Args:
        vocab_size (int): Vocabulary size.
        hidden_size (int): Hidden layer size.
        num_layers (int): Number of transformer layers.
    """

    def __init__(self, vocab_size: int, hidden_size: int, num_layers: int):
        super(PolicyModel, self).__init__()
        self.embedding = nn.Embedding(vocab_size, hidden_size)
        self.transformer = nn.Transformer(
            d_model=hidden_size,
            nhead=8,
            num_encoder_layers=num_layers,
            num_decoder_layers=num_layers,
        )
        self.fc_out = nn.Linear(hidden_size, vocab_size)

    def forward(self, src: Tensor, tgt: Tensor) -> Tensor:
        """
        Forward pass of the policy model.

        Args:
            src (Tensor): Source sequence tensor (prompt), shape (S, N).
            tgt (Tensor): Target sequence tensor (continuation), shape (T, N).

        Returns:
            Tensor: Logits over the vocabulary, shape (T, N, V).
        """
        src_emb = self.embedding(src)  # (S, N, E)
        tgt_emb = self.embedding(tgt)  # (T, N, E)
        memory = self.transformer.encoder(src_emb)
        output = self.transformer.decoder(tgt_emb, memory)
        logits = self.fc_out(output)  # (T, N, V)
        return logits

class RewardModel(nn.Module):
    """
    Reward model R(s) to compute rewards for sequences.

    Args:
        vocab_size (int): Vocabulary size.
        hidden_size (int): Hidden layer size.
        num_layers (int): Number of transformer layers.
    """

    def __init__(self, vocab_size: int, hidden_size: int, num_layers: int):
        super(RewardModel, self).__init__()
        self.embedding = nn.Embedding(vocab_size, hidden_size)
        encoder_layer = nn.TransformerEncoderLayer(
            d_model=hidden_size, nhead=8, dim_feedforward=hidden_size * 4
        )
        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
        self.fc_out = nn.Linear(hidden_size, 1)


RewardModel

    def forward(self, src: Tensor, tgt: Tensor) -> Tensor:
        """
        Forward pass of the policy model.

        Args:
            src (Tensor): Source sequence tensor (prompt), shape (S, N).
            tgt (Tensor): Target sequence tensor (continuation), shape (T, N).

        Returns:
            Tensor: Logits over the vocabulary, shape (T, N, V).
        """
        src_emb = self.embedding(src)  # (S, N, E)
        tgt_emb = self.embedding(tgt)  # (T, N, E)
        memory = self.transformer.encoder(src_emb)
        output = self.transformer.decoder(tgt_emb, memory)
        logits = self.fc_out(output)  # (T, N, V)
        return logits

class RewardModel(nn.Module):
    """
    Reward model R(s) to compute rewards for sequences.

    Args:
        vocab_size (int): Vocabulary size.
        hidden_size (int): Hidden layer size.
        num_layers (int): Number of transformer layers.
    """

    def __init__(self, vocab_size: int, hidden_size: int, num_layers: int):
        super(RewardModel, self).__init__()
        self.embedding = nn.Embedding(vocab_size, hidden_size)
        encoder_layer = nn.TransformerEncoderLayer(
            d_model=hidden_size, nhead=8, dim_feedforward=hidden_size * 4
        )
        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
        self.fc_out = nn.Linear(hidden_size, 1)

    def forward(self, sequence: Tensor) -> Tensor:
        """
        Forward pass of the reward model.

        Args:
            sequence (Tensor): Sequence tensor, shape (S, N).

        Returns:
            Tensor: Scalar reward value, shape (N).
        """
        emb = self.embedding(sequence)  # (S, N, E)
        output = self.transformer(emb)  # (S, N, E)
        # Take the mean over the sequence length
        pooled_output = output.mean(dim=0)  # (N, E)
        reward = self.fc_out(pooled_output)  # (N, 1)
        return reward.squeeze(-1)  # (N)

def sample_sequence(
    model: PolicyModel, context: List[int], max_length: int, eos_token_id: int
) -> List[int]:
    """
    Samples a continuation from the model given the context.

    Args:
        model (PolicyModel): The policy model.
        context (List[int]): The context sequence (list of token ids).
        max_length (int): Maximum length of the continuation.
        eos_token_id (int): End-of-sequence token ID.

    Returns:
        List[int]: Sampled continuation tokens.
    """
    model.eval()
    generated = context.copy()
    with torch.no_grad():


        self.fc_out = nn.Linear(hidden_size, 1)

    def forward(self, sequence: Tensor) -> Tensor:
        """
        Forward pass of the reward model.

        Args:
            sequence (Tensor): Sequence tensor, shape (S, N).

        Returns:
            Tensor: Scalar reward value, shape (N).
        """
        emb = self.embedding(sequence)  # (S, N, E)
        output = self.transformer(emb)  # (S, N, E)
        # Take the mean over the sequence length
        pooled_output = output.mean(dim=0)  # (N, E)
        reward = self.fc_out(pooled_output)  # (N, 1)
        return reward.squeeze(-1)  # (N)

def sample_sequence(
    model: PolicyModel, context: List[int], max_length: int, eos_token_id: int
) -> List[int]:
    """
    Samples a continuation from the model given the context.

    Args:
        model (PolicyModel): The policy model.
        context (List[int]): The context sequence (list of token ids).
        max_length (int): Maximum length of the continuation.
        eos_token_id (int): End-of-sequence token ID.

    Returns:
        List[int]: Sampled continuation tokens.
    """
    model.eval()
    generated = context.copy()
    with torch.no_grad():
        for _ in range(max_length):
            src = torch.tensor(context).unsqueeze(1)  # (S, 1)
            tgt_input = torch.tensor(generated).unsqueeze(1)  # (T, 1)
            logits = model(src, tgt_input)
            next_token_logits = logits[-1, 0, :]  # (V)
            probabilities = torch.softmax(next_token_logits, dim=-1)
            next_token_id = torch.multinomial(probabilities, num_samples=1).item()
            generated.append(next_token_id)
            if next_token_id == eos_token_id:
                break
    continuation = generated[len(context) :]
    return continuation

def compute_log_probability(
    model: PolicyModel, sequence: List[int], requires_grad: bool = False
) -> Tensor:
    """
    Computes the total log probability of the sequence under the model.

    Args:
        model (PolicyModel): The model (policy or reference).
        sequence (List[int]): The sequence of token ids.
        requires_grad (bool): Whether to compute gradients.

    Returns:
        Tensor: Total log probability of the sequence.
    """
    sequence_tensor = torch.tensor(sequence).unsqueeze(1)  # (S, 1)
    src = sequence_tensor[:-1, :]  # (S-1, 1)
    tgt = sequence_tensor[:-1, :]  # (S-1, 1)
    target_ids = sequence_tensor[1:, 0]  # (S-1)


    Returns:
        List[int]: Sampled continuation tokens.
    """
    model.eval()
    generated = context.copy()
    with torch.no_grad():
        for _ in range(max_length):
            src = torch.tensor(context).unsqueeze(1)  # (S, 1)
            tgt_input = torch.tensor(generated).unsqueeze(1)  # (T, 1)
            logits = model(src, tgt_input)
            next_token_logits = logits[-1, 0, :]  # (V)
            probabilities = torch.softmax(next_token_logits, dim=-1)
            next_token_id = torch.multinomial(probabilities, num_samples=1).item()
            generated.append(next_token_id)
            if next_token_id == eos_token_id:
                break
    continuation = generated[len(context) :]
    return continuation

def compute_log_probability(
    model: PolicyModel, sequence: List[int], requires_grad: bool = False
) -> Tensor:
    """
    Computes the total log probability of the sequence under the model.

    Args:
        model (PolicyModel): The model (policy or reference).
        sequence (List[int]): The sequence of token ids.
        requires_grad (bool): Whether to compute gradients.

    Returns:
        Tensor: Total log probability of the sequence.
    """
    sequence_tensor = torch.tensor(sequence).unsqueeze(1)  # (S, 1)
    src = sequence_tensor[:-1, :]  # (S-1, 1)
    tgt = sequence_tensor[:-1, :]  # (S-1, 1)
    target_ids = sequence_tensor[1:, 0]  # (S-1)
    if not requires_grad:
        model.eval()
        with torch.no_grad():
            logits = model(src, tgt)  # (T, N, V)
            logits = logits.squeeze(1)  # (S-1, V)
            log_probs = torch.log_softmax(logits, dim=-1)  # (S-1, V)
            token_logprobs = log_probs[range(len(target_ids)), target_ids]  # (S-1)
            total_logprob = token_logprobs.sum()  # Scalar
    else:
        model.train()
        logits = model(src, tgt)
        logits = logits.squeeze(1)
        log_probs = torch.log_softmax(logits, dim=-1)
        token_logprobs = log_probs[range(len(target_ids)), target_ids]
        total_logprob = token_logprobs.sum()
    return total_logprob

def compute_reward(reward_model: RewardModel, sequence: List[int]) -> float:
    """
    Computes the reward of a sequence using the reward model.

    Args:
        reward_model (RewardModel): The reward model.
        sequence (List[int]): The sequence of token ids.

    Returns:
        float: Reward value.
    """
    reward_model.eval()
    with torch.no_grad():
        input_ids = torch.tensor(sequence).unsqueeze(1)  # (S, 1)
        reward = reward_model(input_ids)  # (1)
        return reward.item()


    tgt = sequence_tensor[:-1, :]  # (S-1, 1)
    target_ids = sequence_tensor[1:, 0]  # (S-1)
    if not requires_grad:
        model.eval()
        with torch.no_grad():
            logits = model(src, tgt)  # (T, N, V)
            logits = logits.squeeze(1)  # (S-1, V)
            log_probs = torch.log_softmax(logits, dim=-1)  # (S-1, V)
            token_logprobs = log_probs[range(len(target_ids)), target_ids]  # (S-1)
            total_logprob = token_logprobs.sum()  # Scalar
    else:
        model.train()
        logits = model(src, tgt)
        logits = logits.squeeze(1)
        log_probs = torch.log_softmax(logits, dim=-1)
        token_logprobs = log_probs[range(len(target_ids)), target_ids]
        total_logprob = token_logprobs.sum()
    return total_logprob

def compute_reward(reward_model: RewardModel, sequence: List[int]) -> float:
    """
    Computes the reward of a sequence using the reward model.

    Args:
        reward_model (RewardModel): The reward model.
        sequence (List[int]): The sequence of token ids.

    Returns:
        float: Reward value.
    """
    reward_model.eval()
    with torch.no_grad():
        input_ids = torch.tensor(sequence).unsqueeze(1)  # (S, 1)
        reward = reward_model(input_ids)  # (1)
        return reward.item()

def train_policy_model(
    policy_model: PolicyModel,
    reward_model: RewardModel,
    prompts: List[List[int]],
    vocab_size: int,
    eos_token_id: int,
    beta: float = 0.1,
    D: int = 3,
    B: int = 2,
    max_length: int = 10,
    learning_rate: float = 1e-4,
    T_max: int = 1000,
    update_reference_model_every: Optional[int] = None,
) -> None:
    """
    Trains the policy model using RLHF with DPO and Monte Carlo Tree of Thoughts.

    Args:


def compute_reward(reward_model: RewardModel, sequence: List[int]) -> float:
    """
    Computes the reward of a sequence using the reward model.

    Args:
        reward_model (RewardModel): The reward model.
        sequence (List[int]): The sequence of token ids.

    Returns:
        float: Reward value.
    """
    reward_model.eval()
    with torch.no_grad():
        input_ids = torch.tensor(sequence).unsqueeze(1)  # (S, 1)
        reward = reward_model(input_ids)  # (1)
        return reward.item()

def train_policy_model(
    policy_model: PolicyModel,
    reward_model: RewardModel,
    prompts: List[List[int]],
    vocab_size: int,
    eos_token_id: int,
    beta: float = 0.1,
    D: int = 3,
    B: int = 2,
    max_length: int = 10,
    learning_rate: float = 1e-4,
    T_max: int = 1000,
    update_reference_model_every: Optional[int] = None,
) -> None:
    """
    Trains the policy model using RLHF with DPO and Monte Carlo Tree of Thoughts.

    Args:
        policy_model (PolicyModel): The policy model to train.
        reward_model (RewardModel): The reward model.
        prompts (List[List[int]]): The training dataset of prompts.
        vocab_size (int): Vocabulary size.
        eos_token_id (int): End-of-sequence token id.
        beta (float, optional): Beta parameter for DPO loss. Defaults to 0.1.
        D (int, optional): Maximum depth of the tree. Defaults to 3.
        B (int, optional): Number of branches per node. Defaults to 2.
        max_length (int, optional): Maximum length of continuations. Defaults to 10.
        learning_rate (float, optional): Learning rate for optimizer. Defaults to 1e-4.
        T_max (int, optional): Maximum number of training iterations. Defaults to 1000.
        update_reference_model_every (Optional[int], optional): Number of iterations after which to update the reference model. If None, keep fixed.

    """
    optimizer = optim.Adam(policy_model.parameters(), lr=learning_rate)
    reference_model = deepcopy(policy_model)
    reference_model.eval()
    for param in reference_model.parameters():
        param.requires_grad = False

    for t in range(1, T_max + 1):
        logger.info(f"Starting iteration {t}/{T_max}")
        for prompt_idx, prompt in enumerate(prompts):
            logger.info(f"Processing prompt {prompt_idx + 1}/{len(prompts)}")
            # Initialize tree with root node
            root_node = Node(sequence=prompt)
            frontier = [root_node]
            # Build the tree up to depth D
            for d in range(1, D + 1):
                logger.debug(f"Depth {d}/{D}")
                new_frontier = []
                for node in frontier:
                    for b in range(B):
                        continuation = sample_sequence(
                            policy_model, node.sequence, max_length, eos_token_id
                        )
                        child_sequence = node.sequence + continuation
                        child_node = Node(sequence=child_sequence)
                        node.add_child(child_node)
                        new_frontier.append(child_node)
                frontier = new_frontier
                if not frontier:
                    logger.debug("Frontier is empty. Breaking out of depth loop.")
                    break
            # Collect leaf nodes
            leaf_nodes = frontier
            if not leaf_nodes:
                logger.warning("No leaf nodes generated for this prompt.")
                continue
            # Evaluate leaf nodes
            rewards = []
            for node in leaf_nodes:
                sequence = node.sequence
                reward = compute_reward(reward_model, sequence)
                rewards.append((node, reward))
            # Rank sequences based on rewards
            rewards.sort(key=lambda x: x[1], reverse=True)
            ranked_nodes = [node for node, reward in rewards]
            # Create preference pairs
            preference_pairs = []
            M = len(ranked_nodes)
            for i in range(M - 1):
                for j in range(i + 1, M):
                    preferred_node = ranked_nodes[i]
                    unpreferred_node = ranked_nodes[j]
                    preference_pairs.append((preferred_node, unpreferred_node))
            # Compute DPO loss
            losses = []
            for preferred_node, unpreferred_node in preference_pairs:
                s_i = preferred_node.sequence
                s_j = unpreferred_node.sequence
                # Compute log probabilities
                policy_logprob_s_i = compute_log_probability(
                    policy_model, s_i, requires_grad=True
                )
                policy_logprob_s_j = compute_log_probability(
                    policy_model, s_j, requires_grad=True
                )
                ref_logprob_s_i = compute_log_probability(
                    reference_model, s_i, requires_grad=False
                )
                ref_logprob_s_j = compute_log_probability(
                    reference_model, s_j, requires_grad=False
                )
                # Compute log ratios
                policy_log_ratio = policy_logprob_s_i - policy_logprob_s_j  # tensor
                ref_log_ratio = ref_logprob_s_i - ref_logprob_s_j  # tensor
                # Compute loss
                loss = -torch.log(
                    torch.sigmoid(beta * (policy_log_ratio - ref_log_ratio))
                )
                losses.append(loss)
            if losses:
                total_loss = torch.stack(losses).mean()
                optimizer.zero_grad()
                total_loss.backward()
                optimizer.step()
                logger.info(f"Loss: {total_loss.item():.4f}")
            else:
                logger.info("No preference pairs generated.")
        # Optionally update the reference model
        if update_reference_model_every is not None and t % update_reference_model_every == 0:
            logger.info("Updating reference model.")
            reference_model = deepcopy(policy_model)
            reference_model.eval()
            for param in reference_model.parameters():
                param.requires_grad = False

# Example usage (you need to define your own data and models)
if __name__ == "__main__":
    # Initialize models with example hyperparameters
    vocab_size = 5000
    hidden_size = 256
    num_layers = 2

    policy_model = PolicyModel(vocab_size, hidden_size, num_layers)
    reward_model = RewardModel(vocab_size, hidden_size, num_layers)

    # Example prompts (list of token IDs)
    prompts = [
        [1, 5, 20],
        [1, 15, 30],
        [1, 25, 40],
    ]

    # Training parameters


            Tensor: Logits over the vocabulary, shape (T, N, V).
        """
        src_emb = self.embedding(src)  # (S, N, E)
        tgt_emb = self.embedding(tgt)  # (T, N, E)
        memory = self.transformer.encoder(src_emb)
        output = self.transformer.decoder(tgt_emb, memory)
        logits = self.fc_out(output)  # (T, N, V)
        return logits

class RewardModel(nn.Module):
    """
    Reward model R(s) to compute rewards for sequences.

    Args:
        vocab_size (int): Vocabulary size.
        hidden_size (int): Hidden layer size.
        num_layers (int): Number of transformer layers.
    """

    def __init__(self, vocab_size: int, hidden_size: int, num_layers: int):
        super(RewardModel, self).__init__()
        self.embedding = nn.Embedding(vocab_size, hidden_size)
        encoder_layer = nn.TransformerEncoderLayer(
            d_model=hidden_size, nhead=8, dim_feedforward=hidden_size * 4
        )
        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
        self.fc_out = nn.Linear(hidden_size, 1)

    def forward(self, sequence: Tensor) -> Tensor:
        """
        Forward pass of the reward model.

        Args:
            sequence (Tensor): Sequence tensor, shape (S, N).

        Returns:
            Tensor: Scalar reward value, shape (N).
        """
        emb = self.embedding(sequence)  # (S, N, E)
        output = self.transformer(emb)  # (S, N, E)
        # Take the mean over the sequence length
        pooled_output = output.mean(dim=0)  # (N, E)
        reward = self.fc_out(pooled_output)  # (N, 1)
        return reward.squeeze(-1)  # (N)

def sample_sequence(


from loguru import logger
from copy import deepcopy

# Define the end-of-sequence token ID (you should set this according to your tokenizer)
eos_token_id = 2  # Example EOS token ID

class Node:
    """
    A class representing a node in the tree of thoughts.

    Attributes:
        sequence (List[int]): The sequence of tokens from the root to this node.
        children (List[Node]): The list of child nodes.
    """

    def __init__(self, sequence: List[int]):
        self.sequence: List[int] = sequence
        self.children: List[Node] = []

    def add_child(self, child_node: Node):
        """Adds a child node to the current node."""
        self.children.append(child_node)

class PolicyModel(nn.Module):
    """
    Policy model π_θ to generate sequences.

    Args:
        vocab_size (int): Vocabulary size.
        hidden_size (int): Hidden layer size.
        num_layers (int): Number of transformer layers.
    """

    def __init__(self, vocab_size: int, hidden_size: int, num_layers: int):
        super(PolicyModel, self).__init__()
        self.embedding = nn.Embedding(vocab_size, hidden_size)
        self.transformer = nn.Transformer(
            d_model=hidden_size,
            nhead=8,
            num_encoder_layers=num_layers,
            num_decoder_layers=num_layers,


forward

class RewardModel(nn.Module):
    """
    Reward model R(s) to compute rewards for sequences.

    Args:
        vocab_size (int): Vocabulary size.
        hidden_size (int): Hidden layer size.
        num_layers (int): Number of transformer layers.
    """

    def __init__(self, vocab_size: int, hidden_size: int, num_layers: int):
        super(RewardModel, self).__init__()
        self.embedding = nn.Embedding(vocab_size, hidden_size)
        encoder_layer = nn.TransformerEncoderLayer(
            d_model=hidden_size, nhead=8, dim_feedforward=hidden_size * 4
        )
        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
        self.fc_out = nn.Linear(hidden_size, 1)

    def forward(self, sequence: Tensor) -> Tensor:
        """
        Forward pass of the reward model.

        Args:
            sequence (Tensor): Sequence tensor, shape (S, N).

        Returns:
            Tensor: Scalar reward value, shape (N).
        """
        emb = self.embedding(sequence)  # (S, N, E)
        output = self.transformer(emb)  # (S, N, E)
        # Take the mean over the sequence length
        pooled_output = output.mean(dim=0)  # (N, E)
        reward = self.fc_out(pooled_output)  # (N, 1)
        return reward.squeeze(-1)  # (N)

def sample_sequence(
    model: PolicyModel, context: List[int], max_length: int, eos_token_id: int
) -> List[int]:
    """
    Samples a continuation from the model given the context.

    Args:
        model (PolicyModel): The policy model.
        context (List[int]): The context sequence (list of token ids).
        max_length (int): Maximum length of the continuation.
        eos_token_id (int): End-of-sequence token ID.

    Returns:
        List[int]: Sampled continuation tokens.
    """
    model.eval()
    generated = context.copy()
    with torch.no_grad():


import torch
import torch.nn as nn
import torch.optim as optim
from torch.distributions import Categorical
from loguru import logger
from typing import List, Tuple

# Set up logging
logger.add("training.log", rotation="500 MB")

# Device configuration
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

class TransformerPolicyNetwork(nn.Module):
    """
    Transformer-based Policy Network that outputs action probabilities given a state sequence.
    """

    def __init__(
        self,
        input_dim: int,
        action_dim: int,
        nhead: int = 8,
        num_layers: int = 6,
        dim_feedforward: int = 2048,
        dropout: float = 0.1,
    ):
        super(TransformerPolicyNetwork, self).__init__()
        self.model_type = "Transformer"

        self.embedding = nn.Linear(input_dim, dim_feedforward)
        encoder_layers = nn.TransformerEncoderLayer(
            d_model=dim_feedforward, nhead=nhead, dropout=dropout
        )
        self.transformer_encoder = nn.TransformerEncoder(
            encoder_layers, num_layers
        )
        self.fc_out = nn.Linear(dim_feedforward, action_dim)

    def forward(self, src: torch.Tensor) -> torch.Tensor:
        """
        Forward pass for the policy network.

        Args:
            src (torch.Tensor): Input tensor of shape (sequence_length, batch_size, input_dim).

        Returns:
            torch.Tensor: Action probabilities of shape (batch_size, action_dim).
        """
        src = self.embedding(src)
        output = self.transformer_encoder(src)
        # Take the output from the last time step
        output = output[-1, :, :]
        action_logits = self.fc_out(output)
        action_probs = torch.softmax(action_logits, dim=-1)
        return action_probs

class TransformerValueNetwork(nn.Module):
    """
    Transformer-based Value Network that estimates the value of a given state sequence.
    """

    def __init__(
        self,
        input_dim: int,
        nhead: int = 8,
        num_layers: int = 6,
        dim_feedforward: int = 2048,
        dropout: float = 0.1,
    ):
        super(TransformerValueNetwork, self).__init__()
        self.model_type = "Transformer"

        self.embedding = nn.Linear(input_dim, dim_feedforward)
        encoder_layers = nn.TransformerEncoderLayer(
            d_model=dim_feedforward, nhead=nhead, dropout=dropout
        )
        self.transformer_encoder = nn.TransformerEncoder(
            encoder_layers, num_layers
        )
        self.fc_out = nn.Linear(dim_feedforward, 1)

    def forward(self, src: torch.Tensor) -> torch.Tensor:
        """
        Forward pass for the value network.

        Args:
            src (torch.Tensor): Input tensor of shape (sequence_length, batch_size, input_dim).

        Returns:
            torch.Tensor: State value of shape (batch_size, 1).
        """
        src = self.embedding(src)
        output = self.transformer_encoder(src)
        # Take the output from the last time step
        output = output[-1, :, :]
        state_value = self.fc_out(output)
        return state_value

class TransformerRewardModel(nn.Module):
    """
    Transformer-based Reward Model that assigns rewards to thought branches.
    """

    def __init__(
        self,
        input_dim: int,
        nhead: int = 8,
        num_layers: int = 6,
        dim_feedforward: int = 2048,
        dropout: float = 0.1,
    ):
        super(TransformerRewardModel, self).__init__()
        self.model_type = "Transformer"

        self.embedding = nn.Linear(input_dim, dim_feedforward)
        encoder_layers = nn.TransformerEncoderLayer(
            d_model=dim_feedforward, nhead=nhead, dropout=dropout
        )
        self.transformer_encoder = nn.TransformerEncoder(
            encoder_layers, num_layers
        )
        self.fc_out = nn.Linear(dim_feedforward, 1)

    def forward(self, src: torch.Tensor) -> torch.Tensor:
        """
        Forward pass for the reward model.

        Args:
            src (torch.Tensor): Input tensor of shape (sequence_length, batch_size, input_dim).

        Returns:
            torch.Tensor: Reward estimate of shape (batch_size, 1).
        """
        src = self.embedding(src)
        output = self.transformer_encoder(src)
        # Take the output from the last time step
        output = output[-1, :, :]
        reward = self.fc_out(output)
        return reward

class ThoughtTree:
    """
    Class representing a tree of thoughts.
    """

    def __init__(self, root_state: torch.Tensor):
        self.root = {"state": root_state, "children": [], "reward": 0}

    def add_child(
        self, parent: dict, child_state: torch.Tensor, reward: float
    ):
        child = {
            "state": child_state,
            "children": [],
            "reward": reward,
        }
        parent["children"].append(child)
        return child

def monte_carlo_rollout(
    policy_net: TransformerPolicyNetwork,
    state_sequence: torch.Tensor,
    depth: int,
    max_depth: int,
    sequence_length: int,
) -> List[Tuple[torch.Tensor, float]]:
    """
    Perform a Monte Carlo rollout to simulate future thoughts.

    Args:
        policy_net (TransformerPolicyNetwork): The policy network.
        state_sequence (torch.Tensor): The current state sequence.
        depth (int): Current depth in the thought tree.
        max_depth (int): Maximum depth for rollouts.
        sequence_length (int): The length of the input sequence.

    Returns:
        List[Tuple[torch.Tensor, float]]: A list of (state_sequence, reward) tuples.
    """
    trajectory = []
    current_sequence = state_sequence.clone()
    for _ in range(depth, max_depth):
        action_probs = policy_net(current_sequence)
        m = Categorical(action_probs)
        action = m.sample()
        next_state = transition(current_sequence[-1], action)
        # Update the sequence by appending the new state
        next_sequence = torch.cat(
            [current_sequence, next_state.unsqueeze(0)], dim=0
        )
        # Ensure the sequence length does not exceed the maximum
        if next_sequence.size(0) > sequence_length:
            next_sequence = next_sequence[1:, :]
        reward = reward_function(next_state)
        trajectory.append((next_sequence, reward))
        current_sequence = next_sequence
    return trajectory

def transition(
    state: torch.Tensor, action: torch.Tensor
) -> torch.Tensor:
    """
    State transition function (placeholder).

    Args:
        state (torch.Tensor): Current state tensor.
        action (torch.Tensor): Action tensor.

    Returns:
        torch.Tensor: Next state tensor.
    """
    # Implement your state transition logic here
    next_state = state + action.float()  # Simplified example
    return next_state

def reward_function(state: torch.Tensor) -> float:
    """
    Reward function (placeholder).

    Args:
        state (torch.Tensor): State tensor.

    Returns:
        float: Reward value.
    """
    # Implement your reward logic here
    reward = -torch.sum(state**2).item()  # Simplified example
    return reward

def train(
    policy_net: TransformerPolicyNetwork,
    value_net: TransformerValueNetwork,
    reward_model: TransformerRewardModel,
    num_iterations: int = 1000,
    episodes_per_iteration: int = 10,
    max_depth: int = 5,
    sequence_length: int = 10,
    gamma: float = 0.99,
    clip_epsilon: float = 0.2,
    policy_lr: float = 1e-4,
    value_lr: float = 1e-3,
):
    """
    Train the policy and value networks using PPO.

    Args:
        policy_net (TransformerPolicyNetwork): The policy network.
        value_net (TransformerValueNetwork): The value network.
        reward_model (TransformerRewardModel): The reward model.
        num_iterations (int): Number of training iterations.
        episodes_per_iteration (int): Episodes per iteration.
        max_depth (int): Maximum depth for Monte Carlo rollouts.
        sequence_length (int): Maximum sequence length for the transformer.
        gamma (float): Discount factor.
        clip_epsilon (float): Clipping epsilon for PPO.
        policy_lr (float): Learning rate for the policy optimizer.
        value_lr (float): Learning rate for the value optimizer.
    """
    policy_optimizer = optim.Adam(
        policy_net.parameters(), lr=policy_lr
    )
    value_optimizer = optim.Adam(value_net.parameters(), lr=value_lr)

    for iteration in range(num_iterations):
        logger.info(
            f"Starting iteration {iteration + 1}/{num_iterations}"
        )
        memory = []

        for episode in range(episodes_per_iteration):
            logger.debug(
                f"Starting episode {episode + 1}/{episodes_per_iteration}"
            )
            # Initialize state sequence with zeros
            state = torch.zeros(policy_net.embedding.in_features).to(
                device
            )
            state_sequence = state.unsqueeze(
                0
            )  # Shape: (1, input_dim)
            thought_tree = ThoughtTree(state_sequence)
            trajectory = []

            # Generate thought branches
            for depth in range(max_depth):
                # Expand dimensions to match (sequence_length, batch_size, input_dim)
                src = state_sequence.unsqueeze(
                    1
                )  # Shape: (sequence_length, 1, input_dim)
                action_probs = policy_net(src)
                m = Categorical(action_probs)
                actions = m.sample((5,))  # Generate multiple branches
                rewards = []

                for action in actions:
                    next_state = transition(
                        state_sequence[-1], action
                    )
                    # Update the sequence by appending the new state
                    next_sequence = torch.cat(
                        [state_sequence, next_state.unsqueeze(0)],
                        dim=0,
                    )
                    # Ensure the sequence length does not exceed the maximum
                    if next_sequence.size(0) > sequence_length:
                        next_sequence = next_sequence[1:, :]
                    rollout = monte_carlo_rollout(
                        policy_net,
                        next_sequence,
                        depth + 1,
                        max_depth,
                        sequence_length,
                    )
                    total_reward = sum([r for _, r in rollout])
                    # Expand dimensions for reward model input
                    reward_input = next_sequence.unsqueeze(1)
                    reward_estimate = reward_model(reward_input)
                    reward = reward_estimate.item() + total_reward
                    rewards.append(reward)

                    # Update thought tree
                    thought_tree.add_child(
                        thought_tree.root, next_sequence, reward
                    )

                # Select the best action based on rewards
                best_action_index = (
                    torch.tensor(rewards).argmax().item()
                )
                best_action = actions[best_action_index]
                best_reward = rewards[best_action_index]

                # Log the selected action and reward
                logger.debug(
                    f"Selected action {best_action.item()} with reward {best_reward}"
                )

                # Store the experience
                trajectory.append(
                    (state_sequence.clone(), best_action, best_reward)
                )

                # Move to the next state sequence
                next_state = transition(
                    state_sequence[-1], best_action
                )
                state_sequence = torch.cat(
                    [state_sequence, next_state.unsqueeze(0)], dim=0
                )
                if state_sequence.size(0) > sequence_length:
                    state_sequence = state_sequence[1:, :]

            # Compute returns and advantages
            returns = []
            advantages = []
            Gt = 0
            for state_seq_t, action_t, reward_t in reversed(
                trajectory
            ):
                Gt = reward_t + gamma * Gt
                returns.insert(0, Gt)
                # Expand dimensions for value network input
                value_input = state_seq_t.unsqueeze(1)
                state_value = value_net(value_input)
                advantage = Gt - state_value.item()
                advantages.insert(0, advantage)

            # Normalize advantages
            advantages_tensor = torch.tensor(
                advantages, dtype=torch.float32
            ).to(device)
            advantages_tensor = (
                advantages_tensor - advantages_tensor.mean()
            ) / (advantages_tensor.std() + 1e-8)

            # Update policy network using PPO
            for i, (state_seq_t, action_t, _) in enumerate(
                trajectory
            ):
                # Expand dimensions to match (sequence_length, batch_size, input_dim)
                src = state_seq_t.unsqueeze(1)
                action_probs = policy_net(src)
                m = Categorical(action_probs)
                log_prob = m.log_prob(action_t)
                old_log_prob = log_prob.detach()
                ratio = torch.exp(log_prob - old_log_prob)
                surr1 = ratio * advantages_tensor[i]
                surr2 = (
                    torch.clamp(
                        ratio, 1 - clip_epsilon, 1 + clip_epsilon
                    )
                    * advantages_tensor[i]
                )
                policy_loss = -torch.min(surr1, surr2)

                policy_optimizer.zero_grad()
                policy_loss.backward()
                policy_optimizer.step()

                # Log the policy loss
                logger.debug(
                    f"Policy loss at step {i}: {policy_loss.item()}"
                )

            # Update value network
            returns_tensor = (
                torch.tensor(returns, dtype=torch.float32)
                .unsqueeze(1)
                .to(device)
            )
            # Prepare inputs for the value network
            value_inputs = torch.stack(
                [s for s, _, _ in trajectory]
            ).transpose(0, 1)
            value_inputs = value_inputs.to(device)
            values = value_net(value_inputs)
            value_loss = nn.MSELoss()(values, returns_tensor)

            value_optimizer.zero_grad()
            value_loss.backward()
            value_optimizer.step()

            # Log the value loss
            logger.debug(f"Value loss: {value_loss.item()}")

        logger.info(
            f"Completed iteration {iteration + 1}/{num_iterations}"
        )

if __name__ == "__main__":
    # Hyperparameters
    input_dim = 10  # Dimension of the input state
    action_dim = 4  # Number of possible actions
    num_iterations = 10
    episodes_per_iteration = 5
    sequence_length = (
        10  # Maximum sequence length for the transformer
    )

    # Initialize networks
    policy_net = TransformerPolicyNetwork(input_dim, action_dim).to(
        device
    )
    value_net = TransformerValueNetwork(input_dim).to(device)
    reward_model = TransformerRewardModel(input_dim).to(device)

    # Start training
    train(
        policy_net,
        value_net,
        reward_model,
        num_iterations=num_iterations,
        episodes_per_iteration=episodes_per_iteration,
        sequence_length=sequence_length,
    )


TransformerPolicyNetwork

import torch
import torch.nn as nn
import torch.optim as optim
from torch.distributions import Categorical
from loguru import logger
from typing import List, Tuple

# Set up logging
logger.add("training.log", rotation="500 MB")

# Device configuration
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

class TransformerPolicyNetwork(nn.Module):
    """
    Transformer-based Policy Network that outputs action probabilities given a state sequence.
    """

    def __init__(
        self,
        input_dim: int,
        action_dim: int,
        nhead: int = 8,
        num_layers: int = 6,
        dim_feedforward: int = 2048,
        dropout: float = 0.1,
    ):
        super(TransformerPolicyNetwork, self).__init__()
        self.model_type = "Transformer"

        self.embedding = nn.Linear(input_dim, dim_feedforward)
        encoder_layers = nn.TransformerEncoderLayer(
            d_model=dim_feedforward, nhead=nhead, dropout=dropout
        )
        self.transformer_encoder = nn.TransformerEncoder(
            encoder_layers, num_layers
        )
        self.fc_out = nn.Linear(dim_feedforward, action_dim)

    def forward(self, src: torch.Tensor) -> torch.Tensor:
        """
        Forward pass for the policy network.

        Args:
            src (torch.Tensor): Input tensor of shape (sequence_length, batch_size, input_dim).

        Returns:
            torch.Tensor: Action probabilities of shape (batch_size, action_dim).
        """
        src = self.embedding(src)
        output = self.transformer_encoder(src)
        # Take the output from the last time step
        output = output[-1, :, :]
        action_logits = self.fc_out(output)
        action_probs = torch.softmax(action_logits, dim=-1)
        return action_probs

class TransformerValueNetwork(nn.Module):
    """
    Transformer-based Value Network that estimates the value of a given state sequence.
    """

    def __init__(
        self,
        input_dim: int,
        nhead: int = 8,
        num_layers: int = 6,
        dim_feedforward: int = 2048,
        dropout: float = 0.1,
    ):
        super(TransformerValueNetwork, self).__init__()
        self.model_type = "Transformer"

        self.embedding = nn.Linear(input_dim, dim_feedforward)
        encoder_layers = nn.TransformerEncoderLayer(


TransformerValueNetwork

    def forward(self, src: torch.Tensor) -> torch.Tensor:
        """
        Forward pass for the policy network.

        Args:
            src (torch.Tensor): Input tensor of shape (sequence_length, batch_size, input_dim).

        Returns:
            torch.Tensor: Action probabilities of shape (batch_size, action_dim).
        """
        src = self.embedding(src)
        output = self.transformer_encoder(src)
        # Take the output from the last time step
        output = output[-1, :, :]
        action_logits = self.fc_out(output)
        action_probs = torch.softmax(action_logits, dim=-1)
        return action_probs

class TransformerValueNetwork(nn.Module):
    """
    Transformer-based Value Network that estimates the value of a given state sequence.
    """

    def __init__(
        self,
        input_dim: int,
        nhead: int = 8,
        num_layers: int = 6,
        dim_feedforward: int = 2048,
        dropout: float = 0.1,
    ):
        super(TransformerValueNetwork, self).__init__()
        self.model_type = "Transformer"

        self.embedding = nn.Linear(input_dim, dim_feedforward)
        encoder_layers = nn.TransformerEncoderLayer(
            d_model=dim_feedforward, nhead=nhead, dropout=dropout
        )
        self.transformer_encoder = nn.TransformerEncoder(
            encoder_layers, num_layers
        )
        self.fc_out = nn.Linear(dim_feedforward, 1)

    def forward(self, src: torch.Tensor) -> torch.Tensor:
        """
        Forward pass for the value network.

        Args:
            src (torch.Tensor): Input tensor of shape (sequence_length, batch_size, input_dim).

        Returns:
            torch.Tensor: State value of shape (batch_size, 1).
        """
        src = self.embedding(src)
        output = self.transformer_encoder(src)
        # Take the output from the last time step
        output = output[-1, :, :]
        state_value = self.fc_out(output)
        return state_value

class TransformerRewardModel(nn.Module):
    """
    Transformer-based Reward Model that assigns rewards to thought branches.
    """

    def __init__(
        self,
        input_dim: int,
        nhead: int = 8,
        num_layers: int = 6,
        dim_feedforward: int = 2048,
        dropout: float = 0.1,
    ):
        super(TransformerRewardModel, self).__init__()
        self.model_type = "Transformer"

        self.embedding = nn.Linear(input_dim, dim_feedforward)
        encoder_layers = nn.TransformerEncoderLayer(


TransformerRewardModel

        self.fc_out = nn.Linear(dim_feedforward, 1)

    def forward(self, src: torch.Tensor) -> torch.Tensor:
        """
        Forward pass for the value network.

        Args:
            src (torch.Tensor): Input tensor of shape (sequence_length, batch_size, input_dim).

        Returns:
            torch.Tensor: State value of shape (batch_size, 1).
        """
        src = self.embedding(src)
        output = self.transformer_encoder(src)
        # Take the output from the last time step
        output = output[-1, :, :]
        state_value = self.fc_out(output)
        return state_value

class TransformerRewardModel(nn.Module):
    """
    Transformer-based Reward Model that assigns rewards to thought branches.
    """

    def __init__(
        self,
        input_dim: int,
        nhead: int = 8,
        num_layers: int = 6,
        dim_feedforward: int = 2048,
        dropout: float = 0.1,
    ):
        super(TransformerRewardModel, self).__init__()
        self.model_type = "Transformer"

        self.embedding = nn.Linear(input_dim, dim_feedforward)
        encoder_layers = nn.TransformerEncoderLayer(
            d_model=dim_feedforward, nhead=nhead, dropout=dropout
        )
        self.transformer_encoder = nn.TransformerEncoder(
            encoder_layers, num_layers
        )
        self.fc_out = nn.Linear(dim_feedforward, 1)

    def forward(self, src: torch.Tensor) -> torch.Tensor:
        """
        Forward pass for the reward model.

        Args:
            src (torch.Tensor): Input tensor of shape (sequence_length, batch_size, input_dim).

        Returns:
            torch.Tensor: Reward estimate of shape (batch_size, 1).
        """
        src = self.embedding(src)
        output = self.transformer_encoder(src)
        # Take the output from the last time step
        output = output[-1, :, :]
        reward = self.fc_out(output)
        return reward

class ThoughtTree:
    """
    Class representing a tree of thoughts.
    """

    def __init__(self, root_state: torch.Tensor):
        self.root = {"state": root_state, "children": [], "reward": 0}

    def add_child(
        self, parent: dict, child_state: torch.Tensor, reward: float
    ):
        child = {
            "state": child_state,
            "children": [],
            "reward": reward,
        }
        parent["children"].append(child)
        return child


ThoughtTree

        self.fc_out = nn.Linear(dim_feedforward, 1)

    def forward(self, src: torch.Tensor) -> torch.Tensor:
        """
        Forward pass for the reward model.

        Args:
            src (torch.Tensor): Input tensor of shape (sequence_length, batch_size, input_dim).

        Returns:
            torch.Tensor: Reward estimate of shape (batch_size, 1).
        """
        src = self.embedding(src)
        output = self.transformer_encoder(src)
        # Take the output from the last time step
        output = output[-1, :, :]
        reward = self.fc_out(output)
        return reward

class ThoughtTree:
    """
    Class representing a tree of thoughts.
    """

    def __init__(self, root_state: torch.Tensor):
        self.root = {"state": root_state, "children": [], "reward": 0}

    def add_child(
        self, parent: dict, child_state: torch.Tensor, reward: float
    ):
        child = {
            "state": child_state,
            "children": [],
            "reward": reward,
        }
        parent["children"].append(child)
        return child

def monte_carlo_rollout(
    policy_net: TransformerPolicyNetwork,
    state_sequence: torch.Tensor,
    depth: int,
    max_depth: int,
    sequence_length: int,
) -> List[Tuple[torch.Tensor, float]]:
    """
    Perform a Monte Carlo rollout to simulate future thoughts.

    Args:
        policy_net (TransformerPolicyNetwork): The policy network.
        state_sequence (torch.Tensor): The current state sequence.
        depth (int): Current depth in the thought tree.
        max_depth (int): Maximum depth for rollouts.
        sequence_length (int): The length of the input sequence.

    Returns:


class ThoughtTree:
    """
    Class representing a tree of thoughts.
    """

    def __init__(self, root_state: torch.Tensor):
        self.root = {"state": root_state, "children": [], "reward": 0}

    def add_child(
        self, parent: dict, child_state: torch.Tensor, reward: float
    ):
        child = {
            "state": child_state,
            "children": [],
            "reward": reward,
        }
        parent["children"].append(child)
        return child

def monte_carlo_rollout(
    policy_net: TransformerPolicyNetwork,
    state_sequence: torch.Tensor,
    depth: int,
    max_depth: int,
    sequence_length: int,
) -> List[Tuple[torch.Tensor, float]]:
    """
    Perform a Monte Carlo rollout to simulate future thoughts.

    Args:
        policy_net (TransformerPolicyNetwork): The policy network.
        state_sequence (torch.Tensor): The current state sequence.
        depth (int): Current depth in the thought tree.
        max_depth (int): Maximum depth for rollouts.
        sequence_length (int): The length of the input sequence.

    Returns:
        List[Tuple[torch.Tensor, float]]: A list of (state_sequence, reward) tuples.
    """
    trajectory = []
    current_sequence = state_sequence.clone()
    for _ in range(depth, max_depth):
        action_probs = policy_net(current_sequence)
        m = Categorical(action_probs)
        action = m.sample()
        next_state = transition(current_sequence[-1], action)
        # Update the sequence by appending the new state
        next_sequence = torch.cat(
            [current_sequence, next_state.unsqueeze(0)], dim=0
        )
        # Ensure the sequence length does not exceed the maximum
        if next_sequence.size(0) > sequence_length:
            next_sequence = next_sequence[1:, :]
        reward = reward_function(next_state)
        trajectory.append((next_sequence, reward))
        current_sequence = next_sequence
    return trajectory

def transition(
    state: torch.Tensor, action: torch.Tensor
) -> torch.Tensor:
    """
    State transition function (placeholder).

    Args:
        state (torch.Tensor): Current state tensor.
        action (torch.Tensor): Action tensor.

    Returns:
        torch.Tensor: Next state tensor.
    """
    # Implement your state transition logic here
    next_state = state + action.float()  # Simplified example
    return next_state


transition

    trajectory = []
    current_sequence = state_sequence.clone()
    for _ in range(depth, max_depth):
        action_probs = policy_net(current_sequence)
        m = Categorical(action_probs)
        action = m.sample()
        next_state = transition(current_sequence[-1], action)
        # Update the sequence by appending the new state
        next_sequence = torch.cat(
            [current_sequence, next_state.unsqueeze(0)], dim=0
        )
        # Ensure the sequence length does not exceed the maximum
        if next_sequence.size(0) > sequence_length:
            next_sequence = next_sequence[1:, :]
        reward = reward_function(next_state)
        trajectory.append((next_sequence, reward))
        current_sequence = next_sequence
    return trajectory

def transition(
    state: torch.Tensor, action: torch.Tensor
) -> torch.Tensor:
    """
    State transition function (placeholder).

    Args:
        state (torch.Tensor): Current state tensor.
        action (torch.Tensor): Action tensor.

    Returns:
        torch.Tensor: Next state tensor.
    """
    # Implement your state transition logic here
    next_state = state + action.float()  # Simplified example
    return next_state

def reward_function(state: torch.Tensor) -> float:
    """
    Reward function (placeholder).

    Args:
        state (torch.Tensor): State tensor.

    Returns:
        float: Reward value.
    """
    # Implement your reward logic here
    reward = -torch.sum(state**2).item()  # Simplified example
    return reward

def train(
    policy_net: TransformerPolicyNetwork,
    value_net: TransformerValueNetwork,


def transition(
    state: torch.Tensor, action: torch.Tensor
) -> torch.Tensor:
    """
    State transition function (placeholder).

    Args:
        state (torch.Tensor): Current state tensor.
        action (torch.Tensor): Action tensor.

    Returns:
        torch.Tensor: Next state tensor.
    """
    # Implement your state transition logic here
    next_state = state + action.float()  # Simplified example
    return next_state

def reward_function(state: torch.Tensor) -> float:
    """
    Reward function (placeholder).

    Args:
        state (torch.Tensor): State tensor.

    Returns:
        float: Reward value.
    """
    # Implement your reward logic here
    reward = -torch.sum(state**2).item()  # Simplified example
    return reward

def train(
    policy_net: TransformerPolicyNetwork,
    value_net: TransformerValueNetwork,
    reward_model: TransformerRewardModel,
    num_iterations: int = 1000,
    episodes_per_iteration: int = 10,
    max_depth: int = 5,
    sequence_length: int = 10,
    gamma: float = 0.99,
    clip_epsilon: float = 0.2,
    policy_lr: float = 1e-4,
    value_lr: float = 1e-3,
):
    """
    Train the policy and value networks using PPO.

    Args:
        policy_net (TransformerPolicyNetwork): The policy network.


train

    # Implement your state transition logic here
    next_state = state + action.float()  # Simplified example
    return next_state

def reward_function(state: torch.Tensor) -> float:
    """
    Reward function (placeholder).

    Args:
        state (torch.Tensor): State tensor.

    Returns:
        float: Reward value.
    """
    # Implement your reward logic here
    reward = -torch.sum(state**2).item()  # Simplified example
    return reward

def train(
    policy_net: TransformerPolicyNetwork,
    value_net: TransformerValueNetwork,
    reward_model: TransformerRewardModel,
    num_iterations: int = 1000,
    episodes_per_iteration: int = 10,
    max_depth: int = 5,
    sequence_length: int = 10,
    gamma: float = 0.99,
    clip_epsilon: float = 0.2,
    policy_lr: float = 1e-4,
    value_lr: float = 1e-3,
):
    """
    Train the policy and value networks using PPO.

    Args:
        policy_net (TransformerPolicyNetwork): The policy network.
        value_net (TransformerValueNetwork): The value network.
        reward_model (TransformerRewardModel): The reward model.
        num_iterations (int): Number of training iterations.
        episodes_per_iteration (int): Episodes per iteration.
        max_depth (int): Maximum depth for Monte Carlo rollouts.
        sequence_length (int): Maximum sequence length for the transformer.
        gamma (float): Discount factor.
        clip_epsilon (float): Clipping epsilon for PPO.
        policy_lr (float): Learning rate for the policy optimizer.
        value_lr (float): Learning rate for the value optimizer.
    """
    policy_optimizer = optim.Adam(
        policy_net.parameters(), lr=policy_lr
    )
    value_optimizer = optim.Adam(value_net.parameters(), lr=value_lr)

    for iteration in range(num_iterations):
        logger.info(
            f"Starting iteration {iteration + 1}/{num_iterations}"
        )
        memory = []

        for episode in range(episodes_per_iteration):
            logger.debug(
                f"Starting episode {episode + 1}/{episodes_per_iteration}"
            )
            # Initialize state sequence with zeros
            state = torch.zeros(policy_net.embedding.in_features).to(
                device
            )
            state_sequence = state.unsqueeze(
                0
            )  # Shape: (1, input_dim)
            thought_tree = ThoughtTree(state_sequence)
            trajectory = []

            # Generate thought branches
            for depth in range(max_depth):
                # Expand dimensions to match (sequence_length, batch_size, input_dim)
                src = state_sequence.unsqueeze(
                    1
                )  # Shape: (sequence_length, 1, input_dim)
                action_probs = policy_net(src)
                m = Categorical(action_probs)
                actions = m.sample((5,))  # Generate multiple branches
                rewards = []

                for action in actions:
                    next_state = transition(
                        state_sequence[-1], action
                    )
                    # Update the sequence by appending the new state
                    next_sequence = torch.cat(
                        [state_sequence, next_state.unsqueeze(0)],
                        dim=0,
                    )
                    # Ensure the sequence length does not exceed the maximum
                    if next_sequence.size(0) > sequence_length:
                        next_sequence = next_sequence[1:, :]
                    rollout = monte_carlo_rollout(
                        policy_net,
                        next_sequence,
                        depth + 1,
                        max_depth,
                        sequence_length,
                    )
                    total_reward = sum([r for _, r in rollout])
                    # Expand dimensions for reward model input
                    reward_input = next_sequence.unsqueeze(1)
                    reward_estimate = reward_model(reward_input)
                    reward = reward_estimate.item() + total_reward
                    rewards.append(reward)

                    # Update thought tree
                    thought_tree.add_child(
                        thought_tree.root, next_sequence, reward
                    )

                # Select the best action based on rewards
                best_action_index = (
                    torch.tensor(rewards).argmax().item()
                )
                best_action = actions[best_action_index]
                best_reward = rewards[best_action_index]

                # Log the selected action and reward
                logger.debug(
                    f"Selected action {best_action.item()} with reward {best_reward}"
                )

                # Store the experience
                trajectory.append(
                    (state_sequence.clone(), best_action, best_reward)
                )

                # Move to the next state sequence
                next_state = transition(
                    state_sequence[-1], best_action
                )
                state_sequence = torch.cat(
                    [state_sequence, next_state.unsqueeze(0)], dim=0
                )
                if state_sequence.size(0) > sequence_length:
                    state_sequence = state_sequence[1:, :]

            # Compute returns and advantages
            returns = []
            advantages = []
            Gt = 0
            for state_seq_t, action_t, reward_t in reversed(
                trajectory
            ):
                Gt = reward_t + gamma * Gt
                returns.insert(0, Gt)
                # Expand dimensions for value network input
                value_input = state_seq_t.unsqueeze(1)
                state_value = value_net(value_input)
                advantage = Gt - state_value.item()
                advantages.insert(0, advantage)

            # Normalize advantages
            advantages_tensor = torch.tensor(
                advantages, dtype=torch.float32
            ).to(device)
            advantages_tensor = (
                advantages_tensor - advantages_tensor.mean()
            ) / (advantages_tensor.std() + 1e-8)

            # Update policy network using PPO
            for i, (state_seq_t, action_t, _) in enumerate(
                trajectory
            ):
                # Expand dimensions to match (sequence_length, batch_size, input_dim)
                src = state_seq_t.unsqueeze(1)
                action_probs = policy_net(src)
                m = Categorical(action_probs)
                log_prob = m.log_prob(action_t)
                old_log_prob = log_prob.detach()
                ratio = torch.exp(log_prob - old_log_prob)
                surr1 = ratio * advantages_tensor[i]
                surr2 = (
                    torch.clamp(
                        ratio, 1 - clip_epsilon, 1 + clip_epsilon
                    )
                    * advantages_tensor[i]
                )
                policy_loss = -torch.min(surr1, surr2)

                policy_optimizer.zero_grad()
                policy_loss.backward()
                policy_optimizer.step()

                # Log the policy loss
                logger.debug(
                    f"Policy loss at step {i}: {policy_loss.item()}"
                )

            # Update value network
            returns_tensor = (
                torch.tensor(returns, dtype=torch.float32)
                .unsqueeze(1)
                .to(device)
            )
            # Prepare inputs for the value network
            value_inputs = torch.stack(
                [s for s, _, _ in trajectory]
            ).transpose(0, 1)
            value_inputs = value_inputs.to(device)
            values = value_net(value_inputs)
            value_loss = nn.MSELoss()(values, returns_tensor)

            value_optimizer.zero_grad()
            value_loss.backward()
            value_optimizer.step()

            # Log the value loss
            logger.debug(f"Value loss: {value_loss.item()}")

        logger.info(
            f"Completed iteration {iteration + 1}/{num_iterations}"
        )

if __name__ == "__main__":
    # Hyperparameters
    input_dim = 10  # Dimension of the input state
    action_dim = 4  # Number of possible actions
    num_iterations = 10
    episodes_per_iteration = 5
    sequence_length = (
        10  # Maximum sequence length for the transformer
    )

    # Initialize networks
    policy_net = TransformerPolicyNetwork(input_dim, action_dim).to(
        device
    )
    value_net = TransformerValueNetwork(input_dim).to(device)
    reward_model = TransformerRewardModel(input_dim).to(device)

    # Start training


        Args:
            src (torch.Tensor): Input tensor of shape (sequence_length, batch_size, input_dim).

        Returns:
            torch.Tensor: Reward estimate of shape (batch_size, 1).
        """
        src = self.embedding(src)
        output = self.transformer_encoder(src)
        # Take the output from the last time step
        output = output[-1, :, :]
        reward = self.fc_out(output)
        return reward

class ThoughtTree:
    """
    Class representing a tree of thoughts.
    """

    def __init__(self, root_state: torch.Tensor):
        self.root = {"state": root_state, "children": [], "reward": 0}

    def add_child(
        self, parent: dict, child_state: torch.Tensor, reward: float
    ):
        child = {
            "state": child_state,
            "children": [],
            "reward": reward,
        }
        parent["children"].append(child)
        return child

def monte_carlo_rollout(
    policy_net: TransformerPolicyNetwork,
    state_sequence: torch.Tensor,
    depth: int,
    max_depth: int,
    sequence_length: int,
) -> List[Tuple[torch.Tensor, float]]:


forward

    def __init__(
        self,
        input_dim: int,
        nhead: int = 8,
        num_layers: int = 6,
        dim_feedforward: int = 2048,
        dropout: float = 0.1,
    ):
        super(TransformerRewardModel, self).__init__()
        self.model_type = "Transformer"

        self.embedding = nn.Linear(input_dim, dim_feedforward)
        encoder_layers = nn.TransformerEncoderLayer(
            d_model=dim_feedforward, nhead=nhead, dropout=dropout
        )
        self.transformer_encoder = nn.TransformerEncoder(
            encoder_layers, num_layers
        )
        self.fc_out = nn.Linear(dim_feedforward, 1)

    def forward(self, src: torch.Tensor) -> torch.Tensor:
        """
        Forward pass for the reward model.

        Args:
            src (torch.Tensor): Input tensor of shape (sequence_length, batch_size, input_dim).

        Returns:
            torch.Tensor: Reward estimate of shape (batch_size, 1).
        """
        src = self.embedding(src)
        output = self.transformer_encoder(src)
        # Take the output from the last time step
        output = output[-1, :, :]
        reward = self.fc_out(output)
        return reward

class ThoughtTree:
    """
    Class representing a tree of thoughts.
    """

    def __init__(self, root_state: torch.Tensor):
        self.root = {"state": root_state, "children": [], "reward": 0}

    def add_child(
        self, parent: dict, child_state: torch.Tensor, reward: float
    ):
        child = {
            "state": child_state,
            "children": [],
            "reward": reward,
        }
        parent["children"].append(child)
        return child


        Returns:
            torch.Tensor: Reward estimate of shape (batch_size, 1).
        """
        src = self.embedding(src)
        output = self.transformer_encoder(src)
        # Take the output from the last time step
        output = output[-1, :, :]
        reward = self.fc_out(output)
        return reward

class ThoughtTree:
    """
    Class representing a tree of thoughts.
    """

    def __init__(self, root_state: torch.Tensor):
        self.root = {"state": root_state, "children": [], "reward": 0}

    def add_child(
        self, parent: dict, child_state: torch.Tensor, reward: float
    ):
        child = {
            "state": child_state,
            "children": [],
            "reward": reward,
        }
        parent["children"].append(child)
        return child

def monte_carlo_rollout(
    policy_net: TransformerPolicyNetwork,
    state_sequence: torch.Tensor,
    depth: int,
    max_depth: int,
    sequence_length: int,
) -> List[Tuple[torch.Tensor, float]]:
    """
    Perform a Monte Carlo rollout to simulate future thoughts.

    Args:
        policy_net (TransformerPolicyNetwork): The policy network.
        state_sequence (torch.Tensor): The current state sequence.
        depth (int): Current depth in the thought tree.
        max_depth (int): Maximum depth for rollouts.
        sequence_length (int): The length of the input sequence.

    Returns:


from collections.abc import Iterator
from copy import deepcopy

import torch
from torch.nn import Module
import torch.nn.functional as F
from x_transformers.x_transformers import TransformerWrapper

from einops import rearrange

# helper functions

def exists(v):
    return v is not None

def freeze_all_layers_(module):
    for param in module.parameters():
        param.requires_grad = False

def log_prob_from_model_and_seq(model, seq):
    logits = model(seq)
    log_prob = logits.log_softmax(dim=-1)
    indices = rearrange(seq, "... -> ... 1")
    log_probs = log_prob.gather(-1, indices)
    return rearrange(log_probs, "... 1 -> ...")

def masked_mean(log_probs, mask=None):
    if not exists(mask):
        return log_probs.mean(dim=-1)

    log_probs = log_probs.masked_fill(~mask, 0.0)
    num = log_probs.sum(dim=-1)
    den = mask.sum(dim=-1)
    return num / den.clamp(min=1e-5)

def maybe_and_mask(*masks):
    masks = [*filter(exists, masks)]
    if len(masks) == 0:
        return None

    mask, *rest_masks = masks
    for rest_mask in rest_masks:
        mask = mask & rest_mask

    return mask

# main class
class DPO(Module):
    """
    DPO (Divergence-based Policy Optimization) model for optimizing policy models based on divergence metrics.

    This class implements the DPO algorithm as described in the paper "Divergence-based Policy Optimization" (https://arxiv.org/abs/2305.18290).
    It takes a TransformerWrapper model as input and computes the divergence between the policy model and a reference model.
    The divergence is used to optimize the policy model parameters.

    Attributes:
        policy_model (TransformerWrapper): The policy model to be optimized.
        ref_model (TransformerWrapper): The reference model used for computing divergence.
        beta (float): The beta parameter for the divergence metric.
        pad_id (int, optional): The padding token ID. Defaults to None.
    """

    def __init__(
        self,
        model: TransformerWrapper,
        *,
        beta: float = 0.1,
        pad_id: int = None,
    ):
        """
        Initializes the DPO model.

        Args:
            model (TransformerWrapper): The policy model to be optimized.
            beta (float, optional): The beta parameter for the divergence metric. Defaults to 0.1.
            pad_id (int, optional): The padding token ID. Defaults to None.
        """
        super().__init__()
        self.policy_model = model

        self.ref_model = deepcopy(model)
        freeze_all_layers_(self.ref_model)

        self.beta = beta
        self.pad_id = pad_id

    def parameters(self) -> Iterator[torch.nn.Parameter]:
        """
        Returns an iterator over the model parameters.

        Returns:
            Iterator[torch.nn.Parameter]: An iterator over the model parameters.
        """
        return self.policy_model.parameters()

    def forward(
        self,
        preferred_seq: torch.Tensor,
        unpreferred_seq: torch.Tensor,
        *,
        prompt_mask: torch.Tensor,
        preferred_seq_mask: torch.Tensor = None,
        unpreferred_seq_mask: torch.Tensor = None,
    ) -> torch.Tensor:
        """
        Computes the DPO loss for the given sequences and masks.

        Args:
            preferred_seq (torch.Tensor): The preferred sequence tensor.
            unpreferred_seq (torch.Tensor): The unpreferred sequence tensor.
            prompt_mask (torch.Tensor): The prompt mask tensor.
            preferred_seq_mask (torch.Tensor, optional): The mask for the preferred sequence. Defaults to None.
            unpreferred_seq_mask (torch.Tensor, optional): The mask for the unpreferred sequence. Defaults to None.

        Returns:
            torch.Tensor: The computed DPO loss tensor.
        """
        assert preferred_seq.ndim == 2
        assert preferred_seq.shape == unpreferred_seq.shape

        if exists(self.pad_id):
            if not exists(preferred_seq_mask):
                preferred_seq_mask = preferred_seq != self.pad_id

            if not exists(unpreferred_seq_mask):
                unpreferred_seq_mask = unpreferred_seq != self.pad_id

        """
        Following Appendix B in https://arxiv.org/abs/2305.18290
        """

        with torch.no_grad():
            self.ref_model.eval()
            ref_preferred_logprob = log_prob_from_model_and_seq(
                self.ref_model, preferred_seq
            )
            ref_unpreferred_logprob = log_prob_from_model_and_seq(
                self.ref_model, unpreferred_seq
            )

        policy_preferred_logprob = log_prob_from_model_and_seq(
            self.policy_model, preferred_seq
        )
        policy_unpreferred_logprob = log_prob_from_model_and_seq(
            self.policy_model, unpreferred_seq
        )

        # masked mean of log probs

        preferred_seq_mask = maybe_and_mask(
            ~prompt_mask, preferred_seq_mask
        )
        unpreferred_seq_mask = maybe_and_mask(
            ~prompt_mask, unpreferred_seq_mask
        )

        ref_preferred_logprob, policy_preferred_logprob = map(
            lambda t: masked_mean(t, preferred_seq_mask),
            (ref_preferred_logprob, policy_preferred_logprob),
        )
        ref_unpreferred_logprob, policy_unpreferred_logprob = map(
            lambda t: masked_mean(t, unpreferred_seq_mask),
            (ref_unpreferred_logprob, policy_unpreferred_logprob),
        )

        # main dpo formula

        policy_logratios = (
            policy_preferred_logprob - policy_unpreferred_logprob
        )
        ref_logratios = (
            ref_preferred_logprob - ref_unpreferred_logprob
        )

        losses = -F.logsigmoid(
            self.beta * (policy_logratios - ref_logratios)
        )

        return losses.mean()


exists

from collections.abc import Iterator
from copy import deepcopy

import torch
from torch.nn import Module
import torch.nn.functional as F
from x_transformers.x_transformers import TransformerWrapper

from einops import rearrange

# helper functions

def exists(v):
    return v is not None

def freeze_all_layers_(module):
    for param in module.parameters():
        param.requires_grad = False

def log_prob_from_model_and_seq(model, seq):
    logits = model(seq)
    log_prob = logits.log_softmax(dim=-1)
    indices = rearrange(seq, "... -> ... 1")
    log_probs = log_prob.gather(-1, indices)
    return rearrange(log_probs, "... 1 -> ...")

def masked_mean(log_probs, mask=None):
    if not exists(mask):
        return log_probs.mean(dim=-1)

    log_probs = log_probs.masked_fill(~mask, 0.0)


from collections.abc import Iterator
from copy import deepcopy

import torch
from torch.nn import Module
import torch.nn.functional as F
from x_transformers.x_transformers import TransformerWrapper

from einops import rearrange

# helper functions

def exists(v):
    return v is not None

def freeze_all_layers_(module):
    for param in module.parameters():
        param.requires_grad = False

def log_prob_from_model_and_seq(model, seq):
    logits = model(seq)
    log_prob = logits.log_softmax(dim=-1)
    indices = rearrange(seq, "... -> ... 1")
    log_probs = log_prob.gather(-1, indices)
    return rearrange(log_probs, "... 1 -> ...")

def masked_mean(log_probs, mask=None):
    if not exists(mask):
        return log_probs.mean(dim=-1)

    log_probs = log_probs.masked_fill(~mask, 0.0)
    num = log_probs.sum(dim=-1)
    den = mask.sum(dim=-1)
    return num / den.clamp(min=1e-5)


import torch
from torch.nn import Module
import torch.nn.functional as F
from x_transformers.x_transformers import TransformerWrapper

from einops import rearrange

# helper functions

def exists(v):
    return v is not None

def freeze_all_layers_(module):
    for param in module.parameters():
        param.requires_grad = False

def log_prob_from_model_and_seq(model, seq):
    logits = model(seq)
    log_prob = logits.log_softmax(dim=-1)
    indices = rearrange(seq, "... -> ... 1")
    log_probs = log_prob.gather(-1, indices)
    return rearrange(log_probs, "... 1 -> ...")

def masked_mean(log_probs, mask=None):
    if not exists(mask):
        return log_probs.mean(dim=-1)

    log_probs = log_probs.masked_fill(~mask, 0.0)
    num = log_probs.sum(dim=-1)
    den = mask.sum(dim=-1)
    return num / den.clamp(min=1e-5)

def maybe_and_mask(*masks):
    masks = [*filter(exists, masks)]
    if len(masks) == 0:
        return None

    mask, *rest_masks = masks
    for rest_mask in rest_masks:
        mask = mask & rest_mask


# helper functions

def exists(v):
    return v is not None

def freeze_all_layers_(module):
    for param in module.parameters():
        param.requires_grad = False

def log_prob_from_model_and_seq(model, seq):
    logits = model(seq)
    log_prob = logits.log_softmax(dim=-1)
    indices = rearrange(seq, "... -> ... 1")
    log_probs = log_prob.gather(-1, indices)
    return rearrange(log_probs, "... 1 -> ...")

def masked_mean(log_probs, mask=None):
    if not exists(mask):
        return log_probs.mean(dim=-1)

    log_probs = log_probs.masked_fill(~mask, 0.0)
    num = log_probs.sum(dim=-1)
    den = mask.sum(dim=-1)
    return num / den.clamp(min=1e-5)

def maybe_and_mask(*masks):
    masks = [*filter(exists, masks)]
    if len(masks) == 0:
        return None

    mask, *rest_masks = masks
    for rest_mask in rest_masks:
        mask = mask & rest_mask

    return mask

# main class
class DPO(Module):
    """
    DPO (Divergence-based Policy Optimization) model for optimizing policy models based on divergence metrics.

    This class implements the DPO algorithm as described in the paper "Divergence-based Policy Optimization" (https://arxiv.org/abs/2305.18290).


def log_prob_from_model_and_seq(model, seq):
    logits = model(seq)
    log_prob = logits.log_softmax(dim=-1)
    indices = rearrange(seq, "... -> ... 1")
    log_probs = log_prob.gather(-1, indices)
    return rearrange(log_probs, "... 1 -> ...")

def masked_mean(log_probs, mask=None):
    if not exists(mask):
        return log_probs.mean(dim=-1)

    log_probs = log_probs.masked_fill(~mask, 0.0)
    num = log_probs.sum(dim=-1)
    den = mask.sum(dim=-1)
    return num / den.clamp(min=1e-5)

def maybe_and_mask(*masks):
    masks = [*filter(exists, masks)]
    if len(masks) == 0:
        return None

    mask, *rest_masks = masks
    for rest_mask in rest_masks:
        mask = mask & rest_mask

    return mask

# main class
class DPO(Module):
    """
    DPO (Divergence-based Policy Optimization) model for optimizing policy models based on divergence metrics.

    This class implements the DPO algorithm as described in the paper "Divergence-based Policy Optimization" (https://arxiv.org/abs/2305.18290).
    It takes a TransformerWrapper model as input and computes the divergence between the policy model and a reference model.
    The divergence is used to optimize the policy model parameters.

    Attributes:
        policy_model (TransformerWrapper): The policy model to be optimized.
        ref_model (TransformerWrapper): The reference model used for computing divergence.
        beta (float): The beta parameter for the divergence metric.
        pad_id (int, optional): The padding token ID. Defaults to None.
    """

    def __init__(
        self,


DPO

    log_probs = log_probs.masked_fill(~mask, 0.0)
    num = log_probs.sum(dim=-1)
    den = mask.sum(dim=-1)
    return num / den.clamp(min=1e-5)

def maybe_and_mask(*masks):
    masks = [*filter(exists, masks)]
    if len(masks) == 0:
        return None

    mask, *rest_masks = masks
    for rest_mask in rest_masks:
        mask = mask & rest_mask

    return mask

# main class
class DPO(Module):
    """
    DPO (Divergence-based Policy Optimization) model for optimizing policy models based on divergence metrics.

    This class implements the DPO algorithm as described in the paper "Divergence-based Policy Optimization" (https://arxiv.org/abs/2305.18290).
    It takes a TransformerWrapper model as input and computes the divergence between the policy model and a reference model.
    The divergence is used to optimize the policy model parameters.

    Attributes:
        policy_model (TransformerWrapper): The policy model to be optimized.
        ref_model (TransformerWrapper): The reference model used for computing divergence.
        beta (float): The beta parameter for the divergence metric.
        pad_id (int, optional): The padding token ID. Defaults to None.
    """

    def __init__(
        self,
        model: TransformerWrapper,
        *,
        beta: float = 0.1,
        pad_id: int = None,
    ):
        """
        Initializes the DPO model.

        Args:
            model (TransformerWrapper): The policy model to be optimized.
            beta (float, optional): The beta parameter for the divergence metric. Defaults to 0.1.
            pad_id (int, optional): The padding token ID. Defaults to None.
        """
        super().__init__()
        self.policy_model = model

        self.ref_model = deepcopy(model)
        freeze_all_layers_(self.ref_model)

        self.beta = beta
        self.pad_id = pad_id

    def parameters(self) -> Iterator[torch.nn.Parameter]:
        """
        Returns an iterator over the model parameters.

        Returns:
            Iterator[torch.nn.Parameter]: An iterator over the model parameters.
        """
        return self.policy_model.parameters()

    def forward(
        self,
        preferred_seq: torch.Tensor,
        unpreferred_seq: torch.Tensor,
        *,
        prompt_mask: torch.Tensor,
        preferred_seq_mask: torch.Tensor = None,
        unpreferred_seq_mask: torch.Tensor = None,
    ) -> torch.Tensor:
        """
        Computes the DPO loss for the given sequences and masks.

        Args:
            preferred_seq (torch.Tensor): The preferred sequence tensor.
            unpreferred_seq (torch.Tensor): The unpreferred sequence tensor.
            prompt_mask (torch.Tensor): The prompt mask tensor.
            preferred_seq_mask (torch.Tensor, optional): The mask for the preferred sequence. Defaults to None.
            unpreferred_seq_mask (torch.Tensor, optional): The mask for the unpreferred sequence. Defaults to None.

        Returns:
            torch.Tensor: The computed DPO loss tensor.
        """
        assert preferred_seq.ndim == 2
        assert preferred_seq.shape == unpreferred_seq.shape

        if exists(self.pad_id):
            if not exists(preferred_seq_mask):
                preferred_seq_mask = preferred_seq != self.pad_id

            if not exists(unpreferred_seq_mask):
                unpreferred_seq_mask = unpreferred_seq != self.pad_id

        """
        Following Appendix B in https://arxiv.org/abs/2305.18290
        """

        with torch.no_grad():
            self.ref_model.eval()
            ref_preferred_logprob = log_prob_from_model_and_seq(
                self.ref_model, preferred_seq
            )
            ref_unpreferred_logprob = log_prob_from_model_and_seq(
                self.ref_model, unpreferred_seq
            )

        policy_preferred_logprob = log_prob_from_model_and_seq(
            self.policy_model, preferred_seq
        )
        policy_unpreferred_logprob = log_prob_from_model_and_seq(
            self.policy_model, unpreferred_seq
        )

        # masked mean of log probs

        preferred_seq_mask = maybe_and_mask(
            ~prompt_mask, preferred_seq_mask
        )
        unpreferred_seq_mask = maybe_and_mask(
            ~prompt_mask, unpreferred_seq_mask
        )

        ref_preferred_logprob, policy_preferred_logprob = map(
            lambda t: masked_mean(t, preferred_seq_mask),
            (ref_preferred_logprob, policy_preferred_logprob),
        )
        ref_unpreferred_logprob, policy_unpreferred_logprob = map(
            lambda t: masked_mean(t, unpreferred_seq_mask),
            (ref_unpreferred_logprob, policy_unpreferred_logprob),
        )

        # main dpo formula

        policy_logratios = (
            policy_preferred_logprob - policy_unpreferred_logprob
        )
        ref_logratios = (
            ref_preferred_logprob - ref_unpreferred_logprob
        )

        losses = -F.logsigmoid(
            self.beta * (policy_logratios - ref_logratios)
        )

        return losses.mean()


    return mask

# main class
class DPO(Module):
    """
    DPO (Divergence-based Policy Optimization) model for optimizing policy models based on divergence metrics.

    This class implements the DPO algorithm as described in the paper "Divergence-based Policy Optimization" (https://arxiv.org/abs/2305.18290).
    It takes a TransformerWrapper model as input and computes the divergence between the policy model and a reference model.
    The divergence is used to optimize the policy model parameters.

    Attributes:
        policy_model (TransformerWrapper): The policy model to be optimized.
        ref_model (TransformerWrapper): The reference model used for computing divergence.
        beta (float): The beta parameter for the divergence metric.
        pad_id (int, optional): The padding token ID. Defaults to None.
    """

    def __init__(
        self,
        model: TransformerWrapper,
        *,
        beta: float = 0.1,
        pad_id: int = None,
    ):
        """
        Initializes the DPO model.

        Args:
            model (TransformerWrapper): The policy model to be optimized.
            beta (float, optional): The beta parameter for the divergence metric. Defaults to 0.1.
            pad_id (int, optional): The padding token ID. Defaults to None.
        """
        super().__init__()
        self.policy_model = model

        self.ref_model = deepcopy(model)
        freeze_all_layers_(self.ref_model)

        self.beta = beta
        self.pad_id = pad_id

    def parameters(self) -> Iterator[torch.nn.Parameter]:
        """
        Returns an iterator over the model parameters.

        Returns:
            Iterator[torch.nn.Parameter]: An iterator over the model parameters.
        """
        return self.policy_model.parameters()

    def forward(
        self,
        preferred_seq: torch.Tensor,
        unpreferred_seq: torch.Tensor,
        *,
        prompt_mask: torch.Tensor,
        preferred_seq_mask: torch.Tensor = None,
        unpreferred_seq_mask: torch.Tensor = None,
    ) -> torch.Tensor:
        """