# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import torch

from .model_configs import configs

class CrossEntropyLossBase:
    def __init__(self, model_name, dtype):
        self.config = configs[model_name]()
        self.dtype = dtype

    def model(self):
        raise NotImplementedError

    def inputs(self):
        hidden_states = torch.randn(
            self.config.batch_size,
            self.config.seq_len,
            self.config.hidden_size,
            device="cuda",
            dtype=self.dtype,
            requires_grad=True,
        )
        labels = torch.randint(
            0,
            self.config.vocab_size,
            (self.config.batch_size, self.config.seq_len),
            device="cuda",
            requires_grad=False,
        )
        return {"hidden_states": hidden_states, "labels": labels}

    def grads(self):
        grad = torch.tensor(1, device="cuda", dtype=self.dtype, requires_grad=False)
        return grad

    # please note that this computes the IO bytes based on the graph/decomposition
    # ===== Backward graph 0 =====
    # <eval_with_key>.3 class GraphModule(torch.nn.Module):
    # def forward(self, primals_1: "i64[8192][1]cuda:0", primals_2: "f32[8192, 3024][3024, 1]cuda:0",
    #        amax: "f32[8192, 1][1, 1]cuda:0", log: "f32[8192, 1][1, 1]cuda:0", convert_element_type: "f32[][]cuda:0", tangents_1: "f32[][]cuda:0"):
    #     # File: /opt/pytorch/nvfuser/test.py:6 in fn, code: return torch.nn.functional.cross_entropy(input, target)
    #    div_1: "f32[][]cuda:0" = torch.ops.aten.div.Tensor(tangents_1, convert_element_type);  tangents_1 = convert_element_type = None
    #    unsqueeze_1: "i64[8192, 1][1, 1]cuda:0" = torch.ops.aten.unsqueeze.default(primals_1, 1);  primals_1 = None
    #    ne_3: "b8[8192, 1][1, 1]cuda:0" = torch.ops.aten.ne.Scalar(unsqueeze_1, -100)
    #    full_default: "i64[][]cuda:0" = torch.ops.aten.full.default([], 0, dtype = torch.int64, layout = torch.strided, device = device(type='cuda', index=0), pin_memory = False)
    #    where_2: "i64[8192, 1][1, 1]cuda:0" = torch.ops.aten.where.self(ne_3, unsqueeze_1, full_default);  unsqueeze_1 = full_default = None
    #    full_default_3: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.full.default([8192, 3024], 0, dtype = torch.float32, layout = torch.strided, device = device(type='cuda', index=0), pin_memory = False)
    #    scatter: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.scatter.value(full_default_3, 1, where_2, -1.0);  full_default_3 = where_2 = None
    #    full_default_1: "f32[][]cuda:0" = torch.ops.aten.full.default([], 0.0, dtype = torch.float32, layout = torch.strided, device = device(type='cuda', index=0), pin_memory = False)
    #    where_3: "f32[8192, 1][1, 1]cuda:0" = torch.ops.aten.where.self(ne_3, div_1, full_default_1);  ne_3 = div_1 = full_default_1 = None
    #    mul: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.mul.Tensor(scatter, where_3);  scatter = where_3 = None
    #    sub: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.sub.Tensor(primals_2, amax);  primals_2 = amax = None
    #    sub_1: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.sub.Tensor(sub, log);  sub = log = None
    #    exp_1: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.exp.default(sub_1);  sub_1 = None
    #    sum_4: "f32[8192, 1][1, 1]cuda:0" = torch.ops.aten.sum.dim_IntList(mul, [1], True)
    #    mul_1: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.mul.Tensor(exp_1, sum_4);  exp_1 = sum_4 = None
    #    sub_2: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.sub.Tensor(mul, mul_1);  mul = mul_1 = None
    #    return (None, sub_2)

    # We account for inputs primals_1, primals_2, amax, log
    # and for the output sub_2
    def grad_iobytes(self):
        n_elements = 0
        # adding size of primals_2 and the output
        n_elements += 2 * (
            self.config.batch_size * self.config.seq_len * self.config.vocab_size
        )
        # adding size of amax and log and primals_1
        n_elements += 3 * self.config.batch_size * self.config.seq_len
        # scale by dtype size
        return n_elements * self.dtype.itemsize

class HfQwen2(CrossEntropyLossBase):
    def __init__(self, dtype):
        super().__init__("hf_qwen2", dtype)

    def model(self):
        from transformers.models.qwen2.modeling_qwen2 import Qwen2PreTrainedModel

        class MyModel(Qwen2PreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )
                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

class HfPhi3(CrossEntropyLossBase):
    def __init__(self, dtype):
        super().__init__("hf_phi3", dtype)

    def model(self):
        from transformers.models.phi3 import Phi3PreTrainedModel

        class MyModel(Phi3PreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )

                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

class HfMistralNemo(CrossEntropyLossBase):
    def __init__(self, dtype):
        super().__init__("hf_mistral_nemo", dtype)

    def model(self):
        from transformers.models.mistral import MistralPreTrainedModel

        class MyModel(MistralPreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )

                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

cross_entropy_loss_setup = {
    "hf_qwen2": HfQwen2,
    "hf_phi3": HfPhi3,
    "hf_mistral_nemo": HfMistralNemo,
}

class SyntheticMiniModel:
    # Vocab sizes from popular models
    sizes_from_models = [
        49152,  # Starcoder
        129280,  # DeepSeek-R1
        128256,  # Llama3
        202048,  # Llama4
        256000,  # Gemma2
        131072,  # Mistral
        152064,  # Qwen2
        32064,  # Phi3.5
        100352,  # Phi4
        50264,  # GPT-2
    ]

    @staticmethod
    def mini_model(logits, labels):
        labels = torch.nn.functional.pad(labels, (0, 1))
        labels = labels[1 : labels.shape[-1]]
        logits = logits.to(dtype=torch.float32)
        logits = logits.squeeze(dim=0)
        return torch.nn.functional.cross_entropy(logits, labels)

    @staticmethod
    def inputs(batch_size, vocab_size):
        input = torch.randn(
            1,
            batch_size,
            vocab_size,
            device="cuda",
            dtype=torch.bfloat16,
            requires_grad=True,
        )
        labels = torch.randint(
            0,
            vocab_size - 1,
            (batch_size,),
            device="cuda",
            requires_grad=False,
        )
        return (input, labels)

    @staticmethod
    def grads():
        grad = torch.tensor(1, device="cuda", dtype=torch.float32, requires_grad=False)
        return grad

    @staticmethod
    def generate_vocab_sizes():
        powers_of_2 = [2**i * 1024 for i in range(4, 9)]

        combined_set = sorted(
            set(SyntheticMiniModel.sizes_from_models) | set(powers_of_2)
        )

        # for each vocab size in the set we increment in steps 64 in +/- 5 directions
        # which gives the total number of vocab sizes to benchmark
        variations = set()
        step = 64
        for num in combined_set:
            for i in range(1, 6):
                variations.add(num + (i * step))
                variations.add(num - (i * step))

            variations.add(num)

        return sorted(variations)


CrossEntropyLossBase

# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import torch

from .model_configs import configs

class CrossEntropyLossBase:
    def __init__(self, model_name, dtype):
        self.config = configs[model_name]()
        self.dtype = dtype

    def model(self):
        raise NotImplementedError

    def inputs(self):
        hidden_states = torch.randn(
            self.config.batch_size,
            self.config.seq_len,
            self.config.hidden_size,
            device="cuda",
            dtype=self.dtype,
            requires_grad=True,
        )
        labels = torch.randint(
            0,
            self.config.vocab_size,
            (self.config.batch_size, self.config.seq_len),
            device="cuda",
            requires_grad=False,
        )
        return {"hidden_states": hidden_states, "labels": labels}

    def grads(self):
        grad = torch.tensor(1, device="cuda", dtype=self.dtype, requires_grad=False)
        return grad

    # please note that this computes the IO bytes based on the graph/decomposition
    # ===== Backward graph 0 =====
    # <eval_with_key>.3 class GraphModule(torch.nn.Module):
    # def forward(self, primals_1: "i64[8192][1]cuda:0", primals_2: "f32[8192, 3024][3024, 1]cuda:0",
    #        amax: "f32[8192, 1][1, 1]cuda:0", log: "f32[8192, 1][1, 1]cuda:0", convert_element_type: "f32[][]cuda:0", tangents_1: "f32[][]cuda:0"):
    #     # File: /opt/pytorch/nvfuser/test.py:6 in fn, code: return torch.nn.functional.cross_entropy(input, target)
    #    div_1: "f32[][]cuda:0" = torch.ops.aten.div.Tensor(tangents_1, convert_element_type);  tangents_1 = convert_element_type = None
    #    unsqueeze_1: "i64[8192, 1][1, 1]cuda:0" = torch.ops.aten.unsqueeze.default(primals_1, 1);  primals_1 = None
    #    ne_3: "b8[8192, 1][1, 1]cuda:0" = torch.ops.aten.ne.Scalar(unsqueeze_1, -100)
    #    full_default: "i64[][]cuda:0" = torch.ops.aten.full.default([], 0, dtype = torch.int64, layout = torch.strided, device = device(type='cuda', index=0), pin_memory = False)
    #    where_2: "i64[8192, 1][1, 1]cuda:0" = torch.ops.aten.where.self(ne_3, unsqueeze_1, full_default);  unsqueeze_1 = full_default = None
    #    full_default_3: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.full.default([8192, 3024], 0, dtype = torch.float32, layout = torch.strided, device = device(type='cuda', index=0), pin_memory = False)
    #    scatter: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.scatter.value(full_default_3, 1, where_2, -1.0);  full_default_3 = where_2 = None
    #    full_default_1: "f32[][]cuda:0" = torch.ops.aten.full.default([], 0.0, dtype = torch.float32, layout = torch.strided, device = device(type='cuda', index=0), pin_memory = False)
    #    where_3: "f32[8192, 1][1, 1]cuda:0" = torch.ops.aten.where.self(ne_3, div_1, full_default_1);  ne_3 = div_1 = full_default_1 = None
    #    mul: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.mul.Tensor(scatter, where_3);  scatter = where_3 = None
    #    sub: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.sub.Tensor(primals_2, amax);  primals_2 = amax = None
    #    sub_1: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.sub.Tensor(sub, log);  sub = log = None
    #    exp_1: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.exp.default(sub_1);  sub_1 = None
    #    sum_4: "f32[8192, 1][1, 1]cuda:0" = torch.ops.aten.sum.dim_IntList(mul, [1], True)
    #    mul_1: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.mul.Tensor(exp_1, sum_4);  exp_1 = sum_4 = None
    #    sub_2: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.sub.Tensor(mul, mul_1);  mul = mul_1 = None
    #    return (None, sub_2)

    # We account for inputs primals_1, primals_2, amax, log
    # and for the output sub_2
    def grad_iobytes(self):
        n_elements = 0
        # adding size of primals_2 and the output
        n_elements += 2 * (
            self.config.batch_size * self.config.seq_len * self.config.vocab_size
        )
        # adding size of amax and log and primals_1
        n_elements += 3 * self.config.batch_size * self.config.seq_len
        # scale by dtype size
        return n_elements * self.dtype.itemsize

class HfQwen2(CrossEntropyLossBase):
    def __init__(self, dtype):
        super().__init__("hf_qwen2", dtype)

    def model(self):
        from transformers.models.qwen2.modeling_qwen2 import Qwen2PreTrainedModel

        class MyModel(Qwen2PreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )
                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)


    #    exp_1: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.exp.default(sub_1);  sub_1 = None
    #    sum_4: "f32[8192, 1][1, 1]cuda:0" = torch.ops.aten.sum.dim_IntList(mul, [1], True)
    #    mul_1: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.mul.Tensor(exp_1, sum_4);  exp_1 = sum_4 = None
    #    sub_2: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.sub.Tensor(mul, mul_1);  mul = mul_1 = None
    #    return (None, sub_2)

    # We account for inputs primals_1, primals_2, amax, log
    # and for the output sub_2
    def grad_iobytes(self):
        n_elements = 0
        # adding size of primals_2 and the output
        n_elements += 2 * (
            self.config.batch_size * self.config.seq_len * self.config.vocab_size
        )
        # adding size of amax and log and primals_1
        n_elements += 3 * self.config.batch_size * self.config.seq_len
        # scale by dtype size
        return n_elements * self.dtype.itemsize

class HfQwen2(CrossEntropyLossBase):
    def __init__(self, dtype):
        super().__init__("hf_qwen2", dtype)

    def model(self):
        from transformers.models.qwen2.modeling_qwen2 import Qwen2PreTrainedModel

        class MyModel(Qwen2PreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )
                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

class HfPhi3(CrossEntropyLossBase):
    def __init__(self, dtype):
        super().__init__("hf_phi3", dtype)

    def model(self):
        from transformers.models.phi3 import Phi3PreTrainedModel

        class MyModel(Phi3PreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )

                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):


        class MyModel(Qwen2PreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )
                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

class HfPhi3(CrossEntropyLossBase):
    def __init__(self, dtype):
        super().__init__("hf_phi3", dtype)

    def model(self):
        from transformers.models.phi3 import Phi3PreTrainedModel

        class MyModel(Phi3PreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )

                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

class HfMistralNemo(CrossEntropyLossBase):
    def __init__(self, dtype):
        super().__init__("hf_mistral_nemo", dtype)

    def model(self):
        from transformers.models.mistral import MistralPreTrainedModel

        class MyModel(MistralPreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )

                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):


HfMistralNemo

        class MyModel(Phi3PreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )

                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

class HfMistralNemo(CrossEntropyLossBase):
    def __init__(self, dtype):
        super().__init__("hf_mistral_nemo", dtype)

    def model(self):
        from transformers.models.mistral import MistralPreTrainedModel

        class MyModel(MistralPreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )

                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

cross_entropy_loss_setup = {
    "hf_qwen2": HfQwen2,
    "hf_phi3": HfPhi3,
    "hf_mistral_nemo": HfMistralNemo,
}

class SyntheticMiniModel:
    # Vocab sizes from popular models
    sizes_from_models = [
        49152,  # Starcoder
        129280,  # DeepSeek-R1
        128256,  # Llama3
        202048,  # Llama4
        256000,  # Gemma2
        131072,  # Mistral
        152064,  # Qwen2
        32064,  # Phi3.5


SyntheticMiniModel

                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

cross_entropy_loss_setup = {
    "hf_qwen2": HfQwen2,
    "hf_phi3": HfPhi3,
    "hf_mistral_nemo": HfMistralNemo,
}

class SyntheticMiniModel:
    # Vocab sizes from popular models
    sizes_from_models = [
        49152,  # Starcoder
        129280,  # DeepSeek-R1
        128256,  # Llama3
        202048,  # Llama4
        256000,  # Gemma2
        131072,  # Mistral
        152064,  # Qwen2
        32064,  # Phi3.5
        100352,  # Phi4
        50264,  # GPT-2
    ]

    @staticmethod
    def mini_model(logits, labels):
        labels = torch.nn.functional.pad(labels, (0, 1))
        labels = labels[1 : labels.shape[-1]]
        logits = logits.to(dtype=torch.float32)
        logits = logits.squeeze(dim=0)
        return torch.nn.functional.cross_entropy(logits, labels)

    @staticmethod
    def inputs(batch_size, vocab_size):
        input = torch.randn(
            1,
            batch_size,
            vocab_size,
            device="cuda",
            dtype=torch.bfloat16,
            requires_grad=True,
        )
        labels = torch.randint(
            0,
            vocab_size - 1,
            (batch_size,),
            device="cuda",
            requires_grad=False,
        )
        return (input, labels)

    @staticmethod
    def grads():
        grad = torch.tensor(1, device="cuda", dtype=torch.float32, requires_grad=False)
        return grad

    @staticmethod
    def generate_vocab_sizes():
        powers_of_2 = [2**i * 1024 for i in range(4, 9)]

        combined_set = sorted(
            set(SyntheticMiniModel.sizes_from_models) | set(powers_of_2)
        )

        # for each vocab size in the set we increment in steps 64 in +/- 5 directions
        # which gives the total number of vocab sizes to benchmark
        variations = set()
        step = 64
        for num in combined_set:
            for i in range(1, 6):
                variations.add(num + (i * step))
                variations.add(num - (i * step))

            variations.add(num)

        return sorted(variations)


                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

class HfMistralNemo(CrossEntropyLossBase):
    def __init__(self, dtype):
        super().__init__("hf_mistral_nemo", dtype)

    def model(self):
        from transformers.models.mistral import MistralPreTrainedModel

        class MyModel(MistralPreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )

                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

cross_entropy_loss_setup = {
    "hf_qwen2": HfQwen2,
    "hf_phi3": HfPhi3,
    "hf_mistral_nemo": HfMistralNemo,
}

class SyntheticMiniModel:
    # Vocab sizes from popular models


model

                    config.hidden_size, config.vocab_size, bias=False
                )

                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

class HfMistralNemo(CrossEntropyLossBase):
    def __init__(self, dtype):
        super().__init__("hf_mistral_nemo", dtype)

    def model(self):
        from transformers.models.mistral import MistralPreTrainedModel

        class MyModel(MistralPreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )

                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

cross_entropy_loss_setup = {
    "hf_qwen2": HfQwen2,
    "hf_phi3": HfPhi3,
    "hf_mistral_nemo": HfMistralNemo,
}

class SyntheticMiniModel:
    # Vocab sizes from popular models
    sizes_from_models = [
        49152,  # Starcoder
        129280,  # DeepSeek-R1
        128256,  # Llama3
        202048,  # Llama4
        256000,  # Gemma2
        131072,  # Mistral
        152064,  # Qwen2
        32064,  # Phi3.5


inputs

        129280,  # DeepSeek-R1
        128256,  # Llama3
        202048,  # Llama4
        256000,  # Gemma2
        131072,  # Mistral
        152064,  # Qwen2
        32064,  # Phi3.5
        100352,  # Phi4
        50264,  # GPT-2
    ]

    @staticmethod
    def mini_model(logits, labels):
        labels = torch.nn.functional.pad(labels, (0, 1))
        labels = labels[1 : labels.shape[-1]]
        logits = logits.to(dtype=torch.float32)
        logits = logits.squeeze(dim=0)
        return torch.nn.functional.cross_entropy(logits, labels)

    @staticmethod
    def inputs(batch_size, vocab_size):
        input = torch.randn(
            1,
            batch_size,
            vocab_size,
            device="cuda",
            dtype=torch.bfloat16,
            requires_grad=True,
        )
        labels = torch.randint(
            0,
            vocab_size - 1,
            (batch_size,),
            device="cuda",
            requires_grad=False,
        )
        return (input, labels)

    @staticmethod
    def grads():
        grad = torch.tensor(1, device="cuda", dtype=torch.float32, requires_grad=False)
        return grad

    @staticmethod
    def generate_vocab_sizes():
        powers_of_2 = [2**i * 1024 for i in range(4, 9)]

        combined_set = sorted(
            set(SyntheticMiniModel.sizes_from_models) | set(powers_of_2)
        )

        # for each vocab size in the set we increment in steps 64 in +/- 5 directions
        # which gives the total number of vocab sizes to benchmark
        variations = set()
        step = 64
        for num in combined_set:
            for i in range(1, 6):


grads

    @staticmethod
    def inputs(batch_size, vocab_size):
        input = torch.randn(
            1,
            batch_size,
            vocab_size,
            device="cuda",
            dtype=torch.bfloat16,
            requires_grad=True,
        )
        labels = torch.randint(
            0,
            vocab_size - 1,
            (batch_size,),
            device="cuda",
            requires_grad=False,
        )
        return (input, labels)

    @staticmethod
    def grads():
        grad = torch.tensor(1, device="cuda", dtype=torch.float32, requires_grad=False)
        return grad

    @staticmethod
    def generate_vocab_sizes():
        powers_of_2 = [2**i * 1024 for i in range(4, 9)]

        combined_set = sorted(
            set(SyntheticMiniModel.sizes_from_models) | set(powers_of_2)
        )

        # for each vocab size in the set we increment in steps 64 in +/- 5 directions
        # which gives the total number of vocab sizes to benchmark
        variations = set()
        step = 64
        for num in combined_set:
            for i in range(1, 6):
                variations.add(num + (i * step))
                variations.add(num - (i * step))

            variations.add(num)


    #    div_1: "f32[][]cuda:0" = torch.ops.aten.div.Tensor(tangents_1, convert_element_type);  tangents_1 = convert_element_type = None
    #    unsqueeze_1: "i64[8192, 1][1, 1]cuda:0" = torch.ops.aten.unsqueeze.default(primals_1, 1);  primals_1 = None
    #    ne_3: "b8[8192, 1][1, 1]cuda:0" = torch.ops.aten.ne.Scalar(unsqueeze_1, -100)
    #    full_default: "i64[][]cuda:0" = torch.ops.aten.full.default([], 0, dtype = torch.int64, layout = torch.strided, device = device(type='cuda', index=0), pin_memory = False)
    #    where_2: "i64[8192, 1][1, 1]cuda:0" = torch.ops.aten.where.self(ne_3, unsqueeze_1, full_default);  unsqueeze_1 = full_default = None
    #    full_default_3: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.full.default([8192, 3024], 0, dtype = torch.float32, layout = torch.strided, device = device(type='cuda', index=0), pin_memory = False)
    #    scatter: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.scatter.value(full_default_3, 1, where_2, -1.0);  full_default_3 = where_2 = None
    #    full_default_1: "f32[][]cuda:0" = torch.ops.aten.full.default([], 0.0, dtype = torch.float32, layout = torch.strided, device = device(type='cuda', index=0), pin_memory = False)
    #    where_3: "f32[8192, 1][1, 1]cuda:0" = torch.ops.aten.where.self(ne_3, div_1, full_default_1);  ne_3 = div_1 = full_default_1 = None
    #    mul: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.mul.Tensor(scatter, where_3);  scatter = where_3 = None
    #    sub: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.sub.Tensor(primals_2, amax);  primals_2 = amax = None
    #    sub_1: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.sub.Tensor(sub, log);  sub = log = None
    #    exp_1: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.exp.default(sub_1);  sub_1 = None
    #    sum_4: "f32[8192, 1][1, 1]cuda:0" = torch.ops.aten.sum.dim_IntList(mul, [1], True)
    #    mul_1: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.mul.Tensor(exp_1, sum_4);  exp_1 = sum_4 = None
    #    sub_2: "f32[8192, 3024][3024, 1]cuda:0" = torch.ops.aten.sub.Tensor(mul, mul_1);  mul = mul_1 = None
    #    return (None, sub_2)

    # We account for inputs primals_1, primals_2, amax, log
    # and for the output sub_2
    def grad_iobytes(self):
        n_elements = 0
        # adding size of primals_2 and the output
        n_elements += 2 * (
            self.config.batch_size * self.config.seq_len * self.config.vocab_size
        )
        # adding size of amax and log and primals_1
        n_elements += 3 * self.config.batch_size * self.config.seq_len
        # scale by dtype size
        return n_elements * self.dtype.itemsize

class HfQwen2(CrossEntropyLossBase):
    def __init__(self, dtype):
        super().__init__("hf_qwen2", dtype)

    def model(self):
        from transformers.models.qwen2.modeling_qwen2 import Qwen2PreTrainedModel

        class MyModel(Qwen2PreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )
                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)


    "hf_mistral_nemo": HfMistralNemo,
}

class SyntheticMiniModel:
    # Vocab sizes from popular models
    sizes_from_models = [
        49152,  # Starcoder
        129280,  # DeepSeek-R1
        128256,  # Llama3
        202048,  # Llama4
        256000,  # Gemma2
        131072,  # Mistral
        152064,  # Qwen2
        32064,  # Phi3.5
        100352,  # Phi4
        50264,  # GPT-2
    ]

    @staticmethod
    def mini_model(logits, labels):
        labels = torch.nn.functional.pad(labels, (0, 1))
        labels = labels[1 : labels.shape[-1]]
        logits = logits.to(dtype=torch.float32)
        logits = logits.squeeze(dim=0)
        return torch.nn.functional.cross_entropy(logits, labels)

    @staticmethod
    def inputs(batch_size, vocab_size):
        input = torch.randn(
            1,
            batch_size,
            vocab_size,
            device="cuda",
            dtype=torch.bfloat16,
            requires_grad=True,
        )
        labels = torch.randint(
            0,
            vocab_size - 1,
            (batch_size,),
            device="cuda",
            requires_grad=False,
        )
        return (input, labels)


            vocab_size,
            device="cuda",
            dtype=torch.bfloat16,
            requires_grad=True,
        )
        labels = torch.randint(
            0,
            vocab_size - 1,
            (batch_size,),
            device="cuda",
            requires_grad=False,
        )
        return (input, labels)

    @staticmethod
    def grads():
        grad = torch.tensor(1, device="cuda", dtype=torch.float32, requires_grad=False)
        return grad

    @staticmethod
    def generate_vocab_sizes():
        powers_of_2 = [2**i * 1024 for i in range(4, 9)]

        combined_set = sorted(
            set(SyntheticMiniModel.sizes_from_models) | set(powers_of_2)
        )

        # for each vocab size in the set we increment in steps 64 in +/- 5 directions
        # which gives the total number of vocab sizes to benchmark
        variations = set()
        step = 64
        for num in combined_set:
            for i in range(1, 6):
                variations.add(num + (i * step))
                variations.add(num - (i * step))

            variations.add(num)

        return sorted(variations)


MyModel

                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

class HfMistralNemo(CrossEntropyLossBase):
    def __init__(self, dtype):
        super().__init__("hf_mistral_nemo", dtype)

    def model(self):
        from transformers.models.mistral import MistralPreTrainedModel

        class MyModel(MistralPreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )

                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

cross_entropy_loss_setup = {
    "hf_qwen2": HfQwen2,
    "hf_phi3": HfPhi3,
    "hf_mistral_nemo": HfMistralNemo,
}

class SyntheticMiniModel:
    # Vocab sizes from popular models
    sizes_from_models = [
        49152,  # Starcoder
        129280,  # DeepSeek-R1
        128256,  # Llama3
        202048,  # Llama4
        256000,  # Gemma2
        131072,  # Mistral


forward

        return MyModel(self.config).cuda().to(self.dtype)

class HfMistralNemo(CrossEntropyLossBase):
    def __init__(self, dtype):
        super().__init__("hf_mistral_nemo", dtype)

    def model(self):
        from transformers.models.mistral import MistralPreTrainedModel

        class MyModel(MistralPreTrainedModel):
            def __init__(self, config):
                super().__init__(config)
                self.lm_head = torch.nn.Linear(
                    config.hidden_size, config.vocab_size, bias=False
                )

                # Initialize weights and apply final processing
                self.post_init()

            def forward(self, hidden_states: torch.Tensor, labels: torch.LongTensor):
                logits = self.lm_head(hidden_states)
                loss = self.loss_function(
                    logits=logits, labels=labels, vocab_size=self.config.vocab_size
                )
                return (loss,)

        return MyModel(self.config).cuda().to(self.dtype)

cross_entropy_loss_setup = {
    "hf_qwen2": HfQwen2,
    "hf_phi3": HfPhi3,
    "hf_mistral_nemo": HfMistralNemo,
}

class SyntheticMiniModel:
    # Vocab sizes from popular models
    sizes_from_models = [
        49152,  # Starcoder
        129280,  # DeepSeek-R1
        128256,  # Llama3
        202048,  # Llama4
        256000,  # Gemma2
        131072,  # Mistral


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
import torch
from .global_params import generate_input_sizes, FLOAT_DTYPES
from .normalization import norm_bwd_nvf_benchmark, norm_bwd_baseline_benchmark

@pytest.mark.parametrize("size", generate_input_sizes(dims=4))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "channels_last",
    [
        pytest.param(True, marks=pytest.mark.outer_persistent),
        pytest.param(False, marks=pytest.mark.inner_persistent),
    ],
)
def test_instancenorm_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    channels_last: bool,
    disable_validation: bool,
    disable_benchmarking: bool,
    eps: float = 1e-5,
):
    norm_bwd_nvf_benchmark(
        benchmark,
        size,
        dtype,
        "instance_norm",
        channels_last,
        disable_validation,
        disable_benchmarking,
    )

@pytest.mark.parametrize("executor", ["eager", "torchcompile"])
@pytest.mark.parametrize("size", generate_input_sizes(dims=4))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "channels_last",
    [
        pytest.param(True, marks=pytest.mark.outer_persistent),
        pytest.param(False, marks=pytest.mark.inner_persistent),
    ],
)
def test_instancenorm_bwd_baseline_benchmark(
    benchmark, size: tuple, dtype: torch.dtype, channels_last: bool, executor: str
):
    norm_bwd_baseline_benchmark(
        benchmark, size, dtype, channels_last, executor, "instance_norm"
    )


        benchmark,
        size,
        dtype,
        "instance_norm",
        channels_last,
        disable_validation,
        disable_benchmarking,
    )

@pytest.mark.parametrize("executor", ["eager", "torchcompile"])
@pytest.mark.parametrize("size", generate_input_sizes(dims=4))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "channels_last",
    [
        pytest.param(True, marks=pytest.mark.outer_persistent),
        pytest.param(False, marks=pytest.mark.inner_persistent),
    ],
)
def test_instancenorm_bwd_baseline_benchmark(
    benchmark, size: tuple, dtype: torch.dtype, channels_last: bool, executor: str
):
    norm_bwd_baseline_benchmark(
        benchmark, size, dtype, channels_last, executor, "instance_norm"
    )


# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

"""Inference benchmark focusing on throughput and latency metrics of prefill and decode phases.

AutoModelForCausalLM from Hugging Face transformers is used for model implementation.

Key metrics:
- Throughput (tokens/second)
- Latency (ms/token)
- Time to First Token (TTFT)
- Time Between Output Tokens (TBOT)

Pulled from the lightning-thunder repo. Reference:
https://github.com/Lightning-AI/lightning-thunder/blob/4d3a3c3a7481efdc6a23cdeea99c3ffd31af5e78/thunder/benchmarks/benchmark_inference.py
"""

# fmt: off

from __future__ import annotations
from contextlib import contextmanager
from dataclasses import dataclass, field
import argparse
import json
import os
import statistics
import time
import warnings
from typing import Any
from collections.abc import Callable
from looseversion import LooseVersion

import torch
import torch.distributed as dist
import torch.nn as nn
from torch.distributed.device_mesh import init_device_mesh
from torch.distributed.tensor.parallel import parallelize_module, RowwiseParallel, ColwiseParallel
from tqdm import tqdm
import transformers
from transformers import AutoConfig, AutoModelForCausalLM
from transformers.cache_utils import HybridChunkedCache, StaticCache
from transformers.models.llama4 import Llama4TextConfig
from transformers.models.llama4.modeling_llama4 import Llama4TextMoe
from torch.distributed.tensor.placement_types import Shard
from torch.distributed.tensor import DTensor

import thunder
from thunder.dynamo.compiler import thunderfx
if __name__ == "__main__":
    # TODO: remove this after folks switch to pytest
    from layers_for_inference_benchmark import (
        GroupedSwiGLU,
        Llama4MoE,
        NVFP4InferenceGroupedSwiGLU,
        nvfuser_f16a_nvfp4weight_scaled_grouped_mm,
    )
else:
    from .layers_for_inference_benchmark import (
        GroupedSwiGLU,
        Llama4MoE,
        NVFP4InferenceGroupedSwiGLU,
        nvfuser_f16a_nvfp4weight_scaled_grouped_mm,
    )
from thunder.tests.distributed.test_moe import GroupedLinearColwiseParallel, GroupedLinearRowwiseParallel
from thunder.transforms.cudagraph import CUDAGraphTransform
from thunder.torch.custom_op import _register_custom_op, _register_nvfuser_translator

RANK = int(os.environ.get("RANK", 0))
LOCAL_RANK = int(os.environ.get("LOCAL_RANK", 0))
WORLD_SIZE = int(os.environ.get("WORLD_SIZE", 1))
MASTER_ADDR = os.environ.get("MASTER_ADDR", "localhost")
MASTER_PORT = os.environ.get("MASTER_PORT", "29500")
os.environ["TORCH_NCCL_ASYNC_ERROR_HANDLING"] = "1"

DEVICE = torch.device("cuda", LOCAL_RANK)
torch.cuda.set_device(DEVICE)

if dist.is_torchelastic_launched():
    mesh = init_device_mesh("cuda", (WORLD_SIZE,), mesh_dim_names=("tp",))
else:
    mesh = None

LLAMA4_MAVERICK_MODEL_ID: str = "meta-llama/Llama-4-Maverick-17B-128E"
llama_4_Maverick_17B_128E_cfg_str = r""" {
  "attention_bias": false,
  "attention_chunk_size": 8192,
  "attention_dropout": 0.0,
  "attn_scale": 0.1,
  "attn_temperature_tuning": true,
  "bos_token_id": 200000,
  "cache_implementation": "hybrid",
  "eos_token_id": [
    200001,
    200007,
    200008
  ],
  "floor_scale": 8192,
  "for_llm_compressor": false,
  "head_dim": 128,
  "hidden_act": "silu",
  "hidden_size": 5120,
  "initializer_range": 0.02,
  "interleave_moe_layer_step": 2,
  "intermediate_size": 8192,
  "intermediate_size_mlp": 16384,
  "max_position_embeddings": 262144,
  "model_type": "llama4_text",
  "moe_layers": [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47],
  "no_rope_layers": [1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0],
  "num_attention_heads": 40,
  "num_experts_per_tok": 1,
  "num_hidden_layers": 48,
  "num_key_value_heads": 8,
  "num_local_experts": 128,
  "output_router_logits": false,
  "pad_token_id": 200018,
  "rms_norm_eps": 1e-05,
  "rope_scaling": null,
  "rope_theta": 500000.0,
  "router_aux_loss_coef": 0.001,
  "router_jitter_noise": 0.0,
  "torch_dtype": "bfloat16",
  "use_cache": true,
  "use_qk_norm": false,
  "vocab_size": 202048
}
"""

# TODO: Add mm quantization once nvfuser implements nvfp4 gemm
# Register nvfp4 custom ops with Thunder and nvFuser
def _register_nvfp4_ops():
    """Register nvfp4 custom operations with Thunder."""
    # Register f16a_nvfp4weight_scaled_grouped_mm with nvfuser translator
    _nvfp4_grouped_mm_symbol = _register_custom_op(nvfuser_f16a_nvfp4weight_scaled_grouped_mm)

    def nvfp4_grouped_mm_translator(
        activation,
        fp4_weight,
        weight_scaling_factor,
        global_scale,
        offsets,
        blockscale_offsets,
        problem_sizes,
        *,
        fd,
        lc_to_nv_map,
    ):
        from nvfuser_direct import DataType
        from thunder.executors.nvfuserex_impl import getnv

        nv_act = getnv(activation, fd, lc_to_nv_map)
        nv_fp4_w = getnv(fp4_weight, fd, lc_to_nv_map)
        nv_sf_w = getnv(weight_scaling_factor, fd, lc_to_nv_map)
        nv_alpha = getnv(global_scale, fd, lc_to_nv_map)
        nv_offsets = getnv(offsets, fd, lc_to_nv_map)
        nv_blocksf_offsets = getnv(blockscale_offsets, fd, lc_to_nv_map)
        nv_problem_sizes = getnv(problem_sizes, fd, lc_to_nv_map)
        fp4_mat1, layout_fp8_scale1 = fd.ops.nv_grouped_block_quantize(nv_act, nv_offsets, nv_blocksf_offsets)
        out = fd.ops.cutlass_nvfp4_grouped_mm(
            fp4_mat1,
            nv_fp4_w,
            layout_fp8_scale1,
            nv_sf_w,
            nv_alpha,
            # NOTE: we might need to call contiguous on problem_sizes
            nv_problem_sizes,
            nv_offsets,
            nv_blocksf_offsets,
            DataType.BFloat16,
        )
        return out

    _register_nvfuser_translator(_nvfp4_grouped_mm_symbol, nvfp4_grouped_mm_translator)

# The logic is based on https://github.com/pytorch/ao/blob/b34c1037/torchao/quantization/quant_api.py#L230
def _replace_with_custom_fn_if_matches_filter_with_name(
    model,
    replacement_fn: Callable[[torch.nn.Module, str], torch.nn.Module],
    filter_fn: Callable[[torch.nn.Module, str], bool],
    cur_fqn="",
) -> None:
    """
    Recursively replaces each child module in `model` with the result of `replacement_fn(child)`

    replacement_fn (Callable[[torch.nn.Module, str], torch.nn.Module]): The function to replace matching modules.
    filter_fn (Callable[[torch.nn.Module, str], bool]): The function to filter matching modules.
    cur_fqn (str): The current fully qualified name of the module.

    Returns:
        None
    """
    if filter_fn(model, cur_fqn[:-1]):
        model = replacement_fn(model, cur_fqn[:-1])
        return model
    else:
        named_children_list = list(model.named_children())
        for name, child in named_children_list:
            new_child = _replace_with_custom_fn_if_matches_filter_with_name(
                child,
                replacement_fn,
                filter_fn,
                f"{cur_fqn}{name}.",
            )
            if new_child is not child:
                setattr(model, name, new_child)
        return model

def _replace_llama4_moe(model: nn.Module) -> None:
    """Replace Llama4TextMoe with Llama4MoE to use grouped gemm."""
    _replace_with_custom_fn_if_matches_filter_with_name(
        model,
        lambda model, cur_fqn: Llama4MoE.from_transformers_llama4textmoe(model),
        lambda model, cur_fqn: isinstance(model, Llama4TextMoe),
    )

def _quantize_llama4(model: nn.Module) -> None:
    """Replace linear and/or MoE with nvfp4 inference version.

    Args:
        model: The model to quantize

    Note: GroupedSwiGLU is always quantized when this function is called.
    """
    # Always quantize GroupedSwiGLU when this function is called
    _replace_with_custom_fn_if_matches_filter_with_name(
        model,
        NVFP4InferenceGroupedSwiGLU.from_grouped_swiglu,
        lambda model, cur_fqn: isinstance(model, GroupedSwiGLU),
    )

@contextmanager
def timer():
    torch.cuda.synchronize()
    t1 = t2 = time.perf_counter()
    yield lambda: (t2 - t1) * 1000  # Convert to ms
    torch.cuda.synchronize()
    t2 = time.perf_counter()

@dataclass
class InferenceBenchmarkConfig:
    """Configuration for inference benchmarking"""

    model_name: str
    batch_size: int
    input_length: int
    output_length: int
    num_layers: int | None
    num_iterations: int
    warmup_iterations: int
    enable_nvfp4: bool  # Enable NVFP4 registration and quantize GroupedSwiGLU in MoE
    fx_report_folder: str | None
    enable_nv_linear: bool
    mode: str
    disable_moe_replacement: bool
    attn_implementation: str | None
    thunder_cache: str | None
    enable_cudagraph: bool
    debug_moe: bool
    use_hardcoded_model: bool

@dataclass
class InferenceMetrics:
    """Metrics collected during inference benchmarking"""

    throughput_tokens_per_sec: float = 0.0
    latency_ms_per_token: float = 0.0
    time_to_first_token_ms: float = 0.0
    time_between_output_tokens_ms: float = 0.0
    total_time_ms: float = 0.0
    memory_used_gb: float = 0.0
    peak_memory_gb: float = 0.0

    # Separate prefill and decode metrics
    prefill_throughput_tokens_per_sec: float = 0.0
    decode_throughput_tokens_per_sec: float = 0.0
    prefill_time_ms: float = 0.0
    decode_time_ms: float = 0.0

    # Per-iteration metrics for variance analysis
    iteration_times: list[float] = field(default_factory=list)
    ttft_times: list[float] = field(default_factory=list)
    prefill_times: list[float] = field(default_factory=list)
    decode_times: list[float] = field(default_factory=list)

class InferenceBenchmark:
    """Main benchmark class"""

    def __init__(self, config: InferenceBenchmarkConfig):
        self.config = config
        self.metrics = InferenceMetrics()
        # profiler_toggle is used to start/stop profiler for moe debugging
        self.profiler_toggle = False

        # NOTE: Model resides on meta device
        model = self._load_model()
        assert all(p.device == torch.device("meta") for p in model.parameters())

        # NOTE: Replacement happens before model is materialized
        #       otherwise, the memory usage will be increased due to
        #       additional parameters materialized from the replacement module
        if not self.config.disable_moe_replacement:
            _replace_llama4_moe(model)
        assert all(p.device == torch.device("meta") for p in model.parameters())

        tp_plan = {
            "*.layers.*.self_attn.q_proj": ColwiseParallel(use_local_output=True),
            "*.layers.*.self_attn.k_proj": ColwiseParallel(use_local_output=True),
            "*.layers.*.self_attn.v_proj": ColwiseParallel(use_local_output=True),
            "*.layers.*.self_attn.o_proj": RowwiseParallel(use_local_output=True),
            "*.layers.*.feed_forward.gate_proj": ColwiseParallel(use_local_output=False),
            "*.layers.*.feed_forward.up_proj": ColwiseParallel(use_local_output=False),
            "*.layers.*.feed_forward.down_proj": RowwiseParallel(use_local_output=True),
        }

        if self.config.disable_moe_replacement:
            tp_plan.update(
                {
                    # HF MoE
                    "*.layers.*.feed_forward.shared_expert.gate_proj": ColwiseParallel(use_local_output=False),
                    "*.layers.*.feed_forward.shared_expert.up_proj": ColwiseParallel(use_local_output=False),
                    "*.layers.*.feed_forward.shared_expert.down_proj": RowwiseParallel(use_local_output=True),
                    # TODO:Need to write ParallelStyle for HF's grouped_mm implementation.
                }
            )

        else:
            tp_plan.update(
                {
                    # Custom MoE
                    "*.layers.*.feed_forward.shared_experts.gate_proj": ColwiseParallel(
                        use_local_output=False, output_layouts=Shard(2)
                    ),
                    "*.layers.*.feed_forward.shared_experts.up_proj": ColwiseParallel(
                        use_local_output=False, output_layouts=Shard(2)
                    ),
                    "*.layers.*.feed_forward.shared_experts.down_proj": RowwiseParallel(),
                    "*.layers.*.feed_forward.routed_experts.gate_proj": GroupedLinearColwiseParallel(
                        use_local_output=False
                    ),
                    "*.layers.*.feed_forward.routed_experts.up_proj": GroupedLinearColwiseParallel(
                        use_local_output=False
                    ),
                    "*.layers.*.feed_forward.routed_experts.down_proj": GroupedLinearRowwiseParallel(),
                }
            )

        if mesh:
            model = parallelize_module(model, mesh, tp_plan)

            # Sanity check
            if not self.config.disable_moe_replacement:
                assert type(model.model.layers[1].feed_forward.shared_experts.gate_proj.weight) == DTensor
                assert type(model.model.layers[1].feed_forward.shared_experts.up_proj.weight) == DTensor
                assert type(model.model.layers[1].feed_forward.shared_experts.down_proj.weight) == DTensor
                assert type(model.model.layers[1].feed_forward.routed_experts.gate_proj.weight) == DTensor
                assert type(model.model.layers[1].feed_forward.routed_experts.up_proj.weight) == DTensor
                assert type(model.model.layers[1].feed_forward.routed_experts.down_proj.weight) == DTensor
            else:
                assert type(model.model.layers[1].feed_forward.shared_expert.gate_proj.weight) == DTensor
                assert type(model.model.layers[1].feed_forward.shared_expert.up_proj.weight) == DTensor
                assert type(model.model.layers[1].feed_forward.shared_expert.down_proj.weight) == DTensor

        # Materialize the model on the device (after Llama4MoE replacement and sharding)
        model.to_empty(device=DEVICE)
        assert all(p.device == DEVICE for p in model.parameters())

        # Required as thunder doesn't understand inference mode
        # And some prims like `prims._grouped_mm` don't have grad rule defined yet.
        for p in model.parameters():
            p.requires_grad_(False)

        assert all(not p.requires_grad for p in model.parameters())

        # `thunderfx` seems to hide the access to vocab_size somewhere so
        # store it here before any compiler is applied.
        self.vocab_size = model.vocab_size

        if self.config.enable_nvfp4:
            _quantize_llama4(model)

        # If debug_moe is on, we'll only compile the moe section, which is much easier to digest.
        if self.config.debug_moe:
            def compile_wrapper(model, jitter):
                class ModelWrapper(torch.nn.Module):

                    def __init__(self, model, jitter):
                        super().__init__()
                        self.model = jitter._compile_model(model)
                        assert not hasattr(jitter.model, "_backend")
                        # store a reference to the compiled backend, so we can print the trace later.
                        if jitter.config.mode == "thunder":
                            jitter.model._backend = self.model._backend
                        self.jitter = jitter

                    def forward(self, *args, **kwargs):
                        if self.jitter.profiler_toggle:
                            torch.cuda.cudart().cudaProfilerStart()
                        out = self.model(*args, **kwargs)
                        if self.jitter.profiler_toggle:
                            torch.cuda.cudart().cudaProfilerStop()
                        return out

                return ModelWrapper(model, jitter)

            self.model = model
            # reuse the `replace` function with compilation.
            _replace_with_custom_fn_if_matches_filter_with_name(
                model,
                lambda model, cur_fqn: compile_wrapper(model, self),
                lambda model, cur_fqn: isinstance(model, Llama4MoE),
            )
        else:
            self.model = self._compile_model(model)

    @property
    def _thunder_jit_options(self) -> dict[str, Any]:
        # `nv_enable_linear=True` might fail with distributed run
        # ref: https://github.com/NVIDIA/Fuser/issues/4507
        res = {"transforms": []}
        if self.config.enable_nv_linear:
            res["nv_enable_linear"] = True
            res["nv_enable_matmul"] = True
        if self.config.mode == "thunderjit":
            from thunder.recipes.hf_transformers import SDPAMaskTransform

            if not hasattr(self, "_mask_transform"):
                self._mask_transform = SDPAMaskTransform()
            res["transforms"].append(self._mask_transform)
            res["executors"] = [self._mask_transform.get_executor(), *thunder.get_default_executors()]
        if self.config.enable_cudagraph:
            res["transforms"].append(CUDAGraphTransform())
        if self.config.thunder_cache is not None:
            res["cache"] = self.config.thunder_cache

        return res

    def _compile_model(self, model):
        match self.config.mode:
            case "eager":
                return model
            case "inductor":
                if (self.config.enable_cudagraph):
                    return torch.compile(model, mode="reduce-overhead")
                else:
                    return torch.compile(model, mode="default")
            case "thunder":
                return thunderfx(model, **self._thunder_jit_options)
            case "thunderjit":
                return thunder.jit(model, **self._thunder_jit_options)
            case _:
                raise ValueError(f"Unknown mode: {self.config.mode}")

    def _load_model(self) -> torch.nn.Module:
        """Load the model based on configuration"""
        model_id = self.config.model_name

        config = None
        if self.config.use_hardcoded_model:
            config = Llama4TextConfig.from_dict(
                json.loads(llama_4_Maverick_17B_128E_cfg_str)
            )
        else:
            config = AutoConfig.from_pretrained(model_id)

        if hasattr(config, "text_config"):
            config = config.text_config
        if self.config.num_layers:
            config.num_hidden_layers = self.config.num_layers

        self.hf_config = config

        with torch.device("meta"):
            model = AutoModelForCausalLM.from_config(
                config, torch_dtype=torch.bfloat16, attn_implementation=self.config.attn_implementation
            )

        return model

    def generate_batch(self) -> tuple[torch.Tensor, HybridChunkedCache]:
        """Generate a batch of input tokens"""
        batch_size = self.config.batch_size
        input_length = self.config.input_length

        input_ids = torch.randint(0, self.vocab_size, (batch_size, input_length), device=DEVICE)
        if LooseVersion(transformers.__version__) >= LooseVersion("4.55"):
            # Transformers deprecated HybridChunkedCache in favour of static in 4.55.x
            # NOTE: tp_size should only reflect tensor parallelism size, not total world size.
            # If 2D parallelism (data parallel + tensor parallel) is added in the future,
            # tp_size should be set to the tensor parallelism size only (e.g., WORLD_SIZE // DP_SIZE),
            # not WORLD_SIZE, so that StaticCache correctly handles sharded KV heads.
            past_key_values = StaticCache(
                config=self.hf_config,
                max_batch_size=input_ids.shape[0],
                max_cache_len=input_ids.shape[1] + self.config.output_length,
                device=DEVICE,
                dtype=torch.bfloat16,
                tp_size=W
# ... truncated ...


  "num_hidden_layers": 48,
  "num_key_value_heads": 8,
  "num_local_experts": 128,
  "output_router_logits": false,
  "pad_token_id": 200018,
  "rms_norm_eps": 1e-05,
  "rope_scaling": null,
  "rope_theta": 500000.0,
  "router_aux_loss_coef": 0.001,
  "router_jitter_noise": 0.0,
  "torch_dtype": "bfloat16",
  "use_cache": true,
  "use_qk_norm": false,
  "vocab_size": 202048
}
"""

# TODO: Add mm quantization once nvfuser implements nvfp4 gemm
# Register nvfp4 custom ops with Thunder and nvFuser
def _register_nvfp4_ops():
    """Register nvfp4 custom operations with Thunder."""
    # Register f16a_nvfp4weight_scaled_grouped_mm with nvfuser translator
    _nvfp4_grouped_mm_symbol = _register_custom_op(nvfuser_f16a_nvfp4weight_scaled_grouped_mm)

    def nvfp4_grouped_mm_translator(
        activation,
        fp4_weight,
        weight_scaling_factor,
        global_scale,
        offsets,
        blockscale_offsets,
        problem_sizes,
        *,
        fd,
        lc_to_nv_map,
    ):
        from nvfuser_direct import DataType
        from thunder.executors.nvfuserex_impl import getnv

        nv_act = getnv(activation, fd, lc_to_nv_map)
        nv_fp4_w = getnv(fp4_weight, fd, lc_to_nv_map)
        nv_sf_w = getnv(weight_scaling_factor, fd, lc_to_nv_map)
        nv_alpha = getnv(global_scale, fd, lc_to_nv_map)
        nv_offsets = getnv(offsets, fd, lc_to_nv_map)
        nv_blocksf_offsets = getnv(blockscale_offsets, fd, lc_to_nv_map)
        nv_problem_sizes = getnv(problem_sizes, fd, lc_to_nv_map)
        fp4_mat1, layout_fp8_scale1 = fd.ops.nv_grouped_block_quantize(nv_act, nv_offsets, nv_blocksf_offsets)
        out = fd.ops.cutlass_nvfp4_grouped_mm(
            fp4_mat1,
            nv_fp4_w,
            layout_fp8_scale1,
            nv_sf_w,
            nv_alpha,
            # NOTE: we might need to call contiguous on problem_sizes
            nv_problem_sizes,
            nv_offsets,
            nv_blocksf_offsets,
            DataType.BFloat16,
        )
        return out

    _register_nvfuser_translator(_nvfp4_grouped_mm_symbol, nvfp4_grouped_mm_translator)

# The logic is based on https://github.com/pytorch/ao/blob/b34c1037/torchao/quantization/quant_api.py#L230
def _replace_with_custom_fn_if_matches_filter_with_name(
    model,
    replacement_fn: Callable[[torch.nn.Module, str], torch.nn.Module],
    filter_fn: Callable[[torch.nn.Module, str], bool],
    cur_fqn="",
) -> None:
    """
    Recursively replaces each child module in `model` with the result of `replacement_fn(child)`

    replacement_fn (Callable[[torch.nn.Module, str], torch.nn.Module]): The function to replace matching modules.
    filter_fn (Callable[[torch.nn.Module, str], bool]): The function to filter matching modules.
    cur_fqn (str): The current fully qualified name of the module.

    Returns:
        None
    """
    if filter_fn(model, cur_fqn[:-1]):


        nv_problem_sizes = getnv(problem_sizes, fd, lc_to_nv_map)
        fp4_mat1, layout_fp8_scale1 = fd.ops.nv_grouped_block_quantize(nv_act, nv_offsets, nv_blocksf_offsets)
        out = fd.ops.cutlass_nvfp4_grouped_mm(
            fp4_mat1,
            nv_fp4_w,
            layout_fp8_scale1,
            nv_sf_w,
            nv_alpha,
            # NOTE: we might need to call contiguous on problem_sizes
            nv_problem_sizes,
            nv_offsets,
            nv_blocksf_offsets,
            DataType.BFloat16,
        )
        return out

    _register_nvfuser_translator(_nvfp4_grouped_mm_symbol, nvfp4_grouped_mm_translator)

# The logic is based on https://github.com/pytorch/ao/blob/b34c1037/torchao/quantization/quant_api.py#L230
def _replace_with_custom_fn_if_matches_filter_with_name(
    model,
    replacement_fn: Callable[[torch.nn.Module, str], torch.nn.Module],
    filter_fn: Callable[[torch.nn.Module, str], bool],
    cur_fqn="",
) -> None:
    """
    Recursively replaces each child module in `model` with the result of `replacement_fn(child)`

    replacement_fn (Callable[[torch.nn.Module, str], torch.nn.Module]): The function to replace matching modules.
    filter_fn (Callable[[torch.nn.Module, str], bool]): The function to filter matching modules.
    cur_fqn (str): The current fully qualified name of the module.

    Returns:
        None
    """
    if filter_fn(model, cur_fqn[:-1]):
        model = replacement_fn(model, cur_fqn[:-1])
        return model
    else:
        named_children_list = list(model.named_children())
        for name, child in named_children_list:
            new_child = _replace_with_custom_fn_if_matches_filter_with_name(
                child,
                replacement_fn,
                filter_fn,
                f"{cur_fqn}{name}.",
            )
            if new_child is not child:
                setattr(model, name, new_child)
        return model

def _replace_llama4_moe(model: nn.Module) -> None:
    """Replace Llama4TextMoe with Llama4MoE to use grouped gemm."""
    _replace_with_custom_fn_if_matches_filter_with_name(
        model,
        lambda model, cur_fqn: Llama4MoE.from_transformers_llama4textmoe(model),
        lambda model, cur_fqn: isinstance(model, Llama4TextMoe),
    )

def _quantize_llama4(model: nn.Module) -> None:
    """Replace linear and/or MoE with nvfp4 inference version.

    Args:
        model: The model to quantize

    Note: GroupedSwiGLU is always quantized when this function is called.
    """
    # Always quantize GroupedSwiGLU when this function is called


    Returns:
        None
    """
    if filter_fn(model, cur_fqn[:-1]):
        model = replacement_fn(model, cur_fqn[:-1])
        return model
    else:
        named_children_list = list(model.named_children())
        for name, child in named_children_list:
            new_child = _replace_with_custom_fn_if_matches_filter_with_name(
                child,
                replacement_fn,
                filter_fn,
                f"{cur_fqn}{name}.",
            )
            if new_child is not child:
                setattr(model, name, new_child)
        return model

def _replace_llama4_moe(model: nn.Module) -> None:
    """Replace Llama4TextMoe with Llama4MoE to use grouped gemm."""
    _replace_with_custom_fn_if_matches_filter_with_name(
        model,
        lambda model, cur_fqn: Llama4MoE.from_transformers_llama4textmoe(model),
        lambda model, cur_fqn: isinstance(model, Llama4TextMoe),
    )

def _quantize_llama4(model: nn.Module) -> None:
    """Replace linear and/or MoE with nvfp4 inference version.

    Args:
        model: The model to quantize

    Note: GroupedSwiGLU is always quantized when this function is called.
    """
    # Always quantize GroupedSwiGLU when this function is called
    _replace_with_custom_fn_if_matches_filter_with_name(
        model,
        NVFP4InferenceGroupedSwiGLU.from_grouped_swiglu,
        lambda model, cur_fqn: isinstance(model, GroupedSwiGLU),
    )

@contextmanager
def timer():


            new_child = _replace_with_custom_fn_if_matches_filter_with_name(
                child,
                replacement_fn,
                filter_fn,
                f"{cur_fqn}{name}.",
            )
            if new_child is not child:
                setattr(model, name, new_child)
        return model

def _replace_llama4_moe(model: nn.Module) -> None:
    """Replace Llama4TextMoe with Llama4MoE to use grouped gemm."""
    _replace_with_custom_fn_if_matches_filter_with_name(
        model,
        lambda model, cur_fqn: Llama4MoE.from_transformers_llama4textmoe(model),
        lambda model, cur_fqn: isinstance(model, Llama4TextMoe),
    )

def _quantize_llama4(model: nn.Module) -> None:
    """Replace linear and/or MoE with nvfp4 inference version.

    Args:
        model: The model to quantize

    Note: GroupedSwiGLU is always quantized when this function is called.
    """
    # Always quantize GroupedSwiGLU when this function is called
    _replace_with_custom_fn_if_matches_filter_with_name(
        model,
        NVFP4InferenceGroupedSwiGLU.from_grouped_swiglu,
        lambda model, cur_fqn: isinstance(model, GroupedSwiGLU),
    )

@contextmanager
def timer():
    torch.cuda.synchronize()
    t1 = t2 = time.perf_counter()
    yield lambda: (t2 - t1) * 1000  # Convert to ms
    torch.cuda.synchronize()
    t2 = time.perf_counter()

@dataclass
class InferenceBenchmarkConfig:
    """Configuration for inference benchmarking"""

    model_name: str
    batch_size: int
    input_length: int
    output_length: int
    num_layers: int | None


timer

    )

def _quantize_llama4(model: nn.Module) -> None:
    """Replace linear and/or MoE with nvfp4 inference version.

    Args:
        model: The model to quantize

    Note: GroupedSwiGLU is always quantized when this function is called.
    """
    # Always quantize GroupedSwiGLU when this function is called
    _replace_with_custom_fn_if_matches_filter_with_name(
        model,
        NVFP4InferenceGroupedSwiGLU.from_grouped_swiglu,
        lambda model, cur_fqn: isinstance(model, GroupedSwiGLU),
    )

@contextmanager
def timer():
    torch.cuda.synchronize()
    t1 = t2 = time.perf_counter()
    yield lambda: (t2 - t1) * 1000  # Convert to ms
    torch.cuda.synchronize()
    t2 = time.perf_counter()

@dataclass
class InferenceBenchmarkConfig:
    """Configuration for inference benchmarking"""

    model_name: str
    batch_size: int
    input_length: int
    output_length: int
    num_layers: int | None
    num_iterations: int
    warmup_iterations: int
    enable_nvfp4: bool  # Enable NVFP4 registration and quantize GroupedSwiGLU in MoE
    fx_report_folder: str | None
    enable_nv_linear: bool
    mode: str
    disable_moe_replacement: bool
    attn_implementation: str | None
    thunder_cache: str | None


InferenceBenchmarkConfig

    Note: GroupedSwiGLU is always quantized when this function is called.
    """
    # Always quantize GroupedSwiGLU when this function is called
    _replace_with_custom_fn_if_matches_filter_with_name(
        model,
        NVFP4InferenceGroupedSwiGLU.from_grouped_swiglu,
        lambda model, cur_fqn: isinstance(model, GroupedSwiGLU),
    )

@contextmanager
def timer():
    torch.cuda.synchronize()
    t1 = t2 = time.perf_counter()
    yield lambda: (t2 - t1) * 1000  # Convert to ms
    torch.cuda.synchronize()
    t2 = time.perf_counter()

@dataclass
class InferenceBenchmarkConfig:
    """Configuration for inference benchmarking"""

    model_name: str
    batch_size: int
    input_length: int
    output_length: int
    num_layers: int | None
    num_iterations: int
    warmup_iterations: int
    enable_nvfp4: bool  # Enable NVFP4 registration and quantize GroupedSwiGLU in MoE
    fx_report_folder: str | None
    enable_nv_linear: bool
    mode: str
    disable_moe_replacement: bool
    attn_implementation: str | None
    thunder_cache: str | None
    enable_cudagraph: bool
    debug_moe: bool
    use_hardcoded_model: bool

@dataclass
class InferenceMetrics:
    """Metrics collected during inference benchmarking"""

    throughput_tokens_per_sec: float = 0.0
    latency_ms_per_token: float = 0.0
    time_to_first_token_ms: float = 0.0
    time_between_output_tokens_ms: float = 0.0
    total_time_ms: float = 0.0
    memory_used_gb: float = 0.0
    peak_memory_gb: float = 0.0

    # Separate prefill and decode metrics
    prefill_throughput_tokens_per_sec: float = 0.0
    decode_throughput_tokens_per_sec: float = 0.0
    prefill_time_ms: float = 0.0
    decode_time_ms: float = 0.0


InferenceMetrics

    model_name: str
    batch_size: int
    input_length: int
    output_length: int
    num_layers: int | None
    num_iterations: int
    warmup_iterations: int
    enable_nvfp4: bool  # Enable NVFP4 registration and quantize GroupedSwiGLU in MoE
    fx_report_folder: str | None
    enable_nv_linear: bool
    mode: str
    disable_moe_replacement: bool
    attn_implementation: str | None
    thunder_cache: str | None
    enable_cudagraph: bool
    debug_moe: bool
    use_hardcoded_model: bool

@dataclass
class InferenceMetrics:
    """Metrics collected during inference benchmarking"""

    throughput_tokens_per_sec: float = 0.0
    latency_ms_per_token: float = 0.0
    time_to_first_token_ms: float = 0.0
    time_between_output_tokens_ms: float = 0.0
    total_time_ms: float = 0.0
    memory_used_gb: float = 0.0
    peak_memory_gb: float = 0.0

    # Separate prefill and decode metrics
    prefill_throughput_tokens_per_sec: float = 0.0
    decode_throughput_tokens_per_sec: float = 0.0
    prefill_time_ms: float = 0.0
    decode_time_ms: float = 0.0

    # Per-iteration metrics for variance analysis
    iteration_times: list[float] = field(default_factory=list)
    ttft_times: list[float] = field(default_factory=list)
    prefill_times: list[float] = field(default_factory=list)
    decode_times: list[float] = field(default_factory=list)

class InferenceBenchmark:
    """Main benchmark class"""

    def __init__(self, config: InferenceBenchmarkConfig):
        self.config = config
        self.metrics = InferenceMetrics()
        # profiler_toggle is used to start/stop profiler for moe debugging
        self.profiler_toggle = False

        # NOTE: Model resides on meta device
        model = self._load_model()
        assert all(p.device == torch.device("meta") for p in model.parameters())

        # NOTE: Replacement happens before model is materialized
        #       otherwise, the memory usage will be increased due to
        #       additional parameters materialized from the replacement module
        if not self.config.disable_moe_replacement:
            _replace_llama4_moe(model)


InferenceBenchmark

    latency_ms_per_token: float = 0.0
    time_to_first_token_ms: float = 0.0
    time_between_output_tokens_ms: float = 0.0
    total_time_ms: float = 0.0
    memory_used_gb: float = 0.0
    peak_memory_gb: float = 0.0

    # Separate prefill and decode metrics
    prefill_throughput_tokens_per_sec: float = 0.0
    decode_throughput_tokens_per_sec: float = 0.0
    prefill_time_ms: float = 0.0
    decode_time_ms: float = 0.0

    # Per-iteration metrics for variance analysis
    iteration_times: list[float] = field(default_factory=list)
    ttft_times: list[float] = field(default_factory=list)
    prefill_times: list[float] = field(default_factory=list)
    decode_times: list[float] = field(default_factory=list)

class InferenceBenchmark:
    """Main benchmark class"""

    def __init__(self, config: InferenceBenchmarkConfig):
        self.config = config
        self.metrics = InferenceMetrics()
        # profiler_toggle is used to start/stop profiler for moe debugging
        self.profiler_toggle = False

        # NOTE: Model resides on meta device
        model = self._load_model()
        assert all(p.device == torch.device("meta") for p in model.parameters())

        # NOTE: Replacement happens before model is materialized
        #       otherwise, the memory usage will be increased due to
        #       additional parameters materialized from the replacement module
        if not self.config.disable_moe_replacement:
            _replace_llama4_moe(model)
        assert all(p.device == torch.device("meta") for p in model.parameters())

        tp_plan = {
            "*.layers.*.self_attn.q_proj": ColwiseParallel(use_local_output=True),
            "*.layers.*.self_attn.k_proj": ColwiseParallel(use_local_output=True),
            "*.layers.*.self_attn.v_proj": ColwiseParallel(use_local_output=True),
            "*.layers.*.self_attn.o_proj": RowwiseParallel(use_local_output=True),
            "*.layers.*.feed_forward.gate_proj": ColwiseParallel(use_local_output=False),
            "*.layers.*.feed_forward.up_proj": ColwiseParallel(use_local_output=False),
            "*.layers.*.feed_forward.down_proj": RowwiseParallel(use_local_output=True),
        }

        if self.config.disable_moe_replacement:
            tp_plan.update(
                {
                    # HF MoE
                    "*.layers.*.feed_forward.shared_expert.gate_proj": ColwiseParallel(use_local_output=False),
                    "*.layers.*.feed_forward.shared_expert.up_proj": ColwiseParallel(use_local_output=False),
                    "*.layers.*.feed_forward.shared_expert.down_proj": RowwiseParallel(use_local_output=True),
                    # TODO:Need to write ParallelStyle for HF's grouped_mm implementation.
                }
            )

        else:
            tp_plan.update(
                {
                    # Custom MoE
                    "*.layers.*.feed_forward.shared_experts.gate_proj": ColwiseParallel(
                        use_local_output=False, output_layouts=Shard(2)
                    ),
                    "*.layers.*.feed_forward.shared_experts.up_proj": ColwiseParallel(
                        use_local_output=False, output_layouts=Shard(2)
                    ),
                    "*.layers.*.feed_forward.shared_experts.down_proj": RowwiseParallel(),
                    "*.layers.*.feed_forward.routed_experts.gate_proj": GroupedLinearColwiseParallel(
                        use_local_output=False
                    ),
                    "*.layers.*.feed_forward.routed_experts.up_proj": GroupedLinearColwiseParallel(
                        use_local_output=False
                    ),
                    "*.layers.*.feed_forward.routed_experts.down_proj": GroupedLinearRowwiseParallel(),
                }
            )

        if mesh:
            model = parallelize_module(model, mesh, tp_plan)

            # Sanity check
            if not self.config.disable_moe_replacement:
                assert type(model.model.layers[1].feed_forward.shared_experts.gate_proj.weight) == DTensor
                assert type(model.model.layers[1].feed_forward.shared_experts.up_proj.weight) == DTensor
                assert type(model.model.layers[1].feed_forward.shared_experts.down_proj.weight) == DTensor
                assert type(model.model.layers[1].feed_forward.routed_experts.gate_proj.weight) == DTensor
                assert type(model.model.layers[1].feed_forward.routed_experts.up_proj.weight) == DTensor
                assert type(model.model.layers[1].feed_forward.routed_experts.down_proj.weight) == DTensor
            else:
                assert type(model.model.layers[1].feed_forward.shared_expert.gate_proj.weight) == DTensor
                assert type(model.model.layers[1].feed_forward.shared_expert.up_proj.weight) == DTensor
                assert type(model.model.layers[1].feed_forward.shared_expert.down_proj.weight) == DTensor

        # Materialize the model on the device (after Llama4MoE replacement and sharding)
        model.to_empty(device=DEVICE)
        assert all(p.device == DEVICE for p in model.parameters())

        # Required as thunder doesn't understand inference mode
        # And some prims like `prims._grouped_mm` don't have grad rule defined yet.
        for p in model.parameters():
            p.requires_grad_(False)

        assert all(not p.requires_grad for p in model.parameters())

        # `thunderfx` seems to hide the access to vocab_size somewhere so
        # store it here before any compiler is applied.
        self.vocab_size = model.vocab_size

        if self.config.enable_nvfp4:
            _quantize_llama4(model)

        # If debug_moe is on, we'll only compile the moe section, which is much easier to digest.
        if self.config.debug_moe:
            def compile_wrapper(model, jitter):
                class ModelWrapper(torch.nn.Module):

                    def __init__(self, model, jitter):
                        super().__init__()
                        self.model = jitter._compile_model(model)
                        assert not hasattr(jitter.model, "_backend")
                        # store a reference to the compiled backend, so we can print the trace later.
                        if jitter.config.mode == "thunder":
                            jitter.model._backend = self.model._backend
                        self.jitter = jitter

                    def forward(self, *args, **kwargs):
                        if self.jitter.profiler_toggle:
                            torch.cuda.cudart().cudaProfilerStart()
                        out = self.model(*args, **kwargs)
                        if self.jitter.profiler_toggle:
                            torch.cuda.cudart().cudaProfilerStop()
                        return out

                return ModelWrapper(model, jitter)

            self.model = model
            # reuse the `replace` function with compilation.
            _replace_with_custom_fn_if_matches_filter_with_name(
                model,
                lambda model, cur_fqn: compile_wrapper(model, self),
                lambda model, cur_fqn: isinstance(model, Llama4MoE),
            )
        else:
            self.model = self._compile_model(model)

    @property
    def _thunder_jit_options(self) -> dict[str, Any]:
        # `nv_enable_linear=True` might fail with distributed run
        # ref: https://github.com/NVIDIA/Fuser/issues/4507
        res = {"transforms": []}
        if self.config.enable_nv_linear:
            res["nv_enable_linear"] = True
            res["nv_enable_matmul"] = True
        if self.config.mode == "thunderjit":
            from thunder.recipes.hf_transformers import SDPAMaskTransform

            if not hasattr(self, "_mask_transform"):
                self._mask_transform = SDPAMaskTransform()
            res["transforms"].append(self._mask_transform)
            res["executors"] = [self._mask_transform.get_executor(), *thunder.get_default_executors()]
        if self.config.enable_cudagraph:
            res["transforms"].append(CUDAGraphTransform())
        if self.config.thunder_cache is not None:
            res["cache"] = self.config.thunder_cache

        return res

    def _compile_model(self, model):
        match self.config.mode:
            case "eager":
                return model
            case "inductor":
                if (self.config.enable_cudagraph):
                    return torch.compile(model, mode="reduce-overhead")
                else:
                    return torch.compile(model, mode="default")
            case "thunder":
                return thunderfx(model, **self._thunder_jit_options)
            case "thunderjit":
                return thunder.jit(model, **self._thunder_jit_options)
            case _:
                raise ValueError(f"Unknown mode: {self.config.mode}")

    def _load_model(self) -> torch.nn.Module:
        """Load the model based on configuration"""
        model_id = self.config.model_name

        config = None
        if self.config.use_hardcoded_model:
            config = Llama4TextConfig.from_dict(
                json.loads(llama_4_Maverick_17B_128E_cfg_str)
            )
        else:
            config = AutoConfig.from_pretrained(model_id)

        if hasattr(config, "text_config"):
            config = config.text_config
        if self.config.num_layers:
            config.num_hidden_layers = self.config.num_layers

        self.hf_config = config

        with torch.device("meta"):
            model = AutoModelForCausalLM.from_config(
                config, torch_dtype=torch.bfloat16, attn_implementation=self.config.attn_implementation
            )

        return model

    def generate_batch(self) -> tuple[torch.Tensor, HybridChunkedCache]:
        """Generate a batch of input tokens"""
        batch_size = self.config.batch_size
        input_length = self.config.input_length

        input_ids = torch.randint(0, self.vocab_size, (batch_size, input_length), device=DEVICE)
        if LooseVersion(transformers.__version__) >= LooseVersion("4.55"):
            # Transformers deprecated HybridChunkedCache in favour of static in 4.55.x
            # NOTE: tp_size should only reflect tensor parallelism size, not total world size.
            # If 2D parallelism (data parallel + tensor parallel) is added in the future,
            # tp_size should be set to the tensor parallelism size only (e.g., WORLD_SIZE // DP_SIZE),
            # not WORLD_SIZE, so that StaticCache correctly handles sharded KV heads.
            past_key_values = StaticCache(
                config=self.hf_config,
                max_batch_size=input_ids.shape[0],
                max_cache_len=input_ids.shape[1] + self.config.output_length,
                device=DEVICE,
                dtype=torch.bfloat16,
                tp_size=WORLD_SIZE,
            )
        else:
            past_key_values = HybridChunkedCache(
                self.hf_config, input_ids.shape[0], input_ids.shape[1] + self.config.output_length
            )
            for layer_idx in range(self.hf_config.num_hidden_layers):
                # key_states.shape[1] is used to retrieve the number of key value heads, all other dimensions can be 1 and ignored
                # https://github.com/huggingface/transformers/blob/9300728665aaeb0ebf4db99f9d9fbce916b4a183/src/transformers/cache_utils.py#L1822
                dummy_key_states = torch.empty(1, self.hf_config.num_key_value_heads // WORLD_SIZE, 1, 1, device=DEVICE)
                past_key_values.initialise_cache_layer(layer_idx, dummy_key_states)

        return input_ids, past_key_values

    def get_next_token(
        self, input_ids: torch.Tensor, past_key_values: HybridChunkedCache | StaticCache
    ) -> torch.Tensor:
        with torch.no_grad():
            outputs = self.model(input_ids, past_key_values=past_key_values, use_cache=True)
        logits = outputs.logits  # [B, seq_len, vocab_size]
        next_token_logits = logits[:, -1, :]
        next_token = torch.argmax(next_token_logits, dim=-1, keepdim=True)
        return next_token

    def prefill(self, input_ids: torch.Tensor, past_key_values: HybridChunkedCache) -> torch.Tensor:
        """
        Prefill phase: Process the entire input prompt at once.
        Returns the next token.

        Similar to: https://github.com/pytorch-labs/gpt-fast/blob/main/generate.py#L68-L82
        """
        return self.get_next_token(input_ids, past_key_values)

    def decode_one_token(self, input_ids: torch.Tensor, past_key_values: HybridChunkedCache) -> torch.Tensor:
        """
        Decode phase: Generate a single token given the current sequence.
        Returns the next token.
        """
        # input_pos: [B, 1] One token at the time
        assert input_ids.shape[-1] == 1, f"Expected shape (B, 1), but found {input_ids.shape}"
        return self.get_next_token(input_ids, past_key_values)

    # TODO: Running `torchrun --nproc-per-node 2 thunder/benchmarks/benchmark_inference.py --input-length 32 --output-length 32 --mode eager --num-iterations 10`
    # with inference mode results in
    # [rank1]:   File "/opt/pytorch/lightning-thunder/thunder/benchmarks/layers_for_inference_benchmark.py", line 358, in grouped_mm
    # [rank1]:     group_outs.append(group_a @ b[idx])
    # [rank1]:                                 ~^^^^^
    # [rank1]: RuntimeError: Cannot set version_counter for inference tensor
    # @torch.inference_mode()
    def generate(
        self, input_ids: torch.Tensor, max_new_tokens: int, past_key_values: HybridChunkedCache
    ) -> dict[str, Any]:
        """
        Generate tokens using separate prefill and decode phases.
        Returns detailed metrics for both phases.
        """
        # Prefill phase - process the entire prompt
        with timer() as prefill_timer:
            torch.cuda.nvtx.range_push("prefill")
            first_token = self.prefill(input_ids, past_key_values)
            torch.cuda.nvtx.range_pop()
        prefill_time = prefill_timer()
        generated_tokens = [first_token]

        # Decode phase - generate remaining tokens one by one
        next_token = first_token
        with timer() as decode_timer:
            for idx in range(max_new_tokens - 1):
                torch.cuda.nvtx.range_push("decode:" + str(idx))
                next_token = self.decode_one_token(next_token, past_key_values)
                generated_tokens.append(next_token)
                torch.cuda.nvtx.range_pop()

        total_decode_time = decode_timer()

        return {
            "prefill_time_ms": prefill_time,
            "decode_time_ms": total_decode_time,
            "generated_tokens": generated_tokens,
            "total_tokens": max_new_tokens,
        }

    def measure_inference_step(
        self, input_ids: torch.Tensor, past_key_values: HybridChunkedCache, max_new_tokens: int
    ) -> dict[str, float]:
        """Measure a single inference step with detailed timing using separate prefill/decode"""
        # Generate tokens with separate prefill/decode tracking
        generation_result = self.generate(input_ids, max_new_tokens, past_key_values)
        total_time = generation_result["prefill_time_ms"] + generation_result["decode_time_ms"]

        # Extract metrics
        ttft = generation_result["prefill_time_ms"]  # Time to first token is the prefill time
        total_decode_time = generation_result["decode_time_ms"]
        avg_tbot = total_decode_time / (max_new_tokens - 1) if max_new_tokens > 1 else 0

        # Calculate throughput
        total_tokens = self.config.output_length * self.config.batch_size
        throughput = (total_tokens / total_time) * 1000  # tokens/second

        # Calculate separate prefill and decode throughput
        prefill_tokens = self.config.input_length * self.config.batch_size
        prefill_throughput = (prefill_tokens / generation_result["prefill_time_ms"]) * 1000

        decode_tokens = (self.config.output_length - 1) * self.config.batch_size
        decode_throughput = (decode_tokens / total_decode_time) * 1000 if total_decode_time > 0 else 0

        return {
            "ttft": ttft,
            "avg_tbot": avg_tbot,
            "total_time": total_time,
            "throughput": throughput,
            "prefill_throughput": prefill_throughput,
            "decode_throughput": decode_throughput,
            "prefill_time": generation_result["prefill_time_ms"],
            "total_decode_time": total_decode_time,
        }

    def run_benchmark(self) -> InferenceMetrics:
        """Run the full benchmark and collect metrics"""
        print(f"Running inference benchmark for {self.config.model_name}")

        print(f"Batch size: {self.config.batch_size}")
        print(f"Input length: {self.config.input_length}")
        print(f"Output length: {self.config.output_length}")
        print(f"Mode: {self.config.mode}")

        print(f"\nWarming up with {self.config.warmup_iterations} iterations...")
        input_ids, past_key_values = self.generate_batch()

        for idx in tqdm(range(self.config.warmup_iterations), disable=LOCAL_RANK != 0):
            past_key_values.reset()
            # Use output_length to warm up sufficiently. Otherwise, Thunder's
            # first-run latency is terribly slow due to lack of dynamic shape
            # support.
            torch.cuda.nvtx.range_push("warmup:" + str(idx))
            _ = self.measure_inference_step(input_ids, past_key_values, self.config.output_length)
            torch.cuda.nvtx.range_pop()

        print(f"\nRunning {self.config.num_iterations} benchmark iterations...")
        all_metrics = []

        if torch.cuda.is_available():
            torch.cuda.reset_peak_memory_stats()

        for idx in tqdm(range(self.config.num_iterations), disable=LOCAL_RANK != 0):
            past_key_values.reset()

            is_under_nsys = bool(os.environ.get("NSYS_PROFILING_SESSION_ID"))
            # Wrap each non-warmup iteration with cudaProfilerStart() and
            # cudaProfilerStop(). This allows the user to run
            # ```shell
            # nsys profile --capture-range=cudaProfilerApi --capture-range-end=repeat:<N> ...
            # ```
            # to record only the non-warmup iterations.
            # Note when debug_moe is on, we'll defer the flag inside the moe wrapper.
            self.profiler_toggle = is_under_nsys
            if is_under_nsys and not self.config.debug_moe:
                torch.cuda.cudart().cudaProfilerStart()
            torch.cuda.nvtx.range_push("step:" + str(idx))
            iter_metrics = self.measure_inference_step(input_ids, past_key_values, self.config.output_length)
            torch.cuda.nvtx.range_pop()
            if is_under_nsys and not self.config.debug_moe:
                torch.cuda.cudart().cudaProfilerStop()

            all_metrics.append(iter_metrics)

            # Track metrics
            self.metrics.iteration_times.append(iter_metrics["total_time"])
            self.metrics.ttft_times.append(iter_metrics["ttft"])
            self.metrics.prefill_times.append(iter_metrics["prefill_time"])
            self.metrics.decode_times.append(iter_metrics["total_decode_time"])

        self._calculate_aggregate_metrics(all_metrics)

        if torch.cuda.is_available():
            self.metrics.memory_used_gb = torch.cuda.memory_allocated() / 1e9
            self.metrics.peak_memory_gb = torch.cuda.max_memory_allocated() / 1e9

        if self.config.fx_report_folder is not None and self.config.mode == "thunder":
            self.model._backend.save_reproducer_to_folder(self.config.fx_report_folder)
            return

        return self.metrics

    def _calculate_aggregate_metrics(self, all_metrics: list[dict[str, Any]]):
        """Calculate aggregate metrics from individual iterations"""
        # Average throughput
        throughputs = [m["throughput"] for m
# ... truncated ...


CustomFormatter

                "prefill_time_ms": self.metrics.prefill_time_ms,
                "decode_time_ms": self.metrics.decode_time_ms,
                "total_time_ms": self.metrics.total_time_ms,
                "memory_used_gb": self.metrics.memory_used_gb,
                "peak_memory_gb": self.metrics.peak_memory_gb,
            },
            "detailed_metrics": {
                "iteration_times": self.metrics.iteration_times,
                "ttft_times": self.metrics.ttft_times,
                "prefill_times": self.metrics.prefill_times,
                "decode_times": self.metrics.decode_times,
            },
        }

        with open(filename, "w") as f:
            json.dump(results, f, indent=2)

        print(f"\nResults saved to {filename}")

class CustomFormatter(argparse.RawDescriptionHelpFormatter, argparse.ArgumentDefaultsHelpFormatter):
    pass

def parse_args() -> argparse.Namespace:
    """Command line interface for the benchmark"""
    parser = argparse.ArgumentParser(
        description="Thunder Inference Benchmark",
        formatter_class=CustomFormatter,
        epilog="""
Examples:
  python benchmark_inference.py --input-length 2048 --output-length 512 --model-name meta-llama/Llama-4-Maverick-17B-128E --mode eager
        """,
    )

    parser.add_argument(
        "--model-name",
        type=str,
        default=LLAMA4_MAVERICK_MODEL_ID,
        help="Model to benchmark",
    )
    parser.add_argument("--batch-size", type=int, default=1, help="Batch size for inference")