# SPDX-FileCopyrightText: Copyright (c) 2023-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

import pytest
from nvfuser_direct import FusionDefinition, DataType
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
from .core import run_benchmark, clear_dynamo_cache, with_executor, DEFAULT_EXECUTORS
import torch
from .global_params import generate_input_sizes, FLOAT_DTYPES, PROMOTE_DTYPES

# test the influence of epilogue on the performance of reduction.
# current reduction scheduler only allows epilogue to be fused with outer reduction without post reduction broadcast.
# So, in this test, only outer reduction is tested. [reduction_axis] is kept to allow the extension to inner reduction.

def reduction_epilogue_fusion(
    fd: FusionDefinition,
    dtype: DataType,
    reduction_axis: int,
) -> None:
    T0 = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=dtype, is_cpu=False
    )
    T1 = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)
    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T1 = fd.ops.cast(T1, dtype=DataType.Float)
    T2 = fd.ops.sum(T0, dims=[reduction_axis], keepdim=False)
    T3 = fd.ops.add(T2, T1)
    if dtype in PROMOTE_DTYPES:
        T3 = fd.ops.cast(T3, dtype=dtype)
    fd.add_output(T3)

def reduction_epilogue_fwd_fn(
    inputs: list,
):  # in_tensor, epilogue_tensor, reduction_axis
    return torch.sum(inputs[0], dim=inputs[2]) + inputs[1]

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize("reduction_axis", [0])
@pytest.mark.reduction
def test_reduction_epilogue_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    reduction_axis: int,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    x = torch.randn(size, device="cuda", dtype=dtype)
    epilogue = torch.randn(size[reduction_axis - 1], device="cuda", dtype=dtype)
    with FusionDefinition() as fd:
        reduction_epilogue_fusion(
            fd, torch_dtype_to_nvfuser_dtype(dtype), reduction_axis
        )

    if not disable_validation:
        eager_output = reduction_epilogue_fwd_fn(
            [x.to(torch.double), epilogue.to(torch.double), reduction_axis]
        )
        fd.validate([x, epilogue], [eager_output.to(dtype)])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, [x, epilogue])

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize("reduction_axis", [0])
@pytest.mark.reduction
def test_reduction_epilogue_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    reduction_axis: int,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    x = torch.randn(size, device="cuda", dtype=dtype)
    epilogue = torch.randn(size[reduction_axis - 1], device="cuda", dtype=dtype)
    # Inputs and outputs are same as nvFuser, no need for manual IOByte computation

    benchmark_fn = with_executor(executor, reduction_epilogue_fwd_fn)

    run_benchmark(
        benchmark,
        benchmark_fn,
        [x, epilogue, reduction_axis],
    )


# SPDX-FileCopyrightText: Copyright (c) 2023-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

import pytest
from nvfuser_direct import FusionDefinition, DataType
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
from .core import run_benchmark, clear_dynamo_cache, with_executor, DEFAULT_EXECUTORS
import torch
from .global_params import generate_input_sizes, FLOAT_DTYPES, PROMOTE_DTYPES

# test the influence of epilogue on the performance of reduction.
# current reduction scheduler only allows epilogue to be fused with outer reduction without post reduction broadcast.
# So, in this test, only outer reduction is tested. [reduction_axis] is kept to allow the extension to inner reduction.

def reduction_epilogue_fusion(
    fd: FusionDefinition,
    dtype: DataType,
    reduction_axis: int,
) -> None:
    T0 = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=dtype, is_cpu=False
    )
    T1 = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)
    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T1 = fd.ops.cast(T1, dtype=DataType.Float)
    T2 = fd.ops.sum(T0, dims=[reduction_axis], keepdim=False)
    T3 = fd.ops.add(T2, T1)
    if dtype in PROMOTE_DTYPES:
        T3 = fd.ops.cast(T3, dtype=dtype)
    fd.add_output(T3)

def reduction_epilogue_fwd_fn(
    inputs: list,
):  # in_tensor, epilogue_tensor, reduction_axis
    return torch.sum(inputs[0], dim=inputs[2]) + inputs[1]

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize("reduction_axis", [0])
@pytest.mark.reduction
def test_reduction_epilogue_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    reduction_axis: int,
    disable_validation: bool,
    disable_benchmarking: bool,
):


def reduction_epilogue_fusion(
    fd: FusionDefinition,
    dtype: DataType,
    reduction_axis: int,
) -> None:
    T0 = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=dtype, is_cpu=False
    )
    T1 = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)
    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T1 = fd.ops.cast(T1, dtype=DataType.Float)
    T2 = fd.ops.sum(T0, dims=[reduction_axis], keepdim=False)
    T3 = fd.ops.add(T2, T1)
    if dtype in PROMOTE_DTYPES:
        T3 = fd.ops.cast(T3, dtype=dtype)
    fd.add_output(T3)

def reduction_epilogue_fwd_fn(
    inputs: list,
):  # in_tensor, epilogue_tensor, reduction_axis
    return torch.sum(inputs[0], dim=inputs[2]) + inputs[1]

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize("reduction_axis", [0])
@pytest.mark.reduction
def test_reduction_epilogue_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    reduction_axis: int,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    x = torch.randn(size, device="cuda", dtype=dtype)
    epilogue = torch.randn(size[reduction_axis - 1], device="cuda", dtype=dtype)
    with FusionDefinition() as fd:
        reduction_epilogue_fusion(
            fd, torch_dtype_to_nvfuser_dtype(dtype), reduction_axis
        )


    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T1 = fd.ops.cast(T1, dtype=DataType.Float)
    T2 = fd.ops.sum(T0, dims=[reduction_axis], keepdim=False)
    T3 = fd.ops.add(T2, T1)
    if dtype in PROMOTE_DTYPES:
        T3 = fd.ops.cast(T3, dtype=dtype)
    fd.add_output(T3)

def reduction_epilogue_fwd_fn(
    inputs: list,
):  # in_tensor, epilogue_tensor, reduction_axis
    return torch.sum(inputs[0], dim=inputs[2]) + inputs[1]

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize("reduction_axis", [0])
@pytest.mark.reduction
def test_reduction_epilogue_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    reduction_axis: int,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    x = torch.randn(size, device="cuda", dtype=dtype)
    epilogue = torch.randn(size[reduction_axis - 1], device="cuda", dtype=dtype)
    with FusionDefinition() as fd:
        reduction_epilogue_fusion(
            fd, torch_dtype_to_nvfuser_dtype(dtype), reduction_axis
        )

    if not disable_validation:
        eager_output = reduction_epilogue_fwd_fn(
            [x.to(torch.double), epilogue.to(torch.double), reduction_axis]
        )
        fd.validate([x, epilogue], [eager_output.to(dtype)])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, [x, epilogue])

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize("reduction_axis", [0])
@pytest.mark.reduction
def test_reduction_epilogue_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    reduction_axis: int,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    x = torch.randn(size, device="cuda", dtype=dtype)
    epilogue = torch.randn(size[reduction_axis - 1], device="cuda", dtype=dtype)
    # Inputs and outputs are same as nvFuser, no need for manual IOByte computation


    with FusionDefinition() as fd:
        reduction_epilogue_fusion(
            fd, torch_dtype_to_nvfuser_dtype(dtype), reduction_axis
        )

    if not disable_validation:
        eager_output = reduction_epilogue_fwd_fn(
            [x.to(torch.double), epilogue.to(torch.double), reduction_axis]
        )
        fd.validate([x, epilogue], [eager_output.to(dtype)])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, [x, epilogue])

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize("reduction_axis", [0])
@pytest.mark.reduction
def test_reduction_epilogue_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    reduction_axis: int,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    x = torch.randn(size, device="cuda", dtype=dtype)
    epilogue = torch.randn(size[reduction_axis - 1], device="cuda", dtype=dtype)
    # Inputs and outputs are same as nvFuser, no need for manual IOByte computation

    benchmark_fn = with_executor(executor, reduction_epilogue_fwd_fn)

    run_benchmark(
        benchmark,
        benchmark_fn,
        [x, epilogue, reduction_axis],
    )


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
from .core import run_benchmark, clear_dynamo_cache, with_executor, DEFAULT_EXECUTORS
import torch
from .global_params import generate_input_sizes, FLOAT_DTYPES, PROMOTE_DTYPES
import numpy as np
from .torch_ops import rmsnorm

def rmsnorm_fwd_fusion(
    fd: FusionDefinition,
    dtype: DataType,
    eps: float = 1e-5,
):
    T0 = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=dtype, is_cpu=False
    )
    T1 = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)
    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T1 = fd.ops.cast(T1, dtype=DataType.Float)
    S3 = fd.define_scalar(2.00000, dtype=DataType.Double)
    T4 = fd.ops.pow(T0, S3)
    T5 = fd.ops.sum(T4, dims=[1], keepdim=False, dtype=DataType.Null)
    T9 = fd.ops.broadcast_in_dim(T5, shape=[T0.size(0), 1], broadcast_dims=[0])
    S11 = fd.ops.reciprocal(T0.size(1))
    T12 = fd.ops.mul(T9, S11)
    S13 = fd.define_scalar(eps, dtype=DataType.Double)
    T14 = fd.ops.add(T12, S13)
    T15 = fd.ops.sqrt(T14)

    T20 = fd.ops.broadcast_in_dim(T15, shape=T0.shape(), broadcast_dims=[0, 1])
    T22 = fd.ops.reciprocal(T20)
    T23 = fd.ops.mul(T0, T22)
    T27 = fd.ops.broadcast_in_dim(T1, shape=T0.shape(), broadcast_dims=[1])
    T29 = fd.ops.mul(T27, T23)
    if dtype in PROMOTE_DTYPES:
        T29 = fd.ops.cast(T29, dtype=dtype)

    fd.add_output(T29)
    fd.add_output(T15)

def rmsnorm_fwd_iobytes(size: tuple, dtype: torch.dtype):
    # Manual IOBytes computation required since nvFuser outputs (out, rms) differs from baselines (out)
    # Total IO bytes = in_tensor (size, dtype) + weights (size[1], dtype) +
    #       rms_eps (size[0], float) + outputs (size, dtype)
    return int(
        dtype.itemsize * (2 * np.prod(size) + size[1]) + torch.float.itemsize * size[0]
    )

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.inner_persistent
def test_rmsnorm_fwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    disable_validation: bool,
    disable_benchmarking: bool,
    eps: float = 1e-5,
):
    inputs = torch.randn(size, device="cuda", dtype=dtype)
    weights = torch.randn(size[1], device="cuda", dtype=dtype)

    with FusionDefinition() as fd:
        rmsnorm_fwd_fusion(fd, torch_dtype_to_nvfuser_dtype(dtype))

    if not disable_validation:
        squared_mean = (inputs.to(torch.float) ** 2).mean(1, keepdim=True)
        rms_eps = torch.sqrt(squared_mean + eps)
        eager_output = weights * (inputs / rms_eps)
        fd.validate([inputs, weights], [eager_output.to(dtype), rms_eps])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, [inputs, weights])

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.inner_persistent
def test_rmsnorm_fwd_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    inputs = torch.randn(size, device="cuda", dtype=dtype, requires_grad=True)
    weights = torch.randn(size[1], device="cuda", dtype=dtype, requires_grad=True)

    benchmark_fn = with_executor(executor, rmsnorm)
    # Manually compute IOBytes: See PR #1725
    run_benchmark(
        benchmark,
        benchmark_fn,
        [inputs, weights],
        iobytes=rmsnorm_fwd_iobytes(size, dtype),
    )


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
from .core import run_benchmark, clear_dynamo_cache, with_executor, DEFAULT_EXECUTORS
import torch
from .global_params import generate_input_sizes, FLOAT_DTYPES, PROMOTE_DTYPES
import numpy as np
from .torch_ops import rmsnorm

def rmsnorm_fwd_fusion(
    fd: FusionDefinition,
    dtype: DataType,
    eps: float = 1e-5,
):
    T0 = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=dtype, is_cpu=False
    )
    T1 = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)
    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T1 = fd.ops.cast(T1, dtype=DataType.Float)
    S3 = fd.define_scalar(2.00000, dtype=DataType.Double)
    T4 = fd.ops.pow(T0, S3)
    T5 = fd.ops.sum(T4, dims=[1], keepdim=False, dtype=DataType.Null)
    T9 = fd.ops.broadcast_in_dim(T5, shape=[T0.size(0), 1], broadcast_dims=[0])
    S11 = fd.ops.reciprocal(T0.size(1))
    T12 = fd.ops.mul(T9, S11)
    S13 = fd.define_scalar(eps, dtype=DataType.Double)
    T14 = fd.ops.add(T12, S13)
    T15 = fd.ops.sqrt(T14)

    T20 = fd.ops.broadcast_in_dim(T15, shape=T0.shape(), broadcast_dims=[0, 1])
    T22 = fd.ops.reciprocal(T20)
    T23 = fd.ops.mul(T0, T22)
    T27 = fd.ops.broadcast_in_dim(T1, shape=T0.shape(), broadcast_dims=[1])
    T29 = fd.ops.mul(T27, T23)
    if dtype in PROMOTE_DTYPES:
        T29 = fd.ops.cast(T29, dtype=dtype)

    fd.add_output(T29)
    fd.add_output(T15)

def rmsnorm_fwd_iobytes(size: tuple, dtype: torch.dtype):
    # Manual IOBytes computation required since nvFuser outputs (out, rms) differs from baselines (out)
    # Total IO bytes = in_tensor (size, dtype) + weights (size[1], dtype) +
    #       rms_eps (size[0], float) + outputs (size, dtype)
    return int(
        dtype.itemsize * (2 * np.prod(size) + size[1]) + torch.float.itemsize * size[0]
    )

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.inner_persistent
def test_rmsnorm_fwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    disable_validation: bool,
    disable_benchmarking: bool,


    T5 = fd.ops.sum(T4, dims=[1], keepdim=False, dtype=DataType.Null)
    T9 = fd.ops.broadcast_in_dim(T5, shape=[T0.size(0), 1], broadcast_dims=[0])
    S11 = fd.ops.reciprocal(T0.size(1))
    T12 = fd.ops.mul(T9, S11)
    S13 = fd.define_scalar(eps, dtype=DataType.Double)
    T14 = fd.ops.add(T12, S13)
    T15 = fd.ops.sqrt(T14)

    T20 = fd.ops.broadcast_in_dim(T15, shape=T0.shape(), broadcast_dims=[0, 1])
    T22 = fd.ops.reciprocal(T20)
    T23 = fd.ops.mul(T0, T22)
    T27 = fd.ops.broadcast_in_dim(T1, shape=T0.shape(), broadcast_dims=[1])
    T29 = fd.ops.mul(T27, T23)
    if dtype in PROMOTE_DTYPES:
        T29 = fd.ops.cast(T29, dtype=dtype)

    fd.add_output(T29)
    fd.add_output(T15)

def rmsnorm_fwd_iobytes(size: tuple, dtype: torch.dtype):
    # Manual IOBytes computation required since nvFuser outputs (out, rms) differs from baselines (out)
    # Total IO bytes = in_tensor (size, dtype) + weights (size[1], dtype) +
    #       rms_eps (size[0], float) + outputs (size, dtype)
    return int(
        dtype.itemsize * (2 * np.prod(size) + size[1]) + torch.float.itemsize * size[0]
    )

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.inner_persistent
def test_rmsnorm_fwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    disable_validation: bool,
    disable_benchmarking: bool,
    eps: float = 1e-5,
):
    inputs = torch.randn(size, device="cuda", dtype=dtype)
    weights = torch.randn(size[1], device="cuda", dtype=dtype)

    with FusionDefinition() as fd:
        rmsnorm_fwd_fusion(fd, torch_dtype_to_nvfuser_dtype(dtype))

    if not disable_validation:


    T29 = fd.ops.mul(T27, T23)
    if dtype in PROMOTE_DTYPES:
        T29 = fd.ops.cast(T29, dtype=dtype)

    fd.add_output(T29)
    fd.add_output(T15)

def rmsnorm_fwd_iobytes(size: tuple, dtype: torch.dtype):
    # Manual IOBytes computation required since nvFuser outputs (out, rms) differs from baselines (out)
    # Total IO bytes = in_tensor (size, dtype) + weights (size[1], dtype) +
    #       rms_eps (size[0], float) + outputs (size, dtype)
    return int(
        dtype.itemsize * (2 * np.prod(size) + size[1]) + torch.float.itemsize * size[0]
    )

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.inner_persistent
def test_rmsnorm_fwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    disable_validation: bool,
    disable_benchmarking: bool,
    eps: float = 1e-5,
):
    inputs = torch.randn(size, device="cuda", dtype=dtype)
    weights = torch.randn(size[1], device="cuda", dtype=dtype)

    with FusionDefinition() as fd:
        rmsnorm_fwd_fusion(fd, torch_dtype_to_nvfuser_dtype(dtype))

    if not disable_validation:
        squared_mean = (inputs.to(torch.float) ** 2).mean(1, keepdim=True)
        rms_eps = torch.sqrt(squared_mean + eps)
        eager_output = weights * (inputs / rms_eps)
        fd.validate([inputs, weights], [eager_output.to(dtype), rms_eps])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, [inputs, weights])

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.inner_persistent
def test_rmsnorm_fwd_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    inputs = torch.randn(size, device="cuda", dtype=dtype, requires_grad=True)
    weights = torch.randn(size[1], device="cuda", dtype=dtype, requires_grad=True)

    benchmark_fn = with_executor(executor, rmsnorm)
    # Manually compute IOBytes: See PR #1725
    run_benchmark(


    inputs = torch.randn(size, device="cuda", dtype=dtype)
    weights = torch.randn(size[1], device="cuda", dtype=dtype)

    with FusionDefinition() as fd:
        rmsnorm_fwd_fusion(fd, torch_dtype_to_nvfuser_dtype(dtype))

    if not disable_validation:
        squared_mean = (inputs.to(torch.float) ** 2).mean(1, keepdim=True)
        rms_eps = torch.sqrt(squared_mean + eps)
        eager_output = weights * (inputs / rms_eps)
        fd.validate([inputs, weights], [eager_output.to(dtype), rms_eps])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, [inputs, weights])

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.inner_persistent
def test_rmsnorm_fwd_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    inputs = torch.randn(size, device="cuda", dtype=dtype, requires_grad=True)
    weights = torch.randn(size[1], device="cuda", dtype=dtype, requires_grad=True)

    benchmark_fn = with_executor(executor, rmsnorm)
    # Manually compute IOBytes: See PR #1725
    run_benchmark(
        benchmark,
        benchmark_fn,
        [inputs, weights],
        iobytes=rmsnorm_fwd_iobytes(size, dtype),
    )


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

import torch
import torch.nn.functional as F

def dropout_layernorm(inputs: list):
    inp1, inp2, weights, bias, dropout_p = inputs
    return F.layer_norm(
        inp2 + torch.nn.functional.dropout(inp1, p=dropout_p),
        normalized_shape=inp1.shape[1:],
        weight=weights,
        bias=bias,
    )

def dropout_rmsnorm(inputs: list):
    inp1, inp2, weights, dropout_p = inputs
    x = inp2 + F.dropout(inp1, p=dropout_p)
    output = weights * x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-5)
    return output

def gelu(inputs: list):
    inp, bias = inputs
    return F.gelu(inp + bias, approximate="tanh")

def huggingface_attn(inputs: list):
    # Reference implementation in Thunder: https://github.com/Lightning-AI/lightning-thunder/blob/888b46324462fba70f93d5017bc0d99025f05091/thunder/tests/hf_bart_self_attn.py#L73-L83
    inp, attention_mask, size, dropout_p = inputs
    batch_size, seq_len, nh, n_embd = size
    attn = (inp + attention_mask).view(batch_size * nh, seq_len, seq_len)
    attn = F.softmax(attn, dim=-1)
    output = F.dropout(attn, p=dropout_p)
    return output

def layernorm(inputs: list):
    inp, weights, bias = inputs
    return F.layer_norm(
        inp,
        inp.shape[1:],
        weight=weights,
        bias=bias,
    )

def nanogpt_attn(inputs: list):
    # Reference implementation from Thunder: https://github.com/Lightning-AI/lightning-thunder/blob/d3da8517bff02a913fd149b4d6559f6b5a4c6c7f/thunder/tests/nanogpt_model.py#L102-L106
    inp, bias, size, dropout_p = inputs
    batch_size, seq_len, nh, n_embd = size
    hs = n_embd // nh
    attn = inp / (hs**0.5)
    attn = attn.masked_fill(bias[:, :, :seq_len, :seq_len] == 0, float("-inf"))
    attn = F.softmax(attn, dim=-1)
    output = F.dropout(attn, p=dropout_p)
    return output

def rmsnorm(inputs: list):
    inp, weights = inputs
    squared_mean = (inp**2).mean(1, keepdim=True)
    rms_eps = torch.sqrt(squared_mean + 1e-5)
    output = weights * (inp / rms_eps)
    return output

# reference implementation from TRT-LLM
# https://github.com/NVIDIA/TensorRT-LLM/blob/334e2cab0d6ca186b98c6c15faeb0336d84a027f/tensorrt_llm/_torch/modules/rms_norm.py#L92
# inputs: x, w, r
# return rmsnorm(x+r, weight), x+r
# different ways to implement rmsnorm, here we use rsqrt based used in TRT-LLM
def rmsnorm_add(inputs: list):
    hidden_states, weight, residual = inputs
    hidden_states = hidden_states + residual
    residual = hidden_states
    variance = hidden_states.pow(2).mean(-1, keepdim=True)
    hidden_states = hidden_states * torch.rsqrt(variance + 1e-5)
    hidden_states = weight * hidden_states
    return hidden_states, residual

def scale_bias_relu(inputs: list):
    inp, scale, bias = inputs
    return F.relu(inp * scale + bias)

def silu_mul(inputs: list):
    x, y = inputs
    return F.silu(x) * y

def softmax(inputs: list):
    inp, reduction_axis = inputs
    return F.softmax(inp, dim=reduction_axis)

def embedding(inputs: list):
    indices, embedding_table = inputs
    return F.embedding(indices, embedding_table)

# deepseek v3 moe scatter
# commit e815299
# https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/modeling_deepseek.py#L599-L607
def scatter_reduce(inputs: list):
    bmm_out: torch.Tensor  # [seq*top_k, hidden]
    idxs: torch.Tensor  # [seq*top_k]
    topk_weight: torch.Tensor  # [seq , top_k]]
    bmm_out, idxs, topk_weight = inputs
    out = bmm_out.index_put([idxs], bmm_out)  # [seq*top_k, hidden]
    out = out.reshape(*topk_weight.shape, -1)  # [seq, top_k, hidden]
    out = out * topk_weight.unsqueeze(-1)  # [seq, top_k, hidden]
    return out.sum(dim=1)  # [seq, hidden]

# simply cross entropy to benchmark quack cross entropy
def cross_entropy(inputs: list):
    logits, labels = inputs
    return F.cross_entropy(logits, labels, reduction="none")


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

import torch
import torch.nn.functional as F

def dropout_layernorm(inputs: list):
    inp1, inp2, weights, bias, dropout_p = inputs
    return F.layer_norm(
        inp2 + torch.nn.functional.dropout(inp1, p=dropout_p),
        normalized_shape=inp1.shape[1:],
        weight=weights,
        bias=bias,
    )

def dropout_rmsnorm(inputs: list):
    inp1, inp2, weights, dropout_p = inputs
    x = inp2 + F.dropout(inp1, p=dropout_p)
    output = weights * x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-5)
    return output

def gelu(inputs: list):
    inp, bias = inputs
    return F.gelu(inp + bias, approximate="tanh")

def huggingface_attn(inputs: list):
    # Reference implementation in Thunder: https://github.com/Lightning-AI/lightning-thunder/blob/888b46324462fba70f93d5017bc0d99025f05091/thunder/tests/hf_bart_self_attn.py#L73-L83
    inp, attention_mask, size, dropout_p = inputs
    batch_size, seq_len, nh, n_embd = size
    attn = (inp + attention_mask).view(batch_size * nh, seq_len, seq_len)
    attn = F.softmax(attn, dim=-1)


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

import torch
import torch.nn.functional as F

def dropout_layernorm(inputs: list):
    inp1, inp2, weights, bias, dropout_p = inputs
    return F.layer_norm(
        inp2 + torch.nn.functional.dropout(inp1, p=dropout_p),
        normalized_shape=inp1.shape[1:],
        weight=weights,
        bias=bias,
    )

def dropout_rmsnorm(inputs: list):
    inp1, inp2, weights, dropout_p = inputs
    x = inp2 + F.dropout(inp1, p=dropout_p)
    output = weights * x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-5)
    return output

def gelu(inputs: list):
    inp, bias = inputs
    return F.gelu(inp + bias, approximate="tanh")

def huggingface_attn(inputs: list):
    # Reference implementation in Thunder: https://github.com/Lightning-AI/lightning-thunder/blob/888b46324462fba70f93d5017bc0d99025f05091/thunder/tests/hf_bart_self_attn.py#L73-L83
    inp, attention_mask, size, dropout_p = inputs
    batch_size, seq_len, nh, n_embd = size
    attn = (inp + attention_mask).view(batch_size * nh, seq_len, seq_len)
    attn = F.softmax(attn, dim=-1)
    output = F.dropout(attn, p=dropout_p)
    return output

def layernorm(inputs: list):
    inp, weights, bias = inputs
    return F.layer_norm(


gelu

import torch.nn.functional as F

def dropout_layernorm(inputs: list):
    inp1, inp2, weights, bias, dropout_p = inputs
    return F.layer_norm(
        inp2 + torch.nn.functional.dropout(inp1, p=dropout_p),
        normalized_shape=inp1.shape[1:],
        weight=weights,
        bias=bias,
    )

def dropout_rmsnorm(inputs: list):
    inp1, inp2, weights, dropout_p = inputs
    x = inp2 + F.dropout(inp1, p=dropout_p)
    output = weights * x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-5)
    return output

def gelu(inputs: list):
    inp, bias = inputs
    return F.gelu(inp + bias, approximate="tanh")

def huggingface_attn(inputs: list):
    # Reference implementation in Thunder: https://github.com/Lightning-AI/lightning-thunder/blob/888b46324462fba70f93d5017bc0d99025f05091/thunder/tests/hf_bart_self_attn.py#L73-L83
    inp, attention_mask, size, dropout_p = inputs
    batch_size, seq_len, nh, n_embd = size
    attn = (inp + attention_mask).view(batch_size * nh, seq_len, seq_len)
    attn = F.softmax(attn, dim=-1)
    output = F.dropout(attn, p=dropout_p)
    return output

def layernorm(inputs: list):
    inp, weights, bias = inputs
    return F.layer_norm(
        inp,
        inp.shape[1:],
        weight=weights,
        bias=bias,
    )


    return F.layer_norm(
        inp2 + torch.nn.functional.dropout(inp1, p=dropout_p),
        normalized_shape=inp1.shape[1:],
        weight=weights,
        bias=bias,
    )

def dropout_rmsnorm(inputs: list):
    inp1, inp2, weights, dropout_p = inputs
    x = inp2 + F.dropout(inp1, p=dropout_p)
    output = weights * x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-5)
    return output

def gelu(inputs: list):
    inp, bias = inputs
    return F.gelu(inp + bias, approximate="tanh")

def huggingface_attn(inputs: list):
    # Reference implementation in Thunder: https://github.com/Lightning-AI/lightning-thunder/blob/888b46324462fba70f93d5017bc0d99025f05091/thunder/tests/hf_bart_self_attn.py#L73-L83
    inp, attention_mask, size, dropout_p = inputs
    batch_size, seq_len, nh, n_embd = size
    attn = (inp + attention_mask).view(batch_size * nh, seq_len, seq_len)
    attn = F.softmax(attn, dim=-1)
    output = F.dropout(attn, p=dropout_p)
    return output

def layernorm(inputs: list):
    inp, weights, bias = inputs
    return F.layer_norm(
        inp,
        inp.shape[1:],
        weight=weights,
        bias=bias,
    )

def nanogpt_attn(inputs: list):
    # Reference implementation from Thunder: https://github.com/Lightning-AI/lightning-thunder/blob/d3da8517bff02a913fd149b4d6559f6b5a4c6c7f/thunder/tests/nanogpt_model.py#L102-L106
    inp, bias, size, dropout_p = inputs
    batch_size, seq_len, nh, n_embd = size
    hs = n_embd // nh
    attn = inp / (hs**0.5)
    attn = attn.masked_fill(bias[:, :, :seq_len, :seq_len] == 0, float("-inf"))
    attn = F.softmax(attn, dim=-1)


layernorm

    x = inp2 + F.dropout(inp1, p=dropout_p)
    output = weights * x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-5)
    return output

def gelu(inputs: list):
    inp, bias = inputs
    return F.gelu(inp + bias, approximate="tanh")

def huggingface_attn(inputs: list):
    # Reference implementation in Thunder: https://github.com/Lightning-AI/lightning-thunder/blob/888b46324462fba70f93d5017bc0d99025f05091/thunder/tests/hf_bart_self_attn.py#L73-L83
    inp, attention_mask, size, dropout_p = inputs
    batch_size, seq_len, nh, n_embd = size
    attn = (inp + attention_mask).view(batch_size * nh, seq_len, seq_len)
    attn = F.softmax(attn, dim=-1)
    output = F.dropout(attn, p=dropout_p)
    return output

def layernorm(inputs: list):
    inp, weights, bias = inputs
    return F.layer_norm(
        inp,
        inp.shape[1:],
        weight=weights,
        bias=bias,
    )

def nanogpt_attn(inputs: list):
    # Reference implementation from Thunder: https://github.com/Lightning-AI/lightning-thunder/blob/d3da8517bff02a913fd149b4d6559f6b5a4c6c7f/thunder/tests/nanogpt_model.py#L102-L106
    inp, bias, size, dropout_p = inputs
    batch_size, seq_len, nh, n_embd = size
    hs = n_embd // nh
    attn = inp / (hs**0.5)
    attn = attn.masked_fill(bias[:, :, :seq_len, :seq_len] == 0, float("-inf"))
    attn = F.softmax(attn, dim=-1)
    output = F.dropout(attn, p=dropout_p)
    return output

def rmsnorm(inputs: list):
    inp, weights = inputs
    squared_mean = (inp**2).mean(1, keepdim=True)
    rms_eps = torch.sqrt(squared_mean + 1e-5)
    output = weights * (inp / rms_eps)
    return output


def huggingface_attn(inputs: list):
    # Reference implementation in Thunder: https://github.com/Lightning-AI/lightning-thunder/blob/888b46324462fba70f93d5017bc0d99025f05091/thunder/tests/hf_bart_self_attn.py#L73-L83
    inp, attention_mask, size, dropout_p = inputs
    batch_size, seq_len, nh, n_embd = size
    attn = (inp + attention_mask).view(batch_size * nh, seq_len, seq_len)
    attn = F.softmax(attn, dim=-1)
    output = F.dropout(attn, p=dropout_p)
    return output

def layernorm(inputs: list):
    inp, weights, bias = inputs
    return F.layer_norm(
        inp,
        inp.shape[1:],
        weight=weights,
        bias=bias,
    )

def nanogpt_attn(inputs: list):
    # Reference implementation from Thunder: https://github.com/Lightning-AI/lightning-thunder/blob/d3da8517bff02a913fd149b4d6559f6b5a4c6c7f/thunder/tests/nanogpt_model.py#L102-L106
    inp, bias, size, dropout_p = inputs
    batch_size, seq_len, nh, n_embd = size
    hs = n_embd // nh
    attn = inp / (hs**0.5)
    attn = attn.masked_fill(bias[:, :, :seq_len, :seq_len] == 0, float("-inf"))
    attn = F.softmax(attn, dim=-1)
    output = F.dropout(attn, p=dropout_p)
    return output

def rmsnorm(inputs: list):
    inp, weights = inputs
    squared_mean = (inp**2).mean(1, keepdim=True)
    rms_eps = torch.sqrt(squared_mean + 1e-5)
    output = weights * (inp / rms_eps)
    return output

# reference implementation from TRT-LLM
# https://github.com/NVIDIA/TensorRT-LLM/blob/334e2cab0d6ca186b98c6c15faeb0336d84a027f/tensorrt_llm/_torch/modules/rms_norm.py#L92
# inputs: x, w, r
# return rmsnorm(x+r, weight), x+r
# different ways to implement rmsnorm, here we use rsqrt based used in TRT-LLM
def rmsnorm_add(inputs: list):
    hidden_states, weight, residual = inputs
    hidden_states = hidden_states + residual
    residual = hidden_states
    variance = hidden_states.pow(2).mean(-1, keepdim=True)


rmsnorm

    return F.layer_norm(
        inp,
        inp.shape[1:],
        weight=weights,
        bias=bias,
    )

def nanogpt_attn(inputs: list):
    # Reference implementation from Thunder: https://github.com/Lightning-AI/lightning-thunder/blob/d3da8517bff02a913fd149b4d6559f6b5a4c6c7f/thunder/tests/nanogpt_model.py#L102-L106
    inp, bias, size, dropout_p = inputs
    batch_size, seq_len, nh, n_embd = size
    hs = n_embd // nh
    attn = inp / (hs**0.5)
    attn = attn.masked_fill(bias[:, :, :seq_len, :seq_len] == 0, float("-inf"))
    attn = F.softmax(attn, dim=-1)
    output = F.dropout(attn, p=dropout_p)
    return output

def rmsnorm(inputs: list):
    inp, weights = inputs
    squared_mean = (inp**2).mean(1, keepdim=True)
    rms_eps = torch.sqrt(squared_mean + 1e-5)
    output = weights * (inp / rms_eps)
    return output

# reference implementation from TRT-LLM
# https://github.com/NVIDIA/TensorRT-LLM/blob/334e2cab0d6ca186b98c6c15faeb0336d84a027f/tensorrt_llm/_torch/modules/rms_norm.py#L92
# inputs: x, w, r
# return rmsnorm(x+r, weight), x+r
# different ways to implement rmsnorm, here we use rsqrt based used in TRT-LLM
def rmsnorm_add(inputs: list):
    hidden_states, weight, residual = inputs
    hidden_states = hidden_states + residual
    residual = hidden_states
    variance = hidden_states.pow(2).mean(-1, keepdim=True)
    hidden_states = hidden_states * torch.rsqrt(variance + 1e-5)
    hidden_states = weight * hidden_states
    return hidden_states, residual

def scale_bias_relu(inputs: list):
    inp, scale, bias = inputs
    return F.relu(inp * scale + bias)


    attn = inp / (hs**0.5)
    attn = attn.masked_fill(bias[:, :, :seq_len, :seq_len] == 0, float("-inf"))
    attn = F.softmax(attn, dim=-1)
    output = F.dropout(attn, p=dropout_p)
    return output

def rmsnorm(inputs: list):
    inp, weights = inputs
    squared_mean = (inp**2).mean(1, keepdim=True)
    rms_eps = torch.sqrt(squared_mean + 1e-5)
    output = weights * (inp / rms_eps)
    return output

# reference implementation from TRT-LLM
# https://github.com/NVIDIA/TensorRT-LLM/blob/334e2cab0d6ca186b98c6c15faeb0336d84a027f/tensorrt_llm/_torch/modules/rms_norm.py#L92
# inputs: x, w, r
# return rmsnorm(x+r, weight), x+r
# different ways to implement rmsnorm, here we use rsqrt based used in TRT-LLM
def rmsnorm_add(inputs: list):
    hidden_states, weight, residual = inputs
    hidden_states = hidden_states + residual
    residual = hidden_states
    variance = hidden_states.pow(2).mean(-1, keepdim=True)
    hidden_states = hidden_states * torch.rsqrt(variance + 1e-5)
    hidden_states = weight * hidden_states
    return hidden_states, residual

def scale_bias_relu(inputs: list):
    inp, scale, bias = inputs
    return F.relu(inp * scale + bias)

def silu_mul(inputs: list):
    x, y = inputs
    return F.silu(x) * y

def softmax(inputs: list):
    inp, reduction_axis = inputs
    return F.softmax(inp, dim=reduction_axis)

def embedding(inputs: list):
    indices, embedding_table = inputs
    return F.embedding(indices, embedding_table)


    rms_eps = torch.sqrt(squared_mean + 1e-5)
    output = weights * (inp / rms_eps)
    return output

# reference implementation from TRT-LLM
# https://github.com/NVIDIA/TensorRT-LLM/blob/334e2cab0d6ca186b98c6c15faeb0336d84a027f/tensorrt_llm/_torch/modules/rms_norm.py#L92
# inputs: x, w, r
# return rmsnorm(x+r, weight), x+r
# different ways to implement rmsnorm, here we use rsqrt based used in TRT-LLM
def rmsnorm_add(inputs: list):
    hidden_states, weight, residual = inputs
    hidden_states = hidden_states + residual
    residual = hidden_states
    variance = hidden_states.pow(2).mean(-1, keepdim=True)
    hidden_states = hidden_states * torch.rsqrt(variance + 1e-5)
    hidden_states = weight * hidden_states
    return hidden_states, residual

def scale_bias_relu(inputs: list):
    inp, scale, bias = inputs
    return F.relu(inp * scale + bias)

def silu_mul(inputs: list):
    x, y = inputs
    return F.silu(x) * y

def softmax(inputs: list):
    inp, reduction_axis = inputs
    return F.softmax(inp, dim=reduction_axis)

def embedding(inputs: list):
    indices, embedding_table = inputs
    return F.embedding(indices, embedding_table)

# deepseek v3 moe scatter
# commit e815299
# https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/modeling_deepseek.py#L599-L607


# reference implementation from TRT-LLM
# https://github.com/NVIDIA/TensorRT-LLM/blob/334e2cab0d6ca186b98c6c15faeb0336d84a027f/tensorrt_llm/_torch/modules/rms_norm.py#L92
# inputs: x, w, r
# return rmsnorm(x+r, weight), x+r
# different ways to implement rmsnorm, here we use rsqrt based used in TRT-LLM
def rmsnorm_add(inputs: list):
    hidden_states, weight, residual = inputs
    hidden_states = hidden_states + residual
    residual = hidden_states
    variance = hidden_states.pow(2).mean(-1, keepdim=True)
    hidden_states = hidden_states * torch.rsqrt(variance + 1e-5)
    hidden_states = weight * hidden_states
    return hidden_states, residual

def scale_bias_relu(inputs: list):
    inp, scale, bias = inputs
    return F.relu(inp * scale + bias)

def silu_mul(inputs: list):
    x, y = inputs
    return F.silu(x) * y

def softmax(inputs: list):
    inp, reduction_axis = inputs
    return F.softmax(inp, dim=reduction_axis)

def embedding(inputs: list):
    indices, embedding_table = inputs
    return F.embedding(indices, embedding_table)

# deepseek v3 moe scatter
# commit e815299
# https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/modeling_deepseek.py#L599-L607
def scatter_reduce(inputs: list):
    bmm_out: torch.Tensor  # [seq*top_k, hidden]
    idxs: torch.Tensor  # [seq*top_k]
    topk_weight: torch.Tensor  # [seq , top_k]]
    bmm_out, idxs, topk_weight = inputs


softmax

def rmsnorm_add(inputs: list):
    hidden_states, weight, residual = inputs
    hidden_states = hidden_states + residual
    residual = hidden_states
    variance = hidden_states.pow(2).mean(-1, keepdim=True)
    hidden_states = hidden_states * torch.rsqrt(variance + 1e-5)
    hidden_states = weight * hidden_states
    return hidden_states, residual

def scale_bias_relu(inputs: list):
    inp, scale, bias = inputs
    return F.relu(inp * scale + bias)

def silu_mul(inputs: list):
    x, y = inputs
    return F.silu(x) * y

def softmax(inputs: list):
    inp, reduction_axis = inputs
    return F.softmax(inp, dim=reduction_axis)

def embedding(inputs: list):
    indices, embedding_table = inputs
    return F.embedding(indices, embedding_table)

# deepseek v3 moe scatter
# commit e815299
# https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/modeling_deepseek.py#L599-L607
def scatter_reduce(inputs: list):
    bmm_out: torch.Tensor  # [seq*top_k, hidden]
    idxs: torch.Tensor  # [seq*top_k]
    topk_weight: torch.Tensor  # [seq , top_k]]
    bmm_out, idxs, topk_weight = inputs
    out = bmm_out.index_put([idxs], bmm_out)  # [seq*top_k, hidden]
    out = out.reshape(*topk_weight.shape, -1)  # [seq, top_k, hidden]
    out = out * topk_weight.unsqueeze(-1)  # [seq, top_k, hidden]
    return out.sum(dim=1)  # [seq, hidden]


embedding

    hidden_states = hidden_states * torch.rsqrt(variance + 1e-5)
    hidden_states = weight * hidden_states
    return hidden_states, residual

def scale_bias_relu(inputs: list):
    inp, scale, bias = inputs
    return F.relu(inp * scale + bias)

def silu_mul(inputs: list):
    x, y = inputs
    return F.silu(x) * y

def softmax(inputs: list):
    inp, reduction_axis = inputs
    return F.softmax(inp, dim=reduction_axis)

def embedding(inputs: list):
    indices, embedding_table = inputs
    return F.embedding(indices, embedding_table)

# deepseek v3 moe scatter
# commit e815299
# https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/modeling_deepseek.py#L599-L607
def scatter_reduce(inputs: list):
    bmm_out: torch.Tensor  # [seq*top_k, hidden]
    idxs: torch.Tensor  # [seq*top_k]
    topk_weight: torch.Tensor  # [seq , top_k]]
    bmm_out, idxs, topk_weight = inputs
    out = bmm_out.index_put([idxs], bmm_out)  # [seq*top_k, hidden]
    out = out.reshape(*topk_weight.shape, -1)  # [seq, top_k, hidden]
    out = out * topk_weight.unsqueeze(-1)  # [seq, top_k, hidden]
    return out.sum(dim=1)  # [seq, hidden]

# simply cross entropy to benchmark quack cross entropy
def cross_entropy(inputs: list):
    logits, labels = inputs
    return F.cross_entropy(logits, labels, reduction="none")


def silu_mul(inputs: list):
    x, y = inputs
    return F.silu(x) * y

def softmax(inputs: list):
    inp, reduction_axis = inputs
    return F.softmax(inp, dim=reduction_axis)

def embedding(inputs: list):
    indices, embedding_table = inputs
    return F.embedding(indices, embedding_table)

# deepseek v3 moe scatter
# commit e815299
# https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/modeling_deepseek.py#L599-L607
def scatter_reduce(inputs: list):
    bmm_out: torch.Tensor  # [seq*top_k, hidden]
    idxs: torch.Tensor  # [seq*top_k]
    topk_weight: torch.Tensor  # [seq , top_k]]
    bmm_out, idxs, topk_weight = inputs
    out = bmm_out.index_put([idxs], bmm_out)  # [seq*top_k, hidden]
    out = out.reshape(*topk_weight.shape, -1)  # [seq, top_k, hidden]
    out = out * topk_weight.unsqueeze(-1)  # [seq, top_k, hidden]
    return out.sum(dim=1)  # [seq, hidden]

# simply cross entropy to benchmark quack cross entropy
def cross_entropy(inputs: list):
    logits, labels = inputs
    return F.cross_entropy(logits, labels, reduction="none")


def embedding(inputs: list):
    indices, embedding_table = inputs
    return F.embedding(indices, embedding_table)

# deepseek v3 moe scatter
# commit e815299
# https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/modeling_deepseek.py#L599-L607
def scatter_reduce(inputs: list):
    bmm_out: torch.Tensor  # [seq*top_k, hidden]
    idxs: torch.Tensor  # [seq*top_k]
    topk_weight: torch.Tensor  # [seq , top_k]]
    bmm_out, idxs, topk_weight = inputs
    out = bmm_out.index_put([idxs], bmm_out)  # [seq*top_k, hidden]
    out = out.reshape(*topk_weight.shape, -1)  # [seq, top_k, hidden]
    out = out * topk_weight.unsqueeze(-1)  # [seq, top_k, hidden]
    return out.sum(dim=1)  # [seq, hidden]

# simply cross entropy to benchmark quack cross entropy
def cross_entropy(inputs: list):
    logits, labels = inputs
    return F.cross_entropy(logits, labels, reduction="none")


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from .core import run_benchmark, with_executor, unary_bwd_torch, clear_dynamo_cache

from .rope_ops import rope_setup, SEQ_LENGTHS

@pytest.mark.parametrize(
    "variation",
    [
        "llama_2_7b_hf",
        "llama_3_8B",
        "hf_qwen2",
        "hf_phi3",
        "hf_mistral_nemo",
        "litgpt-gemma-2-9b",
        "litgpt-mistral-7b",
        "litgpt-meta-llama-3-8B",
        "litgpt-phi3.5-mini",
    ],
)
@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
@pytest.mark.parametrize("seq_length", SEQ_LENGTHS)
@pytest.mark.resize
def test_rope_fwd_benchmark(
    benchmark,
    variation: str,
    executor: str,
    seq_length: int | None,
):
    kwargs = {}
    if executor == "thunder":
        kwargs["nv_enable_matmul"] = True
    elif executor == "torchcompile":
        clear_dynamo_cache()

    model, gen_inputs, _, _ = rope_setup[variation](seq_length)
    inputs = gen_inputs()

    def fwd_call(inp):
        return model(*inp)

    # Compile the fwd fn for torchcompile
    benchmark_fn = with_executor(executor, fwd_call, **kwargs)
    run_benchmark(benchmark, benchmark_fn, inputs)

@pytest.mark.parametrize(
    "variation",
    [
        "llama_2_7b_hf",
        "llama_3_8B",
        "hf_qwen2",
        "hf_phi3",
        "hf_mistral_nemo",
        "litgpt-gemma-2-9b",
        "litgpt-mistral-7b",
        "litgpt-meta-llama-3-8B",
        "litgpt-phi3.5-mini",
    ],
)
@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
@pytest.mark.parametrize("seq_length", SEQ_LENGTHS)
@pytest.mark.resize
def test_rope_bwd_benchmark(
    benchmark,
    variation: str,
    executor: str,
    seq_length: int | None,
):
    kwargs = {}
    if executor == "thunder":
        kwargs["nv_enable_matmul"] = True
    elif executor == "torchcompile":
        clear_dynamo_cache()
    elif executor == "thunder-torchcompile" and variation in [
        "llama_2_7b_hf",
        "llama_3_8B",
    ]:
        # See https://github.com/Lightning-AI/lightning-thunder/issues/2297
        pytest.skip(
            "torch._dynamo.exc.TorchRuntimeError: Dynamo failed to run FX node with fake tensors"
        )

    model, gen_inputs, grad, iobytes = rope_setup[variation](seq_length)
    fwd_inputs = gen_inputs()

    def fwd_call(inp):
        return model(*inp)

    # execute the compiled fwd fn
    fwd_fn = with_executor(executor, fwd_call, **kwargs)
    outputs = fwd_fn(fwd_inputs)

    # accumulate all output, so we can feed a single grad and use the unary bwd function
    output = outputs[0]
    for i in range(1, len(outputs)):
        output += outputs[i]

    # NOTE: the iobytes is computed based on how thunder autograd worked. So this is just
    # a reference point for torchcompile and eager executor for comparison.
    run_benchmark(
        benchmark, unary_bwd_torch, [output, grad(), *fwd_inputs], iobytes=iobytes()
    )


@pytest.mark.parametrize(
    "variation",
    [
        "llama_2_7b_hf",
        "llama_3_8B",
        "hf_qwen2",
        "hf_phi3",
        "hf_mistral_nemo",
        "litgpt-gemma-2-9b",
        "litgpt-mistral-7b",
        "litgpt-meta-llama-3-8B",
        "litgpt-phi3.5-mini",
    ],
)
@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
@pytest.mark.parametrize("seq_length", SEQ_LENGTHS)
@pytest.mark.resize
def test_rope_fwd_benchmark(
    benchmark,
    variation: str,
    executor: str,
    seq_length: int | None,
):
    kwargs = {}
    if executor == "thunder":
        kwargs["nv_enable_matmul"] = True
    elif executor == "torchcompile":
        clear_dynamo_cache()

    model, gen_inputs, _, _ = rope_setup[variation](seq_length)
    inputs = gen_inputs()

    def fwd_call(inp):
        return model(*inp)

    # Compile the fwd fn for torchcompile
    benchmark_fn = with_executor(executor, fwd_call, **kwargs)
    run_benchmark(benchmark, benchmark_fn, inputs)

@pytest.mark.parametrize(
    "variation",
    [
        "llama_2_7b_hf",
        "llama_3_8B",
        "hf_qwen2",
        "hf_phi3",
        "hf_mistral_nemo",
        "litgpt-gemma-2-9b",
        "litgpt-mistral-7b",
        "litgpt-meta-llama-3-8B",
        "litgpt-phi3.5-mini",
    ],
)
@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
@pytest.mark.parametrize("seq_length", SEQ_LENGTHS)


@pytest.mark.parametrize(
    "variation",
    [
        "llama_2_7b_hf",
        "llama_3_8B",
        "hf_qwen2",
        "hf_phi3",
        "hf_mistral_nemo",
        "litgpt-gemma-2-9b",
        "litgpt-mistral-7b",
        "litgpt-meta-llama-3-8B",
        "litgpt-phi3.5-mini",
    ],
)
@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
@pytest.mark.parametrize("seq_length", SEQ_LENGTHS)
@pytest.mark.resize
def test_rope_bwd_benchmark(
    benchmark,
    variation: str,
    executor: str,
    seq_length: int | None,
):
    kwargs = {}
    if executor == "thunder":
        kwargs["nv_enable_matmul"] = True
    elif executor == "torchcompile":
        clear_dynamo_cache()
    elif executor == "thunder-torchcompile" and variation in [
        "llama_2_7b_hf",
        "llama_3_8B",
    ]:
        # See https://github.com/Lightning-AI/lightning-thunder/issues/2297
        pytest.skip(
            "torch._dynamo.exc.TorchRuntimeError: Dynamo failed to run FX node with fake tensors"
        )

    model, gen_inputs, grad, iobytes = rope_setup[variation](seq_length)
    fwd_inputs = gen_inputs()

    def fwd_call(inp):
        return model(*inp)

    # execute the compiled fwd fn
    fwd_fn = with_executor(executor, fwd_call, **kwargs)
    outputs = fwd_fn(fwd_inputs)

    # accumulate all output, so we can feed a single grad and use the unary bwd function
    output = outputs[0]
    for i in range(1, len(outputs)):
        output += outputs[i]

    # NOTE: the iobytes is computed based on how thunder autograd worked. So this is just
    # a reference point for torchcompile and eager executor for comparison.
    run_benchmark(
        benchmark, unary_bwd_torch, [output, grad(), *fwd_inputs], iobytes=iobytes()
    )


    executor: str,
    seq_length: int | None,
):
    kwargs = {}
    if executor == "thunder":
        kwargs["nv_enable_matmul"] = True
    elif executor == "torchcompile":
        clear_dynamo_cache()
    elif executor == "thunder-torchcompile" and variation in [
        "llama_2_7b_hf",
        "llama_3_8B",
    ]:
        # See https://github.com/Lightning-AI/lightning-thunder/issues/2297
        pytest.skip(
            "torch._dynamo.exc.TorchRuntimeError: Dynamo failed to run FX node with fake tensors"
        )

    model, gen_inputs, grad, iobytes = rope_setup[variation](seq_length)
    fwd_inputs = gen_inputs()

    def fwd_call(inp):
        return model(*inp)

    # execute the compiled fwd fn
    fwd_fn = with_executor(executor, fwd_call, **kwargs)
    outputs = fwd_fn(fwd_inputs)

    # accumulate all output, so we can feed a single grad and use the unary bwd function
    output = outputs[0]
    for i in range(1, len(outputs)):
        output += outputs[i]

    # NOTE: the iobytes is computed based on how thunder autograd worked. So this is just
    # a reference point for torchcompile and eager executor for comparison.
    run_benchmark(
        benchmark, unary_bwd_torch, [output, grad(), *fwd_inputs], iobytes=iobytes()
    )


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
from .core import (
    run_benchmark,
    clear_dynamo_cache,
    unary_bwd_torch,
    with_executor,
    DEFAULT_EXECUTORS,
)
import torch
from .global_params import generate_input_sizes, FLOAT_DTYPES, PROMOTE_DTYPES
import numpy as np
from .torch_ops import scale_bias_relu

def sbr_bwd_fusion(
    fd: FusionDefinition,
    dtype: DataType,
):
    T0 = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)
    T1 = fd.define_tensor(
        shape=[-1, -1],
        contiguity=[True, True],
        dtype=DataType.Bool,
        is_cpu=False,
    )
    T2 = fd.define_tensor(
        shape=[-1, -1],
        contiguity=[True, True],
        dtype=dtype,
        is_cpu=False,
    )

    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T2 = fd.ops.cast(T2, dtype=DataType.Float)

    V7 = T2.shape()
    T8 = fd.ops.broadcast_in_dim(T0, shape=V7, broadcast_dims=[1])

    S10 = fd.define_scalar(0.00000, dtype=DataType.Double)
    T11 = fd.ops.where(T1, T2, S10)
    T15 = fd.ops.mul(T11, T8)

    if dtype in PROMOTE_DTYPES:
        T15 = fd.ops.cast(T15, dtype=dtype)
    fd.add_output(T15)

def sbr_bwd_iobytes(size: tuple, dtype: torch.dtype):
    # Total IO bytes = grad_out (dtype, size)+ scale (dtype, size[-1])+ bool_mask (bool, size) + grad_in (dtype, size)
    return int(
        dtype.itemsize * (2 * np.prod(size) + size[-1])
        + torch.bool.itemsize * np.prod(size)
    )

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.pointwise
def test_sbr_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    inputs = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(*size, device="cuda", dtype=dtype)
    scale = torch.ones(size[-1], device="cuda", dtype=dtype)
    bias = torch.ones(size[-1], device="cuda", dtype=dtype)
    bool_mask = torch.gt(inputs * scale + bias, 0.0)

    with FusionDefinition() as fd:
        sbr_bwd_fusion(fd, torch_dtype_to_nvfuser_dtype(dtype))

    if not disable_validation:
        eager_output = torch.nn.functional.relu(inputs * scale + bias)
        eager_output.backward(grads)
        fd.validate([scale, bool_mask, grads], [inputs.grad])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, [scale, bool_mask, grads])

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.pointwise
def test_sbr_bwd_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    inputs = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(*size, device="cuda", dtype=dtype)
    scale = torch.ones(size[-1], device="cuda", dtype=dtype)
    bias = torch.ones(size[-1], device="cuda", dtype=dtype)

    # Compile the fwd fn for torchcompile
    fwd_fn = with_executor(executor, scale_bias_relu)
    fwd_inputs = [inputs, scale, bias]
    outputs = fwd_fn(fwd_inputs)

    run_benchmark(
        benchmark,
        unary_bwd_torch,
        [outputs, grads, *fwd_inputs],
        iobytes=sbr_bwd_iobytes(size, dtype),
    )


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
from .core import (
    run_benchmark,
    clear_dynamo_cache,
    unary_bwd_torch,
    with_executor,
    DEFAULT_EXECUTORS,
)
import torch
from .global_params import generate_input_sizes, FLOAT_DTYPES, PROMOTE_DTYPES
import numpy as np
from .torch_ops import scale_bias_relu

def sbr_bwd_fusion(
    fd: FusionDefinition,
    dtype: DataType,
):
    T0 = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)
    T1 = fd.define_tensor(
        shape=[-1, -1],
        contiguity=[True, True],
        dtype=DataType.Bool,
        is_cpu=False,
    )
    T2 = fd.define_tensor(
        shape=[-1, -1],
        contiguity=[True, True],
        dtype=dtype,
        is_cpu=False,
    )

    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T2 = fd.ops.cast(T2, dtype=DataType.Float)

    V7 = T2.shape()
    T8 = fd.ops.broadcast_in_dim(T0, shape=V7, broadcast_dims=[1])

    S10 = fd.define_scalar(0.00000, dtype=DataType.Double)
    T11 = fd.ops.where(T1, T2, S10)
    T15 = fd.ops.mul(T11, T8)

    if dtype in PROMOTE_DTYPES:
        T15 = fd.ops.cast(T15, dtype=dtype)
    fd.add_output(T15)

def sbr_bwd_iobytes(size: tuple, dtype: torch.dtype):
    # Total IO bytes = grad_out (dtype, size)+ scale (dtype, size[-1])+ bool_mask (bool, size) + grad_in (dtype, size)
    return int(
        dtype.itemsize * (2 * np.prod(size) + size[-1])
        + torch.bool.itemsize * np.prod(size)
    )

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.pointwise
def test_sbr_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    disable_validation: bool,
    disable_benchmarking: bool,
):


        dtype=dtype,
        is_cpu=False,
    )

    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T2 = fd.ops.cast(T2, dtype=DataType.Float)

    V7 = T2.shape()
    T8 = fd.ops.broadcast_in_dim(T0, shape=V7, broadcast_dims=[1])

    S10 = fd.define_scalar(0.00000, dtype=DataType.Double)
    T11 = fd.ops.where(T1, T2, S10)
    T15 = fd.ops.mul(T11, T8)

    if dtype in PROMOTE_DTYPES:
        T15 = fd.ops.cast(T15, dtype=dtype)
    fd.add_output(T15)

def sbr_bwd_iobytes(size: tuple, dtype: torch.dtype):
    # Total IO bytes = grad_out (dtype, size)+ scale (dtype, size[-1])+ bool_mask (bool, size) + grad_in (dtype, size)
    return int(
        dtype.itemsize * (2 * np.prod(size) + size[-1])
        + torch.bool.itemsize * np.prod(size)
    )

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.pointwise
def test_sbr_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    inputs = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(*size, device="cuda", dtype=dtype)
    scale = torch.ones(size[-1], device="cuda", dtype=dtype)
    bias = torch.ones(size[-1], device="cuda", dtype=dtype)
    bool_mask = torch.gt(inputs * scale + bias, 0.0)

    with FusionDefinition() as fd:
        sbr_bwd_fusion(fd, torch_dtype_to_nvfuser_dtype(dtype))


    S10 = fd.define_scalar(0.00000, dtype=DataType.Double)
    T11 = fd.ops.where(T1, T2, S10)
    T15 = fd.ops.mul(T11, T8)

    if dtype in PROMOTE_DTYPES:
        T15 = fd.ops.cast(T15, dtype=dtype)
    fd.add_output(T15)

def sbr_bwd_iobytes(size: tuple, dtype: torch.dtype):
    # Total IO bytes = grad_out (dtype, size)+ scale (dtype, size[-1])+ bool_mask (bool, size) + grad_in (dtype, size)
    return int(
        dtype.itemsize * (2 * np.prod(size) + size[-1])
        + torch.bool.itemsize * np.prod(size)
    )

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.pointwise
def test_sbr_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    inputs = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(*size, device="cuda", dtype=dtype)
    scale = torch.ones(size[-1], device="cuda", dtype=dtype)
    bias = torch.ones(size[-1], device="cuda", dtype=dtype)
    bool_mask = torch.gt(inputs * scale + bias, 0.0)

    with FusionDefinition() as fd:
        sbr_bwd_fusion(fd, torch_dtype_to_nvfuser_dtype(dtype))

    if not disable_validation:
        eager_output = torch.nn.functional.relu(inputs * scale + bias)
        eager_output.backward(grads)
        fd.validate([scale, bool_mask, grads], [inputs.grad])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, [scale, bool_mask, grads])

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.pointwise
def test_sbr_bwd_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    inputs = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(*size, device="cuda", dtype=dtype)
    scale = torch.ones(size[-1], device="cuda", dtype=dtype)
    bias = torch.ones(size[-1], device="cuda", dtype=dtype)

    # Compile the fwd fn for torchcompile


    scale = torch.ones(size[-1], device="cuda", dtype=dtype)
    bias = torch.ones(size[-1], device="cuda", dtype=dtype)
    bool_mask = torch.gt(inputs * scale + bias, 0.0)

    with FusionDefinition() as fd:
        sbr_bwd_fusion(fd, torch_dtype_to_nvfuser_dtype(dtype))

    if not disable_validation:
        eager_output = torch.nn.functional.relu(inputs * scale + bias)
        eager_output.backward(grads)
        fd.validate([scale, bool_mask, grads], [inputs.grad])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, [scale, bool_mask, grads])

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.pointwise
def test_sbr_bwd_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    inputs = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(*size, device="cuda", dtype=dtype)
    scale = torch.ones(size[-1], device="cuda", dtype=dtype)
    bias = torch.ones(size[-1], device="cuda", dtype=dtype)

    # Compile the fwd fn for torchcompile
    fwd_fn = with_executor(executor, scale_bias_relu)
    fwd_inputs = [inputs, scale, bias]
    outputs = fwd_fn(fwd_inputs)

    run_benchmark(
        benchmark,
        unary_bwd_torch,
        [outputs, grads, *fwd_inputs],
        iobytes=sbr_bwd_iobytes(size, dtype),
    )


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
from .core import (
    run_benchmark,
    clear_dynamo_cache,
    unary_bwd_torch,
    with_executor,
    DEFAULT_EXECUTORS,
)
import torch
from .global_params import generate_input_sizes, FLOAT_DTYPES, PROMOTE_DTYPES
import numpy as np
from .torch_ops import silu_mul

def silu_mul_bwd_fusion(fd: FusionDefinition, dtype: DataType):
    T0 = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=dtype, is_cpu=False
    )
    T1 = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=dtype, is_cpu=False
    )
    T2 = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=dtype, is_cpu=False
    )

    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T1 = fd.ops.cast(T1, dtype=DataType.Float)
        T2 = fd.ops.cast(T2, dtype=DataType.Float)

    T3 = fd.ops.neg(T1)
    T4 = fd.ops.exp(T3)
    S5 = fd.define_scalar(1.00000, dtype=DataType.Double)
    T6 = fd.ops.add(S5, T4)
    T7 = fd.ops.reciprocal(T6)
    T8 = fd.ops.mul(T1, T7)
    T9 = fd.ops.mul(T0, T2)
    T10 = fd.ops.mul(T0, T8)
    T11 = fd.ops.mul(T9, T7)
    T12 = fd.ops.mul(T9, T1)
    T13 = fd.ops.neg(T12)
    T14 = fd.ops.mul(T13, T7)
    T15 = fd.ops.mul(T14, T7)
    S16 = fd.define_scalar(1.00000, dtype=DataType.Double)
    T17 = fd.ops.mul(S16, T15)
    T18 = fd.ops.mul(T17, T4)
    T19 = fd.ops.neg(T18)
    T20 = fd.ops.add(T11, T19)
    if dtype in PROMOTE_DTYPES:
        T10 = fd.ops.cast(T10, dtype=dtype)
        T20 = fd.ops.cast(T20, dtype=dtype)
    fd.add_output(T10)
    fd.add_output(T20)

def silu_mul_bwd_iobytes(size: tuple, dtype: torch.dtype):
    # Total IO bytes = grad_out, x, y, grad_x, grad_y
    return int(dtype.itemsize * np.prod(size) * 5)

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.pointwise
def test_silu_mul_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    x = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    y = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(*size, device="cuda", dtype=dtype)
    with FusionDefinition() as fd:
        silu_mul_bwd_fusion(fd, torch_dtype_to_nvfuser_dtype(dtype))
    if not disable_validation:
        eager_output = torch.nn.functional.silu(x) * y
        eager_output.backward(grads)
        fd.validate([grads, x, y], [y.grad, x.grad])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, [grads, x, y])

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.pointwise
def test_silu_mul_bwd_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    x = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    y = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(*size, device="cuda", dtype=dtype)

    # Compile the fwd fn for torchcompile
    fwd_fn = with_executor(executor, silu_mul)
    fwd_inputs = [x, y]
    outputs = fwd_fn(fwd_inputs)

    run_benchmark(
        benchmark,
        unary_bwd_torch,
        [outputs, grads, *fwd_inputs],
        iobytes=silu_mul_bwd_iobytes(size, dtype),
    )


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
from .core import (
    run_benchmark,
    clear_dynamo_cache,
    unary_bwd_torch,
    with_executor,
    DEFAULT_EXECUTORS,
)
import torch
from .global_params import generate_input_sizes, FLOAT_DTYPES, PROMOTE_DTYPES
import numpy as np
from .torch_ops import silu_mul

def silu_mul_bwd_fusion(fd: FusionDefinition, dtype: DataType):
    T0 = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=dtype, is_cpu=False
    )
    T1 = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=dtype, is_cpu=False
    )
    T2 = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=dtype, is_cpu=False
    )

    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T1 = fd.ops.cast(T1, dtype=DataType.Float)
        T2 = fd.ops.cast(T2, dtype=DataType.Float)

    T3 = fd.ops.neg(T1)
    T4 = fd.ops.exp(T3)
    S5 = fd.define_scalar(1.00000, dtype=DataType.Double)
    T6 = fd.ops.add(S5, T4)
    T7 = fd.ops.reciprocal(T6)
    T8 = fd.ops.mul(T1, T7)
    T9 = fd.ops.mul(T0, T2)
    T10 = fd.ops.mul(T0, T8)
    T11 = fd.ops.mul(T9, T7)
    T12 = fd.ops.mul(T9, T1)
    T13 = fd.ops.neg(T12)
    T14 = fd.ops.mul(T13, T7)
    T15 = fd.ops.mul(T14, T7)
    S16 = fd.define_scalar(1.00000, dtype=DataType.Double)
    T17 = fd.ops.mul(S16, T15)
    T18 = fd.ops.mul(T17, T4)
    T19 = fd.ops.neg(T18)
    T20 = fd.ops.add(T11, T19)
    if dtype in PROMOTE_DTYPES:
        T10 = fd.ops.cast(T10, dtype=dtype)
        T20 = fd.ops.cast(T20, dtype=dtype)
    fd.add_output(T10)
    fd.add_output(T20)

def silu_mul_bwd_iobytes(size: tuple, dtype: torch.dtype):
    # Total IO bytes = grad_out, x, y, grad_x, grad_y
    return int(dtype.itemsize * np.prod(size) * 5)

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.pointwise
def test_silu_mul_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    x = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    y = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(*size, device="cuda", dtype=dtype)


    T8 = fd.ops.mul(T1, T7)
    T9 = fd.ops.mul(T0, T2)
    T10 = fd.ops.mul(T0, T8)
    T11 = fd.ops.mul(T9, T7)
    T12 = fd.ops.mul(T9, T1)
    T13 = fd.ops.neg(T12)
    T14 = fd.ops.mul(T13, T7)
    T15 = fd.ops.mul(T14, T7)
    S16 = fd.define_scalar(1.00000, dtype=DataType.Double)
    T17 = fd.ops.mul(S16, T15)
    T18 = fd.ops.mul(T17, T4)
    T19 = fd.ops.neg(T18)
    T20 = fd.ops.add(T11, T19)
    if dtype in PROMOTE_DTYPES:
        T10 = fd.ops.cast(T10, dtype=dtype)
        T20 = fd.ops.cast(T20, dtype=dtype)
    fd.add_output(T10)
    fd.add_output(T20)

def silu_mul_bwd_iobytes(size: tuple, dtype: torch.dtype):
    # Total IO bytes = grad_out, x, y, grad_x, grad_y
    return int(dtype.itemsize * np.prod(size) * 5)

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.pointwise
def test_silu_mul_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    x = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    y = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(*size, device="cuda", dtype=dtype)
    with FusionDefinition() as fd:
        silu_mul_bwd_fusion(fd, torch_dtype_to_nvfuser_dtype(dtype))
    if not disable_validation:
        eager_output = torch.nn.functional.silu(x) * y
        eager_output.backward(grads)


    S16 = fd.define_scalar(1.00000, dtype=DataType.Double)
    T17 = fd.ops.mul(S16, T15)
    T18 = fd.ops.mul(T17, T4)
    T19 = fd.ops.neg(T18)
    T20 = fd.ops.add(T11, T19)
    if dtype in PROMOTE_DTYPES:
        T10 = fd.ops.cast(T10, dtype=dtype)
        T20 = fd.ops.cast(T20, dtype=dtype)
    fd.add_output(T10)
    fd.add_output(T20)

def silu_mul_bwd_iobytes(size: tuple, dtype: torch.dtype):
    # Total IO bytes = grad_out, x, y, grad_x, grad_y
    return int(dtype.itemsize * np.prod(size) * 5)

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.pointwise
def test_silu_mul_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    x = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    y = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(*size, device="cuda", dtype=dtype)
    with FusionDefinition() as fd:
        silu_mul_bwd_fusion(fd, torch_dtype_to_nvfuser_dtype(dtype))
    if not disable_validation:
        eager_output = torch.nn.functional.silu(x) * y
        eager_output.backward(grads)
        fd.validate([grads, x, y], [y.grad, x.grad])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, [grads, x, y])

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.pointwise
def test_silu_mul_bwd_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    x = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    y = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(*size, device="cuda", dtype=dtype)

    # Compile the fwd fn for torchcompile
    fwd_fn = with_executor(executor, silu_mul)


    disable_benchmarking: bool,
):
    x = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    y = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(*size, device="cuda", dtype=dtype)
    with FusionDefinition() as fd:
        silu_mul_bwd_fusion(fd, torch_dtype_to_nvfuser_dtype(dtype))
    if not disable_validation:
        eager_output = torch.nn.functional.silu(x) * y
        eager_output.backward(grads)
        fd.validate([grads, x, y], [y.grad, x.grad])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, [grads, x, y])

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.pointwise
def test_silu_mul_bwd_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    x = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    y = torch.randn(*size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(*size, device="cuda", dtype=dtype)

    # Compile the fwd fn for torchcompile
    fwd_fn = with_executor(executor, silu_mul)
    fwd_inputs = [x, y]
    outputs = fwd_fn(fwd_inputs)

    run_benchmark(
        benchmark,
        unary_bwd_torch,
        [outputs, grads, *fwd_inputs],
        iobytes=silu_mul_bwd_iobytes(size, dtype),
    )


benchmarks.python.core

# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
from collections.abc import Iterable
import pytest_benchmark
import torch
from typing import List, Callable, Union
import numpy as np
from nvfuser_direct import FusionDefinition, PythonProfiler
from nvfuser_direct.pytorch_utils import DEVICE_PROPERTIES
from nvfuser_direct.benchmark_utils import FusionProfileTimer, CuptiTimer
import warnings
import thunder
from thunder.executors.nvfuserex import nvfuserex
import importlib.util

# These variables can be overwritten through CLI commands
# --benchmark-rounds=rounds --benchmark-warmup-rounds=warmup_rounds
# --benchmark-num-inputs=num_inputs
BENCHMARK_CONFIG = {
    "rounds": 10,
    "warmup_rounds": 1,
    "num_inputs": None,
    "with_nsys": False,
}

L2_CACHE_SIZE = DEVICE_PROPERTIES["gpu_l2_bytes"]
PEAK_BANDWIDTH_GBPS = DEVICE_PROPERTIES["gpu_peak_bandwidth_gbps"]

DEFAULT_EXECUTORS = ["eager", "torchcompile"]

def check_module_available(module_name):
    # If spec is not None, the module can be imported.
    spec = importlib.util.find_spec(module_name)
    return spec is not None

def clear_l2_cache() -> None:
    """
    Flushes the L2 cache by creating a buffer of the same size.
    """
    n_elements = L2_CACHE_SIZE // 4
    x = torch.empty(n_elements, dtype=torch.float32, device="cuda", requires_grad=False)
    y = torch.clone(x)

def clear_dynamo_cache() -> None:
    """
    Utility function to enforce re-compilation to avoid different results between
    running a serials of tests and a standalone test due to kernel re-use.
    It slows down the test but ensures the correctness of the benchmark results.
    Ref: https://github.com/pytorch/pytorch/issues/107444
    """
    torch._dynamo.reset()

# Backward function for torch baseline benchmarks.
# The first two inputs are expected to be out and grad_out. The remaining are inputs of the forward pass used to clear grad between subsequent runs to avoid grad accumulation. See setup() in run_benchmark().
def unary_bwd_torch(inputs: List):  # [output, grad_out, fwd_inputs]
    inputs[0].backward(inputs[1], retain_graph=True)

def with_executor(executor: str, fwd_fn: Callable, **kwargs) -> Callable:
    assert executor in ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
    if executor == "eager":
        return fwd_fn
    if executor == "torchcompile":
        return torch.compile(fwd_fn, **kwargs)
    if executor == "thunder":
        return thunder.jit(
            fwd_fn, nv_enable_bookend=False, executors=[nvfuserex], **kwargs
        )
    if executor == "thunder-torchcompile":
        return thunder.jit(fwd_fn, executors=["torchcompile"], **kwargs)

def compute_total_iobytes(
    tensor_props: dict[str, tuple[int | tuple[int, ...], torch.dtype]],
):
    """
    Compute IObytes for baselines from given description:
    Tensor_props has entries of the form: {'tensor_id': (size: tuple, dtype: torch.dtype)}
    """
    iobytes = 0
    for _, tensor_prop in tensor_props.items():
        size, dtype = tensor_prop[0], tensor_prop[1]
        if isinstance(size, tuple):
            iobytes += np.prod(size) * dtype.itemsize
        else:
            iobytes += size * dtype.itemsize
    return int(iobytes)

class NVFBenchmark:
    """
    A wrapper class around pytest-benchmark to support
    torchprofiler-based timer and metric computation.
    """

    def __init__(
        self, benchmark_fixture, device: str = "cuda", precision: float = 1e-6
    ):
        """
        Arguments:
            benchmark_fixture: pytest-benchmark fixture passed to every
                function intended to be run as a benchmark by pytest
            precision: Precision for the torchprofiler-based timer used.
                Set explicitly to avoid timer calibration.

        Class members:
            self.device: Device type -- "cuda" or "host"
            self.benchmark: Underlying pytest-benchmark fixture
        """
        self.device = device
        self._setup_timer(benchmark_fixture, device, precision)
        self.benchmark = benchmark_fixture

    def _setup_timer(self, benchmark_fixture, device: str, precision: float):
        """Setup the appropriate timer based on device type."""
        if BENCHMARK_CONFIG["with_nsys"]:
            warnings.warn(
                "NSYS is enabled. No profiler will be used and pytest will report wall-clock time. "
                "Please refer to the nsys report for accurate timings."
            )
            return

        # Timer selection based on device
        timer_class = CuptiTimer if device == "cuda" else FusionProfileTimer
        benchmark_fixture._timer = timer_class()
        # Externally set the precision to avoid timer calibration. Since the timer uses CUDA times,
        # calibration using subsequent timer calls produces invalid results.
        # https://github.com/ionelmc/pytest-benchmark/blob/728752d2976ef53fde7e40beb3e55f09cf4d4736/src/pytest_benchmark/timers.py#L15
        benchmark_fixture._precisions[benchmark_fixture._timer] = precision

    def __call__(self, function_to_benchmark: Callable, *args, **kwargs):
        return self.benchmark(function_to_benchmark, *args, **kwargs)

    def __getattr__(self, attr):
        if attr not in self.__dict__:
            return getattr(self.benchmark, attr)
        return super().__getattr__(attr)

    def set_fd(self, fd):
        """
        Set the fd object for fusion profiling.
        fd is returned by setup() for host benchmarking.
        """
        if BENCHMARK_CONFIG["with_nsys"]:
            return
        assert isinstance(self._timer, FusionProfileTimer)
        self._timer.set_fd(fd)

    def cleanup(self):
        if not BENCHMARK_CONFIG["with_nsys"]:
            self._timer.cleanup()

    def set_metrics(
        self,
        inputs: Union[torch.Tensor, List],
        outputs: Union[torch.Tensor, List],
        iobytes: int = None,
    ) -> None:
        """
        Compute metrics for the target function when device = "cuda".

        Args:
            inputs: Inputs to the target function
            outputs: Outputs of the target function
            iobytes (Optional): When given, IO bytes computation is skipped
                and this is used to compute the metrics.

        Current metrics:
            IOBytes: Total bytes in inputs + outputs
            BytesPerSecond: IOBytes * total_rounds / total_time
            Bandwdith (GBps): BytesPerSecond / 1e9
            % Peak Bandwidth (SOL): 100 * Bandwidth /PEAK_BANDWIDTH
        """
        if not iobytes:
            if not isinstance(inputs, Iterable) or isinstance(inputs, torch.Tensor):
                inputs = [inputs]
            if not isinstance(outputs, Iterable) or isinstance(outputs, torch.Tensor):
                outputs = [outputs]

            iobytes = 0
            for inp in inputs:
                if isinstance(inp, torch.Tensor):
                    iobytes += inp.element_size() * inp.numel()

            for out in outputs:
                if isinstance(out, torch.Tensor):
                    iobytes += out.element_size() * out.numel()

        self.benchmark.extra_info["IOBytes"] = iobytes
        bandwidth_bps = (
            iobytes * self.benchmark.stats["rounds"]
        ) / self.benchmark.stats["total"]
        self.benchmark.extra_info["Bandwidth (Bps)"] = bandwidth_bps
        self.benchmark.extra_info["Bandwidth (GBps)"] = bandwidth_bps / 1e9
        self.benchmark.extra_info["% Peak Bandwidth (SOL)"] = (
            100 * (bandwidth_bps / 1e9) / PEAK_BANDWIDTH_GBPS
        )

def run_benchmark(
    benchmark: pytest_benchmark.fixture.BenchmarkFixture,
    benchmark_fn: Callable | None,
    inputs: Union[torch.Tensor, List],
    iobytes: int = None,
    device: str = "cuda",
    fusion_fn: Callable = None,
) -> Union[torch.Tensor, List]:
    """
    Benchmarks the target function using torchprofiler and stores metrics as extra information.

    Arguments:
        benchmark: pytest-benchmark fixture
        benchmark_fn: Target function
        inputs: Inputs to the target function
        iobytes (Optional): When given, IO bytes computation is skipped
                and this is used to compute SOL and bandwidth.
        device (Optional): Default: CUDA, Possible values: ["cuda", "host"].
            Using device="host" is only allowed with nvFuser FusionDefinition.
        fusion_fn (Optional): Must be provided if device = "host".
            fusion_fn should only require FusionDefinition() as the input.
            Use functools.partial if fusion_fn accepts additional arguments.
            See test_many_pointwise_ops.py for example.

    Returns:
        outputs: Output of the target function
    """

    # Check that the device is `cuda` or `host:{compile/steady/dynamic}`.
    assert device.split(":")[0] in [
        "cuda",
        "host",
    ], f'Unsupported device type: {device.split(":")[0]}. Use one of cuda/host.'

    host_bench_mode = None

    # Store warmup rounds locally to modify for host:steady/dynamic cases.
    warmup_rounds = BENCHMARK_CONFIG["warmup_rounds"]

    if device.split(":")[0] == "host":
        # Host benchmarking expects a fusion function to generate fusion definitions everytime FusionCache is reset.
        assert fusion_fn is not None and benchmark_fn is None

        # device = 'host:compile', 'host:steady', 'host:dyanamic'
        # Set the host_bench_mode -- The 3 modes require different setup calls.
        host_bench_mode = device.split(":")[-1]
        device = device.split(":")[0]  # device = 'host'

        # Set the warmup rounds if required for `steady/dynamic` host latency measurement.
        if (
            host_bench_mode in ["steady", "dynamic"]
            and BENCHMARK_CONFIG["warmup_rounds"] == 0
        ):
            # By default, warmup_rounds=1. If BENCHMARK_CONFIG['warmup_rounds'] == 0 through --benchmark-warmup-rounds, raise a warning that it was ignored.
            warnings.warn(
                "--benchmark-warmup-rounds=0 is ignored for host:steady/dynamic benchmarking. Setting warmup_rounds=1."
            )
            warmup_rounds = 1

    """
    Setup function: This is called before each benchmarking round. This function is used to:
    1. Clear L2 cache.
    2. For host latency benchmarks, the 3 modes use different setups.
        a) 'compile': FusionCache is reset at every round to measure the first time overhead before instantiating fd.
        b) 'steady': Nothing additional is required. The warmup round avoids including the first time overhead in the measurements.
        c) 'dynamic': We maintain a global counter to track which input is executed. Once all the inputs have been executed once,
        the FusionCache is reset again and we execute fd for the first input to avoid including the first time compile overhead in the dynamic measurement.
    """

    # Counter used in `dynamic` host latency benchmarking, unused in other cases.
    global counter
    counter = 0

    def setup():
        clear_l2_cache()
        if device == "cuda":
            for inp in inputs:
                if isinstance(inp, torch.Tensor):
                    inp.grad = None
            return [inputs], {}

        # Device = 'host'
        # For device='host', we use the host_benchmark_fn below. It expects a list of fusion inputs, and the fd object.
        assert host_bench_mode in [
            "compile",
            "steady",
            "dynamic",
        ], f"Expected host benchmark mode to be one of compile, steady, or dynamic, found {host_bench_mode}"

        # Instantiate the fusion definition
        with FusionDefinition() as fd:
            fusion_fn(fd)

        if host_bench_mode in ["compile"]:
            return [inputs], {"fd": fd}

        if host_bench_mode in ["steady"]:
            # Run once to compile FusionExecutorCache and avoid measuring first time overhead.
            fd.execute(inputs)
            return [inputs], {"fd": fd}

        # For dynamic host latency benchmarking, return a particular input shape, and reset FusionCache if all inputs have been executed.
        global counter
        counter += 1
        # The current input is counter % len(inputs). Execute fd with next input
        # to avoid measuring first time overhead but avoid cache hit with current input.
        fd.execute(inputs[(counter + 1) % len(inputs)])
        return [inputs[counter % len(inputs)]], {"fd": fd}

    # Create an instance of NVFBenchmark
    nvf_benchmark = NVFBenchmark(benchmark, device=device)

    # The host_benchmark_fn uses the `fd` object returned from setup function.
    def host_benchmark_fn(inputs, fd):
        # Set the fd variable used to query the profile object
        nvf_benchmark.set_fd(fd)
        if not BENCHMARK_CONFIG["with_nsys"]:
            with PythonProfiler() as prof:
                return fd.execute(inputs)
        return fd.execute(inputs)

    benchmark_fn = benchmark_fn if benchmark_fn is not None else host_benchmark_fn

    try:
        outputs = nvf_benchmark.pedantic(
            benchmark_fn,
            setup=setup,
            rounds=BENCHMARK_CONFIG["rounds"],
            warmup_rounds=warmup_rounds,
        )
        if device == "cuda":
            # Record additional metrics (IOBytes, Bandwidth)
            nvf_benchmark.set_metrics(inputs, outputs, iobytes)
        return outputs
    except Exception as e:
        raise RuntimeError(
            f"Exception when running {benchmark_fn.__name__}: {e}"
        ) from e
    finally:
        nvf_benchmark.cleanup()


import thunder
from thunder.executors.nvfuserex import nvfuserex
import importlib.util

# These variables can be overwritten through CLI commands
# --benchmark-rounds=rounds --benchmark-warmup-rounds=warmup_rounds
# --benchmark-num-inputs=num_inputs
BENCHMARK_CONFIG = {
    "rounds": 10,
    "warmup_rounds": 1,
    "num_inputs": None,
    "with_nsys": False,
}

L2_CACHE_SIZE = DEVICE_PROPERTIES["gpu_l2_bytes"]
PEAK_BANDWIDTH_GBPS = DEVICE_PROPERTIES["gpu_peak_bandwidth_gbps"]

DEFAULT_EXECUTORS = ["eager", "torchcompile"]

def check_module_available(module_name):
    # If spec is not None, the module can be imported.
    spec = importlib.util.find_spec(module_name)
    return spec is not None

def clear_l2_cache() -> None:
    """
    Flushes the L2 cache by creating a buffer of the same size.
    """
    n_elements = L2_CACHE_SIZE // 4
    x = torch.empty(n_elements, dtype=torch.float32, device="cuda", requires_grad=False)
    y = torch.clone(x)

def clear_dynamo_cache() -> None:
    """
    Utility function to enforce re-compilation to avoid different results between
    running a serials of tests and a standalone test due to kernel re-use.
    It slows down the test but ensures the correctness of the benchmark results.
    Ref: https://github.com/pytorch/pytorch/issues/107444
    """
    torch._dynamo.reset()


# --benchmark-num-inputs=num_inputs
BENCHMARK_CONFIG = {
    "rounds": 10,
    "warmup_rounds": 1,
    "num_inputs": None,
    "with_nsys": False,
}

L2_CACHE_SIZE = DEVICE_PROPERTIES["gpu_l2_bytes"]
PEAK_BANDWIDTH_GBPS = DEVICE_PROPERTIES["gpu_peak_bandwidth_gbps"]

DEFAULT_EXECUTORS = ["eager", "torchcompile"]

def check_module_available(module_name):
    # If spec is not None, the module can be imported.
    spec = importlib.util.find_spec(module_name)
    return spec is not None

def clear_l2_cache() -> None:
    """
    Flushes the L2 cache by creating a buffer of the same size.
    """
    n_elements = L2_CACHE_SIZE // 4
    x = torch.empty(n_elements, dtype=torch.float32, device="cuda", requires_grad=False)
    y = torch.clone(x)

def clear_dynamo_cache() -> None:
    """
    Utility function to enforce re-compilation to avoid different results between
    running a serials of tests and a standalone test due to kernel re-use.
    It slows down the test but ensures the correctness of the benchmark results.
    Ref: https://github.com/pytorch/pytorch/issues/107444
    """
    torch._dynamo.reset()

# Backward function for torch baseline benchmarks.
# The first two inputs are expected to be out and grad_out. The remaining are inputs of the forward pass used to clear grad between subsequent runs to avoid grad accumulation. See setup() in run_benchmark().
def unary_bwd_torch(inputs: List):  # [output, grad_out, fwd_inputs]
    inputs[0].backward(inputs[1], retain_graph=True)

def with_executor(executor: str, fwd_fn: Callable, **kwargs) -> Callable:
    assert executor in ["eager", "torchcompile", "thunder", "thunder-torchcompile"]


PEAK_BANDWIDTH_GBPS = DEVICE_PROPERTIES["gpu_peak_bandwidth_gbps"]

DEFAULT_EXECUTORS = ["eager", "torchcompile"]

def check_module_available(module_name):
    # If spec is not None, the module can be imported.
    spec = importlib.util.find_spec(module_name)
    return spec is not None

def clear_l2_cache() -> None:
    """
    Flushes the L2 cache by creating a buffer of the same size.
    """
    n_elements = L2_CACHE_SIZE // 4
    x = torch.empty(n_elements, dtype=torch.float32, device="cuda", requires_grad=False)
    y = torch.clone(x)

def clear_dynamo_cache() -> None:
    """
    Utility function to enforce re-compilation to avoid different results between
    running a serials of tests and a standalone test due to kernel re-use.
    It slows down the test but ensures the correctness of the benchmark results.
    Ref: https://github.com/pytorch/pytorch/issues/107444
    """
    torch._dynamo.reset()

# Backward function for torch baseline benchmarks.
# The first two inputs are expected to be out and grad_out. The remaining are inputs of the forward pass used to clear grad between subsequent runs to avoid grad accumulation. See setup() in run_benchmark().
def unary_bwd_torch(inputs: List):  # [output, grad_out, fwd_inputs]
    inputs[0].backward(inputs[1], retain_graph=True)

def with_executor(executor: str, fwd_fn: Callable, **kwargs) -> Callable:
    assert executor in ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
    if executor == "eager":
        return fwd_fn
    if executor == "torchcompile":
        return torch.compile(fwd_fn, **kwargs)
    if executor == "thunder":
        return thunder.jit(
            fwd_fn, nv_enable_bookend=False, executors=[nvfuserex], **kwargs
        )
    if executor == "thunder-torchcompile":
        return thunder.jit(fwd_fn, executors=["torchcompile"], **kwargs)


    """
    Flushes the L2 cache by creating a buffer of the same size.
    """
    n_elements = L2_CACHE_SIZE // 4
    x = torch.empty(n_elements, dtype=torch.float32, device="cuda", requires_grad=False)
    y = torch.clone(x)

def clear_dynamo_cache() -> None:
    """
    Utility function to enforce re-compilation to avoid different results between
    running a serials of tests and a standalone test due to kernel re-use.
    It slows down the test but ensures the correctness of the benchmark results.
    Ref: https://github.com/pytorch/pytorch/issues/107444
    """
    torch._dynamo.reset()

# Backward function for torch baseline benchmarks.
# The first two inputs are expected to be out and grad_out. The remaining are inputs of the forward pass used to clear grad between subsequent runs to avoid grad accumulation. See setup() in run_benchmark().
def unary_bwd_torch(inputs: List):  # [output, grad_out, fwd_inputs]
    inputs[0].backward(inputs[1], retain_graph=True)

def with_executor(executor: str, fwd_fn: Callable, **kwargs) -> Callable:
    assert executor in ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
    if executor == "eager":
        return fwd_fn
    if executor == "torchcompile":
        return torch.compile(fwd_fn, **kwargs)
    if executor == "thunder":
        return thunder.jit(
            fwd_fn, nv_enable_bookend=False, executors=[nvfuserex], **kwargs
        )
    if executor == "thunder-torchcompile":
        return thunder.jit(fwd_fn, executors=["torchcompile"], **kwargs)

def compute_total_iobytes(
    tensor_props: dict[str, tuple[int | tuple[int, ...], torch.dtype]],
):
    """


    x = torch.empty(n_elements, dtype=torch.float32, device="cuda", requires_grad=False)
    y = torch.clone(x)

def clear_dynamo_cache() -> None:
    """
    Utility function to enforce re-compilation to avoid different results between
    running a serials of tests and a standalone test due to kernel re-use.
    It slows down the test but ensures the correctness of the benchmark results.
    Ref: https://github.com/pytorch/pytorch/issues/107444
    """
    torch._dynamo.reset()

# Backward function for torch baseline benchmarks.
# The first two inputs are expected to be out and grad_out. The remaining are inputs of the forward pass used to clear grad between subsequent runs to avoid grad accumulation. See setup() in run_benchmark().
def unary_bwd_torch(inputs: List):  # [output, grad_out, fwd_inputs]
    inputs[0].backward(inputs[1], retain_graph=True)

def with_executor(executor: str, fwd_fn: Callable, **kwargs) -> Callable:
    assert executor in ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
    if executor == "eager":
        return fwd_fn
    if executor == "torchcompile":
        return torch.compile(fwd_fn, **kwargs)
    if executor == "thunder":
        return thunder.jit(
            fwd_fn, nv_enable_bookend=False, executors=[nvfuserex], **kwargs
        )
    if executor == "thunder-torchcompile":
        return thunder.jit(fwd_fn, executors=["torchcompile"], **kwargs)

def compute_total_iobytes(
    tensor_props: dict[str, tuple[int | tuple[int, ...], torch.dtype]],
):
    """
    Compute IObytes for baselines from given description:
    Tensor_props has entries of the form: {'tensor_id': (size: tuple, dtype: torch.dtype)}
    """
    iobytes = 0
    for _, tensor_prop in tensor_props.items():
        size, dtype = tensor_prop[0], tensor_prop[1]
        if isinstance(size, tuple):
            iobytes += np.prod(size) * dtype.itemsize
        else:
            iobytes += size * dtype.itemsize
    return int(iobytes)

class NVFBenchmark:


# Backward function for torch baseline benchmarks.
# The first two inputs are expected to be out and grad_out. The remaining are inputs of the forward pass used to clear grad between subsequent runs to avoid grad accumulation. See setup() in run_benchmark().
def unary_bwd_torch(inputs: List):  # [output, grad_out, fwd_inputs]
    inputs[0].backward(inputs[1], retain_graph=True)

def with_executor(executor: str, fwd_fn: Callable, **kwargs) -> Callable:
    assert executor in ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
    if executor == "eager":
        return fwd_fn
    if executor == "torchcompile":
        return torch.compile(fwd_fn, **kwargs)
    if executor == "thunder":
        return thunder.jit(
            fwd_fn, nv_enable_bookend=False, executors=[nvfuserex], **kwargs
        )
    if executor == "thunder-torchcompile":
        return thunder.jit(fwd_fn, executors=["torchcompile"], **kwargs)

def compute_total_iobytes(
    tensor_props: dict[str, tuple[int | tuple[int, ...], torch.dtype]],
):
    """
    Compute IObytes for baselines from given description:
    Tensor_props has entries of the form: {'tensor_id': (size: tuple, dtype: torch.dtype)}
    """
    iobytes = 0
    for _, tensor_prop in tensor_props.items():
        size, dtype = tensor_prop[0], tensor_prop[1]
        if isinstance(size, tuple):
            iobytes += np.prod(size) * dtype.itemsize
        else:
            iobytes += size * dtype.itemsize
    return int(iobytes)

class NVFBenchmark:
    """
    A wrapper class around pytest-benchmark to support
    torchprofiler-based timer and metric computation.
    """

    def __init__(
        self, benchmark_fixture, device: str = "cuda", precision: float = 1e-6
    ):
        """
        Arguments:
            benchmark_fixture: pytest-benchmark fixture passed to every
                function intended to be run as a benchmark by pytest
            precision: Precision for the torchprofiler-based timer used.
                Set explicitly to avoid timer calibration.

        Class members:
            self.device: Device type -- "cuda" or "host"