NVFBenchmark

        return thunder.jit(fwd_fn, executors=["torchcompile"], **kwargs)

def compute_total_iobytes(
    tensor_props: dict[str, tuple[int | tuple[int, ...], torch.dtype]],
):
    """
    Compute IObytes for baselines from given description:
    Tensor_props has entries of the form: {'tensor_id': (size: tuple, dtype: torch.dtype)}
    """
    iobytes = 0
    for _, tensor_prop in tensor_props.items():
        size, dtype = tensor_prop[0], tensor_prop[1]
        if isinstance(size, tuple):
            iobytes += np.prod(size) * dtype.itemsize
        else:
            iobytes += size * dtype.itemsize
    return int(iobytes)

class NVFBenchmark:
    """
    A wrapper class around pytest-benchmark to support
    torchprofiler-based timer and metric computation.
    """

    def __init__(
        self, benchmark_fixture, device: str = "cuda", precision: float = 1e-6
    ):
        """
        Arguments:
            benchmark_fixture: pytest-benchmark fixture passed to every
                function intended to be run as a benchmark by pytest
            precision: Precision for the torchprofiler-based timer used.
                Set explicitly to avoid timer calibration.

        Class members:
            self.device: Device type -- "cuda" or "host"
            self.benchmark: Underlying pytest-benchmark fixture
        """
        self.device = device
        self._setup_timer(benchmark_fixture, device, precision)
        self.benchmark = benchmark_fixture

    def _setup_timer(self, benchmark_fixture, device: str, precision: float):
        """Setup the appropriate timer based on device type."""
        if BENCHMARK_CONFIG["with_nsys"]:
            warnings.warn(
                "NSYS is enabled. No profiler will be used and pytest will report wall-clock time. "
                "Please refer to the nsys report for accurate timings."
            )
            return

        # Timer selection based on device
        timer_class = CuptiTimer if device == "cuda" else FusionProfileTimer
        benchmark_fixture._timer = timer_class()
        # Externally set the precision to avoid timer calibration. Since the timer uses CUDA times,
        # calibration using subsequent timer calls produces invalid results.
        # https://github.com/ionelmc/pytest-benchmark/blob/728752d2976ef53fde7e40beb3e55f09cf4d4736/src/pytest_benchmark/timers.py#L15
        benchmark_fixture._precisions[benchmark_fixture._timer] = precision

    def __call__(self, function_to_benchmark: Callable, *args, **kwargs):
        return self.benchmark(function_to_benchmark, *args, **kwargs)

    def __getattr__(self, attr):
        if attr not in self.__dict__:
            return getattr(self.benchmark, attr)
        return super().__getattr__(attr)

    def set_fd(self, fd):
        """
        Set the fd object for fusion profiling.
        fd is returned by setup() for host benchmarking.
        """
        if BENCHMARK_CONFIG["with_nsys"]:
            return
        assert isinstance(self._timer, FusionProfileTimer)
        self._timer.set_fd(fd)

    def cleanup(self):
        if not BENCHMARK_CONFIG["with_nsys"]:
            self._timer.cleanup()

    def set_metrics(
        self,
        inputs: Union[torch.Tensor, List],
        outputs: Union[torch.Tensor, List],
        iobytes: int = None,
    ) -> None:
        """
        Compute metrics for the target function when device = "cuda".

        Args:
            inputs: Inputs to the target function
            outputs: Outputs of the target function
            iobytes (Optional): When given, IO bytes computation is skipped
                and this is used to compute the metrics.

        Current metrics:
            IOBytes: Total bytes in inputs + outputs
            BytesPerSecond: IOBytes * total_rounds / total_time
            Bandwdith (GBps): BytesPerSecond / 1e9
            % Peak Bandwidth (SOL): 100 * Bandwidth /PEAK_BANDWIDTH
        """
        if not iobytes:
            if not isinstance(inputs, Iterable) or isinstance(inputs, torch.Tensor):
                inputs = [inputs]
            if not isinstance(outputs, Iterable) or isinstance(outputs, torch.Tensor):
                outputs = [outputs]

            iobytes = 0
            for inp in inputs:
                if isinstance(inp, torch.Tensor):
                    iobytes += inp.element_size() * inp.numel()

            for out in outputs:
                if isinstance(out, torch.Tensor):
                    iobytes += out.element_size() * out.numel()

        self.benchmark.extra_info["IOBytes"] = iobytes
        bandwidth_bps = (
            iobytes * self.benchmark.stats["rounds"]
        ) / self.benchmark.stats["total"]
        self.benchmark.extra_info["Bandwidth (Bps)"] = bandwidth_bps
        self.benchmark.extra_info["Bandwidth (GBps)"] = bandwidth_bps / 1e9
        self.benchmark.extra_info["% Peak Bandwidth (SOL)"] = (
            100 * (bandwidth_bps / 1e9) / PEAK_BANDWIDTH_GBPS
        )

def run_benchmark(
    benchmark: pytest_benchmark.fixture.BenchmarkFixture,
    benchmark_fn: Callable | None,
    inputs: Union[torch.Tensor, List],
    iobytes: int = None,
    device: str = "cuda",
    fusion_fn: Callable = None,
) -> Union[torch.Tensor, List]:
    """
    Benchmarks the target function using torchprofiler and stores metrics as extra information.

    Arguments:
        benchmark: pytest-benchmark fixture
        benchmark_fn: Target function
        inputs: Inputs to the target function
        iobytes (Optional): When given, IO bytes computation is skipped
                and this is used to compute SOL and bandwidth.
        device (Optional): Default: CUDA, Possible values: ["cuda", "host"].


            iobytes = 0
            for inp in inputs:
                if isinstance(inp, torch.Tensor):
                    iobytes += inp.element_size() * inp.numel()

            for out in outputs:
                if isinstance(out, torch.Tensor):
                    iobytes += out.element_size() * out.numel()

        self.benchmark.extra_info["IOBytes"] = iobytes
        bandwidth_bps = (
            iobytes * self.benchmark.stats["rounds"]
        ) / self.benchmark.stats["total"]
        self.benchmark.extra_info["Bandwidth (Bps)"] = bandwidth_bps
        self.benchmark.extra_info["Bandwidth (GBps)"] = bandwidth_bps / 1e9
        self.benchmark.extra_info["% Peak Bandwidth (SOL)"] = (
            100 * (bandwidth_bps / 1e9) / PEAK_BANDWIDTH_GBPS
        )

def run_benchmark(
    benchmark: pytest_benchmark.fixture.BenchmarkFixture,
    benchmark_fn: Callable | None,
    inputs: Union[torch.Tensor, List],
    iobytes: int = None,
    device: str = "cuda",
    fusion_fn: Callable = None,
) -> Union[torch.Tensor, List]:
    """
    Benchmarks the target function using torchprofiler and stores metrics as extra information.

    Arguments:
        benchmark: pytest-benchmark fixture
        benchmark_fn: Target function
        inputs: Inputs to the target function
        iobytes (Optional): When given, IO bytes computation is skipped
                and this is used to compute SOL and bandwidth.
        device (Optional): Default: CUDA, Possible values: ["cuda", "host"].
            Using device="host" is only allowed with nvFuser FusionDefinition.
        fusion_fn (Optional): Must be provided if device = "host".
            fusion_fn should only require FusionDefinition() as the input.
            Use functools.partial if fusion_fn accepts additional arguments.
            See test_many_pointwise_ops.py for example.

    Returns:
        outputs: Output of the target function
    """

    # Check that the device is `cuda` or `host:{compile/steady/dynamic}`.
    assert device.split(":")[0] in [
        "cuda",
        "host",
    ], f'Unsupported device type: {device.split(":")[0]}. Use one of cuda/host.'

    host_bench_mode = None

    # Store warmup rounds locally to modify for host:steady/dynamic cases.
    warmup_rounds = BENCHMARK_CONFIG["warmup_rounds"]

    if device.split(":")[0] == "host":
        # Host benchmarking expects a fusion function to generate fusion definitions everytime FusionCache is reset.
        assert fusion_fn is not None and benchmark_fn is None

        # device = 'host:compile', 'host:steady', 'host:dyanamic'
        # Set the host_bench_mode -- The 3 modes require different setup calls.
        host_bench_mode = device.split(":")[-1]
        device = device.split(":")[0]  # device = 'host'

        # Set the warmup rounds if required for `steady/dynamic` host latency measurement.
        if (
            host_bench_mode in ["steady", "dynamic"]
            and BENCHMARK_CONFIG["warmup_rounds"] == 0
        ):
            # By default, warmup_rounds=1. If BENCHMARK_CONFIG['warmup_rounds'] == 0 through --benchmark-warmup-rounds, raise a warning that it was ignored.
            warnings.warn(
                "--benchmark-warmup-rounds=0 is ignored for host:steady/dynamic benchmarking. Setting warmup_rounds=1."
            )
            warmup_rounds = 1

    """
    Setup function: This is called before each benchmarking round. This function is used to:
    1. Clear L2 cache.
    2. For host latency benchmarks, the 3 modes use different setups.
        a) 'compile': FusionCache is reset at every round to measure the first time overhead before instantiating fd.
        b) 'steady': Nothing additional is required. The warmup round avoids including the first time overhead in the measurements.
        c) 'dynamic': We maintain a global counter to track which input is executed. Once all the inputs have been executed once,
        the FusionCache is reset again and we execute fd for the first input to avoid including the first time compile overhead in the dynamic measurement.
    """

    # Counter used in `dynamic` host latency benchmarking, unused in other cases.
    global counter
    counter = 0

    def setup():
        clear_l2_cache()
        if device == "cuda":
            for inp in inputs:
                if isinstance(inp, torch.Tensor):
                    inp.grad = None
            return [inputs], {}

        # Device = 'host'
        # For device='host', we use the host_benchmark_fn below. It expects a list of fusion inputs, and the fd object.
        assert host_bench_mode in [
            "compile",
            "steady",
            "dynamic",
        ], f"Expected host benchmark mode to be one of compile, steady, or dynamic, found {host_bench_mode}"

        # Instantiate the fusion definition
        with FusionDefinition() as fd:
            fusion_fn(fd)

        if host_bench_mode in ["compile"]:
            return [inputs], {"fd": fd}

        if host_bench_mode in ["steady"]:
            # Run once to compile FusionExecutorCache and avoid measuring first time overhead.
            fd.execute(inputs)
            return [inputs], {"fd": fd}

        # For dynamic host latency benchmarking, return a particular input shape, and reset FusionCache if all inputs have been executed.
        global counter
        counter += 1
        # The current input is counter % len(inputs). Execute fd with next input
        # to avoid measuring first time overhead but avoid cache hit with current input.
        fd.execute(inputs[(counter + 1) % len(inputs)])
        return [inputs[counter % len(inputs)]], {"fd": fd}

    # Create an instance of NVFBenchmark
    nvf_benchmark = NVFBenchmark(benchmark, device=device)

    # The host_benchmark_fn uses the `fd` object returned from setup function.
    def host_benchmark_fn(inputs, fd):
        # Set the fd variable used to query the profile object
        nvf_benchmark.set_fd(fd)
        if not BENCHMARK_CONFIG["with_nsys"]:
            with PythonProfiler() as prof:
                return fd.execute(inputs)
        return fd.execute(inputs)

    benchmark_fn = benchmark_fn if benchmark_fn is not None else host_benchmark_fn

    try:
        outputs = nvf_benchmark.pedantic(
            benchmark_fn,
            setup=setup,
            rounds=BENCHMARK_CONFIG["rounds"],
            warmup_rounds=warmup_rounds,
        )
        if device == "cuda":
            # Record additional metrics (IOBytes, Bandwidth)
            nvf_benchmark.set_metrics(inputs, outputs, iobytes)
        return outputs
    except Exception as e:
        raise RuntimeError(
            f"Exception when running {benchmark_fn.__name__}: {e}"
        ) from e
    finally:
        nvf_benchmark.cleanup()


    """
    Compute IObytes for baselines from given description:
    Tensor_props has entries of the form: {'tensor_id': (size: tuple, dtype: torch.dtype)}
    """
    iobytes = 0
    for _, tensor_prop in tensor_props.items():
        size, dtype = tensor_prop[0], tensor_prop[1]
        if isinstance(size, tuple):
            iobytes += np.prod(size) * dtype.itemsize
        else:
            iobytes += size * dtype.itemsize
    return int(iobytes)

class NVFBenchmark:
    """
    A wrapper class around pytest-benchmark to support
    torchprofiler-based timer and metric computation.
    """

    def __init__(
        self, benchmark_fixture, device: str = "cuda", precision: float = 1e-6
    ):
        """
        Arguments:
            benchmark_fixture: pytest-benchmark fixture passed to every
                function intended to be run as a benchmark by pytest
            precision: Precision for the torchprofiler-based timer used.
                Set explicitly to avoid timer calibration.

        Class members:
            self.device: Device type -- "cuda" or "host"
            self.benchmark: Underlying pytest-benchmark fixture
        """
        self.device = device
        self._setup_timer(benchmark_fixture, device, precision)
        self.benchmark = benchmark_fixture

    def _setup_timer(self, benchmark_fixture, device: str, precision: float):
        """Setup the appropriate timer based on device type."""
        if BENCHMARK_CONFIG["with_nsys"]:
            warnings.warn(
                "NSYS is enabled. No profiler will be used and pytest will report wall-clock time. "
                "Please refer to the nsys report for accurate timings."
            )
            return

        # Timer selection based on device
        timer_class = CuptiTimer if device == "cuda" else FusionProfileTimer
        benchmark_fixture._timer = timer_class()
        # Externally set the precision to avoid timer calibration. Since the timer uses CUDA times,
        # calibration using subsequent timer calls produces invalid results.
        # https://github.com/ionelmc/pytest-benchmark/blob/728752d2976ef53fde7e40beb3e55f09cf4d4736/src/pytest_benchmark/timers.py#L15
        benchmark_fixture._precisions[benchmark_fixture._timer] = precision

    def __call__(self, function_to_benchmark: Callable, *args, **kwargs):
        return self.benchmark(function_to_benchmark, *args, **kwargs)


    """

    def __init__(
        self, benchmark_fixture, device: str = "cuda", precision: float = 1e-6
    ):
        """
        Arguments:
            benchmark_fixture: pytest-benchmark fixture passed to every
                function intended to be run as a benchmark by pytest
            precision: Precision for the torchprofiler-based timer used.
                Set explicitly to avoid timer calibration.

        Class members:
            self.device: Device type -- "cuda" or "host"
            self.benchmark: Underlying pytest-benchmark fixture
        """
        self.device = device
        self._setup_timer(benchmark_fixture, device, precision)
        self.benchmark = benchmark_fixture

    def _setup_timer(self, benchmark_fixture, device: str, precision: float):
        """Setup the appropriate timer based on device type."""
        if BENCHMARK_CONFIG["with_nsys"]:
            warnings.warn(
                "NSYS is enabled. No profiler will be used and pytest will report wall-clock time. "
                "Please refer to the nsys report for accurate timings."
            )
            return

        # Timer selection based on device
        timer_class = CuptiTimer if device == "cuda" else FusionProfileTimer
        benchmark_fixture._timer = timer_class()
        # Externally set the precision to avoid timer calibration. Since the timer uses CUDA times,
        # calibration using subsequent timer calls produces invalid results.
        # https://github.com/ionelmc/pytest-benchmark/blob/728752d2976ef53fde7e40beb3e55f09cf4d4736/src/pytest_benchmark/timers.py#L15
        benchmark_fixture._precisions[benchmark_fixture._timer] = precision

    def __call__(self, function_to_benchmark: Callable, *args, **kwargs):
        return self.benchmark(function_to_benchmark, *args, **kwargs)

    def __getattr__(self, attr):
        if attr not in self.__dict__:
            return getattr(self.benchmark, attr)
        return super().__getattr__(attr)

    def set_fd(self, fd):
        """
        Set the fd object for fusion profiling.
        fd is returned by setup() for host benchmarking.
        """
        if BENCHMARK_CONFIG["with_nsys"]:
            return
        assert isinstance(self._timer, FusionProfileTimer)
        self._timer.set_fd(fd)

    def cleanup(self):


        self._setup_timer(benchmark_fixture, device, precision)
        self.benchmark = benchmark_fixture

    def _setup_timer(self, benchmark_fixture, device: str, precision: float):
        """Setup the appropriate timer based on device type."""
        if BENCHMARK_CONFIG["with_nsys"]:
            warnings.warn(
                "NSYS is enabled. No profiler will be used and pytest will report wall-clock time. "
                "Please refer to the nsys report for accurate timings."
            )
            return

        # Timer selection based on device
        timer_class = CuptiTimer if device == "cuda" else FusionProfileTimer
        benchmark_fixture._timer = timer_class()
        # Externally set the precision to avoid timer calibration. Since the timer uses CUDA times,
        # calibration using subsequent timer calls produces invalid results.
        # https://github.com/ionelmc/pytest-benchmark/blob/728752d2976ef53fde7e40beb3e55f09cf4d4736/src/pytest_benchmark/timers.py#L15
        benchmark_fixture._precisions[benchmark_fixture._timer] = precision

    def __call__(self, function_to_benchmark: Callable, *args, **kwargs):
        return self.benchmark(function_to_benchmark, *args, **kwargs)

    def __getattr__(self, attr):
        if attr not in self.__dict__:
            return getattr(self.benchmark, attr)
        return super().__getattr__(attr)

    def set_fd(self, fd):
        """
        Set the fd object for fusion profiling.
        fd is returned by setup() for host benchmarking.
        """
        if BENCHMARK_CONFIG["with_nsys"]:
            return
        assert isinstance(self._timer, FusionProfileTimer)
        self._timer.set_fd(fd)

    def cleanup(self):
        if not BENCHMARK_CONFIG["with_nsys"]:
            self._timer.cleanup()


    def _setup_timer(self, benchmark_fixture, device: str, precision: float):
        """Setup the appropriate timer based on device type."""
        if BENCHMARK_CONFIG["with_nsys"]:
            warnings.warn(
                "NSYS is enabled. No profiler will be used and pytest will report wall-clock time. "
                "Please refer to the nsys report for accurate timings."
            )
            return

        # Timer selection based on device
        timer_class = CuptiTimer if device == "cuda" else FusionProfileTimer
        benchmark_fixture._timer = timer_class()
        # Externally set the precision to avoid timer calibration. Since the timer uses CUDA times,
        # calibration using subsequent timer calls produces invalid results.
        # https://github.com/ionelmc/pytest-benchmark/blob/728752d2976ef53fde7e40beb3e55f09cf4d4736/src/pytest_benchmark/timers.py#L15
        benchmark_fixture._precisions[benchmark_fixture._timer] = precision

    def __call__(self, function_to_benchmark: Callable, *args, **kwargs):
        return self.benchmark(function_to_benchmark, *args, **kwargs)

    def __getattr__(self, attr):
        if attr not in self.__dict__:
            return getattr(self.benchmark, attr)
        return super().__getattr__(attr)

    def set_fd(self, fd):
        """
        Set the fd object for fusion profiling.
        fd is returned by setup() for host benchmarking.
        """
        if BENCHMARK_CONFIG["with_nsys"]:
            return
        assert isinstance(self._timer, FusionProfileTimer)
        self._timer.set_fd(fd)

    def cleanup(self):
        if not BENCHMARK_CONFIG["with_nsys"]:
            self._timer.cleanup()

    def set_metrics(
        self,
        inputs: Union[torch.Tensor, List],
        outputs: Union[torch.Tensor, List],
        iobytes: int = None,


                "Please refer to the nsys report for accurate timings."
            )
            return

        # Timer selection based on device
        timer_class = CuptiTimer if device == "cuda" else FusionProfileTimer
        benchmark_fixture._timer = timer_class()
        # Externally set the precision to avoid timer calibration. Since the timer uses CUDA times,
        # calibration using subsequent timer calls produces invalid results.
        # https://github.com/ionelmc/pytest-benchmark/blob/728752d2976ef53fde7e40beb3e55f09cf4d4736/src/pytest_benchmark/timers.py#L15
        benchmark_fixture._precisions[benchmark_fixture._timer] = precision

    def __call__(self, function_to_benchmark: Callable, *args, **kwargs):
        return self.benchmark(function_to_benchmark, *args, **kwargs)

    def __getattr__(self, attr):
        if attr not in self.__dict__:
            return getattr(self.benchmark, attr)
        return super().__getattr__(attr)

    def set_fd(self, fd):
        """
        Set the fd object for fusion profiling.
        fd is returned by setup() for host benchmarking.
        """
        if BENCHMARK_CONFIG["with_nsys"]:
            return
        assert isinstance(self._timer, FusionProfileTimer)
        self._timer.set_fd(fd)

    def cleanup(self):
        if not BENCHMARK_CONFIG["with_nsys"]:
            self._timer.cleanup()

    def set_metrics(
        self,
        inputs: Union[torch.Tensor, List],
        outputs: Union[torch.Tensor, List],
        iobytes: int = None,
    ) -> None:
        """
        Compute metrics for the target function when device = "cuda".

        Args:
            inputs: Inputs to the target function
            outputs: Outputs of the target function
            iobytes (Optional): When given, IO bytes computation is skipped
                and this is used to compute the metrics.


cleanup

        benchmark_fixture._precisions[benchmark_fixture._timer] = precision

    def __call__(self, function_to_benchmark: Callable, *args, **kwargs):
        return self.benchmark(function_to_benchmark, *args, **kwargs)

    def __getattr__(self, attr):
        if attr not in self.__dict__:
            return getattr(self.benchmark, attr)
        return super().__getattr__(attr)

    def set_fd(self, fd):
        """
        Set the fd object for fusion profiling.
        fd is returned by setup() for host benchmarking.
        """
        if BENCHMARK_CONFIG["with_nsys"]:
            return
        assert isinstance(self._timer, FusionProfileTimer)
        self._timer.set_fd(fd)

    def cleanup(self):
        if not BENCHMARK_CONFIG["with_nsys"]:
            self._timer.cleanup()

    def set_metrics(
        self,
        inputs: Union[torch.Tensor, List],
        outputs: Union[torch.Tensor, List],
        iobytes: int = None,
    ) -> None:
        """
        Compute metrics for the target function when device = "cuda".

        Args:
            inputs: Inputs to the target function
            outputs: Outputs of the target function
            iobytes (Optional): When given, IO bytes computation is skipped
                and this is used to compute the metrics.

        Current metrics:
            IOBytes: Total bytes in inputs + outputs
            BytesPerSecond: IOBytes * total_rounds / total_time
            Bandwdith (GBps): BytesPerSecond / 1e9


    def __getattr__(self, attr):
        if attr not in self.__dict__:
            return getattr(self.benchmark, attr)
        return super().__getattr__(attr)

    def set_fd(self, fd):
        """
        Set the fd object for fusion profiling.
        fd is returned by setup() for host benchmarking.
        """
        if BENCHMARK_CONFIG["with_nsys"]:
            return
        assert isinstance(self._timer, FusionProfileTimer)
        self._timer.set_fd(fd)

    def cleanup(self):
        if not BENCHMARK_CONFIG["with_nsys"]:
            self._timer.cleanup()

    def set_metrics(
        self,
        inputs: Union[torch.Tensor, List],
        outputs: Union[torch.Tensor, List],
        iobytes: int = None,
    ) -> None:
        """
        Compute metrics for the target function when device = "cuda".

        Args:
            inputs: Inputs to the target function
            outputs: Outputs of the target function
            iobytes (Optional): When given, IO bytes computation is skipped
                and this is used to compute the metrics.

        Current metrics:
            IOBytes: Total bytes in inputs + outputs
            BytesPerSecond: IOBytes * total_rounds / total_time
            Bandwdith (GBps): BytesPerSecond / 1e9
            % Peak Bandwidth (SOL): 100 * Bandwidth /PEAK_BANDWIDTH
        """
        if not iobytes:
            if not isinstance(inputs, Iterable) or isinstance(inputs, torch.Tensor):
                inputs = [inputs]
            if not isinstance(outputs, Iterable) or isinstance(outputs, torch.Tensor):
                outputs = [outputs]

            iobytes = 0
            for inp in inputs:
                if isinstance(inp, torch.Tensor):
                    iobytes += inp.element_size() * inp.numel()

            for out in outputs:
                if isinstance(out, torch.Tensor):
                    iobytes += out.element_size() * out.numel()

        self.benchmark.extra_info["IOBytes"] = iobytes
        bandwidth_bps = (
            iobytes * self.benchmark.stats["rounds"]
        ) / self.benchmark.stats["total"]
        self.benchmark.extra_info["Bandwidth (Bps)"] = bandwidth_bps
        self.benchmark.extra_info["Bandwidth (GBps)"] = bandwidth_bps / 1e9
        self.benchmark.extra_info["% Peak Bandwidth (SOL)"] = (
            100 * (bandwidth_bps / 1e9) / PEAK_BANDWIDTH_GBPS
        )

def run_benchmark(
    benchmark: pytest_benchmark.fixture.BenchmarkFixture,
    benchmark_fn: Callable | None,
    inputs: Union[torch.Tensor, List],
    iobytes: int = None,
    device: str = "cuda",
    fusion_fn: Callable = None,
) -> Union[torch.Tensor, List]:
    """
    Benchmarks the target function using torchprofiler and stores metrics as extra information.

    Arguments:
        benchmark: pytest-benchmark fixture
        benchmark_fn: Target function
        inputs: Inputs to the target function
        iobytes (Optional): When given, IO bytes computation is skipped
                and this is used to compute SOL and bandwidth.
        device (Optional): Default: CUDA, Possible values: ["cuda", "host"].


setup

            # By default, warmup_rounds=1. If BENCHMARK_CONFIG['warmup_rounds'] == 0 through --benchmark-warmup-rounds, raise a warning that it was ignored.
            warnings.warn(
                "--benchmark-warmup-rounds=0 is ignored for host:steady/dynamic benchmarking. Setting warmup_rounds=1."
            )
            warmup_rounds = 1

    """
    Setup function: This is called before each benchmarking round. This function is used to:
    1. Clear L2 cache.
    2. For host latency benchmarks, the 3 modes use different setups.
        a) 'compile': FusionCache is reset at every round to measure the first time overhead before instantiating fd.
        b) 'steady': Nothing additional is required. The warmup round avoids including the first time overhead in the measurements.
        c) 'dynamic': We maintain a global counter to track which input is executed. Once all the inputs have been executed once,
        the FusionCache is reset again and we execute fd for the first input to avoid including the first time compile overhead in the dynamic measurement.
    """

    # Counter used in `dynamic` host latency benchmarking, unused in other cases.
    global counter
    counter = 0

    def setup():
        clear_l2_cache()
        if device == "cuda":
            for inp in inputs:
                if isinstance(inp, torch.Tensor):
                    inp.grad = None
            return [inputs], {}

        # Device = 'host'
        # For device='host', we use the host_benchmark_fn below. It expects a list of fusion inputs, and the fd object.
        assert host_bench_mode in [
            "compile",
            "steady",
            "dynamic",
        ], f"Expected host benchmark mode to be one of compile, steady, or dynamic, found {host_bench_mode}"

        # Instantiate the fusion definition
        with FusionDefinition() as fd:
            fusion_fn(fd)

        if host_bench_mode in ["compile"]:
            return [inputs], {"fd": fd}

        if host_bench_mode in ["steady"]:
            # Run once to compile FusionExecutorCache and avoid measuring first time overhead.
            fd.execute(inputs)
            return [inputs], {"fd": fd}

        # For dynamic host latency benchmarking, return a particular input shape, and reset FusionCache if all inputs have been executed.
        global counter
        counter += 1
        # The current input is counter % len(inputs). Execute fd with next input
        # to avoid measuring first time overhead but avoid cache hit with current input.
        fd.execute(inputs[(counter + 1) % len(inputs)])
        return [inputs[counter % len(inputs)]], {"fd": fd}

    # Create an instance of NVFBenchmark
    nvf_benchmark = NVFBenchmark(benchmark, device=device)

    # The host_benchmark_fn uses the `fd` object returned from setup function.
    def host_benchmark_fn(inputs, fd):
        # Set the fd variable used to query the profile object
        nvf_benchmark.set_fd(fd)
        if not BENCHMARK_CONFIG["with_nsys"]:
            with PythonProfiler() as prof:
                return fd.execute(inputs)
        return fd.execute(inputs)

    benchmark_fn = benchmark_fn if benchmark_fn is not None else host_benchmark_fn

    try:
        outputs = nvf_benchmark.pedantic(
            benchmark_fn,
            setup=setup,
            rounds=BENCHMARK_CONFIG["rounds"],


        if host_bench_mode in ["compile"]:
            return [inputs], {"fd": fd}

        if host_bench_mode in ["steady"]:
            # Run once to compile FusionExecutorCache and avoid measuring first time overhead.
            fd.execute(inputs)
            return [inputs], {"fd": fd}

        # For dynamic host latency benchmarking, return a particular input shape, and reset FusionCache if all inputs have been executed.
        global counter
        counter += 1
        # The current input is counter % len(inputs). Execute fd with next input
        # to avoid measuring first time overhead but avoid cache hit with current input.
        fd.execute(inputs[(counter + 1) % len(inputs)])
        return [inputs[counter % len(inputs)]], {"fd": fd}

    # Create an instance of NVFBenchmark
    nvf_benchmark = NVFBenchmark(benchmark, device=device)

    # The host_benchmark_fn uses the `fd` object returned from setup function.
    def host_benchmark_fn(inputs, fd):
        # Set the fd variable used to query the profile object
        nvf_benchmark.set_fd(fd)
        if not BENCHMARK_CONFIG["with_nsys"]:
            with PythonProfiler() as prof:
                return fd.execute(inputs)
        return fd.execute(inputs)

    benchmark_fn = benchmark_fn if benchmark_fn is not None else host_benchmark_fn

    try:
        outputs = nvf_benchmark.pedantic(
            benchmark_fn,
            setup=setup,
            rounds=BENCHMARK_CONFIG["rounds"],
            warmup_rounds=warmup_rounds,
        )
        if device == "cuda":
            # Record additional metrics (IOBytes, Bandwidth)
            nvf_benchmark.set_metrics(inputs, outputs, iobytes)
        return outputs
    except Exception as e:
        raise RuntimeError(
            f"Exception when running {benchmark_fn.__name__}: {e}"
        ) from e
    finally:
        nvf_benchmark.cleanup()


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
from .core import (
    run_benchmark,
    clear_dynamo_cache,
    unary_bwd_torch,
    with_executor,
    DEFAULT_EXECUTORS,
)
import torch
from .global_params import generate_input_sizes, FLOAT_DTYPES
import numpy as np
from .torch_ops import softmax

def softmax_bwd_fusion(
    fd: FusionDefinition, dtype: DataType, reduction_axis: int
) -> None:
    T0 = fd.define_tensor(
        shape=[-1, -1],
        contiguity=[True, True],
        dtype=dtype,
        is_cpu=False,
    )
    T1 = fd.define_tensor(
        shape=[-1, -1],
        contiguity=[True, True],
        dtype=dtype,
        is_cpu=False,
    )

    if dtype is not DataType.Float:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T1 = fd.ops.cast(T1, dtype=DataType.Float)

    T4 = fd.ops.mul(T0, T1)
    T5 = fd.ops.sum(T4, dims=[reduction_axis], keepdim=False, dtype=DataType.Null)

    if reduction_axis:
        shape_v9 = [T0.size(0), 1]
    else:
        shape_v9 = [1, T0.size(1)]
    bcast_dim = 1 - reduction_axis

    T10 = fd.ops.broadcast_in_dim(T5, shape=shape_v9, broadcast_dims=[bcast_dim])

    if dtype is not DataType.Float:
        T10 = fd.ops.cast(T10, dtype=dtype)

    T16 = fd.ops.broadcast_in_dim(
        T10, shape=[T0.size(0), T0.size(1)], broadcast_dims=[0, 1]
    )

    if dtype is not DataType.Float:
        T16 = fd.ops.cast(T16, dtype=DataType.Float)

    T18 = fd.ops.sub(T1, T16)
    T19 = fd.ops.mul(T0, T18)

    if dtype is not DataType.Float:
        T19 = fd.ops.cast(T19, dtype=dtype)
    fd.add_output(T19)

def softmax_bwd_iobytes(size: tuple, dtype: torch.dtype):
    # Total IO bytes = output + grad_out + grad_input
    return int(np.prod(size) * dtype.itemsize * 3)

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "reduction_axis",
    [
        pytest.param(0, marks=pytest.mark.outer_persistent),
        pytest.param(1, marks=pytest.mark.inner_persistent),
    ],
)
def test_softmax_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    reduction_axis: int,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    inputs = [
        torch.randn(size, device="cuda", dtype=dtype, requires_grad=True),
        torch.randn(size, device="cuda", dtype=dtype),
    ]

    with FusionDefinition() as fd:
        softmax_bwd_fusion(fd, torch_dtype_to_nvfuser_dtype(dtype), reduction_axis)

    if not disable_validation:
        eager_output = torch.nn.functional.softmax(inputs[0], dim=reduction_axis)
        eager_output.backward(inputs[1])
        fd.validate([eager_output, inputs[1]], [inputs[0].grad])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, inputs)

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "reduction_axis",
    [
        pytest.param(0, marks=pytest.mark.outer_persistent),
        pytest.param(1, marks=pytest.mark.inner_persistent),
    ],
)
def test_softmax_bwd_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    reduction_axis: int,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    inputs = torch.randn(size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(size, device="cuda", dtype=dtype)

    fwd_fn = with_executor(executor, softmax)
    fwd_inputs = [inputs, reduction_axis]
    outputs = fwd_fn(fwd_inputs)

    run_benchmark(
        benchmark,
        unary_bwd_torch,
        [outputs, grads, *fwd_inputs],
        iobytes=softmax_bwd_iobytes(size, dtype),
    )


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
from .core import (
    run_benchmark,
    clear_dynamo_cache,
    unary_bwd_torch,
    with_executor,
    DEFAULT_EXECUTORS,
)
import torch
from .global_params import generate_input_sizes, FLOAT_DTYPES
import numpy as np
from .torch_ops import softmax

def softmax_bwd_fusion(
    fd: FusionDefinition, dtype: DataType, reduction_axis: int
) -> None:
    T0 = fd.define_tensor(
        shape=[-1, -1],
        contiguity=[True, True],
        dtype=dtype,
        is_cpu=False,
    )
    T1 = fd.define_tensor(
        shape=[-1, -1],
        contiguity=[True, True],
        dtype=dtype,
        is_cpu=False,
    )

    if dtype is not DataType.Float:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T1 = fd.ops.cast(T1, dtype=DataType.Float)

    T4 = fd.ops.mul(T0, T1)
    T5 = fd.ops.sum(T4, dims=[reduction_axis], keepdim=False, dtype=DataType.Null)

    if reduction_axis:
        shape_v9 = [T0.size(0), 1]
    else:
        shape_v9 = [1, T0.size(1)]
    bcast_dim = 1 - reduction_axis

    T10 = fd.ops.broadcast_in_dim(T5, shape=shape_v9, broadcast_dims=[bcast_dim])

    if dtype is not DataType.Float:
        T10 = fd.ops.cast(T10, dtype=dtype)

    T16 = fd.ops.broadcast_in_dim(
        T10, shape=[T0.size(0), T0.size(1)], broadcast_dims=[0, 1]
    )

    if dtype is not DataType.Float:
        T16 = fd.ops.cast(T16, dtype=DataType.Float)

    T18 = fd.ops.sub(T1, T16)
    T19 = fd.ops.mul(T0, T18)

    if dtype is not DataType.Float:
        T19 = fd.ops.cast(T19, dtype=dtype)
    fd.add_output(T19)

def softmax_bwd_iobytes(size: tuple, dtype: torch.dtype):
    # Total IO bytes = output + grad_out + grad_input
    return int(np.prod(size) * dtype.itemsize * 3)

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "reduction_axis",
    [
        pytest.param(0, marks=pytest.mark.outer_persistent),
        pytest.param(1, marks=pytest.mark.inner_persistent),
    ],
)
def test_softmax_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,


    T10 = fd.ops.broadcast_in_dim(T5, shape=shape_v9, broadcast_dims=[bcast_dim])

    if dtype is not DataType.Float:
        T10 = fd.ops.cast(T10, dtype=dtype)

    T16 = fd.ops.broadcast_in_dim(
        T10, shape=[T0.size(0), T0.size(1)], broadcast_dims=[0, 1]
    )

    if dtype is not DataType.Float:
        T16 = fd.ops.cast(T16, dtype=DataType.Float)

    T18 = fd.ops.sub(T1, T16)
    T19 = fd.ops.mul(T0, T18)

    if dtype is not DataType.Float:
        T19 = fd.ops.cast(T19, dtype=dtype)
    fd.add_output(T19)

def softmax_bwd_iobytes(size: tuple, dtype: torch.dtype):
    # Total IO bytes = output + grad_out + grad_input
    return int(np.prod(size) * dtype.itemsize * 3)

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "reduction_axis",
    [
        pytest.param(0, marks=pytest.mark.outer_persistent),
        pytest.param(1, marks=pytest.mark.inner_persistent),
    ],
)
def test_softmax_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    reduction_axis: int,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    inputs = [


    if dtype is not DataType.Float:
        T19 = fd.ops.cast(T19, dtype=dtype)
    fd.add_output(T19)

def softmax_bwd_iobytes(size: tuple, dtype: torch.dtype):
    # Total IO bytes = output + grad_out + grad_input
    return int(np.prod(size) * dtype.itemsize * 3)

@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "reduction_axis",
    [
        pytest.param(0, marks=pytest.mark.outer_persistent),
        pytest.param(1, marks=pytest.mark.inner_persistent),
    ],
)
def test_softmax_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    reduction_axis: int,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    inputs = [
        torch.randn(size, device="cuda", dtype=dtype, requires_grad=True),
        torch.randn(size, device="cuda", dtype=dtype),
    ]

    with FusionDefinition() as fd:
        softmax_bwd_fusion(fd, torch_dtype_to_nvfuser_dtype(dtype), reduction_axis)

    if not disable_validation:
        eager_output = torch.nn.functional.softmax(inputs[0], dim=reduction_axis)
        eager_output.backward(inputs[1])
        fd.validate([eager_output, inputs[1]], [inputs[0].grad])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, inputs)

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "reduction_axis",
    [
        pytest.param(0, marks=pytest.mark.outer_persistent),
        pytest.param(1, marks=pytest.mark.inner_persistent),
    ],
)
def test_softmax_bwd_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    reduction_axis: int,
    executor: str,
):
    if executor == "torchcompile":


    if not disable_validation:
        eager_output = torch.nn.functional.softmax(inputs[0], dim=reduction_axis)
        eager_output.backward(inputs[1])
        fd.validate([eager_output, inputs[1]], [inputs[0].grad])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, inputs)

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size", generate_input_sizes(dims=2))
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "reduction_axis",
    [
        pytest.param(0, marks=pytest.mark.outer_persistent),
        pytest.param(1, marks=pytest.mark.inner_persistent),
    ],
)
def test_softmax_bwd_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    reduction_axis: int,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    inputs = torch.randn(size, device="cuda", dtype=dtype, requires_grad=True)
    grads = torch.randn(size, device="cuda", dtype=dtype)

    fwd_fn = with_executor(executor, softmax)
    fwd_inputs = [inputs, reduction_axis]
    outputs = fwd_fn(fwd_inputs)

    run_benchmark(
        benchmark,
        unary_bwd_torch,
        [outputs, grads, *fwd_inputs],
        iobytes=softmax_bwd_iobytes(size, dtype),
    )


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest

import torch
from nvfuser_direct import FusionDefinition, DataType
from .core import (
    run_benchmark,
    with_executor,
    unary_bwd_torch,
    clear_dynamo_cache,
    check_module_available,
)
from .cross_entropy_loss import (
    cross_entropy_loss_setup,
    SyntheticMiniModel,
)
from .torch_ops import cross_entropy as torch_cross_entropy_fwd

if check_module_available("quack"):
    from quack.cross_entropy import cross_entropy_fwd as quack_cross_entropy_fwd

@pytest.mark.parametrize(
    "variation",
    [
        "hf_qwen2",
        "hf_phi3",
        "hf_mistral_nemo",
    ],
)
@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
def test_cross_entropy_fwd_benchmark(
    benchmark,
    variation: str,
    executor: str,
):
    kwargs = {}
    if executor == "torchcompile":
        clear_dynamo_cache()

    test_case = cross_entropy_loss_setup[variation](dtype=torch.bfloat16)
    inputs = test_case.inputs()
    model = test_case.model()

    def fwd_call(inp):
        return model(**inp)

    # Compile the fwd fn for torchcompile
    benchmark_fn = with_executor(executor, fwd_call, **kwargs)
    run_benchmark(benchmark, benchmark_fn, inputs)

@pytest.mark.parametrize(
    "variation",
    [
        "hf_qwen2",
        "hf_phi3",
        "hf_mistral_nemo",
    ],
)
@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
def test_cross_entropy_bwd_benchmark(
    benchmark,
    variation: str,
    executor: str,
):
    kwargs = {}
    if executor == "torchcompile":
        clear_dynamo_cache()

    test_case = cross_entropy_loss_setup[variation](dtype=torch.bfloat16)
    fwd_inputs = test_case.inputs()
    model = test_case.model()

    def fwd_call(inp):
        return model(**inp)

    # execute the compiled fwd fn
    fwd_fn = with_executor(executor, fwd_call, **kwargs)
    outputs = fwd_fn(fwd_inputs)

    assert len(outputs) == 1

    run_benchmark(
        benchmark,
        unary_bwd_torch,
        [outputs[0], test_case.grads(), *fwd_inputs, *list(model.parameters())],
        iobytes=test_case.grad_iobytes(),
    )

@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
@pytest.mark.parametrize("vocab_size", SyntheticMiniModel.generate_vocab_sizes())
def test_cross_entropy_mini_benchmark_fwd(benchmark, executor: str, vocab_size: int):
    if executor == "torchcompile":
        clear_dynamo_cache()

    # picking a value that doesn't OOM for large vocab sizes
    batch_size = 4096

    def fwd_call(inp):
        return SyntheticMiniModel.mini_model(*inp)

    inputs = SyntheticMiniModel.inputs(int(batch_size), int(vocab_size))

    fwd_fn = with_executor(executor, fwd_call)
    run_benchmark(benchmark, fwd_fn, inputs)

@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
@pytest.mark.parametrize("vocab_size", SyntheticMiniModel.generate_vocab_sizes())
def test_cross_entropy_mini_benchmark_bwd(benchmark, executor: str, vocab_size: int):
    if executor == "torchcompile":
        clear_dynamo_cache()

    # picking a value that doesn't OOM for large vocab sizes
    batch_size = 4096

    def fwd_call(inp):
        return SyntheticMiniModel.mini_model(*inp)

    inputs = SyntheticMiniModel.inputs(batch_size, vocab_size)

    fwd_fn = with_executor(executor, fwd_call)
    outputs = fwd_fn(inputs)
    grads = SyntheticMiniModel.grads()
    run_benchmark(benchmark, unary_bwd_torch, [outputs, grads, *inputs])

# Simple test of F.cross_entropy without final reduction
# thunder won't use nvFuser for this fusion, here we manually define the fusion
def nvfuser_cross_entropy_fusion(
    fd: FusionDefinition, batch_size: int, vocab_size: int
) -> None:
    """
    NvFuser fusion definition for torch.nn.functional.cross_entropy(logits, labels, reduction='none')
    1. Compute LSE (log-sum-exp) for each row
    2. Access target logit directly: logits[i, labels[i]]
    3. Compute loss: loss = lse - target_logit
    """
    # Input tensors
    T0 = fd.define_tensor(
        shape=[batch_size, vocab_size],
        contiguity=[True, True],
        dtype=DataType.BFloat16,
        is_cpu=False,
        stride_order=[1, 0],
    )
    T1 = fd.define_tensor(
        shape=[batch_size],
        contiguity=[True],
        dtype=DataType.Int,
        is_cpu=False,
        stride_order=[0],
    )
    T2 = fd.ops.cast(T0, dtype=DataType.Float)
    T3 = fd.ops.max(T2, dims=[1], keepdim=False, dtype=DataType.Null)
    T4 = fd.ops.broadcast_in_dim(T3, shape=[batch_size, 1], broadcast_dims=[0])
    T11 = fd.ops.broadcast_in_dim(
        T4, shape=[batch_size, vocab_size], broadcast_dims=[0, 1]
    )
    T12 = fd.ops.sub(T2, T11)
    T13 = fd.ops.exp(T12)
    T14 = fd.ops.sum(T13, dims=[1], keepdim=False, dtype=DataType.Null)
    T15 = fd.ops.log(T14)
    T16 = fd.ops.add(T15, T3)
    T17 = fd.ops.broadcast_in_dim(T1, shape=[batch_size, 1], broadcast_dims=[0])
    T18 = fd.ops.gather(T0, T17, dim=1)
    T19 = fd.ops.squeeze(T18, dims=[1])
    T20 = fd.ops.cast(T19, dtype=DataType.Float)
    T21 = fd.ops.sub(T16, T20)
    T22 = fd.ops.cast(T21, dtype=DataType.BFloat16)
    fd.add_output(T22)

@pytest.mark.parametrize("vocab_size", SyntheticMiniModel.sizes_from_models)
def test_function_cross_entropy_fwd_nvf_benchmark(
    benchmark, vocab_size: int, disable_validation: bool, disable_benchmarking: bool
):
    batch_size = 4096
    inputs = [
        0.1
        * torch.randn(
            batch_size,
            vocab_size,
            device="cuda",
            dtype=torch.bfloat16,
            requires_grad=False,
        ),
        torch.randint(
            0,
            vocab_size,
            (batch_size,),
            device="cuda",
            dtype=torch.int64,
            requires_grad=False,
        ),
    ]
    with FusionDefinition() as fd:
        nvfuser_cross_entropy_fusion(fd, batch_size, vocab_size)

    if not disable_validation:
        fd.validate(inputs, [torch_cross_entropy_fwd(inputs)])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, inputs)

def quack_cross_entropy_fwd_wrapper(inputs: list):
    return quack_cross_entropy_fwd(*inputs, return_dx=False)

# cross entropy in thunder is split into log_softmax and nll_loss without
# using nvFuser, the performance using nvFuser should be measured using
# test_function_cross_entropy_fwd_nvf_benchmark
@pytest.mark.parametrize(
    "executor",
    [
        "eager",
        "torchcompile",
        "thunder-torchcompile",
        pytest.param(
            "quack",
            marks=pytest.mark.skipif(
                not check_module_available("quack"),
                reason="quack executor is not available on this device",
            ),
        ),
    ],
)
@pytest.mark.parametrize("vocab_size", SyntheticMiniModel.sizes_from_models)
def test_function_cross_entropy_fwd_benchmark(
    benchmark, executor: str, vocab_size: int
):
    if executor == "torchcompile":
        clear_dynamo_cache()

    batch_size = 4096
    # vocab_size is large, scale by 0.1 to avoid overflow and represent realistic values, same used in quack test
    logits = 0.1 * torch.randn(
        batch_size, vocab_size, device="cuda", dtype=torch.bfloat16, requires_grad=False
    )
    labels = torch.randint(
        0,
        vocab_size,
        (batch_size,),
        device="cuda",
        dtype=torch.int64,
        requires_grad=False,
    )

    if executor == "quack":
        fwd_fn = quack_cross_entropy_fwd_wrapper
    else:
        fwd_fn = with_executor(executor, torch_cross_entropy_fwd)

    run_benchmark(benchmark, fwd_fn, [logits, labels])


    SyntheticMiniModel,
)
from .torch_ops import cross_entropy as torch_cross_entropy_fwd

if check_module_available("quack"):
    from quack.cross_entropy import cross_entropy_fwd as quack_cross_entropy_fwd

@pytest.mark.parametrize(
    "variation",
    [
        "hf_qwen2",
        "hf_phi3",
        "hf_mistral_nemo",
    ],
)
@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
def test_cross_entropy_fwd_benchmark(
    benchmark,
    variation: str,
    executor: str,
):
    kwargs = {}
    if executor == "torchcompile":
        clear_dynamo_cache()

    test_case = cross_entropy_loss_setup[variation](dtype=torch.bfloat16)
    inputs = test_case.inputs()
    model = test_case.model()

    def fwd_call(inp):
        return model(**inp)

    # Compile the fwd fn for torchcompile
    benchmark_fn = with_executor(executor, fwd_call, **kwargs)
    run_benchmark(benchmark, benchmark_fn, inputs)

@pytest.mark.parametrize(
    "variation",
    [
        "hf_qwen2",
        "hf_phi3",
        "hf_mistral_nemo",
    ],
)
@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
def test_cross_entropy_bwd_benchmark(
    benchmark,
    variation: str,
    executor: str,
):
    kwargs = {}
    if executor == "torchcompile":


    def fwd_call(inp):
        return model(**inp)

    # Compile the fwd fn for torchcompile
    benchmark_fn = with_executor(executor, fwd_call, **kwargs)
    run_benchmark(benchmark, benchmark_fn, inputs)

@pytest.mark.parametrize(
    "variation",
    [
        "hf_qwen2",
        "hf_phi3",
        "hf_mistral_nemo",
    ],
)
@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
def test_cross_entropy_bwd_benchmark(
    benchmark,
    variation: str,
    executor: str,
):
    kwargs = {}
    if executor == "torchcompile":
        clear_dynamo_cache()

    test_case = cross_entropy_loss_setup[variation](dtype=torch.bfloat16)
    fwd_inputs = test_case.inputs()
    model = test_case.model()

    def fwd_call(inp):
        return model(**inp)

    # execute the compiled fwd fn
    fwd_fn = with_executor(executor, fwd_call, **kwargs)
    outputs = fwd_fn(fwd_inputs)

    assert len(outputs) == 1

    run_benchmark(
        benchmark,
        unary_bwd_torch,
        [outputs[0], test_case.grads(), *fwd_inputs, *list(model.parameters())],
        iobytes=test_case.grad_iobytes(),
    )

@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
@pytest.mark.parametrize("vocab_size", SyntheticMiniModel.generate_vocab_sizes())
def test_cross_entropy_mini_benchmark_fwd(benchmark, executor: str, vocab_size: int):
    if executor == "torchcompile":
        clear_dynamo_cache()

    # picking a value that doesn't OOM for large vocab sizes
    batch_size = 4096

    def fwd_call(inp):
        return SyntheticMiniModel.mini_model(*inp)

    inputs = SyntheticMiniModel.inputs(int(batch_size), int(vocab_size))

    fwd_fn = with_executor(executor, fwd_call)
    run_benchmark(benchmark, fwd_fn, inputs)


        return model(**inp)

    # execute the compiled fwd fn
    fwd_fn = with_executor(executor, fwd_call, **kwargs)
    outputs = fwd_fn(fwd_inputs)

    assert len(outputs) == 1

    run_benchmark(
        benchmark,
        unary_bwd_torch,
        [outputs[0], test_case.grads(), *fwd_inputs, *list(model.parameters())],
        iobytes=test_case.grad_iobytes(),
    )

@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
@pytest.mark.parametrize("vocab_size", SyntheticMiniModel.generate_vocab_sizes())
def test_cross_entropy_mini_benchmark_fwd(benchmark, executor: str, vocab_size: int):
    if executor == "torchcompile":
        clear_dynamo_cache()

    # picking a value that doesn't OOM for large vocab sizes
    batch_size = 4096

    def fwd_call(inp):
        return SyntheticMiniModel.mini_model(*inp)

    inputs = SyntheticMiniModel.inputs(int(batch_size), int(vocab_size))

    fwd_fn = with_executor(executor, fwd_call)
    run_benchmark(benchmark, fwd_fn, inputs)

@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
@pytest.mark.parametrize("vocab_size", SyntheticMiniModel.generate_vocab_sizes())
def test_cross_entropy_mini_benchmark_bwd(benchmark, executor: str, vocab_size: int):
    if executor == "torchcompile":
        clear_dynamo_cache()

    # picking a value that doesn't OOM for large vocab sizes
    batch_size = 4096

    def fwd_call(inp):
        return SyntheticMiniModel.mini_model(*inp)

    inputs = SyntheticMiniModel.inputs(batch_size, vocab_size)

    fwd_fn = with_executor(executor, fwd_call)
    outputs = fwd_fn(inputs)


def test_cross_entropy_mini_benchmark_fwd(benchmark, executor: str, vocab_size: int):
    if executor == "torchcompile":
        clear_dynamo_cache()

    # picking a value that doesn't OOM for large vocab sizes
    batch_size = 4096

    def fwd_call(inp):
        return SyntheticMiniModel.mini_model(*inp)

    inputs = SyntheticMiniModel.inputs(int(batch_size), int(vocab_size))

    fwd_fn = with_executor(executor, fwd_call)
    run_benchmark(benchmark, fwd_fn, inputs)

@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
@pytest.mark.parametrize("vocab_size", SyntheticMiniModel.generate_vocab_sizes())
def test_cross_entropy_mini_benchmark_bwd(benchmark, executor: str, vocab_size: int):
    if executor == "torchcompile":
        clear_dynamo_cache()

    # picking a value that doesn't OOM for large vocab sizes
    batch_size = 4096

    def fwd_call(inp):
        return SyntheticMiniModel.mini_model(*inp)

    inputs = SyntheticMiniModel.inputs(batch_size, vocab_size)

    fwd_fn = with_executor(executor, fwd_call)
    outputs = fwd_fn(inputs)
    grads = SyntheticMiniModel.grads()
    run_benchmark(benchmark, unary_bwd_torch, [outputs, grads, *inputs])

# Simple test of F.cross_entropy without final reduction
# thunder won't use nvFuser for this fusion, here we manually define the fusion
def nvfuser_cross_entropy_fusion(
    fd: FusionDefinition, batch_size: int, vocab_size: int
) -> None:
    """
    NvFuser fusion definition for torch.nn.functional.cross_entropy(logits, labels, reduction='none')
    1. Compute LSE (log-sum-exp) for each row
    2. Access target logit directly: logits[i, labels[i]]
    3. Compute loss: loss = lse - target_logit
    """
    # Input tensors
    T0 = fd.define_tensor(
        shape=[batch_size, vocab_size],
        contiguity=[True, True],
        dtype=DataType.BFloat16,
        is_cpu=False,
        stride_order=[1, 0],


def test_cross_entropy_mini_benchmark_bwd(benchmark, executor: str, vocab_size: int):
    if executor == "torchcompile":
        clear_dynamo_cache()

    # picking a value that doesn't OOM for large vocab sizes
    batch_size = 4096

    def fwd_call(inp):
        return SyntheticMiniModel.mini_model(*inp)

    inputs = SyntheticMiniModel.inputs(batch_size, vocab_size)

    fwd_fn = with_executor(executor, fwd_call)
    outputs = fwd_fn(inputs)
    grads = SyntheticMiniModel.grads()
    run_benchmark(benchmark, unary_bwd_torch, [outputs, grads, *inputs])

# Simple test of F.cross_entropy without final reduction
# thunder won't use nvFuser for this fusion, here we manually define the fusion
def nvfuser_cross_entropy_fusion(
    fd: FusionDefinition, batch_size: int, vocab_size: int
) -> None:
    """
    NvFuser fusion definition for torch.nn.functional.cross_entropy(logits, labels, reduction='none')
    1. Compute LSE (log-sum-exp) for each row
    2. Access target logit directly: logits[i, labels[i]]
    3. Compute loss: loss = lse - target_logit
    """
    # Input tensors
    T0 = fd.define_tensor(
        shape=[batch_size, vocab_size],
        contiguity=[True, True],
        dtype=DataType.BFloat16,
        is_cpu=False,
        stride_order=[1, 0],
    )
    T1 = fd.define_tensor(
        shape=[batch_size],
        contiguity=[True],
        dtype=DataType.Int,
        is_cpu=False,
        stride_order=[0],
    )
    T2 = fd.ops.cast(T0, dtype=DataType.Float)
    T3 = fd.ops.max(T2, dims=[1], keepdim=False, dtype=DataType.Null)
    T4 = fd.ops.broadcast_in_dim(T3, shape=[batch_size, 1], broadcast_dims=[0])
    T11 = fd.ops.broadcast_in_dim(
        T4, shape=[batch_size, vocab_size], broadcast_dims=[0, 1]
    )
    T12 = fd.ops.sub(T2, T11)
    T13 = fd.ops.exp(T12)
    T14 = fd.ops.sum(T13, dims=[1], keepdim=False, dtype=DataType.Null)
    T15 = fd.ops.log(T14)
    T16 = fd.ops.add(T15, T3)
    T17 = fd.ops.broadcast_in_dim(T1, shape=[batch_size, 1], broadcast_dims=[0])
    T18 = fd.ops.gather(T0, T17, dim=1)
    T19 = fd.ops.squeeze(T18, dims=[1])
    T20 = fd.ops.cast(T19, dtype=DataType.Float)
    T21 = fd.ops.sub(T16, T20)
    T22 = fd.ops.cast(T21, dtype=DataType.BFloat16)
    fd.add_output(T22)

@pytest.mark.parametrize("vocab_size", SyntheticMiniModel.sizes_from_models)
def test_function_cross_entropy_fwd_nvf_benchmark(
    benchmark, vocab_size: int, disable_validation: bool, disable_benchmarking: bool
):
    batch_size = 4096
    inputs = [
        0.1
        * torch.randn(
            batch_size,
            vocab_size,
            device="cuda",
            dtype=torch.bfloat16,
            requires_grad=False,
        ),
        torch.randint(
            0,
            vocab_size,
            (batch_size,),


    T3 = fd.ops.max(T2, dims=[1], keepdim=False, dtype=DataType.Null)
    T4 = fd.ops.broadcast_in_dim(T3, shape=[batch_size, 1], broadcast_dims=[0])
    T11 = fd.ops.broadcast_in_dim(
        T4, shape=[batch_size, vocab_size], broadcast_dims=[0, 1]
    )
    T12 = fd.ops.sub(T2, T11)
    T13 = fd.ops.exp(T12)
    T14 = fd.ops.sum(T13, dims=[1], keepdim=False, dtype=DataType.Null)
    T15 = fd.ops.log(T14)
    T16 = fd.ops.add(T15, T3)
    T17 = fd.ops.broadcast_in_dim(T1, shape=[batch_size, 1], broadcast_dims=[0])
    T18 = fd.ops.gather(T0, T17, dim=1)
    T19 = fd.ops.squeeze(T18, dims=[1])
    T20 = fd.ops.cast(T19, dtype=DataType.Float)
    T21 = fd.ops.sub(T16, T20)
    T22 = fd.ops.cast(T21, dtype=DataType.BFloat16)
    fd.add_output(T22)

@pytest.mark.parametrize("vocab_size", SyntheticMiniModel.sizes_from_models)
def test_function_cross_entropy_fwd_nvf_benchmark(
    benchmark, vocab_size: int, disable_validation: bool, disable_benchmarking: bool
):
    batch_size = 4096
    inputs = [
        0.1
        * torch.randn(
            batch_size,
            vocab_size,
            device="cuda",
            dtype=torch.bfloat16,
            requires_grad=False,
        ),
        torch.randint(
            0,
            vocab_size,
            (batch_size,),
            device="cuda",
            dtype=torch.int64,
            requires_grad=False,
        ),
    ]
    with FusionDefinition() as fd:
        nvfuser_cross_entropy_fusion(fd, batch_size, vocab_size)

    if not disable_validation:
        fd.validate(inputs, [torch_cross_entropy_fwd(inputs)])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, inputs)

def quack_cross_entropy_fwd_wrapper(inputs: list):
    return quack_cross_entropy_fwd(*inputs, return_dx=False)

# cross entropy in thunder is split into log_softmax and nll_loss without
# using nvFuser, the performance using nvFuser should be measured using
# test_function_cross_entropy_fwd_nvf_benchmark
@pytest.mark.parametrize(
    "executor",
    [
        "eager",
        "torchcompile",
        "thunder-torchcompile",
        pytest.param(
            "quack",
            marks=pytest.mark.skipif(
                not check_module_available("quack"),
                reason="quack executor is not available on this device",


        ),
        torch.randint(
            0,
            vocab_size,
            (batch_size,),
            device="cuda",
            dtype=torch.int64,
            requires_grad=False,
        ),
    ]
    with FusionDefinition() as fd:
        nvfuser_cross_entropy_fusion(fd, batch_size, vocab_size)

    if not disable_validation:
        fd.validate(inputs, [torch_cross_entropy_fwd(inputs)])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, inputs)

def quack_cross_entropy_fwd_wrapper(inputs: list):
    return quack_cross_entropy_fwd(*inputs, return_dx=False)

# cross entropy in thunder is split into log_softmax and nll_loss without
# using nvFuser, the performance using nvFuser should be measured using
# test_function_cross_entropy_fwd_nvf_benchmark
@pytest.mark.parametrize(
    "executor",
    [
        "eager",
        "torchcompile",
        "thunder-torchcompile",
        pytest.param(
            "quack",
            marks=pytest.mark.skipif(
                not check_module_available("quack"),
                reason="quack executor is not available on this device",
            ),
        ),
    ],
)


# cross entropy in thunder is split into log_softmax and nll_loss without
# using nvFuser, the performance using nvFuser should be measured using
# test_function_cross_entropy_fwd_nvf_benchmark
@pytest.mark.parametrize(
    "executor",
    [
        "eager",
        "torchcompile",
        "thunder-torchcompile",
        pytest.param(
            "quack",
            marks=pytest.mark.skipif(
                not check_module_available("quack"),
                reason="quack executor is not available on this device",
            ),
        ),
    ],
)
@pytest.mark.parametrize("vocab_size", SyntheticMiniModel.sizes_from_models)
def test_function_cross_entropy_fwd_benchmark(
    benchmark, executor: str, vocab_size: int
):
    if executor == "torchcompile":
        clear_dynamo_cache()

    batch_size = 4096
    # vocab_size is large, scale by 0.1 to avoid overflow and represent realistic values, same used in quack test
    logits = 0.1 * torch.randn(
        batch_size, vocab_size, device="cuda", dtype=torch.bfloat16, requires_grad=False
    )
    labels = torch.randint(
        0,
        vocab_size,
        (batch_size,),
        device="cuda",
        dtype=torch.int64,
        requires_grad=False,
    )

    if executor == "quack":
        fwd_fn = quack_cross_entropy_fwd_wrapper
    else:
        fwd_fn = with_executor(executor, torch_cross_entropy_fwd)

    run_benchmark(benchmark, fwd_fn, [logits, labels])


    def fwd_call(inp):
        return SyntheticMiniModel.mini_model(*inp)

    inputs = SyntheticMiniModel.inputs(int(batch_size), int(vocab_size))

    fwd_fn = with_executor(executor, fwd_call)
    run_benchmark(benchmark, fwd_fn, inputs)

@pytest.mark.parametrize(
    "executor", ["eager", "torchcompile", "thunder", "thunder-torchcompile"]
)
@pytest.mark.parametrize("vocab_size", SyntheticMiniModel.generate_vocab_sizes())
def test_cross_entropy_mini_benchmark_bwd(benchmark, executor: str, vocab_size: int):
    if executor == "torchcompile":
        clear_dynamo_cache()

    # picking a value that doesn't OOM for large vocab sizes
    batch_size = 4096

    def fwd_call(inp):
        return SyntheticMiniModel.mini_model(*inp)

    inputs = SyntheticMiniModel.inputs(batch_size, vocab_size)

    fwd_fn = with_executor(executor, fwd_call)
    outputs = fwd_fn(inputs)
    grads = SyntheticMiniModel.grads()
    run_benchmark(benchmark, unary_bwd_torch, [outputs, grads, *inputs])

# Simple test of F.cross_entropy without final reduction
# thunder won't use nvFuser for this fusion, here we manually define the fusion
def nvfuser_cross_entropy_fusion(
    fd: FusionDefinition, batch_size: int, vocab_size: int
) -> None:
    """
    NvFuser fusion definition for torch.nn.functional.cross_entropy(logits, labels, reduction='none')
    1. Compute LSE (log-sum-exp) for each row
    2. Access target logit directly: logits[i, labels[i]]
    3. Compute loss: loss = lse - target_logit
    """


benchmarks.python.normalization

# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
from nvfuser_direct import FusionDefinition, DataType
from .global_params import PROMOTE_DTYPES
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
import torch
from .core import run_benchmark, unary_bwd_torch, clear_dynamo_cache, with_executor
import numpy as np
from functools import reduce

def norm_fwd_fusion(
    fd: FusionDefinition,
    dtype: DataType,
    norm: str,
    num_dims: int,
    channels_last: bool,
    eps: float = 1e-5,
    momentum: float = 0.01,
) -> None:
    """
    Fusion definition for batch norm and instance norm forward in training mode.
    """
    batch_dim = 0
    channel_dim = 1 if not channels_last else num_dims - 1
    bcast_mask = [True if i != channel_dim else False for i in range(num_dims)]
    channels_only_bcast_mask = [
        True if i != channel_dim else False for i in range(num_dims)
    ]

    reduction_axes = [i for i in range(num_dims) if i != channel_dim]
    if norm == "instance_norm":
        reduction_axes.remove(batch_dim)
        bcast_mask[batch_dim] = False

    input = fd.define_tensor(
        shape=[-1] * num_dims, contiguity=[True] * num_dims, dtype=dtype, is_cpu=False
    )
    weight = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)
    bias = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)

    running_mean = fd.define_tensor(
        shape=[-1], contiguity=[True], dtype=DataType.Float, is_cpu=False
    )
    running_var = fd.define_tensor(
        shape=[-1], contiguity=[True], dtype=DataType.Float, is_cpu=False
    )

    if dtype in PROMOTE_DTYPES:
        input = fd.ops.cast(input, dtype=DataType.Float)
        weight = fd.ops.cast(weight, dtype=DataType.Float)
        bias = fd.ops.cast(bias, dtype=DataType.Float)

    var, mean = fd.ops.var_mean(input, dims=reduction_axes, correction=0, keepdim=False)

    eps = fd.define_scalar(eps, dtype=DataType.Double)
    var_eps = fd.ops.add(var, eps)
    invstd = fd.ops.rsqrt(var_eps)

    invstd_bcast = fd.ops.broadcast(invstd, bcast_mask)
    mean_bcast = fd.ops.broadcast(mean, bcast_mask)
    x_sub_mean = fd.ops.sub(input, mean_bcast)

    x_norm = fd.ops.mul(x_sub_mean, invstd_bcast)

    weight = fd.ops.broadcast(weight, channels_only_bcast_mask)
    x_scaled = fd.ops.mul(x_norm, weight)
    bias = fd.ops.broadcast(bias, channels_only_bcast_mask)
    output = fd.ops.add(x_scaled, bias)

    rev_momentum = fd.define_scalar(1 - momentum, dtype=DataType.Double)
    momentum = fd.define_scalar(momentum, dtype=DataType.Double)

    updated_mean = fd.ops.add(
        fd.ops.mul(momentum, mean), fd.ops.mul(rev_momentum, running_mean)
    )
    updated_var = fd.ops.add(
        fd.ops.mul(momentum, var), fd.ops.mul(rev_momentum, running_var)
    )

    if batch_dim not in reduction_axes:
        inverse_batch_size = fd.ops.reciprocal(input.size(0))
        updated_mean = fd.ops.mul(
            fd.ops.sum(updated_mean, batch_dim), inverse_batch_size
        )
        updated_var = fd.ops.mul(fd.ops.sum(updated_var, batch_dim), inverse_batch_size)

    if dtype in PROMOTE_DTYPES:
        output = fd.ops.cast(output, dtype=dtype)

    fd.add_output(output)
    fd.add_output(mean)
    fd.add_output(invstd)

def norm_bwd_fusion(
    fd: FusionDefinition,
    dtype: DataType,
    norm: str,
    num_dims: int,
    channels_last: bool,
    eps: float = 1e-5,
) -> None:
    """
    Fusion definition for batch norm and instance norm backward in training mode.
    """
    batch_dim = 0
    channel_dim = 1 if not channels_last else num_dims - 1
    bcast_mask = [True if i != channel_dim else False for i in range(num_dims)]
    channels_only_bcast_mask = [
        True if i != channel_dim else False for i in range(num_dims)
    ]

    reduction_axes = [i for i in range(num_dims) if i != channel_dim]
    if norm == "instance_norm":
        reduction_axes.remove(batch_dim)
        bcast_mask[batch_dim] = False

    num_spatial_dims = num_dims - len(reduction_axes)

    input = fd.define_tensor(
        shape=[-1] * num_dims, contiguity=[True] * num_dims, dtype=dtype, is_cpu=False
    )
    grad = fd.define_tensor(
        shape=[-1] * num_dims, contiguity=[True] * num_dims, dtype=dtype, is_cpu=False
    )
    weight = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)

    running_mean = fd.define_tensor(
        shape=[-1], contiguity=[True], dtype=DataType.Float, is_cpu=False
    )
    running_var = fd.define_tensor(
        shape=[-1], contiguity=[True], dtype=DataType.Float, is_cpu=False
    )

    mean = fd.define_tensor(
        shape=[-1] * num_spatial_dims,
        contiguity=[True] * num_spatial_dims,
        dtype=DataType.Float,
        is_cpu=False,
    )
    invstd = fd.define_tensor(
        shape=[-1] * num_spatial_dims,
        contiguity=[True] * num_spatial_dims,
        dtype=DataType.Float,
        is_cpu=False,
    )

    if dtype in PROMOTE_DTYPES:
        input = fd.ops.cast(input, dtype=DataType.Float)
        grad = fd.ops.cast(grad, dtype=DataType.Float)
        weight = fd.ops.cast(weight, dtype=DataType.Float)

    num_features = reduce(fd.ops.mul, [input.size(ax) for ax in reduction_axes], 1)

    norm = fd.ops.reciprocal(num_features)

    mean = fd.ops.broadcast(mean, bcast_mask)

    grad_sum = fd.ops.sum(grad, dims=reduction_axes, keepdim=False)

    x_sub_mean = fd.ops.sub(input, mean)
    dot_p = fd.ops.sum(fd.ops.mul(grad, x_sub_mean), dims=reduction_axes, keepdim=False)

    grad_mean = fd.ops.broadcast(fd.ops.mul(grad_sum, norm), bcast_mask)
    proj_scale = fd.ops.mul(fd.ops.mul(dot_p, norm), fd.ops.mul(invstd, invstd))
    proj_scale = fd.ops.broadcast(proj_scale, bcast_mask)

    invstd_bcast = fd.ops.broadcast(invstd, bcast_mask)
    weight = fd.ops.broadcast(weight, channels_only_bcast_mask)
    grad_scale = fd.ops.mul(weight, invstd_bcast)
    proj = fd.ops.mul(proj_scale, x_sub_mean)

    grad_input = fd.ops.mul(fd.ops.sub(fd.ops.sub(grad, proj), grad_mean), grad_scale)
    grad_weight = fd.ops.mul(dot_p, invstd)
    grad_bias = grad_sum

    if batch_dim not in reduction_axes:
        # Weights and bias are channel-only
        grad_weight = fd.ops.sum(grad_weight, batch_dim, keepdim=False)
        grad_bias = fd.ops.sum(grad_bias, batch_dim, keepdim=False)

    if dtype in PROMOTE_DTYPES:
        grad_input = fd.ops.cast(grad_input, dtype=dtype)
        grad_weight = fd.ops.cast(grad_weight, dtype=dtype)
        grad_bias = fd.ops.cast(grad_bias, dtype=dtype)

    fd.add_output(grad_input)
    fd.add_output(grad_weight)
    fd.add_output(grad_bias)

def norm_fwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    norm: str,
    channels_last: bool,
    disable_validation: bool,
    disable_benchmarking: bool,
    eps: float = 1e-5,
):
    """
    Common benchmark setup for batchnorm/instance forward call in training mode.
    """

    assert norm in ["batch_norm", "instance_norm"], NotImplementedError

    # Size is assumed to be in the order N, C, ...
    num_dims = len(size)

    at_inputs = torch.randn(size, device="cuda", dtype=dtype, requires_grad=True)
    weight = torch.randn(size[1], device="cuda", dtype=dtype)
    bias = torch.randn(size[1], device="cuda", dtype=dtype)

    # CPP benchmarks assume mean and variance to be of type Float
    running_mean = torch.zeros(size[1], device="cuda", dtype=torch.float)
    running_var = torch.ones(size[1], device="cuda", dtype=torch.float)

    if channels_last:
        at_inputs = at_inputs.to(memory_format=torch.channels_last)
        inputs = at_inputs.clone().detach().permute((0, *range(2, num_dims), 1))
    else:
        inputs = at_inputs

    with FusionDefinition() as fd:
        norm_fwd_fusion(
            fd=fd,
            dtype=torch_dtype_to_nvfuser_dtype(dtype),
            norm=norm,
            num_dims=num_dims,
            channels_last=channels_last,
        )

    if not disable_validation:
        # PyTorch expects running mean and variance to be of same type as input.
        if norm == "batch_norm":
            eager_output = torch.nn.functional.batch_norm(
                at_inputs,
                running_mean.to(dtype),
                running_var.to(dtype),
                weight=weight,
                bias=bias,
                training=True,
            )
        elif norm == "instance_norm":
            eager_output = torch.nn.functional.instance_norm(
                at_inputs,
                running_mean.to(dtype),
                running_var.to(dtype),
                weight=weight,
                bias=bias,
            )
        if channels_last:
            eager_output = eager_output.permute((0, *range(2, num_dims), 1))

        batch_dim = 0
        channel_dim = 1 if not channels_last else num_dims - 1
        reduction_axes = [i for i in range(len(size)) if i != channel_dim]
        if norm == "instance_norm":
            reduction_axes.remove(batch_dim)

        mean = inputs.to(torch.float).mean(dim=reduction_axes)
        var = inputs.to(torch.float).var(dim=reduction_axes, unbiased=False)
        invstd = 1.0 / torch.sqrt(var + eps)

        fd.validate(
            [inputs, weight, bias, running_mean, running_var],
            [eager_output, mean, invstd],
        )

    if not disable_benchmarking:
        run_benchmark(
            benchmark, fd.execute, [inputs, weight, bias, running_mean, running_var]
        )

def norm_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    norm: str,
    channels_last: bool,
    disable_validation: bool,
    disable_benchmarking: bool,
    eps: float = 1e-5,
):
    """
    Common benchmark setup for batchnorm/instance forward call in training mode.
    """

    assert norm in ["batch_norm", "instance_norm"], NotImplementedError

    # Size is assumed to be in the order N, C, ...

    num_dims = len(size)

    at_inputs = torch.randn(size, device="cuda", dtype=dtype, requires_grad=True)
    at_grads = torch.randn(size, device="cuda", dtype=dtype)
    weight = torch.randn(size[1], device="cuda", dtype=dtype, requires_grad=True)
    bias = torch.randn(size[1], device="cuda", dtype=dtype, requires_grad=True)

    # CPP benchmarks assume mean and variance to be of type Float
    running_mean = torch.zeros(size[1], device="cuda", dtype=torch.float)
    running_var = torch.ones(size[1], device="cuda", dtype=torch.float)

    if channels_last:
        at_inputs = at_inputs.to(memory_format=torch.channels_last)
        at_inputs.retain_grad()
        at_grads = at_grads.to(memory_format=torch.channels_last)

        inputs = at_inputs.clone().detach().permute((0, *range(2, num_dims), 1))
        grads = at_grads.clone().detach().permute((0, *range(2, num_dims), 1))

    else:
        inputs = at_inputs
        grads = at_grads

    batch_dim = 0
    channel_dim = 1 if not channels_last else num_dims - 1
    reduction_axes = [i for i in range(len(size)) if i != channel_dim]
    if norm == "instance_norm":
        reduction_axes.remove(batch_dim)
    mean = inputs.to(torch.float).mean(dim=reduction_axes)
    var = inputs.to(torch.float).var(dim=reduction_axes, unbiased=False)
    invstd = 1.0 / torch.sqrt(var + eps)

    with FusionDefinition() as fd:
        norm_bwd_fusion(
            fd=fd,
            dtype=torch_dtype_to_nvfuser_dtype(dtype),
            norm=norm,
            num_dims=num_dims,
            channels_last=channels_last,
        )

    if not disable_validation:
        # PyTorch expects running mean and variance to be of same type as input.
        if norm == "batch_norm":
            eager_output = torch.nn.functional.batch_norm(
                at_inputs.to(torch.double),
                running_mean.to(torch.double),
                running_var.to(torch.double),
                weight=weight.to(torch.double),
                bias=bias.to(torch.double),
                training=True,
            )
        elif norm == "instance_norm":
            eager_output = torch.nn.functional.instance_norm(
                at_inputs.to(torch.double),
                running_mean.to(torch.double),
                running_var.to(torch.double),
                weight=weight.to(torch.double),
                bias=bias.to(torch.double),
            )

        eager_output.backward(at_grads.to(torch.double))

        if channels_last:
            eager_grad = at_inputs.grad.permute((0, *range(2, num_dims), 1))
        else:
            eager_grad = at_inputs.grad

        fd.validate(
            [inputs, grads, weight, running_mean, running_var, mean, invstd],
            [eager_grad, weight.grad, bias.grad],
        )

    if not disable_benchmarking:
        run_benchmark(
            benchmark,
            fd.execute,
            [inputs, grads, weight, running_mean, running_var, mean, invstd],
        )

def batchnorm_fwd_fn(inputs: list):
    input, weight, bias, running_mean, running_var = inputs
    return torch.nn.functional.batch_norm(
        input,
        running_mean,
        running_var,
        weight=weight,
        bias=bias,
        training=True,
    )

def instancenorm_fwd_fn(inputs: list):
    input, weight, bias, running_mean, running_var = inputs
    return torch.nn.functional.instance_norm(
        input,
        running_mean,
        running_var,
        weight=weight,
        bias=bias,
    )

def norm_fwd_iobytes(size: tuple, dtype: torch.dtype, norm: str):
    # Manual IOBytes computation required since nvFuser outputs (out, mean, invstd) differs from baselines (out)
    # size = [N, C, H, W]
    # Total IO bytes = in_tensor (size, dtype) + weight (size[1], dtype) + bias (size[1], dtype) +
    #           running_mean (size[1], float) + running_var (size[1], float) + output (size, dtype) +
    #           mean ([C]/[N, C] , float) + invstd ([C]/[N, C] , float)
    stat_size = (
        size[1] if norm == "batch_norm" else size[0] * size[1]
    )  # size of mean/invstd
    return int(
        dtype.itemsize * 2 * (np.prod(size) + size[1])
        + torch.float.itemsize * 2 * (size[1] + stat_size)
    )

def norm_bwd_iobytes(size: tuple, dtype: torch.dtype, norm: str):
    # Manual IOBytes computation since nvfuser input/outputs (in_tensor, grad_out, mean, invstd, weigts, grad_in, grad_weight, grad_bias) differ from baselines (out, grad_out)
    # size = [N, C, H, W]
    # Total IO bytes = in_tensor (size, dtype) + weight (size[1], dtype) +
    #           running_mean (size[1], float) + running_var (size[1], float) +
    #           mean ([C]/[N, C] , float) + invstd ([C]/[N, C] , float) + grad_out (size, dtype) +
    #           grad_in (size, dtype) + grad_weight (size[1], dtype) + grad_bias (size[1], dtype)
    stat_size = size[1] if norm == "batch_norm" else size[0] * size[1]
    return int(
        dtype.itemsize * 3 * (np.prod(size) + size[1])
        + torch.float.itemsize * 2 * (size[1] + stat_size)
    )

def norm_fwd_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    channels_last: bool,
    executor: str,
    norm: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()

    assert norm in ["batch_norm", "instance_norm"], NotImplementedError

    # Size is assumed to be in the order N, C, ...
    inputs = torch.randn(size, device="cuda", dtype=dtype, requires_grad=True)
    weight = torch.randn(size[1], device="cuda", dtype=dtype, requires_grad=True)
    bias = torch.randn(size[1], device="cuda", dtype=dtype, requires_grad=True)

    running_mean = torch.zeros(size[1], device="cuda", dtype=dtype)
    running_var = torch.ones(size[1], device="cuda", dtype=dtype)
    if channels_last:
        inputs = inputs.to(memory_format=torch.channels_last)

    norm_fwd_fn = batchnorm_fwd_fn if norm == "batch_norm" else instancenorm_fwd_fn

    benchmark_fn = with_executor(executor, norm_fwd_fn)

    # Manually compute IOBytes: See PR #1725
    run_benchmark(
        benchmark,
        benchmark_fn,
        [inputs, weight, bias, running_mean, running_var],
        iobytes=norm_fwd_iobytes(size, dtype, norm),
    )

def norm_bwd_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    channels_last: bool,
    executor: str,
    norm: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()

    assert norm in ["batch_norm", "instance_norm"], NotImplementedError

    # Size is assumed to be in the order N, C, ...
    inputs = torch.randn(size, device="cuda", dtype=dtype, requires_grad=True)
    weight = torch.randn(size[1], device="cuda", dtype=dtype, requires_grad=True)
    bias = torch.randn(size[1], device="cuda", dtype=dtype, requires_grad=True)

    running_mean = torch.zeros(size[1], device="cuda", dtype=dtype)
    running_var = torch.ones(size[1], device="cuda", dtype=dtype)
    grads = torch.randn(size, device="cuda", dtype=dtype)

    if channels_last:
        inputs = inputs.to(memory_format=torch.channels_last)
        grads = grads.to(memory_format=torch.channels_last)

    norm_fwd_fn = batchnorm_fwd_fn if norm == "batch_norm" else instancenorm_fwd_fn

    fwd_fn = with_executor(executor, norm_fwd_fn)
    fwd_inputs = [inputs, weight, bias, running_mean, running_var]
    outputs = fwd_fn(fwd_inputs)

    # Manually compute IOBytes: See PR #1725
    run_benchmark(
        benchmark,
        unary_bwd_torch,
        [outputs, grads, *fwd_inputs],
        iobytes=norm_bwd_iobytes(size, dtype, norm),
    )


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
from nvfuser_direct import FusionDefinition, DataType
from .global_params import PROMOTE_DTYPES
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
import torch
from .core import run_benchmark, unary_bwd_torch, clear_dynamo_cache, with_executor
import numpy as np
from functools import reduce

def norm_fwd_fusion(
    fd: FusionDefinition,
    dtype: DataType,
    norm: str,
    num_dims: int,
    channels_last: bool,
    eps: float = 1e-5,
    momentum: float = 0.01,
) -> None:
    """
    Fusion definition for batch norm and instance norm forward in training mode.
    """
    batch_dim = 0
    channel_dim = 1 if not channels_last else num_dims - 1
    bcast_mask = [True if i != channel_dim else False for i in range(num_dims)]
    channels_only_bcast_mask = [
        True if i != channel_dim else False for i in range(num_dims)
    ]

    reduction_axes = [i for i in range(num_dims) if i != channel_dim]
    if norm == "instance_norm":
        reduction_axes.remove(batch_dim)
        bcast_mask[batch_dim] = False

    input = fd.define_tensor(
        shape=[-1] * num_dims, contiguity=[True] * num_dims, dtype=dtype, is_cpu=False
    )
    weight = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)
    bias = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)

    running_mean = fd.define_tensor(
        shape=[-1], contiguity=[True], dtype=DataType.Float, is_cpu=False
    )
    running_var = fd.define_tensor(
        shape=[-1], contiguity=[True], dtype=DataType.Float, is_cpu=False
    )

    if dtype in PROMOTE_DTYPES:
        input = fd.ops.cast(input, dtype=DataType.Float)
        weight = fd.ops.cast(weight, dtype=DataType.Float)
        bias = fd.ops.cast(bias, dtype=DataType.Float)

    var, mean = fd.ops.var_mean(input, dims=reduction_axes, correction=0, keepdim=False)

    eps = fd.define_scalar(eps, dtype=DataType.Double)
    var_eps = fd.ops.add(var, eps)
    invstd = fd.ops.rsqrt(var_eps)

    invstd_bcast = fd.ops.broadcast(invstd, bcast_mask)
    mean_bcast = fd.ops.broadcast(mean, bcast_mask)
    x_sub_mean = fd.ops.sub(input, mean_bcast)

    x_norm = fd.ops.mul(x_sub_mean, invstd_bcast)

    weight = fd.ops.broadcast(weight, channels_only_bcast_mask)
    x_scaled = fd.ops.mul(x_norm, weight)
    bias = fd.ops.broadcast(bias, channels_only_bcast_mask)
    output = fd.ops.add(x_scaled, bias)

    rev_momentum = fd.define_scalar(1 - momentum, dtype=DataType.Double)
    momentum = fd.define_scalar(momentum, dtype=DataType.Double)

    updated_mean = fd.ops.add(
        fd.ops.mul(momentum, mean), fd.ops.mul(rev_momentum, running_mean)
    )
    updated_var = fd.ops.add(
        fd.ops.mul(momentum, var), fd.ops.mul(rev_momentum, running_var)
    )

    if batch_dim not in reduction_axes:
        inverse_batch_size = fd.ops.reciprocal(input.size(0))
        updated_mean = fd.ops.mul(
            fd.ops.sum(updated_mean, batch_dim), inverse_batch_size
        )
        updated_var = fd.ops.mul(fd.ops.sum(updated_var, batch_dim), inverse_batch_size)

    if dtype in PROMOTE_DTYPES:
        output = fd.ops.cast(output, dtype=dtype)

    fd.add_output(output)
    fd.add_output(mean)
    fd.add_output(invstd)

def norm_bwd_fusion(
    fd: FusionDefinition,
    dtype: DataType,
    norm: str,
    num_dims: int,
    channels_last: bool,
    eps: float = 1e-5,
) -> None:
    """
    Fusion definition for batch norm and instance norm backward in training mode.
    """
    batch_dim = 0
    channel_dim = 1 if not channels_last else num_dims - 1
    bcast_mask = [True if i != channel_dim else False for i in range(num_dims)]
    channels_only_bcast_mask = [
        True if i != channel_dim else False for i in range(num_dims)
    ]


    )
    updated_var = fd.ops.add(
        fd.ops.mul(momentum, var), fd.ops.mul(rev_momentum, running_var)
    )

    if batch_dim not in reduction_axes:
        inverse_batch_size = fd.ops.reciprocal(input.size(0))
        updated_mean = fd.ops.mul(
            fd.ops.sum(updated_mean, batch_dim), inverse_batch_size
        )
        updated_var = fd.ops.mul(fd.ops.sum(updated_var, batch_dim), inverse_batch_size)

    if dtype in PROMOTE_DTYPES:
        output = fd.ops.cast(output, dtype=dtype)

    fd.add_output(output)
    fd.add_output(mean)
    fd.add_output(invstd)

def norm_bwd_fusion(
    fd: FusionDefinition,
    dtype: DataType,
    norm: str,
    num_dims: int,
    channels_last: bool,
    eps: float = 1e-5,
) -> None:
    """
    Fusion definition for batch norm and instance norm backward in training mode.
    """
    batch_dim = 0
    channel_dim = 1 if not channels_last else num_dims - 1
    bcast_mask = [True if i != channel_dim else False for i in range(num_dims)]
    channels_only_bcast_mask = [
        True if i != channel_dim else False for i in range(num_dims)
    ]

    reduction_axes = [i for i in range(num_dims) if i != channel_dim]
    if norm == "instance_norm":
        reduction_axes.remove(batch_dim)
        bcast_mask[batch_dim] = False

    num_spatial_dims = num_dims - len(reduction_axes)

    input = fd.define_tensor(
        shape=[-1] * num_dims, contiguity=[True] * num_dims, dtype=dtype, is_cpu=False
    )
    grad = fd.define_tensor(
        shape=[-1] * num_dims, contiguity=[True] * num_dims, dtype=dtype, is_cpu=False
    )
    weight = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)

    running_mean = fd.define_tensor(
        shape=[-1], contiguity=[True], dtype=DataType.Float, is_cpu=False
    )
    running_var = fd.define_tensor(
        shape=[-1], contiguity=[True], dtype=DataType.Float, is_cpu=False
    )

    mean = fd.define_tensor(
        shape=[-1] * num_spatial_dims,
        contiguity=[True] * num_spatial_dims,
        dtype=DataType.Float,
        is_cpu=False,
    )
    invstd = fd.define_tensor(
        shape=[-1] * num_spatial_dims,
        contiguity=[True] * num_spatial_dims,
        dtype=DataType.Float,
        is_cpu=False,
    )

    if dtype in PROMOTE_DTYPES:
        input = fd.ops.cast(input, dtype=DataType.Float)
        grad = fd.ops.cast(grad, dtype=DataType.Float)
        weight = fd.ops.cast(weight, dtype=DataType.Float)

    num_features = reduce(fd.ops.mul, [input.size(ax) for ax in reduction_axes], 1)

    norm = fd.ops.reciprocal(num_features)

    mean = fd.ops.broadcast(mean, bcast_mask)

    grad_sum = fd.ops.sum(grad, dims=reduction_axes, keepdim=False)

    x_sub_mean = fd.ops.sub(input, mean)
    dot_p = fd.ops.sum(fd.ops.mul(grad, x_sub_mean), dims=reduction_axes, keepdim=False)

    grad_mean = fd.ops.broadcast(fd.ops.mul(grad_sum, norm), bcast_mask)
    proj_scale = fd.ops.mul(fd.ops.mul(dot_p, norm), fd.ops.mul(invstd, invstd))
    proj_scale = fd.ops.broadcast(proj_scale, bcast_mask)

    invstd_bcast = fd.ops.broadcast(invstd, bcast_mask)
    weight = fd.ops.broadcast(weight, channels_only_bcast_mask)
    grad_scale = fd.ops.mul(weight, invstd_bcast)
    proj = fd.ops.mul(proj_scale, x_sub_mean)

    grad_input = fd.ops.mul(fd.ops.sub(fd.ops.sub(grad, proj), grad_mean), grad_scale)
    grad_weight = fd.ops.mul(dot_p, invstd)
    grad_bias = grad_sum

    if batch_dim not in reduction_axes:
        # Weights and bias are channel-only
        grad_weight = fd.ops.sum(grad_weight, batch_dim, keepdim=False)
        grad_bias = fd.ops.sum(grad_bias, batch_dim, keepdim=False)

    if dtype in PROMOTE_DTYPES:
        grad_input = fd.ops.cast(grad_input, dtype=dtype)
        grad_weight = fd.ops.cast(grad_weight, dtype=dtype)
        grad_bias = fd.ops.cast(grad_bias, dtype=dtype)

    fd.add_output(grad_input)
    fd.add_output(grad_weight)
    fd.add_output(grad_bias)

def norm_fwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    norm: str,
    channels_last: bool,
    disable_validation: bool,
    disable_benchmarking: bool,
    eps: float = 1e-5,
):
    """
    Common benchmark setup for batchnorm/instance forward call in training mode.
    """

    assert norm in ["batch_norm", "instance_norm"], NotImplementedError

    # Size is assumed to be in the order N, C, ...
    num_dims = len(size)


    grad_input = fd.ops.mul(fd.ops.sub(fd.ops.sub(grad, proj), grad_mean), grad_scale)
    grad_weight = fd.ops.mul(dot_p, invstd)
    grad_bias = grad_sum

    if batch_dim not in reduction_axes:
        # Weights and bias are channel-only
        grad_weight = fd.ops.sum(grad_weight, batch_dim, keepdim=False)
        grad_bias = fd.ops.sum(grad_bias, batch_dim, keepdim=False)

    if dtype in PROMOTE_DTYPES:
        grad_input = fd.ops.cast(grad_input, dtype=dtype)
        grad_weight = fd.ops.cast(grad_weight, dtype=dtype)
        grad_bias = fd.ops.cast(grad_bias, dtype=dtype)

    fd.add_output(grad_input)
    fd.add_output(grad_weight)
    fd.add_output(grad_bias)

def norm_fwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    norm: str,
    channels_last: bool,
    disable_validation: bool,
    disable_benchmarking: bool,
    eps: float = 1e-5,
):
    """
    Common benchmark setup for batchnorm/instance forward call in training mode.
    """

    assert norm in ["batch_norm", "instance_norm"], NotImplementedError

    # Size is assumed to be in the order N, C, ...
    num_dims = len(size)

    at_inputs = torch.randn(size, device="cuda", dtype=dtype, requires_grad=True)
    weight = torch.randn(size[1], device="cuda", dtype=dtype)
    bias = torch.randn(size[1], device="cuda", dtype=dtype)

    # CPP benchmarks assume mean and variance to be of type Float
    running_mean = torch.zeros(size[1], device="cuda", dtype=torch.float)
    running_var = torch.ones(size[1], device="cuda", dtype=torch.float)

    if channels_last:
        at_inputs = at_inputs.to(memory_format=torch.channels_last)
        inputs = at_inputs.clone().detach().permute((0, *range(2, num_dims), 1))
    else:
        inputs = at_inputs

    with FusionDefinition() as fd:
        norm_fwd_fusion(
            fd=fd,
            dtype=torch_dtype_to_nvfuser_dtype(dtype),
            norm=norm,
            num_dims=num_dims,
            channels_last=channels_last,
        )

    if not disable_validation:
        # PyTorch expects running mean and variance to be of same type as input.
        if norm == "batch_norm":
            eager_output = torch.nn.functional.batch_norm(
                at_inputs,
                running_mean.to(dtype),
                running_var.to(dtype),
                weight=weight,
                bias=bias,
                training=True,
            )
        elif norm == "instance_norm":
            eager_output = torch.nn.functional.instance_norm(
                at_inputs,
                running_mean.to(dtype),
                running_var.to(dtype),
                weight=weight,
                bias=bias,
            )
        if channels_last:
            eager_output = eager_output.permute((0, *range(2, num_dims), 1))

        batch_dim = 0
        channel_dim = 1 if not channels_last else num_dims - 1
        reduction_axes = [i for i in range(len(size)) if i != channel_dim]
        if norm == "instance_norm":
            reduction_axes.remove(batch_dim)

        mean = inputs.to(torch.float).mean(dim=reduction_axes)
        var = inputs.to(torch.float).var(dim=reduction_axes, unbiased=False)
        invstd = 1.0 / torch.sqrt(var + eps)

        fd.validate(
            [inputs, weight, bias, running_mean, running_var],
            [eager_output, mean, invstd],
        )

    if not disable_benchmarking:
        run_benchmark(
            benchmark, fd.execute, [inputs, weight, bias, running_mean, running_var]
        )

def norm_bwd_nvf_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    norm: str,
    channels_last: bool,
    disable_validation: bool,
    disable_benchmarking: bool,
    eps: float = 1e-5,
):
    """
    Common benchmark setup for batchnorm/instance forward call in training mode.
    """

    assert norm in ["batch_norm", "instance_norm"], NotImplementedError

    # Size is assumed to be in the order N, C, ...