# SPDX-FileCopyrightText: Copyright (c) 2023-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import pytest
import torch
from torch.testing import make_tensor
from enum import Enum, auto
from typing import Optional
from functools import wraps

try:
    # flake8: noqa
    import jax

    JAX_AVAILABLE = True
except ImportError as e:
    JAX_AVAILABLE = False
    pass

def requiresJAX(fn):
    @wraps(fn)
    def _fn(*args, **kwargs):
        if not JAX_AVAILABLE:
            pytest.xfail("Requires JAX")
        return fn(*args, **kwargs)

    return _fn

class ArgumentType(Enum):
    # a symbolic value requires an input argument during kernel execution
    Symbolic = auto()
    # scalar with constant value
    ConstantScalar = auto()
    # python number - int, float, complex, bool
    Constant = auto()

bool_dtypes = (torch.bool,)

int_dtypes = (
    torch.int32,
    torch.int64,
)

half_precision_float_dtypes = (
    torch.bfloat16,
    torch.float16,
)

full_precision_float_dtypes = (
    torch.float32,
    torch.float64,
)

complex_dtypes = (
    torch.complex64,
    torch.complex128,
)

# Half-precision float dtypes bf16, fp16 are skipped because nvfuser upcasts those dtypes to fp32
# but does not return the original type.
bool_int_dtypes = bool_dtypes + int_dtypes
float_dtypes = half_precision_float_dtypes + full_precision_float_dtypes
int_float_dtypes = int_dtypes + full_precision_float_dtypes
float_complex_dtypes = full_precision_float_dtypes + complex_dtypes
all_dtypes_except_reduced = int_dtypes + full_precision_float_dtypes + complex_dtypes
all_dtypes_except_bool = all_dtypes_except_reduced + half_precision_float_dtypes
all_dtypes = all_dtypes_except_bool + bool_dtypes

map_dtype_to_str = {
    torch.bool: "bool",
    torch.uint8: "uint8",
    torch.int8: "int8",
    torch.int16: "int16",
    torch.int32: "int32",
    torch.int64: "int64",
    torch.float8_e4m3fn: "float8_e4m3fn",
    torch.float8_e5m2: "float8_e5m2",
    torch.float8_e8m0fnu: "float8_e8m0fnu",
    torch.bfloat16: "bfloat16",
    torch.float16: "float16",
    torch.float32: "float32",
    torch.float64: "float64",
    torch.complex64: "complex64",
    torch.complex128: "complex128",
}

if hasattr(torch, "float4_e2m1fn_x2"):
    map_dtype_to_str[torch.float4_e2m1fn_x2] = "float4_e2m1fn_x2"

torch_to_jax_dtype_map = None
if JAX_AVAILABLE:
    import jax.numpy as jnp

    torch_to_jax_dtype_map = {
        torch.bool: jnp.bool_,
        torch.uint8: jnp.uint8,
        torch.int8: jnp.int8,
        torch.int16: jnp.int16,
        torch.int32: jnp.int32,
        torch.int64: jnp.int64,
        torch.bfloat16: jnp.bfloat16,
        torch.float16: jnp.float16,
        torch.float32: jnp.float32,
        torch.float64: jnp.float64,
        torch.complex64: jnp.complex64,
        torch.complex128: jnp.complex128,
    }

torch_to_python_dtype_map = {
    torch.bool: bool,
    torch.uint8: int,
    torch.int8: int,
    torch.int16: int,
    torch.int32: int,
    torch.int64: int,
    torch.bfloat16: float,
    torch.float16: float,
    torch.float32: float,
    torch.float64: float,
    torch.complex64: complex,
    torch.complex128: complex,
}

def make_tensor_like(a):
    # type: (torch.Tensor) -> torch.Tensor
    """Returns a tensor with the same properties as the given tensor.

    Args:
        a (torch.Tensor): The tensor to copy properties from.

    Returns:
        torch.Tensor: A tensor with the same properties as :attr:`a`.
    """
    return torch.testing.make_tensor(
        a.shape, device=a.device, dtype=a.dtype, requires_grad=a.requires_grad
    )

def make_number(
    dtype: torch.dtype, low: Optional[float] = None, high: Optional[float] = None
):
    """Returns a random number with desired dtype

    Args:
        dtype (torch.dtype): Desired dtype for number.
        low (Optional[Number]): Sets the lower limit (inclusive) of the given range.
        high (Optional[Number]): Sets the upper limit (exclusive) of the given range.

    Returns:
        (Scalar): The scalar number with specified dtype.
    """
    return make_tensor([1], device="cpu", dtype=dtype, low=low, high=high).item()

def find_nonmatching_dtype(dtype: torch.dtype):
    if dtype in int_float_dtypes:
        return torch.complex128
    elif dtype in complex_dtypes:
        return torch.double
    elif dtype is torch.bool:
        return torch.float32
    return None

def is_complex_dtype(dtype: torch.dtype):
    return dtype in complex_dtypes

def is_floating_dtype(dtype: torch.dtype):
    return dtype in float_dtypes

def is_integer_dtype(dtype: torch.dtype):
    return dtype in int_dtypes

def is_tensor(a):
    return isinstance(a, torch.Tensor)


requiresJAX

# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import pytest
import torch
from torch.testing import make_tensor
from enum import Enum, auto
from typing import Optional
from functools import wraps

try:
    # flake8: noqa
    import jax

    JAX_AVAILABLE = True
except ImportError as e:
    JAX_AVAILABLE = False
    pass

def requiresJAX(fn):
    @wraps(fn)
    def _fn(*args, **kwargs):
        if not JAX_AVAILABLE:
            pytest.xfail("Requires JAX")
        return fn(*args, **kwargs)

    return _fn

class ArgumentType(Enum):
    # a symbolic value requires an input argument during kernel execution
    Symbolic = auto()
    # scalar with constant value
    ConstantScalar = auto()
    # python number - int, float, complex, bool
    Constant = auto()

bool_dtypes = (torch.bool,)

int_dtypes = (
    torch.int32,
    torch.int64,
)

half_precision_float_dtypes = (
    torch.bfloat16,


ArgumentType

try:
    # flake8: noqa
    import jax

    JAX_AVAILABLE = True
except ImportError as e:
    JAX_AVAILABLE = False
    pass

def requiresJAX(fn):
    @wraps(fn)
    def _fn(*args, **kwargs):
        if not JAX_AVAILABLE:
            pytest.xfail("Requires JAX")
        return fn(*args, **kwargs)

    return _fn

class ArgumentType(Enum):
    # a symbolic value requires an input argument during kernel execution
    Symbolic = auto()
    # scalar with constant value
    ConstantScalar = auto()
    # python number - int, float, complex, bool
    Constant = auto()

bool_dtypes = (torch.bool,)

int_dtypes = (
    torch.int32,
    torch.int64,
)

half_precision_float_dtypes = (
    torch.bfloat16,
    torch.float16,
)

full_precision_float_dtypes = (
    torch.float32,
    torch.float64,
)

complex_dtypes = (


        torch.complex64: jnp.complex64,
        torch.complex128: jnp.complex128,
    }

torch_to_python_dtype_map = {
    torch.bool: bool,
    torch.uint8: int,
    torch.int8: int,
    torch.int16: int,
    torch.int32: int,
    torch.int64: int,
    torch.bfloat16: float,
    torch.float16: float,
    torch.float32: float,
    torch.float64: float,
    torch.complex64: complex,
    torch.complex128: complex,
}

def make_tensor_like(a):
    # type: (torch.Tensor) -> torch.Tensor
    """Returns a tensor with the same properties as the given tensor.

    Args:
        a (torch.Tensor): The tensor to copy properties from.

    Returns:
        torch.Tensor: A tensor with the same properties as :attr:`a`.
    """
    return torch.testing.make_tensor(
        a.shape, device=a.device, dtype=a.dtype, requires_grad=a.requires_grad
    )

def make_number(
    dtype: torch.dtype, low: Optional[float] = None, high: Optional[float] = None
):
    """Returns a random number with desired dtype

    Args:
        dtype (torch.dtype): Desired dtype for number.
        low (Optional[Number]): Sets the lower limit (inclusive) of the given range.
        high (Optional[Number]): Sets the upper limit (exclusive) of the given range.

    Returns:
        (Scalar): The scalar number with specified dtype.
    """
    return make_tensor([1], device="cpu", dtype=dtype, low=low, high=high).item()

def find_nonmatching_dtype(dtype: torch.dtype):
    if dtype in int_float_dtypes:


    torch.complex64: complex,
    torch.complex128: complex,
}

def make_tensor_like(a):
    # type: (torch.Tensor) -> torch.Tensor
    """Returns a tensor with the same properties as the given tensor.

    Args:
        a (torch.Tensor): The tensor to copy properties from.

    Returns:
        torch.Tensor: A tensor with the same properties as :attr:`a`.
    """
    return torch.testing.make_tensor(
        a.shape, device=a.device, dtype=a.dtype, requires_grad=a.requires_grad
    )

def make_number(
    dtype: torch.dtype, low: Optional[float] = None, high: Optional[float] = None
):
    """Returns a random number with desired dtype

    Args:
        dtype (torch.dtype): Desired dtype for number.
        low (Optional[Number]): Sets the lower limit (inclusive) of the given range.
        high (Optional[Number]): Sets the upper limit (exclusive) of the given range.

    Returns:
        (Scalar): The scalar number with specified dtype.
    """
    return make_tensor([1], device="cpu", dtype=dtype, low=low, high=high).item()

def find_nonmatching_dtype(dtype: torch.dtype):
    if dtype in int_float_dtypes:
        return torch.complex128
    elif dtype in complex_dtypes:
        return torch.double
    elif dtype is torch.bool:
        return torch.float32
    return None

def is_complex_dtype(dtype: torch.dtype):
    return dtype in complex_dtypes

def is_floating_dtype(dtype: torch.dtype):
    return dtype in float_dtypes


        a.shape, device=a.device, dtype=a.dtype, requires_grad=a.requires_grad
    )

def make_number(
    dtype: torch.dtype, low: Optional[float] = None, high: Optional[float] = None
):
    """Returns a random number with desired dtype

    Args:
        dtype (torch.dtype): Desired dtype for number.
        low (Optional[Number]): Sets the lower limit (inclusive) of the given range.
        high (Optional[Number]): Sets the upper limit (exclusive) of the given range.

    Returns:
        (Scalar): The scalar number with specified dtype.
    """
    return make_tensor([1], device="cpu", dtype=dtype, low=low, high=high).item()

def find_nonmatching_dtype(dtype: torch.dtype):
    if dtype in int_float_dtypes:
        return torch.complex128
    elif dtype in complex_dtypes:
        return torch.double
    elif dtype is torch.bool:
        return torch.float32
    return None

def is_complex_dtype(dtype: torch.dtype):
    return dtype in complex_dtypes

def is_floating_dtype(dtype: torch.dtype):
    return dtype in float_dtypes

def is_integer_dtype(dtype: torch.dtype):
    return dtype in int_dtypes

def is_tensor(a):
    return isinstance(a, torch.Tensor)


        dtype (torch.dtype): Desired dtype for number.
        low (Optional[Number]): Sets the lower limit (inclusive) of the given range.
        high (Optional[Number]): Sets the upper limit (exclusive) of the given range.

    Returns:
        (Scalar): The scalar number with specified dtype.
    """
    return make_tensor([1], device="cpu", dtype=dtype, low=low, high=high).item()

def find_nonmatching_dtype(dtype: torch.dtype):
    if dtype in int_float_dtypes:
        return torch.complex128
    elif dtype in complex_dtypes:
        return torch.double
    elif dtype is torch.bool:
        return torch.float32
    return None

def is_complex_dtype(dtype: torch.dtype):
    return dtype in complex_dtypes

def is_floating_dtype(dtype: torch.dtype):
    return dtype in float_dtypes

def is_integer_dtype(dtype: torch.dtype):
    return dtype in int_dtypes

def is_tensor(a):
    return isinstance(a, torch.Tensor)


    Returns:
        (Scalar): The scalar number with specified dtype.
    """
    return make_tensor([1], device="cpu", dtype=dtype, low=low, high=high).item()

def find_nonmatching_dtype(dtype: torch.dtype):
    if dtype in int_float_dtypes:
        return torch.complex128
    elif dtype in complex_dtypes:
        return torch.double
    elif dtype is torch.bool:
        return torch.float32
    return None

def is_complex_dtype(dtype: torch.dtype):
    return dtype in complex_dtypes

def is_floating_dtype(dtype: torch.dtype):
    return dtype in float_dtypes

def is_integer_dtype(dtype: torch.dtype):
    return dtype in int_dtypes

def is_tensor(a):
    return isinstance(a, torch.Tensor)


def find_nonmatching_dtype(dtype: torch.dtype):
    if dtype in int_float_dtypes:
        return torch.complex128
    elif dtype in complex_dtypes:
        return torch.double
    elif dtype is torch.bool:
        return torch.float32
    return None

def is_complex_dtype(dtype: torch.dtype):
    return dtype in complex_dtypes

def is_floating_dtype(dtype: torch.dtype):
    return dtype in float_dtypes

def is_integer_dtype(dtype: torch.dtype):
    return dtype in int_dtypes

def is_tensor(a):
    return isinstance(a, torch.Tensor)


        return torch.complex128
    elif dtype in complex_dtypes:
        return torch.double
    elif dtype is torch.bool:
        return torch.float32
    return None

def is_complex_dtype(dtype: torch.dtype):
    return dtype in complex_dtypes

def is_floating_dtype(dtype: torch.dtype):
    return dtype in float_dtypes

def is_integer_dtype(dtype: torch.dtype):
    return dtype in int_dtypes

def is_tensor(a):
    return isinstance(a, torch.Tensor)


import pytest
import torch
from torch.testing import make_tensor
from enum import Enum, auto
from typing import Optional
from functools import wraps

try:
    # flake8: noqa
    import jax

    JAX_AVAILABLE = True
except ImportError as e:
    JAX_AVAILABLE = False
    pass

def requiresJAX(fn):
    @wraps(fn)
    def _fn(*args, **kwargs):
        if not JAX_AVAILABLE:
            pytest.xfail("Requires JAX")
        return fn(*args, **kwargs)

    return _fn

class ArgumentType(Enum):
    # a symbolic value requires an input argument during kernel execution
    Symbolic = auto()
    # scalar with constant value
    ConstantScalar = auto()
    # python number - int, float, complex, bool
    Constant = auto()

bool_dtypes = (torch.bool,)

int_dtypes = (
    torch.int32,
    torch.int64,
)


tests.python.direct.conftest

# SPDX-FileCopyrightText: Copyright (c) 2023-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import pytest
from copy import deepcopy
import torch
from torch.testing._internal.common_utils import TestCase

from nvfuser_direct import FusionDefinition, LRUCache
from python.direct_utils import is_pre_volta, check_captured_python_definition

class NVFuserTest(TestCase):
    def __init__(self, cache=None):
        super().__init__()
        self.cache = cache

    # Helper function to verify the nvfuser output and make sure the string
    # definition based on the FusionDefinition is executable and matches the
    # original definition
    def exec_nvfuser(
        self,
        fusion_func,
        inputs,
        *,
        new_fusion_expected=True,
        expected_fd_str=None,
        device=None,
        enable_options=None,
        disable_options=None,
        validate_results=False,
    ):
        # Copy inputs because aliased outputs can modify inputs when running
        # FusionDefinition
        inputs_captured = deepcopy(inputs)

        if self.cache is not None:
            prev_size = self.cache.num_fusions()

            # Run compilation twice to test lru cache; The number of fusions
            # should not increase during the second round.
            for _ in range(2):
                with FusionDefinition() as fd:
                    fusion_func(fd)

                fd.fec = self.cache.cache_compile(fd.fusion)
                del fd._fusion

            # The LRU cache is shared across all tests. If you run all the tests
            # together, a fusion can be cached from an earlier test. If you run
            # a test standalone, then the fusion will not exist. Skip this
            # assertion for this case but check that cache_compile works
            # correctly.
            if new_fusion_expected is not None:
                assert self.cache.num_fusions() == prev_size + int(new_fusion_expected)
        else:
            with FusionDefinition() as fd:
                fusion_func(fd)

        # Execute a fusion function and capture the string python definition
        # Set manual_seed before running fusion to avoid mismatch results
        # with rng kernels
        torch.manual_seed(0)
        if validate_results:
            out = fd.validate(inputs)
        else:
            if enable_options is None:
                enable_options = []
            if disable_options is None:
                disable_options = []
            out = fd.execute(
                inputs,
                device=device,
                _enable_options=enable_options,
                _disable_options=disable_options,
            )

        assert check_captured_python_definition(
            out, fd, inputs_captured, device, enable_options, disable_options
        )
        assert expected_fd_str is None or expected_fd_str in repr(fd)
        return out, fd

# Migrated tests to new direct python bindings use this.
@pytest.fixture(params=["lru_cache", "eager"])
def nvfuser_direct_test(request):
    if is_pre_volta():
        pytest.skip("Only supported on Volta and newer devices.")

    cache_type = request.param
    if cache_type == "lru_cache":
        if not hasattr(nvfuser_direct_test, "cache"):
            nvfuser_direct_test.cache = LRUCache(max_fusions=16384)
        yield NVFuserTest(nvfuser_direct_test.cache)
    else:
        yield NVFuserTest()


            if enable_options is None:
                enable_options = []
            if disable_options is None:
                disable_options = []
            out = fd.execute(
                inputs,
                device=device,
                _enable_options=enable_options,
                _disable_options=disable_options,
            )

        assert check_captured_python_definition(
            out, fd, inputs_captured, device, enable_options, disable_options
        )
        assert expected_fd_str is None or expected_fd_str in repr(fd)
        return out, fd

# Migrated tests to new direct python bindings use this.
@pytest.fixture(params=["lru_cache", "eager"])
def nvfuser_direct_test(request):
    if is_pre_volta():
        pytest.skip("Only supported on Volta and newer devices.")

    cache_type = request.param
    if cache_type == "lru_cache":
        if not hasattr(nvfuser_direct_test, "cache"):
            nvfuser_direct_test.cache = LRUCache(max_fusions=16384)
        yield NVFuserTest(nvfuser_direct_test.cache)
    else:
        yield NVFuserTest()


# SPDX-FileCopyrightText: Copyright (c) 2023-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import pytest
from copy import deepcopy
import torch
from torch.testing._internal.common_utils import TestCase

from nvfuser_direct import FusionDefinition, LRUCache
from python.direct_utils import is_pre_volta, check_captured_python_definition

class NVFuserTest(TestCase):
    def __init__(self, cache=None):
        super().__init__()
        self.cache = cache

    # Helper function to verify the nvfuser output and make sure the string
    # definition based on the FusionDefinition is executable and matches the
    # original definition
    def exec_nvfuser(
        self,
        fusion_func,
        inputs,
        *,
        new_fusion_expected=True,
        expected_fd_str=None,
        device=None,
        enable_options=None,
        disable_options=None,
        validate_results=False,
    ):
        # Copy inputs because aliased outputs can modify inputs when running
        # FusionDefinition
        inputs_captured = deepcopy(inputs)


# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import pytest
from copy import deepcopy
import torch
from torch.testing._internal.common_utils import TestCase

from nvfuser_direct import FusionDefinition, LRUCache
from python.direct_utils import is_pre_volta, check_captured_python_definition

class NVFuserTest(TestCase):
    def __init__(self, cache=None):
        super().__init__()
        self.cache = cache

    # Helper function to verify the nvfuser output and make sure the string
    # definition based on the FusionDefinition is executable and matches the
    # original definition
    def exec_nvfuser(
        self,
        fusion_func,
        inputs,
        *,
        new_fusion_expected=True,
        expected_fd_str=None,
        device=None,
        enable_options=None,
        disable_options=None,
        validate_results=False,
    ):
        # Copy inputs because aliased outputs can modify inputs when running
        # FusionDefinition
        inputs_captured = deepcopy(inputs)

        if self.cache is not None:
            prev_size = self.cache.num_fusions()

            # Run compilation twice to test lru cache; The number of fusions
            # should not increase during the second round.
            for _ in range(2):
                with FusionDefinition() as fd:
                    fusion_func(fd)

                fd.fec = self.cache.cache_compile(fd.fusion)
                del fd._fusion

            # The LRU cache is shared across all tests. If you run all the tests
            # together, a fusion can be cached from an earlier test. If you run
            # a test standalone, then the fusion will not exist. Skip this
            # assertion for this case but check that cache_compile works
            # correctly.
            if new_fusion_expected is not None:
                assert self.cache.num_fusions() == prev_size + int(new_fusion_expected)
        else:
            with FusionDefinition() as fd:
                fusion_func(fd)

        # Execute a fusion function and capture the string python definition
        # Set manual_seed before running fusion to avoid mismatch results
        # with rng kernels
        torch.manual_seed(0)
        if validate_results:
            out = fd.validate(inputs)
        else:
            if enable_options is None:
                enable_options = []
            if disable_options is None:
                disable_options = []
            out = fd.execute(
                inputs,
                device=device,
                _enable_options=enable_options,
                _disable_options=disable_options,
            )

        assert check_captured_python_definition(
            out, fd, inputs_captured, device, enable_options, disable_options
        )
        assert expected_fd_str is None or expected_fd_str in repr(fd)
        return out, fd

# Migrated tests to new direct python bindings use this.
@pytest.fixture(params=["lru_cache", "eager"])
def nvfuser_direct_test(request):
    if is_pre_volta():
        pytest.skip("Only supported on Volta and newer devices.")

    cache_type = request.param
    if cache_type == "lru_cache":
        if not hasattr(nvfuser_direct_test, "cache"):
            nvfuser_direct_test.cache = LRUCache(max_fusions=16384)
        yield NVFuserTest(nvfuser_direct_test.cache)
    else:
        yield NVFuserTest()


# SPDX-FileCopyrightText: Copyright (c) 2023-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import torch

from nvfuser_direct import (
    FusionDefinition,
    DataType,
)
from python.direct_utils import (
    FLOAT4_E2M1_MAX,
    FLOAT8_E4M3_EPS,
    FLOAT8_E4M3_MAX,
    pytorch_nvfp4_quantize,
    microarchitecture_is,
    linear_to_swizzled_128_4,
    round_up,
    activation_scale_to_nvfp4,
)

import pytest

# FIXME: this test needs to be merged back into test_narrow_precision.py.
# We have indexer issue: https://github.com/NVIDIA/Fuser/issues/5200, which
# forces the adoption of environment variable in order to avoid codegen
# assertion. Having this as a separate test file would avoid environment
# variable contamination from others.
@pytest.mark.skipif(
    not microarchitecture_is(10, 0), reason="Only supported on compute 10.0"
)
@pytest.mark.parametrize("config", [[1024, 128, 256]])
@pytest.mark.parametrize("tokens_per_expert_neg_one", [[115, 144, 8]])
@pytest.mark.parametrize("out_dtype", [torch.bfloat16])
def test_layout_op_and_cutlass_nvfp4_grouped_mm(
    nvfuser_direct_test,
    config,
    tokens_per_expert_neg_one,
    out_dtype,
):
    BLOCK_SIZE = 16

    # k dimension is multiple of 4 * 16 to avoid padding on block scaling factor
    m, n, k = config
    assert k % 64 == 0
    tokens_per_expert = list(tokens_per_expert_neg_one)
    tokens_per_expert.append(m - sum(tokens_per_expert))
    g = len(tokens_per_expert)

    mat1 = torch.testing.make_tensor((m, k), dtype=torch.float32, device="cuda:0")
    # format is g, n, k instead of g, k, n
    mat2 = torch.testing.make_tensor((g, n, k), dtype=torch.float32, device="cuda:0")

    offsets = torch.empty((g,), dtype=torch.int32, device="cuda:0")
    blockscale_offsets = torch.empty((g,), dtype=torch.int32, device="cuda:0")
    problem_sizes = torch.empty((g, 3), dtype=torch.int32, device="cuda:0")

    # prepare quantization for mat2
    mat2_gs = torch.empty((g,), dtype=torch.float32, device="cuda:0")
    scale2 = torch.empty(
        (g, n, k // BLOCK_SIZE), dtype=torch.float8_e4m3fn, device="cuda:0"
    )

    acc_tokens = 0
    rounded_acc_tokens = 0
    mat2_scaled = torch.empty(
        (g, n, k // 2), dtype=torch.float4_e2m1fn_x2, device="cuda:0"
    )

    for i in range(g):
        global_sf = FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX / mat2[i].max()
        offsets[i] = acc_tokens
        blockscale_offsets[i] = rounded_acc_tokens
        acc_tokens += tokens_per_expert[i]
        # Note: we technically don't need to round up, since k is perfectly sized.
        rounded_acc_tokens += round_up(tokens_per_expert[i], 128)

        problem_sizes[i][0] = tokens_per_expert[i]
        problem_sizes[i][1] = n
        problem_sizes[i][2] = k

        scaled_mat2_i, bs_mat2_i = pytorch_nvfp4_quantize(mat2[i], global_sf)
        mat2_gs[i] = 1.0 / global_sf
        mat2_scaled[i] = scaled_mat2_i
        scale2[i] = linear_to_swizzled_128_4(bs_mat2_i)

    def nvfuser_fusion_id0(fd: FusionDefinition) -> None:
        mat1 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=True,
            dtype=DataType.Float,
            is_cpu=False,
        )
        mat2 = fd.define_tensor(
            shape=[-1, -1, -1],
            contiguity=True,
            dtype=DataType.Float4_e2m1fn,
            is_cpu=False,
            stride_order=[2, 0, 1],
        )
        scale2 = fd.define_tensor(
            shape=[-1, -1, -1],
            contiguity=True,
            dtype=DataType.Float8_e4m3fn,
            is_cpu=False,
        )
        alpha = fd.define_tensor(
            shape=[-1], contiguity=True, dtype=DataType.Float, is_cpu=False
        )
        problem_sizes = fd.define_tensor(
            shape=[-1, -1], contiguity=True, dtype=DataType.Int32, is_cpu=False
        )
        offsets = fd.define_tensor(
            shape=[-1], contiguity=True, dtype=DataType.Int32, is_cpu=False
        )
        blockscale_offsets = fd.define_tensor(
            shape=[-1], contiguity=True, dtype=DataType.Int32, is_cpu=False
        )
        # TODO: fix dynamic shape in issue https://github.com/NVIDIA/Fuser/issues/5199
        # m_size = fd.ops.size(mat1, 0)
        # k_size = fd.ops.size(mat1, 1)
        # k_tile_size = fd.ops.div(k_size, 16)
        # use static shape as a temporary WAR.
        m_size = m
        k_size = k
        k_tile_size = k_size // 16
        # using primitive operations to handle quantization
        reshaped_mat1 = fd.ops.reshape(mat1, [m_size, k_tile_size, 16])

        # quantization math to compute block scaling factor
        scale1 = fd.ops.abs(reshaped_mat1)
        scale1 = fd.ops.max(scale1, 2)
        scale1 = fd.ops.div(scale1, FLOAT4_E2M1_MAX)
        scale1 = fd.ops.clamp(scale1, FLOAT8_E4M3_EPS, FLOAT8_E4M3_MAX)
        broadcast_scale1 = fd.ops.broadcast(scale1, [False, False, True])
        reshaped_scaled_mat1 = fd.ops.div(reshaped_mat1, broadcast_scale1)
        reshaped_scaled_mat1 = fd.ops.clamp(
            reshaped_scaled_mat1, -FLOAT8_E4M3_MAX, FLOAT8_E4M3_MAX
        )

        scaled_mat1 = fd.ops.reshape(reshaped_scaled_mat1, [m_size, k_size])

        # cast the quantized tv and block sf to proper dtype
        fp4_mat1 = fd.ops.cast(scaled_mat1, DataType.Float4_e2m1fn)
        fp8_scale1 = fd.ops.cast(scale1, DataType.Float8_e4m3fn)

        # swizzle & pad block sf
        layout_fp8_scale1 = fd.ops.preprocess_grouped_matmul_input_sf(
            fp8_scale1, offsets, blockscale_offsets
        )
        out = fd.ops.cutlass_nvfp4_grouped_mm(
            fp4_mat1,
            mat2,
            layout_fp8_scale1,
            scale2,
            alpha,
            problem_sizes,
            offsets,
            blockscale_offsets,
            DataType.BFloat16,
        )
        fd.add_output(out)

    inputs = [
        mat1,
        mat2_scaled.view(torch.float4_e2m1fn_x2).transpose(-1, -2),
        scale2,
        mat2_gs,
        problem_sizes,
        offsets,
        blockscale_offsets,
    ]

    # FIXME: force indexing to use IdModel indexer to avoid indexing error.
    # see issue: https://github.com/NVIDIA/Fuser/issues/5200
    # NOTE: IdModel indexer is now enabled by default, so this is no longer needed.
    o, _ = nvfuser_direct_test.exec_nvfuser(nvfuser_fusion_id0, inputs)

    # quantization for activation is needed for reference.
    # note: following sglang implementation, not computing global scaling factor for mat1
    #       similarly, we don't need to apply mat1_gs to alpha
    mat1_gs = torch.ones((g,), dtype=torch.float32, device="cuda:0")
    mat1_fp4, scale1 = activation_scale_to_nvfp4(
        mat1, mat1_gs, offsets, blockscale_offsets, BLOCK_SIZE
    )
    o_decomposed_ref = torch.empty(m, n, dtype=torch.bfloat16, device="cuda:0")
    for i in range(g):
        l = offsets[i]
        l_sf = blockscale_offsets[i]
        if i == g - 1:
            r = m
        else:
            r = offsets[i + 1]
        r_sf = round_up(tokens_per_expert[i], 128) + l_sf
        # For some reason I cannot feed mat2_gs[i] as alpha in the torch kernel.
        # This triggers a cublas invalid value error.
        o_decomposed_ref[l:r] = (
            torch._scaled_mm(
                mat1_fp4[l:r],
                mat2_scaled[i].transpose(-1, -2),
                scale1[l_sf:r_sf],
                scale2[i],
                None,
                None,
                torch.bfloat16,
            )
            * mat2_gs[i]
        )

    torch.testing.assert_close(o_decomposed_ref, o[0], atol=1e-2, rtol=1e-2)


    microarchitecture_is,
    linear_to_swizzled_128_4,
    round_up,
    activation_scale_to_nvfp4,
)

import pytest

# FIXME: this test needs to be merged back into test_narrow_precision.py.
# We have indexer issue: https://github.com/NVIDIA/Fuser/issues/5200, which
# forces the adoption of environment variable in order to avoid codegen
# assertion. Having this as a separate test file would avoid environment
# variable contamination from others.
@pytest.mark.skipif(
    not microarchitecture_is(10, 0), reason="Only supported on compute 10.0"
)
@pytest.mark.parametrize("config", [[1024, 128, 256]])
@pytest.mark.parametrize("tokens_per_expert_neg_one", [[115, 144, 8]])
@pytest.mark.parametrize("out_dtype", [torch.bfloat16])
def test_layout_op_and_cutlass_nvfp4_grouped_mm(
    nvfuser_direct_test,
    config,
    tokens_per_expert_neg_one,
    out_dtype,
):
    BLOCK_SIZE = 16

    # k dimension is multiple of 4 * 16 to avoid padding on block scaling factor
    m, n, k = config
    assert k % 64 == 0
    tokens_per_expert = list(tokens_per_expert_neg_one)
    tokens_per_expert.append(m - sum(tokens_per_expert))
    g = len(tokens_per_expert)

    mat1 = torch.testing.make_tensor((m, k), dtype=torch.float32, device="cuda:0")
    # format is g, n, k instead of g, k, n
    mat2 = torch.testing.make_tensor((g, n, k), dtype=torch.float32, device="cuda:0")

    offsets = torch.empty((g,), dtype=torch.int32, device="cuda:0")
    blockscale_offsets = torch.empty((g,), dtype=torch.int32, device="cuda:0")
    problem_sizes = torch.empty((g, 3), dtype=torch.int32, device="cuda:0")

    # prepare quantization for mat2
    mat2_gs = torch.empty((g,), dtype=torch.float32, device="cuda:0")
    scale2 = torch.empty(
        (g, n, k // BLOCK_SIZE), dtype=torch.float8_e4m3fn, device="cuda:0"
    )

    acc_tokens = 0
    rounded_acc_tokens = 0
    mat2_scaled = torch.empty(
        (g, n, k // 2), dtype=torch.float4_e2m1fn_x2, device="cuda:0"
    )

    for i in range(g):
        global_sf = FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX / mat2[i].max()
        offsets[i] = acc_tokens
        blockscale_offsets[i] = rounded_acc_tokens
        acc_tokens += tokens_per_expert[i]
        # Note: we technically don't need to round up, since k is perfectly sized.
        rounded_acc_tokens += round_up(tokens_per_expert[i], 128)

        problem_sizes[i][0] = tokens_per_expert[i]
        problem_sizes[i][1] = n
        problem_sizes[i][2] = k

        scaled_mat2_i, bs_mat2_i = pytorch_nvfp4_quantize(mat2[i], global_sf)
        mat2_gs[i] = 1.0 / global_sf
        mat2_scaled[i] = scaled_mat2_i
        scale2[i] = linear_to_swizzled_128_4(bs_mat2_i)

    def nvfuser_fusion_id0(fd: FusionDefinition) -> None:
        mat1 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=True,
            dtype=DataType.Float,
            is_cpu=False,
        )
        mat2 = fd.define_tensor(
            shape=[-1, -1, -1],
            contiguity=True,
            dtype=DataType.Float4_e2m1fn,
            is_cpu=False,
            stride_order=[2, 0, 1],
        )
        scale2 = fd.define_tensor(
            shape=[-1, -1, -1],
            contiguity=True,
            dtype=DataType.Float8_e4m3fn,
            is_cpu=False,
        )
        alpha = fd.define_tensor(
            shape=[-1], contiguity=True, dtype=DataType.Float, is_cpu=False
        )
        problem_sizes = fd.define_tensor(
            shape=[-1, -1], contiguity=True, dtype=DataType.Int32, is_cpu=False
        )
        offsets = fd.define_tensor(
            shape=[-1], contiguity=True, dtype=DataType.Int32, is_cpu=False
        )
        blockscale_offsets = fd.define_tensor(
            shape=[-1], contiguity=True, dtype=DataType.Int32, is_cpu=False
        )
        # TODO: fix dynamic shape in issue https://github.com/NVIDIA/Fuser/issues/5199
        # m_size = fd.ops.size(mat1, 0)
        # k_size = fd.ops.size(mat1, 1)
        # k_tile_size = fd.ops.div(k_size, 16)
        # use static shape as a temporary WAR.
        m_size = m
        k_size = k
        k_tile_size = k_size // 16
        # using primitive operations to handle quantization
        reshaped_mat1 = fd.ops.reshape(mat1, [m_size, k_tile_size, 16])

        # quantization math to compute block scaling factor
        scale1 = fd.ops.abs(reshaped_mat1)
        scale1 = fd.ops.max(scale1, 2)
        scale1 = fd.ops.div(scale1, FLOAT4_E2M1_MAX)
        scale1 = fd.ops.clamp(scale1, FLOAT8_E4M3_EPS, FLOAT8_E4M3_MAX)
        broadcast_scale1 = fd.ops.broadcast(scale1, [False, False, True])
        reshaped_scaled_mat1 = fd.ops.div(reshaped_mat1, broadcast_scale1)
        reshaped_scaled_mat1 = fd.ops.clamp(
            reshaped_scaled_mat1, -FLOAT8_E4M3_MAX, FLOAT8_E4M3_MAX
        )

        scaled_mat1 = fd.ops.reshape(reshaped_scaled_mat1, [m_size, k_size])

        # cast the quantized tv and block sf to proper dtype
        fp4_mat1 = fd.ops.cast(scaled_mat1, DataType.Float4_e2m1fn)
        fp8_scale1 = fd.ops.cast(scale1, DataType.Float8_e4m3fn)

        # swizzle & pad block sf
        layout_fp8_scale1 = fd.ops.preprocess_grouped_matmul_input_sf(
            fp8_scale1, offsets, blockscale_offsets
        )
        out = fd.ops.cutlass_nvfp4_grouped_mm(
            fp4_mat1,
            mat2,
            layout_fp8_scale1,
            scale2,
            alpha,
            problem_sizes,
            offsets,
            blockscale_offsets,
            DataType.BFloat16,
        )
        fd.add_output(out)

    inputs = [
        mat1,
        mat2_scaled.view(torch.float4_e2m1fn_x2).transpose(-1, -2),
        scale2,
        mat2_gs,
        problem_sizes,
        offsets,
        blockscale_offsets,
    ]

    # FIXME: force indexing to use IdModel indexer to avoid indexing error.
    # see issue: https://github.com/NVIDIA/Fuser/issues/5200
    # NOTE: IdModel indexer is now enabled by default, so this is no longer needed.
    o, _ = nvfuser_direct_test.exec_nvfuser(nvfuser_fusion_id0, inputs)

    # quantization for activation is needed for reference.
    # note: following sglang implementation, not computing global scaling factor for mat1
    #       similarly, we don't need to apply mat1_gs to alpha
    mat1_gs = torch.ones((g,), dtype=torch.float32, device="cuda:0")
    mat1_fp4, scale1 = activation_scale_to_nvfp4(
        mat1, mat1_gs, offsets, blockscale_offsets, BLOCK_SIZE
    )
    o_decomposed_ref = torch.empty(m, n, dtype=torch.bfloat16, device="cuda:0")
    for i in range(g):
        l = offsets[i]
        l_sf = blockscale_offsets[i]
        if i == g - 1:
            r = m
        else:
            r = offsets[i + 1]
        r_sf = round_up(tokens_per_expert[i], 128) + l_sf
        # For some reason I cannot feed mat2_gs[i] as alpha in the torch kernel.
        # This triggers a cublas invalid value error.
        o_decomposed_ref[l:r] = (
            torch._scaled_mm(
                mat1_fp4[l:r],
                mat2_scaled[i].transpose(-1, -2),
                scale1[l_sf:r_sf],
                scale2[i],
                None,
                None,
                torch.bfloat16,
            )
            * mat2_gs[i]
        )

    torch.testing.assert_close(o_decomposed_ref, o[0], atol=1e-2, rtol=1e-2)


        (g, n, k // 2), dtype=torch.float4_e2m1fn_x2, device="cuda:0"
    )

    for i in range(g):
        global_sf = FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX / mat2[i].max()
        offsets[i] = acc_tokens
        blockscale_offsets[i] = rounded_acc_tokens
        acc_tokens += tokens_per_expert[i]
        # Note: we technically don't need to round up, since k is perfectly sized.
        rounded_acc_tokens += round_up(tokens_per_expert[i], 128)

        problem_sizes[i][0] = tokens_per_expert[i]
        problem_sizes[i][1] = n
        problem_sizes[i][2] = k

        scaled_mat2_i, bs_mat2_i = pytorch_nvfp4_quantize(mat2[i], global_sf)
        mat2_gs[i] = 1.0 / global_sf
        mat2_scaled[i] = scaled_mat2_i
        scale2[i] = linear_to_swizzled_128_4(bs_mat2_i)

    def nvfuser_fusion_id0(fd: FusionDefinition) -> None:
        mat1 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=True,
            dtype=DataType.Float,
            is_cpu=False,
        )
        mat2 = fd.define_tensor(
            shape=[-1, -1, -1],
            contiguity=True,
            dtype=DataType.Float4_e2m1fn,
            is_cpu=False,
            stride_order=[2, 0, 1],
        )
        scale2 = fd.define_tensor(
            shape=[-1, -1, -1],
            contiguity=True,
            dtype=DataType.Float8_e4m3fn,
            is_cpu=False,
        )
        alpha = fd.define_tensor(
            shape=[-1], contiguity=True, dtype=DataType.Float, is_cpu=False
        )
        problem_sizes = fd.define_tensor(
            shape=[-1, -1], contiguity=True, dtype=DataType.Int32, is_cpu=False
        )
        offsets = fd.define_tensor(
            shape=[-1], contiguity=True, dtype=DataType.Int32, is_cpu=False
        )
        blockscale_offsets = fd.define_tensor(
            shape=[-1], contiguity=True, dtype=DataType.Int32, is_cpu=False
        )
        # TODO: fix dynamic shape in issue https://github.com/NVIDIA/Fuser/issues/5199
        # m_size = fd.ops.size(mat1, 0)
        # k_size = fd.ops.size(mat1, 1)
        # k_tile_size = fd.ops.div(k_size, 16)
        # use static shape as a temporary WAR.
        m_size = m
        k_size = k
        k_tile_size = k_size // 16
        # using primitive operations to handle quantization
        reshaped_mat1 = fd.ops.reshape(mat1, [m_size, k_tile_size, 16])

        # quantization math to compute block scaling factor
        scale1 = fd.ops.abs(reshaped_mat1)
        scale1 = fd.ops.max(scale1, 2)
        scale1 = fd.ops.div(scale1, FLOAT4_E2M1_MAX)
        scale1 = fd.ops.clamp(scale1, FLOAT8_E4M3_EPS, FLOAT8_E4M3_MAX)
        broadcast_scale1 = fd.ops.broadcast(scale1, [False, False, True])
        reshaped_scaled_mat1 = fd.ops.div(reshaped_mat1, broadcast_scale1)
        reshaped_scaled_mat1 = fd.ops.clamp(
            reshaped_scaled_mat1, -FLOAT8_E4M3_MAX, FLOAT8_E4M3_MAX
        )

        scaled_mat1 = fd.ops.reshape(reshaped_scaled_mat1, [m_size, k_size])

        # cast the quantized tv and block sf to proper dtype
        fp4_mat1 = fd.ops.cast(scaled_mat1, DataType.Float4_e2m1fn)
        fp8_scale1 = fd.ops.cast(scale1, DataType.Float8_e4m3fn)

        # swizzle & pad block sf
        layout_fp8_scale1 = fd.ops.preprocess_grouped_matmul_input_sf(
            fp8_scale1, offsets, blockscale_offsets
        )
        out = fd.ops.cutlass_nvfp4_grouped_mm(
            fp4_mat1,
            mat2,
            layout_fp8_scale1,
            scale2,
            alpha,
            problem_sizes,
            offsets,
            blockscale_offsets,
            DataType.BFloat16,
        )
        fd.add_output(out)

    inputs = [
        mat1,
        mat2_scaled.view(torch.float4_e2m1fn_x2).transpose(-1, -2),
        scale2,
        mat2_gs,
        problem_sizes,
        offsets,
        blockscale_offsets,
    ]

    # FIXME: force indexing to use IdModel indexer to avoid indexing error.
    # see issue: https://github.com/NVIDIA/Fuser/issues/5200
    # NOTE: IdModel indexer is now enabled by default, so this is no longer needed.
    o, _ = nvfuser_direct_test.exec_nvfuser(nvfuser_fusion_id0, inputs)

    # quantization for activation is needed for reference.
    # note: following sglang implementation, not computing global scaling factor for mat1
    #       similarly, we don't need to apply mat1_gs to alpha
    mat1_gs = torch.ones((g,), dtype=torch.float32, device="cuda:0")


# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

def test_import_correct():
    try:
        import nvfuser_direct  # noqa: F401
    except Exception as e:
        raise RuntimeError("Failed to import nvfuser_direct.")


# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import pytest
import torch
from nvfuser_direct import nvf_cutlass
from python.direct_utils import microarchitecture_is

if not microarchitecture_is(10, 0):
    pytest.skip(
        reason="MxFp8 Requires compute capability 10.",
        allow_module_level=True,
    )

from python.direct_utils import (
    linear_to_swizzled_128_4,
    swizzled_to_linear_128_4,
)

def dequantize_mxfp8(tensor_fp8, tensor_sf):
    """Dequantize the fp8 tensor back to high precision."""
    m, k = tensor_fp8.shape
    BLOCK_SIZE = 32
    tensor_sf_linear = swizzled_to_linear_128_4(tensor_sf, m, k)
    # Apply scale factor to all elements in the same block
    sf = tensor_sf_linear.repeat_interleave(BLOCK_SIZE, dim=1).to(torch.float32)
    dqx = tensor_fp8.to(torch.float32)
    # Account for padding of scale factor
    sf = sf[: dqx.shape[0], : dqx.shape[1]]
    dequant = dqx * sf
    return dequant.reshape(m, k)

def to_fp8(tensor: torch.Tensor) -> torch.Tensor:
    # The fn suffix means that fp8 is a finite type without infinite support.
    # Clamp values above 464 to avoid casting values to NaN.
    finfo = torch.finfo(torch.float8_e4m3fn)
    return torch.round(tensor.clamp(min=finfo.min, max=finfo.max)).to(
        dtype=torch.float8_e4m3fn
    )

def pytorch_mxfp8_quantize(a):
    BLOCK_SIZE = 32
    assert (
        a.size(-1) % BLOCK_SIZE == 0
    ), "The inner-most dim must be divisible by block_size; Padding is not implemented."
    assert a.is_contiguous(), "Only contiguous tensors are supported."

    # Find absolute maximum along blockwise dimension
    original_shape = a.shape
    a_fp32 = a.float().reshape(original_shape[0], -1, BLOCK_SIZE)
    max_abs = torch.amax(torch.abs(a_fp32), dim=-1)

    # Get fp32 block scale factor for fp8
    FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
    block_scale_fp32 = (max_abs / FLOAT8_E4M3_MAX).float()

    # Clamp scale factor within UE8M0
    FLOAT8_UE8M0_EPS = torch.finfo(torch.float8_e8m0fnu).tiny
    FLOAT8_UE8M0_MAX = torch.finfo(torch.float8_e8m0fnu).max
    block_scale_fp32 = torch.clamp(
        block_scale_fp32, min=FLOAT8_UE8M0_EPS, max=FLOAT8_UE8M0_MAX
    )

    # Apply block conversion factor
    a_scaled = a_fp32 / block_scale_fp32.unsqueeze(-1)
    a_scaled = a_scaled.view(original_shape)

    return to_fp8(a_scaled), block_scale_fp32.to(torch.float8_e8m0fnu)

def get_ref_results(
    a_fp8,
    b_fp8,
    a_sf,
    b_sf,
    m,
    n,
):
    _, m_k = a_fp8.shape
    _, n_k = b_fp8.shape
    assert m_k == n_k
    a_in_dtype = dequantize_mxfp8(a_fp8, a_sf)
    b_in_dtype = dequantize_mxfp8(b_fp8, b_sf)
    return torch.matmul(a_in_dtype, b_in_dtype.t())

@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize(
    "shape", [(128, 128, 128), (128, 128, 256), (256, 128, 128), (128, 256, 256)]
)
@torch.inference_mode()
def test_mxfp8_gemm(
    dtype: torch.dtype,
    shape: tuple[int, int, int],
) -> None:
    m, n, k = shape
    block_size = 32
    a_dtype = torch.randn((m, k), dtype=dtype, device="cuda")
    b_dtype = torch.randn((n, k), dtype=dtype, device="cuda")

    alpha = torch.tensor(1.0, device="cuda")
    a_fp8, a_scale_linear = pytorch_mxfp8_quantize(a_dtype)
    b_fp8, b_scale_linear = pytorch_mxfp8_quantize(b_dtype)
    a_scale_interleaved = linear_to_swizzled_128_4(a_scale_linear)
    b_scale_interleaved = linear_to_swizzled_128_4(b_scale_linear)

    expected_out = get_ref_results(
        a_fp8,
        b_fp8,
        a_scale_interleaved,
        b_scale_interleaved,
        m,
        n,
    )
    out = nvf_cutlass.mxfp8_scaled_mm(
        a_fp8, b_fp8, a_scale_interleaved, b_scale_interleaved, alpha, dtype
    )

    torch.testing.assert_close(out, expected_out.to(dtype=dtype))


# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import pytest
import torch
from nvfuser_direct import nvf_cutlass
from python.direct_utils import microarchitecture_is

if not microarchitecture_is(10, 0):
    pytest.skip(
        reason="MxFp8 Requires compute capability 10.",
        allow_module_level=True,
    )

from python.direct_utils import (
    linear_to_swizzled_128_4,
    swizzled_to_linear_128_4,
)

def dequantize_mxfp8(tensor_fp8, tensor_sf):
    """Dequantize the fp8 tensor back to high precision."""
    m, k = tensor_fp8.shape
    BLOCK_SIZE = 32
    tensor_sf_linear = swizzled_to_linear_128_4(tensor_sf, m, k)
    # Apply scale factor to all elements in the same block
    sf = tensor_sf_linear.repeat_interleave(BLOCK_SIZE, dim=1).to(torch.float32)
    dqx = tensor_fp8.to(torch.float32)
    # Account for padding of scale factor
    sf = sf[: dqx.shape[0], : dqx.shape[1]]
    dequant = dqx * sf
    return dequant.reshape(m, k)

def to_fp8(tensor: torch.Tensor) -> torch.Tensor:
    # The fn suffix means that fp8 is a finite type without infinite support.
    # Clamp values above 464 to avoid casting values to NaN.
    finfo = torch.finfo(torch.float8_e4m3fn)
    return torch.round(tensor.clamp(min=finfo.min, max=finfo.max)).to(
        dtype=torch.float8_e4m3fn
    )

def pytorch_mxfp8_quantize(a):
    BLOCK_SIZE = 32
    assert (
        a.size(-1) % BLOCK_SIZE == 0
    ), "The inner-most dim must be divisible by block_size; Padding is not implemented."
    assert a.is_contiguous(), "Only contiguous tensors are supported."

    # Find absolute maximum along blockwise dimension
    original_shape = a.shape


from python.direct_utils import (
    linear_to_swizzled_128_4,
    swizzled_to_linear_128_4,
)

def dequantize_mxfp8(tensor_fp8, tensor_sf):
    """Dequantize the fp8 tensor back to high precision."""
    m, k = tensor_fp8.shape
    BLOCK_SIZE = 32
    tensor_sf_linear = swizzled_to_linear_128_4(tensor_sf, m, k)
    # Apply scale factor to all elements in the same block
    sf = tensor_sf_linear.repeat_interleave(BLOCK_SIZE, dim=1).to(torch.float32)
    dqx = tensor_fp8.to(torch.float32)
    # Account for padding of scale factor
    sf = sf[: dqx.shape[0], : dqx.shape[1]]
    dequant = dqx * sf
    return dequant.reshape(m, k)

def to_fp8(tensor: torch.Tensor) -> torch.Tensor:
    # The fn suffix means that fp8 is a finite type without infinite support.
    # Clamp values above 464 to avoid casting values to NaN.
    finfo = torch.finfo(torch.float8_e4m3fn)
    return torch.round(tensor.clamp(min=finfo.min, max=finfo.max)).to(
        dtype=torch.float8_e4m3fn
    )

def pytorch_mxfp8_quantize(a):
    BLOCK_SIZE = 32
    assert (
        a.size(-1) % BLOCK_SIZE == 0
    ), "The inner-most dim must be divisible by block_size; Padding is not implemented."
    assert a.is_contiguous(), "Only contiguous tensors are supported."

    # Find absolute maximum along blockwise dimension
    original_shape = a.shape
    a_fp32 = a.float().reshape(original_shape[0], -1, BLOCK_SIZE)
    max_abs = torch.amax(torch.abs(a_fp32), dim=-1)

    # Get fp32 block scale factor for fp8
    FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
    block_scale_fp32 = (max_abs / FLOAT8_E4M3_MAX).float()

    # Clamp scale factor within UE8M0
    FLOAT8_UE8M0_EPS = torch.finfo(torch.float8_e8m0fnu).tiny


    BLOCK_SIZE = 32
    tensor_sf_linear = swizzled_to_linear_128_4(tensor_sf, m, k)
    # Apply scale factor to all elements in the same block
    sf = tensor_sf_linear.repeat_interleave(BLOCK_SIZE, dim=1).to(torch.float32)
    dqx = tensor_fp8.to(torch.float32)
    # Account for padding of scale factor
    sf = sf[: dqx.shape[0], : dqx.shape[1]]
    dequant = dqx * sf
    return dequant.reshape(m, k)

def to_fp8(tensor: torch.Tensor) -> torch.Tensor:
    # The fn suffix means that fp8 is a finite type without infinite support.
    # Clamp values above 464 to avoid casting values to NaN.
    finfo = torch.finfo(torch.float8_e4m3fn)
    return torch.round(tensor.clamp(min=finfo.min, max=finfo.max)).to(
        dtype=torch.float8_e4m3fn
    )

def pytorch_mxfp8_quantize(a):
    BLOCK_SIZE = 32
    assert (
        a.size(-1) % BLOCK_SIZE == 0
    ), "The inner-most dim must be divisible by block_size; Padding is not implemented."
    assert a.is_contiguous(), "Only contiguous tensors are supported."

    # Find absolute maximum along blockwise dimension
    original_shape = a.shape
    a_fp32 = a.float().reshape(original_shape[0], -1, BLOCK_SIZE)
    max_abs = torch.amax(torch.abs(a_fp32), dim=-1)

    # Get fp32 block scale factor for fp8
    FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
    block_scale_fp32 = (max_abs / FLOAT8_E4M3_MAX).float()

    # Clamp scale factor within UE8M0
    FLOAT8_UE8M0_EPS = torch.finfo(torch.float8_e8m0fnu).tiny
    FLOAT8_UE8M0_MAX = torch.finfo(torch.float8_e8m0fnu).max
    block_scale_fp32 = torch.clamp(
        block_scale_fp32, min=FLOAT8_UE8M0_EPS, max=FLOAT8_UE8M0_MAX
    )

    # Apply block conversion factor
    a_scaled = a_fp32 / block_scale_fp32.unsqueeze(-1)
    a_scaled = a_scaled.view(original_shape)

    return to_fp8(a_scaled), block_scale_fp32.to(torch.float8_e8m0fnu)

def get_ref_results(
    a_fp8,
    b_fp8,
    a_sf,
    b_sf,
    m,
    n,
):
    _, m_k = a_fp8.shape
    _, n_k = b_fp8.shape
    assert m_k == n_k
    a_in_dtype = dequantize_mxfp8(a_fp8, a_sf)
    b_in_dtype = dequantize_mxfp8(b_fp8, b_sf)
    return torch.matmul(a_in_dtype, b_in_dtype.t())

@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize(


    max_abs = torch.amax(torch.abs(a_fp32), dim=-1)

    # Get fp32 block scale factor for fp8
    FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
    block_scale_fp32 = (max_abs / FLOAT8_E4M3_MAX).float()

    # Clamp scale factor within UE8M0
    FLOAT8_UE8M0_EPS = torch.finfo(torch.float8_e8m0fnu).tiny
    FLOAT8_UE8M0_MAX = torch.finfo(torch.float8_e8m0fnu).max
    block_scale_fp32 = torch.clamp(
        block_scale_fp32, min=FLOAT8_UE8M0_EPS, max=FLOAT8_UE8M0_MAX
    )

    # Apply block conversion factor
    a_scaled = a_fp32 / block_scale_fp32.unsqueeze(-1)
    a_scaled = a_scaled.view(original_shape)

    return to_fp8(a_scaled), block_scale_fp32.to(torch.float8_e8m0fnu)

def get_ref_results(
    a_fp8,
    b_fp8,
    a_sf,
    b_sf,
    m,
    n,
):
    _, m_k = a_fp8.shape
    _, n_k = b_fp8.shape
    assert m_k == n_k
    a_in_dtype = dequantize_mxfp8(a_fp8, a_sf)
    b_in_dtype = dequantize_mxfp8(b_fp8, b_sf)
    return torch.matmul(a_in_dtype, b_in_dtype.t())

@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize(
    "shape", [(128, 128, 128), (128, 128, 256), (256, 128, 128), (128, 256, 256)]
)
@torch.inference_mode()
def test_mxfp8_gemm(
    dtype: torch.dtype,
    shape: tuple[int, int, int],
) -> None:
    m, n, k = shape
    block_size = 32
    a_dtype = torch.randn((m, k), dtype=dtype, device="cuda")
    b_dtype = torch.randn((n, k), dtype=dtype, device="cuda")

    alpha = torch.tensor(1.0, device="cuda")
    a_fp8, a_scale_linear = pytorch_mxfp8_quantize(a_dtype)
    b_fp8, b_scale_linear = pytorch_mxfp8_quantize(b_dtype)
    a_scale_interleaved = linear_to_swizzled_128_4(a_scale_linear)


    a_fp8,
    b_fp8,
    a_sf,
    b_sf,
    m,
    n,
):
    _, m_k = a_fp8.shape
    _, n_k = b_fp8.shape
    assert m_k == n_k
    a_in_dtype = dequantize_mxfp8(a_fp8, a_sf)
    b_in_dtype = dequantize_mxfp8(b_fp8, b_sf)
    return torch.matmul(a_in_dtype, b_in_dtype.t())

@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize(
    "shape", [(128, 128, 128), (128, 128, 256), (256, 128, 128), (128, 256, 256)]
)
@torch.inference_mode()
def test_mxfp8_gemm(
    dtype: torch.dtype,
    shape: tuple[int, int, int],
) -> None:
    m, n, k = shape
    block_size = 32
    a_dtype = torch.randn((m, k), dtype=dtype, device="cuda")
    b_dtype = torch.randn((n, k), dtype=dtype, device="cuda")

    alpha = torch.tensor(1.0, device="cuda")
    a_fp8, a_scale_linear = pytorch_mxfp8_quantize(a_dtype)
    b_fp8, b_scale_linear = pytorch_mxfp8_quantize(b_dtype)
    a_scale_interleaved = linear_to_swizzled_128_4(a_scale_linear)
    b_scale_interleaved = linear_to_swizzled_128_4(b_scale_linear)

    expected_out = get_ref_results(
        a_fp8,
        b_fp8,
        a_scale_interleaved,
        b_scale_interleaved,
        m,
        n,
    )
    out = nvf_cutlass.mxfp8_scaled_mm(
        a_fp8, b_fp8, a_scale_interleaved, b_scale_interleaved, alpha, dtype
    )

    torch.testing.assert_close(out, expected_out.to(dtype=dtype))


# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import pytest
import torch
from python.direct_utils import microarchitecture_is
from nvfuser_direct import nvf_cutlass

# GPU Compute Capability: https://developer.nvidia.com/cuda/gpus
# tested on blackwell compute 10.0 (B200 and GB200)
# doesn't support 12.0 (RTX PRO 6000 and RTX 50XX)
# Not tested on 10.3 (B300 and GB300)
# Not tested on 12.1 (DGX Spark)
@pytest.mark.skipif(
    not microarchitecture_is(10, 0),
    reason="Does not support blackwell compute 12.0, other arches are not tested.",
)
@pytest.mark.parametrize("config", [[1024, 128, 256], [267, 128, 256]])
@pytest.mark.parametrize("tokens_per_expert_neg_one", [[115, 144, 8], [5, 7, 9]])
@pytest.mark.parametrize("tensor_dtype", [torch.bfloat16, torch.float16])
def test_grouped_mm(
    config,
    tokens_per_expert_neg_one,
    tensor_dtype,
):
    # k dimension is multiple of 128 to avoid padding
    m, n, k = config
    tokens_per_expert = list(tokens_per_expert_neg_one)
    tokens_per_expert.append(m - sum(tokens_per_expert))
    g = len(tokens_per_expert)

    mat1 = torch.testing.make_tensor((m, k), dtype=tensor_dtype, device="cuda:0")
    mat2 = torch.testing.make_tensor((g, n, k), dtype=tensor_dtype, device="cuda:0")
    ab_strides = torch.full((g,), k, dtype=torch.int64, device="cuda:0")
    c_strides = torch.full((g,), n, dtype=torch.int64, device="cuda:0")

    # offsets represents the lhs of slice in nvfuser
    offsets = torch.empty((g,), dtype=torch.int32, device="cuda:0")
    # aten_offsets represents the rhs of slice of torch._grouped_mm(A[m,k], B[g, n, k])
    aten_offsets = torch.empty((g,), dtype=torch.int32, device="cuda:0")
    problem_sizes = torch.empty((g, 3), dtype=torch.int32, device="cuda:0")

    accumulated_tokens = 0
    # Use tokens_per_expert to calculate offsets into m dimension of input tensor.
    for i in range(g):
        offsets[i] = accumulated_tokens
        accumulated_tokens += tokens_per_expert[i]
        aten_offsets[i] = accumulated_tokens

        problem_sizes[i][0] = tokens_per_expert[i]
        problem_sizes[i][1] = n
        problem_sizes[i][2] = k

    out = nvf_cutlass.grouped_mm(
        mat1,
        mat2,
        ab_strides,
        c_strides,
        problem_sizes,
        offsets,
    )

    # Create pytorch expected output reference
    # For each expert, apply gemm. Slice the input matrix given the tokens_per_expert.
    # C[start:stop] = A[start:stop] @ B[expert].
    out_ref = torch._grouped_mm(mat1, mat2.transpose(-1, -2), aten_offsets)
    torch.testing.assert_close(out_ref, out, atol=1e-2, rtol=1e-2)


# Owner(s): ["module: nvfuser"]

import pytest
import torch
from python.direct_utils import microarchitecture_is
from nvfuser_direct import nvf_cutlass

# GPU Compute Capability: https://developer.nvidia.com/cuda/gpus
# tested on blackwell compute 10.0 (B200 and GB200)
# doesn't support 12.0 (RTX PRO 6000 and RTX 50XX)
# Not tested on 10.3 (B300 and GB300)
# Not tested on 12.1 (DGX Spark)
@pytest.mark.skipif(
    not microarchitecture_is(10, 0),
    reason="Does not support blackwell compute 12.0, other arches are not tested.",
)
@pytest.mark.parametrize("config", [[1024, 128, 256], [267, 128, 256]])
@pytest.mark.parametrize("tokens_per_expert_neg_one", [[115, 144, 8], [5, 7, 9]])
@pytest.mark.parametrize("tensor_dtype", [torch.bfloat16, torch.float16])
def test_grouped_mm(
    config,
    tokens_per_expert_neg_one,
    tensor_dtype,
):
    # k dimension is multiple of 128 to avoid padding
    m, n, k = config
    tokens_per_expert = list(tokens_per_expert_neg_one)
    tokens_per_expert.append(m - sum(tokens_per_expert))
    g = len(tokens_per_expert)

    mat1 = torch.testing.make_tensor((m, k), dtype=tensor_dtype, device="cuda:0")
    mat2 = torch.testing.make_tensor((g, n, k), dtype=tensor_dtype, device="cuda:0")
    ab_strides = torch.full((g,), k, dtype=torch.int64, device="cuda:0")
    c_strides = torch.full((g,), n, dtype=torch.int64, device="cuda:0")

    # offsets represents the lhs of slice in nvfuser
    offsets = torch.empty((g,), dtype=torch.int32, device="cuda:0")
    # aten_offsets represents the rhs of slice of torch._grouped_mm(A[m,k], B[g, n, k])
    aten_offsets = torch.empty((g,), dtype=torch.int32, device="cuda:0")
    problem_sizes = torch.empty((g, 3), dtype=torch.int32, device="cuda:0")

    accumulated_tokens = 0
    # Use tokens_per_expert to calculate offsets into m dimension of input tensor.
    for i in range(g):
        offsets[i] = accumulated_tokens
        accumulated_tokens += tokens_per_expert[i]
        aten_offsets[i] = accumulated_tokens

        problem_sizes[i][0] = tokens_per_expert[i]
        problem_sizes[i][1] = n
        problem_sizes[i][2] = k

    out = nvf_cutlass.grouped_mm(
        mat1,
        mat2,
        ab_strides,
        c_strides,
        problem_sizes,
        offsets,
    )

    # Create pytorch expected output reference
    # For each expert, apply gemm. Slice the input matrix given the tokens_per_expert.
    # C[start:stop] = A[start:stop] @ B[expert].
    out_ref = torch._grouped_mm(mat1, mat2.transpose(-1, -2), aten_offsets)
    torch.testing.assert_close(out_ref, out, atol=1e-2, rtol=1e-2)


# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import torch
from nvfuser_direct import FusionDefinition, ParallelType, DataType

def test_matmul():
    c = 3

    with FusionDefinition() as fd:
        inp = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        w = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        out = fd.ops.matmul(inp, w)
        fd.add_output(out)

        out.outer_split(1, c)
        out.axis(1).parallelize(ParallelType.stream)
        # With NVFUSER_DUMP=host_ir, you'll see the host IR container like the
        # following:
        #
        # %HostIrContainer { (T0_g_float[iS0{i0}, iS1{i2}], T1_g_float[istreamIdx7{3}, iS11{i2}, iS8{( ceilDiv(i4, 3) )}]) -> (T2_g_float[istreamIdx9{3}, iS4{i0}, iS10{( ceilDiv(i4, 3) )}, rS6{i2}]) :
        #   FOR i18 from 0 to 3:
        #     T2_g_float[istreamIdx9{3}, iS4{i0}, iS10{( ceilDiv(i4, 3) )}, rS6{i2}]
        #        = matmul(T0_g_float[iS0{i0}, iS1{i2}],
        #                 T1_g_float[istreamIdx7{3}, iS11{i2}, iS8{( ceilDiv(i4, 3) )}])
        # } // %HostIrContainer

    inp = torch.testing.make_tensor(5, 7, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    w = torch.testing.make_tensor(7, c * 2, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    ref = torch.matmul(inp, w)

    with torch.profiler.profile(record_shapes=True) as profile:
        (out,) = fd.execute(
            [inp, w],
            _enable_options=["host_ir_lowering"],
            _disable_options=["infer_contiguity"],
        )
    torch.testing.assert_close(out, ref)

    matmul_events = [event for event in profile.events() if event.name == "aten::mm"]
    assert len(matmul_events) == c
    for event in matmul_events:
        assert event.input_shapes == [[5, 7], [7, 2], [5, 2]]

def test_two_matmuls_inlinable():
    c = 3

    with FusionDefinition() as fd:
        inp = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        w1 = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        w2 = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        out = fd.ops.matmul(inp, w1)
        out = fd.ops.matmul(out, w2)
        fd.add_output(out)

        inp.outer_split(0, c)
        inp.axis(0).parallelize(ParallelType.stream)
        # With NVFUSER_DUMP=host_ir, you'll see the host IR container like the
        # following:
        #
        # %HostIrContainer { (T0_g_float[istreamIdx12{3}, iS13{( ceilDiv(i0, 3) )}, iS1{i2}], T1_g_float[iS14{i2}, iS3{i4}], T2_g_float[iS15{i4}, iS5{i6}]) -> (T4_g_float[istreamIdx18{3}, iS19{( ceilDiv(i0, 3) )}, iS10{i6}, rS11{i4}]) :
        #   T4_g_float[istreamIdx18{3}, iS19{( ceilDiv(i0, 3) )}, iS10{i6}, rS11{i4}] = ALLOCATE(buffer=T4_g_float[istreamIdx18{3}, iS19{( ceilDiv(i0, 3) )}, iS10{i6}, rS11{i4}], mem_type=global, size=( i0 * i6 ), zero_init=false, resets_to_zero=false)
        #   FOR i99 from 0 to 3:
        #     T5_l_float[istreamIdx22{3}, iS23{( ceilDiv(i0, 3) )}, iS21{i2}] = ShardByStream(T0_g_float[istreamIdx12{3}, iS13{( ceilDiv(i0, 3) )}, iS1{i2}], stream_index = i99)
        #     T3_g_float[istreamIdx16{3}, iS17{( ceilDiv(i0, 3) )}, iS7{i4}, rS8{i2}]
        #        = matmul(T5_l_float[istreamIdx22{3}, iS23{( ceilDiv(i0, 3) )}, iS21{i2}],
        #                 T1_g_float[iS14{i2}, iS3{i4}])
        #     T6_l_float[istreamIdx26{3}, iS27{( ceilDiv(i0, 3) )}, iS25{i6}] = ShardByStream(T4_g_float[istreamIdx18{3}, iS19{( ceilDiv(i0, 3) )}, iS10{i6}, rS11{i4}], stream_index = i99)
        #     T6_l_float[istreamIdx26{3}, iS27{( ceilDiv(i0, 3) )}, iS25{i6}]
        #        = matmul(T3_g_float[istreamIdx16{3}, iS17{( ceilDiv(i0, 3) )}, iS7{i4}, rS8{i2}],
        #                 T2_g_float[iS15{i4}, iS5{i6}])
        # } // %HostIrContainer

    inp = torch.testing.make_tensor(c * 2, 3, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    w1 = torch.testing.make_tensor(3, 5, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    w2 = torch.testing.make_tensor(5, 3, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    ref = torch.matmul(torch.matmul(inp, w1), w2)

    with torch.profiler.profile(record_shapes=True) as profile:
        (out,) = fd.execute(
            [inp, w1, w2],
            _enable_options=["host_ir_lowering"],
            _disable_options=["infer_contiguity"],
        )
    torch.testing.assert_close(out, ref)

    matmul_events = [event for event in profile.events() if event.name == "aten::mm"]
    assert len(matmul_events) == c * 2
    for event in matmul_events:
        # The `m` dimension is split into `c` chunks, so each chunk will have `m == 2`.
        assert event.input_shapes[0][0] == 2

def test_two_matmuls_not_inlinable():
    c = 3

    with FusionDefinition() as fd:
        inp = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        w1 = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        w2 = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        out = fd.ops.matmul(inp, w1)
        out = fd.ops.matmul(out, w2)
        fd.add_output(out)

        w1.split(1, c, inner_split=False)
        w1.axis(1).parallelize(ParallelType.stream)
        out.split(0, c, inner_split=False)
        out.axis(0).parallelize(ParallelType.stream)

    # After sharding propagation, the IR looks like the following:
    #
    # in: [m, k]      w1: [k, n]
    #                        /\
    #                       c
    #             |
    #             | matmul
    #             v
    #          [m, n]      w2: [n, k]
    #             /\
    #            c
    #                  |
    #                  | matmul
    #                  v
    #           out: [m, k]
    #                /\
    #               c
    #
    # The first matmul is column-wise (dimension n) parallel, and the second
    # row-wise (dimension m) parallel. They have to stay in different loops.
    # Therefore, the output of the first matmul (of shape [m, n]) has to be
    # fully allocated.

    inp = torch.testing.make_tensor(c * 2, 3, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    w1 = torch.testing.make_tensor(3, c * 5, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    w2 = torch.testing.make_tensor(c * 5, 3, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    ref = torch.matmul(torch.matmul(inp, w1), w2)

    (out,) = fd.execute([inp, w1, w2], _enable_options=["host_ir_lowering"])
    torch.testing.assert_close(out, ref)


# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import torch
from nvfuser_direct import FusionDefinition, ParallelType, DataType

def test_matmul():
    c = 3

    with FusionDefinition() as fd:
        inp = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        w = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        out = fd.ops.matmul(inp, w)
        fd.add_output(out)

        out.outer_split(1, c)
        out.axis(1).parallelize(ParallelType.stream)
        # With NVFUSER_DUMP=host_ir, you'll see the host IR container like the
        # following:
        #
        # %HostIrContainer { (T0_g_float[iS0{i0}, iS1{i2}], T1_g_float[istreamIdx7{3}, iS11{i2}, iS8{( ceilDiv(i4, 3) )}]) -> (T2_g_float[istreamIdx9{3}, iS4{i0}, iS10{( ceilDiv(i4, 3) )}, rS6{i2}]) :
        #   FOR i18 from 0 to 3:
        #     T2_g_float[istreamIdx9{3}, iS4{i0}, iS10{( ceilDiv(i4, 3) )}, rS6{i2}]
        #        = matmul(T0_g_float[iS0{i0}, iS1{i2}],
        #                 T1_g_float[istreamIdx7{3}, iS11{i2}, iS8{( ceilDiv(i4, 3) )}])
        # } // %HostIrContainer

    inp = torch.testing.make_tensor(5, 7, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    w = torch.testing.make_tensor(7, c * 2, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    ref = torch.matmul(inp, w)

    with torch.profiler.profile(record_shapes=True) as profile:
        (out,) = fd.execute(
            [inp, w],
            _enable_options=["host_ir_lowering"],
            _disable_options=["infer_contiguity"],
        )
    torch.testing.assert_close(out, ref)

    matmul_events = [event for event in profile.events() if event.name == "aten::mm"]
    assert len(matmul_events) == c
    for event in matmul_events:
        assert event.input_shapes == [[5, 7], [7, 2], [5, 2]]

def test_two_matmuls_inlinable():
    c = 3

    with FusionDefinition() as fd:
        inp = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        w1 = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        w2 = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        out = fd.ops.matmul(inp, w1)
        out = fd.ops.matmul(out, w2)
        fd.add_output(out)

        inp.outer_split(0, c)
        inp.axis(0).parallelize(ParallelType.stream)
        # With NVFUSER_DUMP=host_ir, you'll see the host IR container like the
        # following:
        #
        # %HostIrContainer { (T0_g_float[istreamIdx12{3}, iS13{( ceilDiv(i0, 3) )}, iS1{i2}], T1_g_float[iS14{i2}, iS3{i4}], T2_g_float[iS15{i4}, iS5{i6}]) -> (T4_g_float[istreamIdx18{3}, iS19{( ceilDiv(i0, 3) )}, iS10{i6}, rS11{i4}]) :
        #   T4_g_float[istreamIdx18{3}, iS19{( ceilDiv(i0, 3) )}, iS10{i6}, rS11{i4}] = ALLOCATE(buffer=T4_g_float[istreamIdx18{3}, iS19{( ceilDiv(i0, 3) )}, iS10{i6}, rS11{i4}], mem_type=global, size=( i0 * i6 ), zero_init=false, resets_to_zero=false)


    )
    w = torch.testing.make_tensor(7, c * 2, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    ref = torch.matmul(inp, w)

    with torch.profiler.profile(record_shapes=True) as profile:
        (out,) = fd.execute(
            [inp, w],
            _enable_options=["host_ir_lowering"],
            _disable_options=["infer_contiguity"],
        )
    torch.testing.assert_close(out, ref)

    matmul_events = [event for event in profile.events() if event.name == "aten::mm"]
    assert len(matmul_events) == c
    for event in matmul_events:
        assert event.input_shapes == [[5, 7], [7, 2], [5, 2]]

def test_two_matmuls_inlinable():
    c = 3

    with FusionDefinition() as fd:
        inp = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        w1 = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        w2 = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        out = fd.ops.matmul(inp, w1)
        out = fd.ops.matmul(out, w2)
        fd.add_output(out)

        inp.outer_split(0, c)
        inp.axis(0).parallelize(ParallelType.stream)
        # With NVFUSER_DUMP=host_ir, you'll see the host IR container like the
        # following:
        #
        # %HostIrContainer { (T0_g_float[istreamIdx12{3}, iS13{( ceilDiv(i0, 3) )}, iS1{i2}], T1_g_float[iS14{i2}, iS3{i4}], T2_g_float[iS15{i4}, iS5{i6}]) -> (T4_g_float[istreamIdx18{3}, iS19{( ceilDiv(i0, 3) )}, iS10{i6}, rS11{i4}]) :
        #   T4_g_float[istreamIdx18{3}, iS19{( ceilDiv(i0, 3) )}, iS10{i6}, rS11{i4}] = ALLOCATE(buffer=T4_g_float[istreamIdx18{3}, iS19{( ceilDiv(i0, 3) )}, iS10{i6}, rS11{i4}], mem_type=global, size=( i0 * i6 ), zero_init=false, resets_to_zero=false)
        #   FOR i99 from 0 to 3:
        #     T5_l_float[istreamIdx22{3}, iS23{( ceilDiv(i0, 3) )}, iS21{i2}] = ShardByStream(T0_g_float[istreamIdx12{3}, iS13{( ceilDiv(i0, 3) )}, iS1{i2}], stream_index = i99)
        #     T3_g_float[istreamIdx16{3}, iS17{( ceilDiv(i0, 3) )}, iS7{i4}, rS8{i2}]
        #        = matmul(T5_l_float[istreamIdx22{3}, iS23{( ceilDiv(i0, 3) )}, iS21{i2}],
        #                 T1_g_float[iS14{i2}, iS3{i4}])
        #     T6_l_float[istreamIdx26{3}, iS27{( ceilDiv(i0, 3) )}, iS25{i6}] = ShardByStream(T4_g_float[istreamIdx18{3}, iS19{( ceilDiv(i0, 3) )}, iS10{i6}, rS11{i4}], stream_index = i99)
        #     T6_l_float[istreamIdx26{3}, iS27{( ceilDiv(i0, 3) )}, iS25{i6}]
        #        = matmul(T3_g_float[istreamIdx16{3}, iS17{( ceilDiv(i0, 3) )}, iS7{i4}, rS8{i2}],
        #                 T2_g_float[iS15{i4}, iS5{i6}])
        # } // %HostIrContainer

    inp = torch.testing.make_tensor(c * 2, 3, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    w1 = torch.testing.make_tensor(3, 5, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    w2 = torch.testing.make_tensor(5, 3, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    ref = torch.matmul(torch.matmul(inp, w1), w2)

    with torch.profiler.profile(record_shapes=True) as profile:
        (out,) = fd.execute(
            [inp, w1, w2],
            _enable_options=["host_ir_lowering"],
            _disable_options=["infer_contiguity"],
        )
    torch.testing.assert_close(out, ref)

    matmul_events = [event for event in profile.events() if event.name == "aten::mm"]
    assert len(matmul_events) == c * 2
    for event in matmul_events:
        # The `m` dimension is split into `c` chunks, so each chunk will have `m == 2`.
        assert event.input_shapes[0][0] == 2

def test_two_matmuls_not_inlinable():
    c = 3

    with FusionDefinition() as fd:
        inp = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        w1 = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        w2 = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        out = fd.ops.matmul(inp, w1)
        out = fd.ops.matmul(out, w2)
        fd.add_output(out)

        w1.split(1, c, inner_split=False)
        w1.axis(1).parallelize(ParallelType.stream)
        out.split(0, c, inner_split=False)
        out.axis(0).parallelize(ParallelType.stream)

    # After sharding propagation, the IR looks like the following:
    #


    w2 = torch.testing.make_tensor(5, 3, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    ref = torch.matmul(torch.matmul(inp, w1), w2)

    with torch.profiler.profile(record_shapes=True) as profile:
        (out,) = fd.execute(
            [inp, w1, w2],
            _enable_options=["host_ir_lowering"],
            _disable_options=["infer_contiguity"],
        )
    torch.testing.assert_close(out, ref)

    matmul_events = [event for event in profile.events() if event.name == "aten::mm"]
    assert len(matmul_events) == c * 2
    for event in matmul_events:
        # The `m` dimension is split into `c` chunks, so each chunk will have `m == 2`.
        assert event.input_shapes[0][0] == 2

def test_two_matmuls_not_inlinable():
    c = 3

    with FusionDefinition() as fd:
        inp = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        w1 = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        w2 = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        out = fd.ops.matmul(inp, w1)
        out = fd.ops.matmul(out, w2)
        fd.add_output(out)

        w1.split(1, c, inner_split=False)
        w1.axis(1).parallelize(ParallelType.stream)
        out.split(0, c, inner_split=False)
        out.axis(0).parallelize(ParallelType.stream)

    # After sharding propagation, the IR looks like the following:
    #
    # in: [m, k]      w1: [k, n]
    #                        /\
    #                       c
    #             |
    #             | matmul
    #             v
    #          [m, n]      w2: [n, k]
    #             /\
    #            c
    #                  |
    #                  | matmul
    #                  v
    #           out: [m, k]
    #                /\
    #               c
    #
    # The first matmul is column-wise (dimension n) parallel, and the second
    # row-wise (dimension m) parallel. They have to stay in different loops.
    # Therefore, the output of the first matmul (of shape [m, n]) has to be
    # fully allocated.

    inp = torch.testing.make_tensor(c * 2, 3, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    w1 = torch.testing.make_tensor(3, c * 5, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    w2 = torch.testing.make_tensor(c * 5, 3, dtype=torch.int32, device="cuda").to(
        torch.float32
    )
    ref = torch.matmul(torch.matmul(inp, w1), w2)

    (out,) = fd.execute([inp, w1, w2], _enable_options=["host_ir_lowering"])
    torch.testing.assert_close(out, ref)


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import itertools
import math
import pytest
import torch
import torch.nn.functional as F
from enum import Enum, auto
from functools import partial

from nvfuser_direct import (
    FusionDefinition,
    DataType,
)
from python.direct_utils import (
    is_pre_ampere,
    define_sdpa_rng_state,
    verify_stride_order,
)

@pytest.mark.skipif(
    is_pre_ampere(),
    reason="Flash Attention is only supported on Ampere and newer devices.",
)
def test_softmax_logsumexp(nvfuser_direct_test):
    def fusion_func(fd: FusionDefinition) -> None:
        q = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            dtype=DataType.BFloat16,
        )
        k = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            dtype=DataType.BFloat16,
        )
        v = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            dtype=DataType.BFloat16,
        )
        (
            _,
            lse,
            *_,
        ) = fd.ops.sdpfa_fwd(q, k, v, dropout_p=None, is_causal=None, scale=None)
        fd.add_output(lse)

    n, h, l, s, e = 1, 1, 4, 4, 8
    inputs = [
        torch.ones((n, h, l, e), dtype=torch.bfloat16, device="cuda"),
        torch.ones((n, h, s, e), dtype=torch.bfloat16, device="cuda"),
        torch.ones((n, h, s, e), dtype=torch.bfloat16, device="cuda"),
    ]

    from torch.nn.attention import SDPBackend, sdpa_kernel

    with sdpa_kernel(SDPBackend.FLASH_ATTENTION):
        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(
            fusion_func,
            inputs,
        )
    # Ignoring size-1 dimensions, `q @ k^T / sqrt(e)` generates a `l`x`s`
    # matrix full of `sqrt(e)`s.  Therefore, the logsumexp of each row is
    # expected to be log(exp(sqrt(e)) * s) = log(s) + sqrt(e).
    torch.testing.assert_close(
        nvf_out[0].cpu(), torch.full((n, h, l), math.log(s) + e**0.5)
    )

@pytest.mark.skipif(
    is_pre_ampere(),
    reason="Flash Attention is only supported on Ampere and newer devices.",
)
def test_sdpa_fwd(nvfuser_direct_test):
    def fusion_func(
        fd: FusionDefinition, has_dropout: bool, has_causal: bool, has_scale: bool
    ):
        q = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        k = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        v = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        dropout_p, is_causal, scale = None, None, None
        if has_dropout:
            dropout_p = fd.define_scalar(value=None, dtype=DataType.Double)
        if has_causal:
            is_causal = fd.define_scalar(value=None, dtype=DataType.Bool)
        if has_scale:
            scale = fd.define_scalar(value=None, dtype=DataType.Double)
        attn, *_ = fd.ops.sdpfa_fwd(
            q, k, v, dropout_p=dropout_p, is_causal=is_causal, scale=scale
        )
        fd.add_output(attn)

    N, H, L, S, E = 4, 8, 16, 16, 8
    qkv = [
        torch.randn((N, H, L, E), dtype=torch.bfloat16, device="cuda"),
        torch.randn((N, H, S, E), dtype=torch.bfloat16, device="cuda"),
        torch.randn((N, H, S, E), dtype=torch.bfloat16, device="cuda"),
    ]

    dropout_vals = [None, 0.0, 0.2]
    is_causal_vals = [None, True, False]
    scale_vals = [None, 1 / E**0.5, 1e-3]
    # TODO: Try to move this to pytest_ops.py. Currently, it does not work since the API between nvFuser and torch differs.
    for dropout_p, is_causal, scale in itertools.product(
        dropout_vals, is_causal_vals, scale_vals
    ):
        with nvfuser_direct_test.subTest(
            dropout_p=dropout_p, is_causal=is_causal, scale=scale
        ):
            from torch.nn.attention import SDPBackend, sdpa_kernel

            has_dropout = True if dropout_p is not None else False
            has_causal = True if is_causal is not None else False
            has_scale = True if scale is not None else False
            inputs = [*qkv]
            for param in [dropout_p, is_causal, scale]:
                if param is not None:
                    inputs.append(param)
            nvf_out, _ = nvfuser_direct_test.exec_nvfuser(
                partial(
                    fusion_func,
                    has_dropout=has_dropout,
                    has_causal=has_causal,
                    has_scale=has_scale,
                ),
                inputs,
                new_fusion_expected=None,
            )

            # Torch does not accept NoneType dropout_p, is_causal.
            dropout_p = 0.0 if dropout_p is None else dropout_p
            is_causal = False if is_causal is None else is_causal

            with sdpa_kernel(SDPBackend.FLASH_ATTENTION):
                torch.manual_seed(0)
                ref_out = F.scaled_dot_product_attention(
                    *qkv, dropout_p=dropout_p, is_causal=is_causal, scale=scale
                )
            torch.testing.assert_close(nvf_out[0], ref_out)

# Memory layout of query, key, value and output tensors.
class Layout(Enum):
    NHSE = auto()
    NSHE = auto()

@pytest.mark.skipif(
    is_pre_ampere(),
    reason="Flash Attention is only supported on Ampere and newer devices.",
)
@pytest.mark.parametrize("layout", [Layout.NHSE, Layout.NSHE])
def test_sdpa_fwd_bias_mask(nvfuser_direct_test, layout: Layout):
    match layout:
        case Layout.NHSE:
            stride_order = [3, 2, 1, 0]
        case Layout.NSHE:
            stride_order = [3, 1, 2, 0]

    with FusionDefinition() as fd:
        q = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            stride_order=stride_order,
        )
        k = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            stride_order=stride_order,
        )
        v = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            stride_order=stride_order,
        )
        bias = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
        )
        mask = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.Bool,
        )
        attn, *_ = fd.ops.sdpfa_fwd(q, k, v, bias=bias, mask=mask)
        fd.add_output(attn)

    N, H, L, S, E = 11, 7, 5, 3, 2
    match layout:
        case Layout.NHSE:
            q = torch.randn((N, H, L, E), dtype=torch.bfloat16, device="cuda")
            k = torch.randn((N, H, S, E), dtype=torch.bfloat16, device="cuda")
            v = torch.randn((N, H, S, E), dtype=torch.bfloat16, device="cuda")
        case Layout.NSHE:
            q = torch.randn(
                (N, L, H, E), dtype=torch.bfloat16, device="cuda"
            ).transpose(1, 2)
            k = torch.randn(
                (N, S, H, E), dtype=torch.bfloat16, device="cuda"
            ).transpose(1, 2)
            v = torch.randn(
                (N, S, H, E), dtype=torch.bfloat16, device="cuda"
            ).transpose(1, 2)
    bias = torch.randn((N, H, L, S), dtype=torch.bfloat16, device="cuda")
    mask = torch.rand((N, H, L, S), device="cuda") > 0.3

    (out,) = fd.execute([q, k, v, bias, mask])

    attn_mask = (bias + torch.where(mask, 0.0, float("-inf"))).to(dtype=bias.dtype)
    ref_out = torch.nn.functional.scaled_dot_product_attention(
        q, k, v, attn_mask=attn_mask
    )
    torch.testing.assert_close(out, ref_out)
    verify_stride_order(out.stride(), stride_order)

def test_sdpa_bwd(nvfuser_direct_test):
    N, H, L, S, E = 4, 8, 16, 16, 8

    grad_output = torch.randn((N, H, L, E), dtype=torch.bfloat16, device="cuda")
    q = torch.randn((N, H, L, E), dtype=torch.bfloat16, device="cuda")
    k = torch.randn((N, H, S, E), dtype=torch.bfloat16, device="cuda")
    v = torch.randn((N, H, S, E), dtype=torch.bfloat16, device="cuda")

    dropout_vals = [None, 0.0, 0.2]
    is_causal_vals = [None, True, False]
    scale_vals = [None, 1 / E**0.5, 1e-3]

    def fusion_func(
        fd: FusionDefinition, has_dropout: bool, has_causal: bool, has_scale: bool
    ):
        grad_output = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        q = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        k = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        v = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        output = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        logsumexp = fd.define_tensor(
            shape=[-1, -1, -1],
            contiguity=True,
            dtype=DataType.Float,
            is_cpu=False,
        )
        philox_seed, philox_offset = define_sdpa_rng_state(fd)

        dropout_p, is_causal, scale = None, None, None
        if has_dropout:
            dropout_p = fd.define_scalar(value=None, dtype=DataType.Double)
        if has_causal:
            is_causal = fd.define_scalar(value=None, dtype=DataType.Bool)
        if has_scale:
            scale = fd.define_scalar(value=None, dtype=DataType.Double)

        grad_query, grad_key, grad_value = fd.ops.sdpfa_bwd(
            grad_output,
            q,
            k,
            v,
            output,
            logsumexp,
            dropout_p,
            is_causal,
            philox_seed,
            philox_offset,
            scale,
        )
        fd.add_output(grad_query)
        fd.add_output(grad_key)
        fd.add_output(grad_value)

    for dropout_p, is_causal, scale in itertools.product(
        dropout_vals, is_causal_vals, scale_vals
    ):
        with nvfuser_direct_test.subTest(
            dropout_p=dropout_p, is_causal=is_causal, scale=scale
        ):
            # Torch does not accept NoneType dropout_p, is_causal.
            at_dropout_p = 0.0 if dropout_p is None else dropout_p
            at_is_causal = False if is_causal is None else is_causal

            (
                output,
                logsumexp,
                cum_seq_q,
                cum_seq_k,
                query_seq_len,
                key_seq_len,
                philox_seed,
                philox_offset,
                _,
            ) = torch.ops.aten._scaled_dot_product_flash_attention(
                q,
                k,
                v,
                at_dropout_p,
                at_is_causal,
                return_debug_mask=False,
                scale=scale,
            )
            ref_grad = torch.ops.aten._scaled_dot_product_flash_attention_backward(
                grad_output,
                q,
                k,
                v,
                output,
                logsumexp,
                cum_seq_q,
                cum_seq_k,
                query_seq_len,
                key_seq_len,
                at_dropout_p,
                at_is_causal,
                philox_seed,
                philox_offset,
                scale=scale,
            )

            has_dropout = True if dropout_p is not None else False
            has_causal = True if is_causal is not None else False
            has_scale = True if scale is not None else False

            inputs = [
                grad_output,
                q,
                k,
                v,
                output,
                logsumexp,
                philox_seed,
                philox_offset,
            ]
            for param in [dropout_p, is_causal, scale]:
                if param is not None:
                    inputs.append(param)

            nvf_out, _ = nvfuser_direct_test.exec_nvfuser(
                partial(
                    fusion_func,
                    has_dropout=has_dropout,
                    has_causal=has_causal,
                    has_scale=has_scale,
                ),
                inputs,
                new_fusion_expected=None,
            )
            torch.testing.assert_close(nvf_out[0], ref_grad[0])
            torch.testing.assert_close(nvf_out[1], ref_grad[1])
            torch.testing.assert_close(nvf_out[2], ref_grad[2])

def test_sdpa_fwd_bwd(nvfuser_direct_test):
    N, H, L, S, E = 4, 8, 16, 16, 8

    dropout_vals = [None, 0.0, 0.2]
    is_causal_vals = [None, True, False]
    scale_vals = [None, 1 / E**0.5, 1e-3]

    def fusion_func(
        fd: FusionDefinition, has_dropout: bool, has_causal: bool, has_scale: bool
    ):
        q = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        k = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        v = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        grad_out = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            is_cpu=False,
        )

        dropout_p, is_causal, scale = None, None, None
        if has_dropout:
            dropout_p = fd.define_scalar(value=None, dtype=DataType.Double)
        if has_causal:
            is_causal = fd.define_scalar(value=None, dtype=DataType.Bool)
        if has_scale:
            scale = fd.define_scalar(value=None, dtype=DataType.Double)

        output, logsumexp, philox_seed, philox_offset = fd.ops.sdpfa_fwd(
            q, k, v, dropout_p=dropout_p, is_causal=is_causal, scale=scale
        )
        grad_query, grad_key, grad_value = fd.ops.sdpfa_bwd(
            grad_out,
            q,
            k,
            v,
            output,
            logsumexp,
            dropout_p,
            is_causal,
            philox_seed,
            philox_offset,
            scale,
        )

        fd.add_output(output)
        fd.add_output(grad_query)
        fd.add_output(grad_key)
        fd.add_output(grad_value)

    for dropout_p, is_causal, scale in itertools.product(
        dropout_vals, is_causal_vals, scale_vals
    ):
        with nvfuser_direct_test.subTest(
            dropout_p=dropout_p, is_causal=is_causal, scale=scale
        ):
            from torch.nn.attention import SDPBackend, sdpa_kernel

            q = torch.randn(
                (N, H, L, E),
                dtype=torch.bfloat16,
                device="cuda",
                requires_grad=True,
            )
            k = torch.randn(
                (N, H, S, E),
                dtype=torch.bfloat16,
                device="cuda",
                requires_grad=True,
            )
            v = torch.randn(
                (N, H, S, E),
                dtype=torch.bfloat16,
                device="cuda",
                requires_grad=True,
            )
            grad_output = torch.randn((N, H, L, E), dtype=torch.bfloat16, device="cuda")

            has_dropout = True if dropout_p is not None else False
            has_causal = True if is_causal is not None else False
            has_scale = True if scale is not None else False

            inputs = [q, k, v, grad_output]
            for param in [dropout_p, is_causal, scale]:
                if param is not None:
                    inputs.append(param)

            # Torch does not accept NoneType dropout_p, is_causal.
            dropout_p = 0.0 if dropout_p is None else dropout_p
            is_causal = False if is_causal is None else is_causal

            with sdpa_kernel(SDPBackend.FLASH_ATTENTION):
                torch.manual_seed(0)
                ref_out = F.scaled_dot_product_attention(
                    q, k, v, dropout_p=dropout_p, is_causal=is_causal, scale=scale
                )
                ref_out.backward(grad_output)

            nvf_out, _ = nvfuser_direct_test.exec_nvfuser(
                partial(
                    fusion_func,
                    has_dropout=has_dropout,
                    has_causal=has_causal,
                    has_scale=has_scale,
                ),
                inputs,
                new_fusion_expected=None,
            )
            torch.testing.assert_close(nvf_out[0], ref_out)
            torch.testing.assert_close(nvf_out[1], q.grad)
            torch.testing.assert_close(nvf_out[2], k.grad)
            torch.testing.assert_close(nvf_out[3], v.grad)


import torch
import torch.nn.functional as F
from enum import Enum, auto
from functools import partial

from nvfuser_direct import (
    FusionDefinition,
    DataType,
)
from python.direct_utils import (
    is_pre_ampere,
    define_sdpa_rng_state,
    verify_stride_order,
)

@pytest.mark.skipif(
    is_pre_ampere(),
    reason="Flash Attention is only supported on Ampere and newer devices.",
)
def test_softmax_logsumexp(nvfuser_direct_test):
    def fusion_func(fd: FusionDefinition) -> None:
        q = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            dtype=DataType.BFloat16,
        )
        k = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            dtype=DataType.BFloat16,
        )
        v = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            dtype=DataType.BFloat16,
        )
        (
            _,
            lse,
            *_,
        ) = fd.ops.sdpfa_fwd(q, k, v, dropout_p=None, is_causal=None, scale=None)
        fd.add_output(lse)

    n, h, l, s, e = 1, 1, 4, 4, 8
    inputs = [
        torch.ones((n, h, l, e), dtype=torch.bfloat16, device="cuda"),
        torch.ones((n, h, s, e), dtype=torch.bfloat16, device="cuda"),
        torch.ones((n, h, s, e), dtype=torch.bfloat16, device="cuda"),
    ]

    from torch.nn.attention import SDPBackend, sdpa_kernel

    with sdpa_kernel(SDPBackend.FLASH_ATTENTION):
        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(
            fusion_func,
            inputs,
        )
    # Ignoring size-1 dimensions, `q @ k^T / sqrt(e)` generates a `l`x`s`
    # matrix full of `sqrt(e)`s.  Therefore, the logsumexp of each row is
    # expected to be log(exp(sqrt(e)) * s) = log(s) + sqrt(e).
    torch.testing.assert_close(
        nvf_out[0].cpu(), torch.full((n, h, l), math.log(s) + e**0.5)
    )

@pytest.mark.skipif(
    is_pre_ampere(),
    reason="Flash Attention is only supported on Ampere and newer devices.",
)
def test_sdpa_fwd(nvfuser_direct_test):
    def fusion_func(
        fd: FusionDefinition, has_dropout: bool, has_causal: bool, has_scale: bool
    ):
        q = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        k = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=True,
            dtype=DataType.BFloat16,