# 2. apply swizzle
def linear_to_swizzled_128_4(a_sf_linear: torch.Tensor):
    mn, sf_k = a_sf_linear.shape
    m_tiles = (mn + 128 - 1) // 128
    mn_padded = m_tiles * 128
    k_tiles = (sf_k + 4 - 1) // 4
    k_padded = k_tiles * 4
    if mn_padded != mn or k_padded != sf_k:
        a_sf_padded = torch.empty(
            mn_padded, k_padded, dtype=a_sf_linear.dtype, device=a_sf_linear.device
        )
        a_sf_padded[0:mn, 0:sf_k] = a_sf_linear
    else:
        a_sf_padded = a_sf_linear
    # details about layout requirement on block-wise scaling factor
    # https://docs.nvidia.com/cutlass/media/docs/cpp/blackwell_functionality.html#scale-factor-layouts
    tmp = torch.reshape(a_sf_padded, (m_tiles, 4, 32, k_tiles, 4))
    return tmp.transpose(1, 3).reshape(mn_padded, k_padded)

def unpack_fp4(x: torch.Tensor) -> torch.Tensor:
    repeated = x.repeat_interleave(2, dim=1)
    repeated[:, 0::2] &= 0x0F
    repeated[:, 1::2] >>= 4
    return repeated

_FP4_LUT = torch.tensor(
    [
        0.0,  # 0: 0000 - zero
        0.5,  # 1: 0001 - smallest positive normal
        1.0,  # 2: 0010
        1.5,  # 3: 0011
        2.0,  # 4: 0100
        3.0,  # 5: 0101
        4.0,  # 6: 0110
        6.0,  # 7: 0111 - largest positive normal
        -0.0,  # 8: 1000 - negative zero
        -0.5,  # 9: 1001 - smallest negative normal
        -1.0,  # 10: 1010
        -1.5,  # 11: 1011
        -2.0,  # 12: 1100
        -3.0,  # 13: 1101
        -4.0,  # 14: 1110
        -6.0,  # 15: 1111 - largest negative normal


        0.5,  # 1: 0001 - smallest positive normal
        1.0,  # 2: 0010
        1.5,  # 3: 0011
        2.0,  # 4: 0100
        3.0,  # 5: 0101
        4.0,  # 6: 0110
        6.0,  # 7: 0111 - largest positive normal
        -0.0,  # 8: 1000 - negative zero
        -0.5,  # 9: 1001 - smallest negative normal
        -1.0,  # 10: 1010
        -1.5,  # 11: 1011
        -2.0,  # 12: 1100
        -3.0,  # 13: 1101
        -4.0,  # 14: 1110
        -6.0,  # 15: 1111 - largest negative normal
    ],
    dtype=torch.float32,
)

def fp4_to_fp32(fp4: torch.Tensor) -> torch.Tensor:
    # Convert FP4 indices to their corresponding floating point values
    # Each index (0-15) represents a 4-bit FP4 value in E2M1 format
    # Values based on the FP4 E2M1 specification
    fp4_lut = _FP4_LUT.to(fp4.device)
    return fp4_lut[fp4.to(torch.long)]

def dequantize_fp4(
    qx: torch.Tensor, sx: torch.Tensor, amax: torch.Tensor
) -> torch.Tensor:
    sf = sx.repeat_interleave(16, dim=1).view(torch.float8_e4m3fn).to(torch.float32)
    dqx = fp4_to_fp32(unpack_fp4(qx))
    sf = sf[: dqx.shape[0], : dqx.shape[1]]
    dequant = dqx * sf * (amax / (6.0 * 448))
    return dequant

def dequantize_to_dtype(
    tensor_fp4, tensor_sf, global_scale, dtype, device, block_size=16
):
    """Dequantize the fp4 tensor back to high precision."""
    # Two fp4 values are packed into one uint8.
    m, packed_k = tensor_fp4.shape
    k = packed_k * 2
    tensor_sf = tensor_sf.view(torch.float8_e4m3fn)


        -0.5,  # 9: 1001 - smallest negative normal
        -1.0,  # 10: 1010
        -1.5,  # 11: 1011
        -2.0,  # 12: 1100
        -3.0,  # 13: 1101
        -4.0,  # 14: 1110
        -6.0,  # 15: 1111 - largest negative normal
    ],
    dtype=torch.float32,
)

def fp4_to_fp32(fp4: torch.Tensor) -> torch.Tensor:
    # Convert FP4 indices to their corresponding floating point values
    # Each index (0-15) represents a 4-bit FP4 value in E2M1 format
    # Values based on the FP4 E2M1 specification
    fp4_lut = _FP4_LUT.to(fp4.device)
    return fp4_lut[fp4.to(torch.long)]

def dequantize_fp4(
    qx: torch.Tensor, sx: torch.Tensor, amax: torch.Tensor
) -> torch.Tensor:
    sf = sx.repeat_interleave(16, dim=1).view(torch.float8_e4m3fn).to(torch.float32)
    dqx = fp4_to_fp32(unpack_fp4(qx))
    sf = sf[: dqx.shape[0], : dqx.shape[1]]
    dequant = dqx * sf * (amax / (6.0 * 448))
    return dequant

def dequantize_to_dtype(
    tensor_fp4, tensor_sf, global_scale, dtype, device, block_size=16
):
    """Dequantize the fp4 tensor back to high precision."""
    # Two fp4 values are packed into one uint8.
    m, packed_k = tensor_fp4.shape
    k = packed_k * 2
    tensor_sf = tensor_sf.view(torch.float8_e4m3fn)
    tensor_sf = swizzled_to_linear_128_4(tensor_sf, m, k)
    out = dequantize_fp4(
        tensor_fp4.view(torch.uint8), tensor_sf, (6.0 * 448.0) / global_scale
    )
    return out.reshape(m, k)

# NOTE: This is from pytorch nvfp4 gemm tests.
def to_fp4(x):
    def down_size(size):


def fp4_to_fp32(fp4: torch.Tensor) -> torch.Tensor:
    # Convert FP4 indices to their corresponding floating point values
    # Each index (0-15) represents a 4-bit FP4 value in E2M1 format
    # Values based on the FP4 E2M1 specification
    fp4_lut = _FP4_LUT.to(fp4.device)
    return fp4_lut[fp4.to(torch.long)]

def dequantize_fp4(
    qx: torch.Tensor, sx: torch.Tensor, amax: torch.Tensor
) -> torch.Tensor:
    sf = sx.repeat_interleave(16, dim=1).view(torch.float8_e4m3fn).to(torch.float32)
    dqx = fp4_to_fp32(unpack_fp4(qx))
    sf = sf[: dqx.shape[0], : dqx.shape[1]]
    dequant = dqx * sf * (amax / (6.0 * 448))
    return dequant

def dequantize_to_dtype(
    tensor_fp4, tensor_sf, global_scale, dtype, device, block_size=16
):
    """Dequantize the fp4 tensor back to high precision."""
    # Two fp4 values are packed into one uint8.
    m, packed_k = tensor_fp4.shape
    k = packed_k * 2
    tensor_sf = tensor_sf.view(torch.float8_e4m3fn)
    tensor_sf = swizzled_to_linear_128_4(tensor_sf, m, k)
    out = dequantize_fp4(
        tensor_fp4.view(torch.uint8), tensor_sf, (6.0 * 448.0) / global_scale
    )
    return out.reshape(m, k)

# NOTE: This is from pytorch nvfp4 gemm tests.
def to_fp4(x):
    def down_size(size):
        assert size[-1] % 2 == 0, f"{size} last dim not divisible by two"
        return (*size[:-1], size[-1] // 2)

    def pack_uint4(uint8_data) -> torch.Tensor:
        # converting to uint8 for operations
        shape = uint8_data.shape
        assert shape[-1] % 2 == 0
        uint8_data = uint8_data.contiguous().view(-1)
        return (uint8_data[1::2] << 4 | uint8_data[::2]).view(down_size(shape))

    from torch.testing._internal.common_quantized import _f32_to_floatx_unpacked

    x = _f32_to_floatx_unpacked(x.float(), ebits=2, mbits=1)
    x = pack_uint4(x)
    x = x.view(torch.float4_e2m1fn_x2)


    dequant = dqx * sf * (amax / (6.0 * 448))
    return dequant

def dequantize_to_dtype(
    tensor_fp4, tensor_sf, global_scale, dtype, device, block_size=16
):
    """Dequantize the fp4 tensor back to high precision."""
    # Two fp4 values are packed into one uint8.
    m, packed_k = tensor_fp4.shape
    k = packed_k * 2
    tensor_sf = tensor_sf.view(torch.float8_e4m3fn)
    tensor_sf = swizzled_to_linear_128_4(tensor_sf, m, k)
    out = dequantize_fp4(
        tensor_fp4.view(torch.uint8), tensor_sf, (6.0 * 448.0) / global_scale
    )
    return out.reshape(m, k)

# NOTE: This is from pytorch nvfp4 gemm tests.
def to_fp4(x):
    def down_size(size):
        assert size[-1] % 2 == 0, f"{size} last dim not divisible by two"
        return (*size[:-1], size[-1] // 2)

    def pack_uint4(uint8_data) -> torch.Tensor:
        # converting to uint8 for operations
        shape = uint8_data.shape
        assert shape[-1] % 2 == 0
        uint8_data = uint8_data.contiguous().view(-1)
        return (uint8_data[1::2] << 4 | uint8_data[::2]).view(down_size(shape))

    from torch.testing._internal.common_quantized import _f32_to_floatx_unpacked

    x = _f32_to_floatx_unpacked(x.float(), ebits=2, mbits=1)
    x = pack_uint4(x)
    x = x.view(torch.float4_e2m1fn_x2)
    return x

def pytorch_nvfp4_quantize(a, a_global_scale):
    BLOCK_SIZE = 16
    assert (
        a.size(-1) % BLOCK_SIZE == 0
    ), "The inner-most dim must be divisible by block_size; Padding is not implemented."
    assert a.is_contiguous(), "Only contiguous tensors are supported."

    original_shape = a.shape
    a_fp32 = a.float().reshape(original_shape[0], -1, BLOCK_SIZE)

    # Find absolute maximum along blockwise dimension
    max_abs = torch.amax(torch.abs(a_fp32), dim=-1)
    block_scale_fp32 = (max_abs / FLOAT4_E2M1_MAX).float()

    scaled_block_scale_fp32 = block_scale_fp32 * a_global_scale
    scaled_block_scale_fp32 = torch.clamp(
        scaled_block_scale_fp32, min=FLOAT8_E4M3_EPS, max=FLOAT8_E4M3_MAX
    )


def to_fp4(x):
    def down_size(size):
        assert size[-1] % 2 == 0, f"{size} last dim not divisible by two"
        return (*size[:-1], size[-1] // 2)

    def pack_uint4(uint8_data) -> torch.Tensor:
        # converting to uint8 for operations
        shape = uint8_data.shape
        assert shape[-1] % 2 == 0
        uint8_data = uint8_data.contiguous().view(-1)
        return (uint8_data[1::2] << 4 | uint8_data[::2]).view(down_size(shape))

    from torch.testing._internal.common_quantized import _f32_to_floatx_unpacked

    x = _f32_to_floatx_unpacked(x.float(), ebits=2, mbits=1)
    x = pack_uint4(x)
    x = x.view(torch.float4_e2m1fn_x2)
    return x

def pytorch_nvfp4_quantize(a, a_global_scale):
    BLOCK_SIZE = 16
    assert (
        a.size(-1) % BLOCK_SIZE == 0
    ), "The inner-most dim must be divisible by block_size; Padding is not implemented."
    assert a.is_contiguous(), "Only contiguous tensors are supported."

    original_shape = a.shape
    a_fp32 = a.float().reshape(original_shape[0], -1, BLOCK_SIZE)

    # Find absolute maximum along blockwise dimension
    max_abs = torch.amax(torch.abs(a_fp32), dim=-1)
    block_scale_fp32 = (max_abs / FLOAT4_E2M1_MAX).float()

    scaled_block_scale_fp32 = block_scale_fp32 * a_global_scale
    scaled_block_scale_fp32 = torch.clamp(
        scaled_block_scale_fp32, min=FLOAT8_E4M3_EPS, max=FLOAT8_E4M3_MAX
    )
    scaled_block_scale_fp8 = scaled_block_scale_fp32.to(torch.float8_e4m3fn)
    total_scale = scaled_block_scale_fp32 / a_global_scale
    a_scaled = a_fp32 / total_scale.unsqueeze(-1)
    a_scaled = torch.clamp(a_scaled, -FLOAT4_E2M1_MAX, FLOAT4_E2M1_MAX)
    a_scaled = a_scaled.view(original_shape)
    return to_fp4(a_scaled), scaled_block_scale_fp8

def round_up(x, y):
    return (x + y - 1) // y * y

def activation_scale_to_nvfp4(x, g_sf, offsets, blockscale_offsets, block_size):
    m = x.size(0)
    k = x.size(1)
    g = g_sf.size(0)
    padded_m_size = blockscale_offsets[g - 1] + round_up(m - offsets[g - 1], 128)
    block_scale = torch.empty(
        (padded_m_size, k // block_size), dtype=torch.float8_e4m3fn, device="cuda:0"
    )
    v_scaled = torch.empty((m, k // 2), dtype=torch.float4_e2m1fn_x2, device="cuda:0")
    for i in range(len(g_sf)):
        l = offsets[i]
        if i == g - 1:
            r = m
        else:


    original_shape = a.shape
    a_fp32 = a.float().reshape(original_shape[0], -1, BLOCK_SIZE)

    # Find absolute maximum along blockwise dimension
    max_abs = torch.amax(torch.abs(a_fp32), dim=-1)
    block_scale_fp32 = (max_abs / FLOAT4_E2M1_MAX).float()

    scaled_block_scale_fp32 = block_scale_fp32 * a_global_scale
    scaled_block_scale_fp32 = torch.clamp(
        scaled_block_scale_fp32, min=FLOAT8_E4M3_EPS, max=FLOAT8_E4M3_MAX
    )
    scaled_block_scale_fp8 = scaled_block_scale_fp32.to(torch.float8_e4m3fn)
    total_scale = scaled_block_scale_fp32 / a_global_scale
    a_scaled = a_fp32 / total_scale.unsqueeze(-1)
    a_scaled = torch.clamp(a_scaled, -FLOAT4_E2M1_MAX, FLOAT4_E2M1_MAX)
    a_scaled = a_scaled.view(original_shape)
    return to_fp4(a_scaled), scaled_block_scale_fp8

def round_up(x, y):
    return (x + y - 1) // y * y

def activation_scale_to_nvfp4(x, g_sf, offsets, blockscale_offsets, block_size):
    m = x.size(0)
    k = x.size(1)
    g = g_sf.size(0)
    padded_m_size = blockscale_offsets[g - 1] + round_up(m - offsets[g - 1], 128)
    block_scale = torch.empty(
        (padded_m_size, k // block_size), dtype=torch.float8_e4m3fn, device="cuda:0"
    )
    v_scaled = torch.empty((m, k // 2), dtype=torch.float4_e2m1fn_x2, device="cuda:0")
    for i in range(len(g_sf)):
        l = offsets[i]
        if i == g - 1:
            r = m
        else:
            r = offsets[i + 1]
        l_sf = blockscale_offsets[i]
        r_sf = l_sf + (r - l + 127) // 128 * 128
        v, b_sf = pytorch_nvfp4_quantize(x[l:r], g_sf[i])


    # Find absolute maximum along blockwise dimension
    max_abs = torch.amax(torch.abs(a_fp32), dim=-1)
    block_scale_fp32 = (max_abs / FLOAT4_E2M1_MAX).float()

    scaled_block_scale_fp32 = block_scale_fp32 * a_global_scale
    scaled_block_scale_fp32 = torch.clamp(
        scaled_block_scale_fp32, min=FLOAT8_E4M3_EPS, max=FLOAT8_E4M3_MAX
    )
    scaled_block_scale_fp8 = scaled_block_scale_fp32.to(torch.float8_e4m3fn)
    total_scale = scaled_block_scale_fp32 / a_global_scale
    a_scaled = a_fp32 / total_scale.unsqueeze(-1)
    a_scaled = torch.clamp(a_scaled, -FLOAT4_E2M1_MAX, FLOAT4_E2M1_MAX)
    a_scaled = a_scaled.view(original_shape)
    return to_fp4(a_scaled), scaled_block_scale_fp8

def round_up(x, y):
    return (x + y - 1) // y * y

def activation_scale_to_nvfp4(x, g_sf, offsets, blockscale_offsets, block_size):
    m = x.size(0)
    k = x.size(1)
    g = g_sf.size(0)
    padded_m_size = blockscale_offsets[g - 1] + round_up(m - offsets[g - 1], 128)
    block_scale = torch.empty(
        (padded_m_size, k // block_size), dtype=torch.float8_e4m3fn, device="cuda:0"
    )
    v_scaled = torch.empty((m, k // 2), dtype=torch.float4_e2m1fn_x2, device="cuda:0")
    for i in range(len(g_sf)):
        l = offsets[i]
        if i == g - 1:
            r = m
        else:
            r = offsets[i + 1]
        l_sf = blockscale_offsets[i]
        r_sf = l_sf + (r - l + 127) // 128 * 128
        v, b_sf = pytorch_nvfp4_quantize(x[l:r], g_sf[i])
        v_scaled[l:r] = v
        block_scale[l_sf:r_sf] = linear_to_swizzled_128_4(b_sf)

    return v_scaled, block_scale


    return dequant

def dequantize_to_dtype(
    tensor_fp4, tensor_sf, global_scale, dtype, device, block_size=16
):
    """Dequantize the fp4 tensor back to high precision."""
    # Two fp4 values are packed into one uint8.
    m, packed_k = tensor_fp4.shape
    k = packed_k * 2
    tensor_sf = tensor_sf.view(torch.float8_e4m3fn)
    tensor_sf = swizzled_to_linear_128_4(tensor_sf, m, k)
    out = dequantize_fp4(
        tensor_fp4.view(torch.uint8), tensor_sf, (6.0 * 448.0) / global_scale
    )
    return out.reshape(m, k)

# NOTE: This is from pytorch nvfp4 gemm tests.
def to_fp4(x):
    def down_size(size):
        assert size[-1] % 2 == 0, f"{size} last dim not divisible by two"
        return (*size[:-1], size[-1] // 2)

    def pack_uint4(uint8_data) -> torch.Tensor:
        # converting to uint8 for operations
        shape = uint8_data.shape
        assert shape[-1] % 2 == 0
        uint8_data = uint8_data.contiguous().view(-1)
        return (uint8_data[1::2] << 4 | uint8_data[::2]).view(down_size(shape))

    from torch.testing._internal.common_quantized import _f32_to_floatx_unpacked

    x = _f32_to_floatx_unpacked(x.float(), ebits=2, mbits=1)
    x = pack_uint4(x)
    x = x.view(torch.float4_e2m1fn_x2)
    return x

def pytorch_nvfp4_quantize(a, a_global_scale):
    BLOCK_SIZE = 16
    assert (
        a.size(-1) % BLOCK_SIZE == 0


    tensor_fp4, tensor_sf, global_scale, dtype, device, block_size=16
):
    """Dequantize the fp4 tensor back to high precision."""
    # Two fp4 values are packed into one uint8.
    m, packed_k = tensor_fp4.shape
    k = packed_k * 2
    tensor_sf = tensor_sf.view(torch.float8_e4m3fn)
    tensor_sf = swizzled_to_linear_128_4(tensor_sf, m, k)
    out = dequantize_fp4(
        tensor_fp4.view(torch.uint8), tensor_sf, (6.0 * 448.0) / global_scale
    )
    return out.reshape(m, k)

# NOTE: This is from pytorch nvfp4 gemm tests.
def to_fp4(x):
    def down_size(size):
        assert size[-1] % 2 == 0, f"{size} last dim not divisible by two"
        return (*size[:-1], size[-1] // 2)

    def pack_uint4(uint8_data) -> torch.Tensor:
        # converting to uint8 for operations
        shape = uint8_data.shape
        assert shape[-1] % 2 == 0
        uint8_data = uint8_data.contiguous().view(-1)
        return (uint8_data[1::2] << 4 | uint8_data[::2]).view(down_size(shape))

    from torch.testing._internal.common_quantized import _f32_to_floatx_unpacked

    x = _f32_to_floatx_unpacked(x.float(), ebits=2, mbits=1)
    x = pack_uint4(x)
    x = x.view(torch.float4_e2m1fn_x2)
    return x

def pytorch_nvfp4_quantize(a, a_global_scale):
    BLOCK_SIZE = 16
    assert (
        a.size(-1) % BLOCK_SIZE == 0
    ), "The inner-most dim must be divisible by block_size; Padding is not implemented."
    assert a.is_contiguous(), "Only contiguous tensors are supported."

    original_shape = a.shape
    a_fp32 = a.float().reshape(original_shape[0], -1, BLOCK_SIZE)

    # Find absolute maximum along blockwise dimension


# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import torch
import pytest
from nvfuser_direct import FusionDefinition, DataType, TensorView
from looseversion import LooseVersion

def microarchitecture_is(major, minor):
    prop = torch.cuda.get_device_properties(torch.cuda.current_device())
    return prop.major == major and prop.minor == minor

def microarchitecture_is_pre(major):
    prop = torch.cuda.get_device_properties(torch.cuda.current_device())
    return prop.major < major

def is_pre_volta():
    return microarchitecture_is_pre(7)

def is_pre_ampere():
    return microarchitecture_is_pre(8)

def is_pre_hopper():
    return microarchitecture_is_pre(9)

def is_pre_blackwell():
    return microarchitecture_is_pre(10)

# Get string representation for FusionDefinition
# Run captured python definition
# Check that the result of captured python definition matches original results
def check_captured_python_definition(
    reference_outputs,
    fd,
    inputs,
    device=None,
    enable_options=None,
    disable_options=None,
):
    try:
        fd_str = fd.__repr__()
        func_name = "nvfuser_fusion"
        exec(fd_str)

        # Execute the python definition that was captured
        with FusionDefinition() as fd_cap:
            eval(func_name)(fd_cap)

        torch.manual_seed(0)
        if enable_options is None:
            enable_options = []
        if disable_options is None:
            disable_options = []
        captured_outputs = fd_cap.execute(
            inputs,
            device=device,
            _enable_options=enable_options,
            _disable_options=disable_options,
        )

        if len(reference_outputs) != len(captured_outputs):
            return False

        # Check that the values of all outputs match
        for ref_out, cap_out in zip(reference_outputs, captured_outputs):
            # torch.allclose does not work with fp8 datatype, so cast to fp64.
            # However, casting complex values to real discards the imaginary
            # part, so skip complex dtypes.
            # Similarly, packed fp4 dtype cannot be compared neither, we view
            # it as int8 and run comparison as-is.
            if ref_out.dtype == torch.float4_e2m1fn_x2:
                ref_out = ref_out.view(torch.int8)
            elif not ref_out.dtype.is_complex:
                ref_out = ref_out.to(torch.float64)
            if cap_out.dtype == torch.float4_e2m1fn_x2:
                cap_out = cap_out.view(torch.int8)
            elif not cap_out.dtype.is_complex:
                cap_out = cap_out.to(torch.float64)
            if not torch.allclose(ref_out, cap_out, equal_nan=True):
                return False

        # Check that the stride of all outputs match
        return all(
            [
                ref_out.stride() == cap_out.stride()
                for ref_out, cap_out in zip(reference_outputs, captured_outputs)
            ]
        )
    except Exception as err:
        print("\nException For Printed FusionDefinition:")
        print(
            "(A failure here suggests a mismatch in functionality between the original definition and the printed definition.)"
        )
        if "fd_str" in locals():
            print(fd_str)
        raise err

def verify_stride_order(output_strides, stride_order):
    sorted_stride = list(output_strides)
    rank = len(output_strides)
    for idx, axis in enumerate(stride_order):
        sorted_stride[rank - 1 - axis] = output_strides[idx]
    assert sorted(sorted_stride, reverse=True) == sorted_stride

UPDATED_SDPA = LooseVersion(torch.__version__) >= LooseVersion("2.7.0")

def define_sdpa_rng_state(fd: FusionDefinition) -> tuple[TensorView, TensorView]:
    dtype = DataType.UInt64 if UPDATED_SDPA else DataType.Int
    is_cpu = False if UPDATED_SDPA else True
    philox_shape = [2] if UPDATED_SDPA else []
    philox_seed = fd.define_tensor(
        shape=philox_shape,
        dtype=dtype,
        is_cpu=is_cpu,
    )
    philox_offset = fd.define_tensor(
        shape=[],
        dtype=dtype,
        is_cpu=is_cpu,
    )
    return philox_seed, philox_offset

def create_sdpa_rng_tensors() -> tuple[torch.Tensor, torch.Tensor]:
    dtype = torch.uint64 if UPDATED_SDPA else torch.int64
    device = "cuda" if UPDATED_SDPA else "cpu"
    philox_shape = (2,) if UPDATED_SDPA else ()
    philox_seed = torch.testing.make_tensor(philox_shape, device=device, dtype=dtype)
    philox_offset = torch.testing.make_tensor((), device=device, dtype=dtype)
    return philox_seed, philox_offset

def skip_if_global_memory_below_gb(min_gb: int, gpu_id: int = 0):
    device_properties = torch.cuda.get_device_properties(gpu_id)
    total_memory_bytes = device_properties.total_memory
    min_bytes = min_gb * (1024**3)

    if total_memory_bytes < min_bytes:
        pytest.skip(
            f"Insufficient GPU global memory: requires ~{min_bytes} B, "
            f"but only {total_memory_bytes} B available"
        )


# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import torch
import pytest
from nvfuser_direct import FusionDefinition, DataType, TensorView
from looseversion import LooseVersion

def microarchitecture_is(major, minor):
    prop = torch.cuda.get_device_properties(torch.cuda.current_device())
    return prop.major == major and prop.minor == minor

def microarchitecture_is_pre(major):
    prop = torch.cuda.get_device_properties(torch.cuda.current_device())
    return prop.major < major

def is_pre_volta():
    return microarchitecture_is_pre(7)

def is_pre_ampere():
    return microarchitecture_is_pre(8)

def is_pre_hopper():
    return microarchitecture_is_pre(9)

def is_pre_blackwell():


# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import torch
import pytest
from nvfuser_direct import FusionDefinition, DataType, TensorView
from looseversion import LooseVersion

def microarchitecture_is(major, minor):
    prop = torch.cuda.get_device_properties(torch.cuda.current_device())
    return prop.major == major and prop.minor == minor

def microarchitecture_is_pre(major):
    prop = torch.cuda.get_device_properties(torch.cuda.current_device())
    return prop.major < major

def is_pre_volta():
    return microarchitecture_is_pre(7)

def is_pre_ampere():
    return microarchitecture_is_pre(8)

def is_pre_hopper():
    return microarchitecture_is_pre(9)

def is_pre_blackwell():
    return microarchitecture_is_pre(10)

# Get string representation for FusionDefinition
# Run captured python definition


# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import torch
import pytest
from nvfuser_direct import FusionDefinition, DataType, TensorView
from looseversion import LooseVersion

def microarchitecture_is(major, minor):
    prop = torch.cuda.get_device_properties(torch.cuda.current_device())
    return prop.major == major and prop.minor == minor

def microarchitecture_is_pre(major):
    prop = torch.cuda.get_device_properties(torch.cuda.current_device())
    return prop.major < major

def is_pre_volta():
    return microarchitecture_is_pre(7)

def is_pre_ampere():
    return microarchitecture_is_pre(8)

def is_pre_hopper():
    return microarchitecture_is_pre(9)

def is_pre_blackwell():
    return microarchitecture_is_pre(10)

# Get string representation for FusionDefinition
# Run captured python definition
# Check that the result of captured python definition matches original results
def check_captured_python_definition(
    reference_outputs,
    fd,


import torch
import pytest
from nvfuser_direct import FusionDefinition, DataType, TensorView
from looseversion import LooseVersion

def microarchitecture_is(major, minor):
    prop = torch.cuda.get_device_properties(torch.cuda.current_device())
    return prop.major == major and prop.minor == minor

def microarchitecture_is_pre(major):
    prop = torch.cuda.get_device_properties(torch.cuda.current_device())
    return prop.major < major

def is_pre_volta():
    return microarchitecture_is_pre(7)

def is_pre_ampere():
    return microarchitecture_is_pre(8)

def is_pre_hopper():
    return microarchitecture_is_pre(9)

def is_pre_blackwell():
    return microarchitecture_is_pre(10)

# Get string representation for FusionDefinition
# Run captured python definition
# Check that the result of captured python definition matches original results
def check_captured_python_definition(
    reference_outputs,
    fd,
    inputs,
    device=None,
    enable_options=None,
    disable_options=None,


def microarchitecture_is(major, minor):
    prop = torch.cuda.get_device_properties(torch.cuda.current_device())
    return prop.major == major and prop.minor == minor

def microarchitecture_is_pre(major):
    prop = torch.cuda.get_device_properties(torch.cuda.current_device())
    return prop.major < major

def is_pre_volta():
    return microarchitecture_is_pre(7)

def is_pre_ampere():
    return microarchitecture_is_pre(8)

def is_pre_hopper():
    return microarchitecture_is_pre(9)

def is_pre_blackwell():
    return microarchitecture_is_pre(10)

# Get string representation for FusionDefinition
# Run captured python definition
# Check that the result of captured python definition matches original results
def check_captured_python_definition(
    reference_outputs,
    fd,
    inputs,
    device=None,
    enable_options=None,
    disable_options=None,
):
    try:
        fd_str = fd.__repr__()
        func_name = "nvfuser_fusion"


    return prop.major == major and prop.minor == minor

def microarchitecture_is_pre(major):
    prop = torch.cuda.get_device_properties(torch.cuda.current_device())
    return prop.major < major

def is_pre_volta():
    return microarchitecture_is_pre(7)

def is_pre_ampere():
    return microarchitecture_is_pre(8)

def is_pre_hopper():
    return microarchitecture_is_pre(9)

def is_pre_blackwell():
    return microarchitecture_is_pre(10)

# Get string representation for FusionDefinition
# Run captured python definition
# Check that the result of captured python definition matches original results
def check_captured_python_definition(
    reference_outputs,
    fd,
    inputs,
    device=None,
    enable_options=None,
    disable_options=None,
):
    try:
        fd_str = fd.__repr__()
        func_name = "nvfuser_fusion"
        exec(fd_str)

        # Execute the python definition that was captured
        with FusionDefinition() as fd_cap:


def is_pre_volta():
    return microarchitecture_is_pre(7)

def is_pre_ampere():
    return microarchitecture_is_pre(8)

def is_pre_hopper():
    return microarchitecture_is_pre(9)

def is_pre_blackwell():
    return microarchitecture_is_pre(10)

# Get string representation for FusionDefinition
# Run captured python definition
# Check that the result of captured python definition matches original results
def check_captured_python_definition(
    reference_outputs,
    fd,
    inputs,
    device=None,
    enable_options=None,
    disable_options=None,
):
    try:
        fd_str = fd.__repr__()
        func_name = "nvfuser_fusion"
        exec(fd_str)

        # Execute the python definition that was captured
        with FusionDefinition() as fd_cap:
            eval(func_name)(fd_cap)

        torch.manual_seed(0)
        if enable_options is None:
            enable_options = []
        if disable_options is None:
            disable_options = []
        captured_outputs = fd_cap.execute(
            inputs,
            device=device,
            _enable_options=enable_options,
            _disable_options=disable_options,
        )

        if len(reference_outputs) != len(captured_outputs):
            return False

        # Check that the values of all outputs match
        for ref_out, cap_out in zip(reference_outputs, captured_outputs):
            # torch.allclose does not work with fp8 datatype, so cast to fp64.
            # However, casting complex values to real discards the imaginary
            # part, so skip complex dtypes.
            # Similarly, packed fp4 dtype cannot be compared neither, we view
            # it as int8 and run comparison as-is.
            if ref_out.dtype == torch.float4_e2m1fn_x2:
                ref_out = ref_out.view(torch.int8)
            elif not ref_out.dtype.is_complex:
                ref_out = ref_out.to(torch.float64)
            if cap_out.dtype == torch.float4_e2m1fn_x2:
                cap_out = cap_out.view(torch.int8)
            elif not cap_out.dtype.is_complex:
                cap_out = cap_out.to(torch.float64)
            if not torch.allclose(ref_out, cap_out, equal_nan=True):
                return False

        # Check that the stride of all outputs match
        return all(
            [
                ref_out.stride() == cap_out.stride()
                for ref_out, cap_out in zip(reference_outputs, captured_outputs)
            ]
        )
    except Exception as err:
        print("\nException For Printed FusionDefinition:")
        print(
            "(A failure here suggests a mismatch in functionality between the original definition and the printed definition.)"
        )
        if "fd_str" in locals():
            print(fd_str)
        raise err

def verify_stride_order(output_strides, stride_order):
    sorted_stride = list(output_strides)
    rank = len(output_strides)
    for idx, axis in enumerate(stride_order):
        sorted_stride[rank - 1 - axis] = output_strides[idx]
    assert sorted(sorted_stride, reverse=True) == sorted_stride

UPDATED_SDPA = LooseVersion(torch.__version__) >= LooseVersion("2.7.0")

def define_sdpa_rng_state(fd: FusionDefinition) -> tuple[TensorView, TensorView]:
    dtype = DataType.UInt64 if UPDATED_SDPA else DataType.Int
    is_cpu = False if UPDATED_SDPA else True
    philox_shape = [2] if UPDATED_SDPA else []
    philox_seed = fd.define_tensor(
        shape=philox_shape,
        dtype=dtype,


            if not torch.allclose(ref_out, cap_out, equal_nan=True):
                return False

        # Check that the stride of all outputs match
        return all(
            [
                ref_out.stride() == cap_out.stride()
                for ref_out, cap_out in zip(reference_outputs, captured_outputs)
            ]
        )
    except Exception as err:
        print("\nException For Printed FusionDefinition:")
        print(
            "(A failure here suggests a mismatch in functionality between the original definition and the printed definition.)"
        )
        if "fd_str" in locals():
            print(fd_str)
        raise err

def verify_stride_order(output_strides, stride_order):
    sorted_stride = list(output_strides)
    rank = len(output_strides)
    for idx, axis in enumerate(stride_order):
        sorted_stride[rank - 1 - axis] = output_strides[idx]
    assert sorted(sorted_stride, reverse=True) == sorted_stride

UPDATED_SDPA = LooseVersion(torch.__version__) >= LooseVersion("2.7.0")

def define_sdpa_rng_state(fd: FusionDefinition) -> tuple[TensorView, TensorView]:
    dtype = DataType.UInt64 if UPDATED_SDPA else DataType.Int
    is_cpu = False if UPDATED_SDPA else True
    philox_shape = [2] if UPDATED_SDPA else []
    philox_seed = fd.define_tensor(
        shape=philox_shape,
        dtype=dtype,
        is_cpu=is_cpu,
    )
    philox_offset = fd.define_tensor(
        shape=[],
        dtype=dtype,
        is_cpu=is_cpu,
    )
    return philox_seed, philox_offset


        print("\nException For Printed FusionDefinition:")
        print(
            "(A failure here suggests a mismatch in functionality between the original definition and the printed definition.)"
        )
        if "fd_str" in locals():
            print(fd_str)
        raise err

def verify_stride_order(output_strides, stride_order):
    sorted_stride = list(output_strides)
    rank = len(output_strides)
    for idx, axis in enumerate(stride_order):
        sorted_stride[rank - 1 - axis] = output_strides[idx]
    assert sorted(sorted_stride, reverse=True) == sorted_stride

UPDATED_SDPA = LooseVersion(torch.__version__) >= LooseVersion("2.7.0")

def define_sdpa_rng_state(fd: FusionDefinition) -> tuple[TensorView, TensorView]:
    dtype = DataType.UInt64 if UPDATED_SDPA else DataType.Int
    is_cpu = False if UPDATED_SDPA else True
    philox_shape = [2] if UPDATED_SDPA else []
    philox_seed = fd.define_tensor(
        shape=philox_shape,
        dtype=dtype,
        is_cpu=is_cpu,
    )
    philox_offset = fd.define_tensor(
        shape=[],
        dtype=dtype,
        is_cpu=is_cpu,
    )
    return philox_seed, philox_offset

def create_sdpa_rng_tensors() -> tuple[torch.Tensor, torch.Tensor]:
    dtype = torch.uint64 if UPDATED_SDPA else torch.int64
    device = "cuda" if UPDATED_SDPA else "cpu"
    philox_shape = (2,) if UPDATED_SDPA else ()
    philox_seed = torch.testing.make_tensor(philox_shape, device=device, dtype=dtype)
    philox_offset = torch.testing.make_tensor((), device=device, dtype=dtype)
    return philox_seed, philox_offset

def skip_if_global_memory_below_gb(min_gb: int, gpu_id: int = 0):
    device_properties = torch.cuda.get_device_properties(gpu_id)
    total_memory_bytes = device_properties.total_memory
    min_bytes = min_gb * (1024**3)

    if total_memory_bytes < min_bytes:
        pytest.skip(
            f"Insufficient GPU global memory: requires ~{min_bytes} B, "
            f"but only {total_memory_bytes} B available"


UPDATED_SDPA = LooseVersion(torch.__version__) >= LooseVersion("2.7.0")

def define_sdpa_rng_state(fd: FusionDefinition) -> tuple[TensorView, TensorView]:
    dtype = DataType.UInt64 if UPDATED_SDPA else DataType.Int
    is_cpu = False if UPDATED_SDPA else True
    philox_shape = [2] if UPDATED_SDPA else []
    philox_seed = fd.define_tensor(
        shape=philox_shape,
        dtype=dtype,
        is_cpu=is_cpu,
    )
    philox_offset = fd.define_tensor(
        shape=[],
        dtype=dtype,
        is_cpu=is_cpu,
    )
    return philox_seed, philox_offset

def create_sdpa_rng_tensors() -> tuple[torch.Tensor, torch.Tensor]:
    dtype = torch.uint64 if UPDATED_SDPA else torch.int64
    device = "cuda" if UPDATED_SDPA else "cpu"
    philox_shape = (2,) if UPDATED_SDPA else ()
    philox_seed = torch.testing.make_tensor(philox_shape, device=device, dtype=dtype)
    philox_offset = torch.testing.make_tensor((), device=device, dtype=dtype)
    return philox_seed, philox_offset

def skip_if_global_memory_below_gb(min_gb: int, gpu_id: int = 0):
    device_properties = torch.cuda.get_device_properties(gpu_id)
    total_memory_bytes = device_properties.total_memory
    min_bytes = min_gb * (1024**3)

    if total_memory_bytes < min_bytes:
        pytest.skip(
            f"Insufficient GPU global memory: requires ~{min_bytes} B, "
            f"but only {total_memory_bytes} B available"
        )


        dtype=dtype,
        is_cpu=is_cpu,
    )
    philox_offset = fd.define_tensor(
        shape=[],
        dtype=dtype,
        is_cpu=is_cpu,
    )
    return philox_seed, philox_offset

def create_sdpa_rng_tensors() -> tuple[torch.Tensor, torch.Tensor]:
    dtype = torch.uint64 if UPDATED_SDPA else torch.int64
    device = "cuda" if UPDATED_SDPA else "cpu"
    philox_shape = (2,) if UPDATED_SDPA else ()
    philox_seed = torch.testing.make_tensor(philox_shape, device=device, dtype=dtype)
    philox_offset = torch.testing.make_tensor((), device=device, dtype=dtype)
    return philox_seed, philox_offset

def skip_if_global_memory_below_gb(min_gb: int, gpu_id: int = 0):
    device_properties = torch.cuda.get_device_properties(gpu_id)
    total_memory_bytes = device_properties.total_memory
    min_bytes = min_gb * (1024**3)

    if total_memory_bytes < min_bytes:
        pytest.skip(
            f"Insufficient GPU global memory: requires ~{min_bytes} B, "
            f"but only {total_memory_bytes} B available"
        )


# SPDX-FileCopyrightText: Copyright (c) 2023-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import os
import pathlib
import importlib.util
from setuptools import setup

try:
    from torch.utils.cpp_extension import BuildExtension, CUDAExtension
except ImportError:
    raise RuntimeError("PyTorch cpp_extension is required for building this package.")

nvfuser_csrc_dir = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), "..", "..", "csrc"
)
dynamic_type_dir = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), "..", "..", "lib", "dynamic_type", "src"
)
flatbuffers_dir = os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    "..",
    "..",
    "third_party",
    "flatbuffers",
    "include",
)

# Ensure nvfuser is installed before trying to find its path
try:
    nvfuser_spec = importlib.util.find_spec("nvfuser_common")
    if nvfuser_spec is None or nvfuser_spec.origin is None:
        raise ImportError("Could not find nvfuser. Is it installed?")
    nvfuser_lib_dir = str(pathlib.Path(nvfuser_spec.origin).parent / "lib")
except ImportError as e:
    print(f"Error finding nvfuser path: {e}")
    print("Ensure 'nvfuser' is installed in the build environment.")
    raise e

# Inherit from PyTorch BuildExtension
# Modify build_extension from setuptools.command.build_ext to move shared
# library to nvfuser_extension package
class custom_build_ext(BuildExtension):
    def build_extension(self, ext):
        # Handle different os and cpu arch
        def _find_library_path():
            for item in os.listdir("build"):
                if item.startswith("lib"):
                    return item
            raise RuntimeException("Cannot find lib in build directory")

        # Call PyTorch BuildExtension first that overloads
        # setuptools.command.build_ext
        super().build_extension(ext)
        if ext.name == "nvfuser_extension":
            # Copy files on necessity.
            filename = self.get_ext_filename(self.get_ext_fullname(ext.name))
            fileext = os.path.splitext(filename)[1]

            extension_path = os.path.join("./build/", _find_library_path(), filename)
            assert os.path.exists(extension_path)

            install_dst = os.path.join("nvfuser_extension", filename)
            if not os.path.exists(os.path.dirname(install_dst)):
                os.makedirs(os.path.dirname(install_dst))
            self.copy_file(extension_path, install_dst)

setup(
    # Name is now in pyproject.toml
    # Version is now in pyproject.toml
    ext_modules=[
        CUDAExtension(
            name="nvfuser_extension",  # The name of the *compiled* module
            # pkg tells setuptools where the compiled module should go.
            # Assumes you have a Python package directory named 'nvfuser_extension'
            pkg="nvfuser_extension",
            include_dirs=[nvfuser_csrc_dir, dynamic_type_dir, flatbuffers_dir],
            libraries=["nvfuser_codegen"],
            library_dirs=[nvfuser_lib_dir],
            extra_link_args=[f"-Wl,-rpath,{nvfuser_lib_dir}"],
            sources=["main.cpp"],
            extra_compile_args={"cxx": ["-std=c++20"]},
        )
    ],
    # Keep cmdclass to use custom build extension logic
    cmdclass={"build_ext": custom_build_ext},
)


    "third_party",
    "flatbuffers",
    "include",
)

# Ensure nvfuser is installed before trying to find its path
try:
    nvfuser_spec = importlib.util.find_spec("nvfuser_common")
    if nvfuser_spec is None or nvfuser_spec.origin is None:
        raise ImportError("Could not find nvfuser. Is it installed?")
    nvfuser_lib_dir = str(pathlib.Path(nvfuser_spec.origin).parent / "lib")
except ImportError as e:
    print(f"Error finding nvfuser path: {e}")
    print("Ensure 'nvfuser' is installed in the build environment.")
    raise e

# Inherit from PyTorch BuildExtension
# Modify build_extension from setuptools.command.build_ext to move shared
# library to nvfuser_extension package
class custom_build_ext(BuildExtension):
    def build_extension(self, ext):
        # Handle different os and cpu arch
        def _find_library_path():
            for item in os.listdir("build"):
                if item.startswith("lib"):
                    return item
            raise RuntimeException("Cannot find lib in build directory")

        # Call PyTorch BuildExtension first that overloads
        # setuptools.command.build_ext
        super().build_extension(ext)
        if ext.name == "nvfuser_extension":
            # Copy files on necessity.
            filename = self.get_ext_filename(self.get_ext_fullname(ext.name))
            fileext = os.path.splitext(filename)[1]

            extension_path = os.path.join("./build/", _find_library_path(), filename)
            assert os.path.exists(extension_path)

            install_dst = os.path.join("nvfuser_extension", filename)
            if not os.path.exists(os.path.dirname(install_dst)):
                os.makedirs(os.path.dirname(install_dst))
            self.copy_file(extension_path, install_dst)

setup(
    # Name is now in pyproject.toml
    # Version is now in pyproject.toml
    ext_modules=[
        CUDAExtension(
            name="nvfuser_extension",  # The name of the *compiled* module
            # pkg tells setuptools where the compiled module should go.
            # Assumes you have a Python package directory named 'nvfuser_extension'
            pkg="nvfuser_extension",
            include_dirs=[nvfuser_csrc_dir, dynamic_type_dir, flatbuffers_dir],
            libraries=["nvfuser_codegen"],
            library_dirs=[nvfuser_lib_dir],
            extra_link_args=[f"-Wl,-rpath,{nvfuser_lib_dir}"],
            sources=["main.cpp"],
            extra_compile_args={"cxx": ["-std=c++20"]},
        )
    ],
    # Keep cmdclass to use custom build extension logic


    "flatbuffers",
    "include",
)

# Ensure nvfuser is installed before trying to find its path
try:
    nvfuser_spec = importlib.util.find_spec("nvfuser_common")
    if nvfuser_spec is None or nvfuser_spec.origin is None:
        raise ImportError("Could not find nvfuser. Is it installed?")
    nvfuser_lib_dir = str(pathlib.Path(nvfuser_spec.origin).parent / "lib")
except ImportError as e:
    print(f"Error finding nvfuser path: {e}")
    print("Ensure 'nvfuser' is installed in the build environment.")
    raise e

# Inherit from PyTorch BuildExtension
# Modify build_extension from setuptools.command.build_ext to move shared
# library to nvfuser_extension package
class custom_build_ext(BuildExtension):
    def build_extension(self, ext):
        # Handle different os and cpu arch
        def _find_library_path():
            for item in os.listdir("build"):
                if item.startswith("lib"):
                    return item
            raise RuntimeException("Cannot find lib in build directory")

        # Call PyTorch BuildExtension first that overloads
        # setuptools.command.build_ext
        super().build_extension(ext)
        if ext.name == "nvfuser_extension":
            # Copy files on necessity.
            filename = self.get_ext_filename(self.get_ext_fullname(ext.name))
            fileext = os.path.splitext(filename)[1]

            extension_path = os.path.join("./build/", _find_library_path(), filename)
            assert os.path.exists(extension_path)

            install_dst = os.path.join("nvfuser_extension", filename)
            if not os.path.exists(os.path.dirname(install_dst)):
                os.makedirs(os.path.dirname(install_dst))
            self.copy_file(extension_path, install_dst)

setup(
    # Name is now in pyproject.toml
    # Version is now in pyproject.toml
    ext_modules=[
        CUDAExtension(
            name="nvfuser_extension",  # The name of the *compiled* module
            # pkg tells setuptools where the compiled module should go.
            # Assumes you have a Python package directory named 'nvfuser_extension'
            pkg="nvfuser_extension",
            include_dirs=[nvfuser_csrc_dir, dynamic_type_dir, flatbuffers_dir],
            libraries=["nvfuser_codegen"],
            library_dirs=[nvfuser_lib_dir],
            extra_link_args=[f"-Wl,-rpath,{nvfuser_lib_dir}"],
            sources=["main.cpp"],
            extra_compile_args={"cxx": ["-std=c++20"]},
        )
    ],
    # Keep cmdclass to use custom build extension logic


)

# Ensure nvfuser is installed before trying to find its path
try:
    nvfuser_spec = importlib.util.find_spec("nvfuser_common")
    if nvfuser_spec is None or nvfuser_spec.origin is None:
        raise ImportError("Could not find nvfuser. Is it installed?")
    nvfuser_lib_dir = str(pathlib.Path(nvfuser_spec.origin).parent / "lib")
except ImportError as e:
    print(f"Error finding nvfuser path: {e}")
    print("Ensure 'nvfuser' is installed in the build environment.")
    raise e

# Inherit from PyTorch BuildExtension
# Modify build_extension from setuptools.command.build_ext to move shared
# library to nvfuser_extension package
class custom_build_ext(BuildExtension):
    def build_extension(self, ext):
        # Handle different os and cpu arch
        def _find_library_path():
            for item in os.listdir("build"):
                if item.startswith("lib"):
                    return item
            raise RuntimeException("Cannot find lib in build directory")

        # Call PyTorch BuildExtension first that overloads
        # setuptools.command.build_ext
        super().build_extension(ext)
        if ext.name == "nvfuser_extension":
            # Copy files on necessity.
            filename = self.get_ext_filename(self.get_ext_fullname(ext.name))
            fileext = os.path.splitext(filename)[1]

            extension_path = os.path.join("./build/", _find_library_path(), filename)
            assert os.path.exists(extension_path)

            install_dst = os.path.join("nvfuser_extension", filename)
            if not os.path.exists(os.path.dirname(install_dst)):
                os.makedirs(os.path.dirname(install_dst))
            self.copy_file(extension_path, install_dst)

setup(
    # Name is now in pyproject.toml


# SPDX-FileCopyrightText: Copyright (c) 2023-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import torch
import nvfuser_extension  # noqa: F401

torch.manual_seed(0)
t = torch.randn((5, 5), device="cuda")
expected = torch.sinh(t)
output = torch.ops.myop.sinh_nvfuser(t)

print("Expected:", expected)
print("Output:", output)

assert torch.allclose(output, expected)
print("They match!")


# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# TODO Update script to use nvfuser_direct module
from nvfuser import (
    FusionDefinition,
    SchedulerType,
    ClusterDims,
    MatMulTileOptions,
    GemmTile,
    MmaMacroEncode,
    MmaMacroArch,
    MatmulTileRasterizationOrder,
    MatmulCircularBufferingStrategy,
)
import torch
from torch.autograd import DeviceType
from torch.profiler import profile, record_function, ProfilerActivity
import math
import itertools
from enum import IntEnum

class Layout(IntEnum):
    NN = 0
    NT = 1
    TN = 2
    TT = 3

def estimate_matmul_size(config, dtype):
    def _estimate_size(shape, dtype):
        return math.prod(shape) * dtype.itemsize

    m, n, k, layout = config
    total_in_gbs = 0
    for shape in [[m, k], [n, k], [m, n]]:
        total_in_gbs += _estimate_size(shape, dtype)
    return total_in_gbs

def get_kernel_time(prof_averages: torch.autograd.profiler_util.EventList):
    elapsed_cuda_time = 0
    has_cuda_event = False
    for event in prof_averages:
        if event.device_type != DeviceType.CUDA:
            continue
        has_cuda_event = True
        elapsed_cuda_time = (
            elapsed_cuda_time + event.self_device_time_total
            if hasattr(event, "self_device_time_total")
            else event.self_cuda_time_total
        )
    assert has_cuda_event, "No CUDA events found"
    return elapsed_cuda_time / 1e3

def matmul_fusion(fd: FusionDefinition, inputs: list[torch.Tensor]) -> None:
    a = fd.from_pytorch(inputs[0])
    b = fd.from_pytorch(inputs[1])
    out = fd.ops.matmul(a, b)
    fd.add_output(out)

class MatmulDefinition(FusionDefinition):
    def __init__(self, inputs, config, verbose=False):
        super().__init__()
        self.inputs = inputs
        self.config = config
        self.verbose = verbose

    def definition(self):
        matmul_fusion(self, self.inputs)

    def schedule(self):
        assert self.config is not None
        status, error = self.sched.can_schedule(SchedulerType.matmul)
        assert status, error

        schedule_params = self.sched.compute_matmul_heuristics()

        # Modify original parameters
        tile_sizes, macro, tile_order, cluster_dims, stages = self.config
        schedule_params.tile_sizes = tile_sizes
        schedule_params.mma_macro = macro.mma_macro()
        schedule_params.cta_order = tile_order
        schedule_params.cluster_dims = cluster_dims
        schedule_params.circular_buffering_strategy = (
            MatmulCircularBufferingStrategy.warp_specialized
        )
        schedule_params.circular_buffer_options.circular_buffer_smem_write = stages > 1
        schedule_params.circular_buffer_options.smem_circular_buffer_stage = stages
        if self.verbose:
            print(schedule_params)

        # Schedule fusion
        self.sched.schedule()

def test_matmul_nvf(
    problem_config: tuple,
    schedule_config: tuple,
    verbose: bool = False,
    validate: bool = False,
):
    m, n, k, layout = problem_config
    dtype = torch.bfloat16

    # NOTE reduced precision accumulation is not supported in nvFuser
    torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = False
    torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False

    a = torch.randn(m, k, device="cuda", dtype=dtype)
    b = torch.randn(k, n, device="cuda", dtype=dtype)

    if layout == "NT" or layout == "NN":
        a = a.as_strided(size=[m, k], stride=[1, m])
    if layout == "TN" or layout == "NN":
        b = b.as_strided(size=[k, n], stride=[1, k])

    scheduled_fd = MatmulDefinition([a, b], schedule_config, verbose)

    try:
        nvf_outputs = scheduled_fd.execute([a, b], profile=True)
    except Exception as e:
        if verbose:
            print(e)
        return -1

    with profile(activities=[ProfilerActivity.CUDA], record_shapes=True) as prof:
        with record_function("matmul"):
            baseline_output = torch.matmul(a, b)
    baseline_time = get_kernel_time(prof.key_averages())

    if validate:
        tolerance = k * 1e-6
        assert torch.allclose(
            nvf_outputs[0], baseline_output, atol=tolerance, rtol=tolerance
        )

    prof = scheduled_fd.profile()
    nvf_time = prof.kernel_profiles[0].time_ms
    return baseline_time, nvf_time

def main():
    import argparse

    parser = argparse.ArgumentParser(
        description="""Run through a combination of matmul parameters and compare relative performance against nvjet for a single problem.""",
        epilog="""How to run script: NVFUSER_ENABLE=fuse_matmul NVFUSER_DISABLE=matmul_expr_eval python single_matmul.py nvjet_pybench.json 1752 4720 584 NN --verbose --validate""",
    )
    parser.add_argument("m", type=int, help="The size of M dimension")
    parser.add_argument("n", type=int, help="The size of N dimension")
    parser.add_argument("k", type=int, help="The size of K dimension")
    parser.add_argument(
        "layout",
        type=str,
        choices=[layout.name for layout in Layout],
        help="The layout for matmul problem.",
    )
    parser.add_argument(
        "--verbose",
        action="store_true",
        help="Print matmul parameters and exceptions.",
    )
    parser.add_argument(
        "--validate",
        action="store_true",
        help="Validate nvfuser against pytorch matmul.",
    )
    args = parser.parse_args()

    problem_config = (args.m, args.n, args.k, args.layout)
    device_properties = torch.cuda.get_device_properties(0)
    # short-circuit: problem does not fit on device
    if (
        estimate_matmul_size(problem_config, torch.bfloat16)
        >= device_properties.total_memory
    ):
        assert False

    # These are the parameters we'll optimize
    parameter_configurations = {
        "tile_sizes": [
            MatMulTileOptions(GemmTile(128, 256, 64), GemmTile(64, 256, 64))
        ],
        "mma_macro": [MmaMacroEncode(MmaMacroArch.hopper, 64, 256, 16)],
        "tile_order": [MatmulTileRasterizationOrder.column_major],
        "cluster_dims": [ClusterDims(1, 1, 1)],
        "circular_buffer_stages": [4],
    }

    print(
        f"problem configuration, m: {args.m}, n: {args.n}, k: {args.k}, layout: {args.layout}"
    )
    for idx, scheduler_config in enumerate(
        itertools.product(*parameter_configurations.values())
    ):
        baseline_result, nvf_result = test_matmul_nvf(
            problem_config, scheduler_config, args.verbose, args.validate
        )
        normalized_result = baseline_result / nvf_result
        print(
            f"index: {idx}, baseline(us): {baseline_result: .3e}, "
            f"nvfuser(us): {nvf_result: 3e}, normalized(us): {normalized_result: 2f}"
        )

if __name__ == "__main__":
    main()


Layout

# TODO Update script to use nvfuser_direct module
from nvfuser import (
    FusionDefinition,
    SchedulerType,
    ClusterDims,
    MatMulTileOptions,
    GemmTile,
    MmaMacroEncode,
    MmaMacroArch,
    MatmulTileRasterizationOrder,
    MatmulCircularBufferingStrategy,
)
import torch
from torch.autograd import DeviceType
from torch.profiler import profile, record_function, ProfilerActivity
import math
import itertools
from enum import IntEnum

class Layout(IntEnum):
    NN = 0
    NT = 1
    TN = 2
    TT = 3

def estimate_matmul_size(config, dtype):
    def _estimate_size(shape, dtype):
        return math.prod(shape) * dtype.itemsize

    m, n, k, layout = config
    total_in_gbs = 0
    for shape in [[m, k], [n, k], [m, n]]:
        total_in_gbs += _estimate_size(shape, dtype)
    return total_in_gbs

def get_kernel_time(prof_averages: torch.autograd.profiler_util.EventList):
    elapsed_cuda_time = 0
    has_cuda_event = False
    for event in prof_averages:
        if event.device_type != DeviceType.CUDA:
            continue
        has_cuda_event = True


    MmaMacroEncode,
    MmaMacroArch,
    MatmulTileRasterizationOrder,
    MatmulCircularBufferingStrategy,
)
import torch
from torch.autograd import DeviceType
from torch.profiler import profile, record_function, ProfilerActivity
import math
import itertools
from enum import IntEnum

class Layout(IntEnum):
    NN = 0
    NT = 1
    TN = 2
    TT = 3

def estimate_matmul_size(config, dtype):
    def _estimate_size(shape, dtype):
        return math.prod(shape) * dtype.itemsize

    m, n, k, layout = config
    total_in_gbs = 0
    for shape in [[m, k], [n, k], [m, n]]:
        total_in_gbs += _estimate_size(shape, dtype)
    return total_in_gbs

def get_kernel_time(prof_averages: torch.autograd.profiler_util.EventList):
    elapsed_cuda_time = 0
    has_cuda_event = False
    for event in prof_averages:
        if event.device_type != DeviceType.CUDA:
            continue
        has_cuda_event = True
        elapsed_cuda_time = (
            elapsed_cuda_time + event.self_device_time_total
            if hasattr(event, "self_device_time_total")
            else event.self_cuda_time_total
        )
    assert has_cuda_event, "No CUDA events found"
    return elapsed_cuda_time / 1e3

def matmul_fusion(fd: FusionDefinition, inputs: list[torch.Tensor]) -> None:
    a = fd.from_pytorch(inputs[0])


class Layout(IntEnum):
    NN = 0
    NT = 1
    TN = 2
    TT = 3

def estimate_matmul_size(config, dtype):
    def _estimate_size(shape, dtype):
        return math.prod(shape) * dtype.itemsize

    m, n, k, layout = config
    total_in_gbs = 0
    for shape in [[m, k], [n, k], [m, n]]:
        total_in_gbs += _estimate_size(shape, dtype)
    return total_in_gbs

def get_kernel_time(prof_averages: torch.autograd.profiler_util.EventList):
    elapsed_cuda_time = 0
    has_cuda_event = False
    for event in prof_averages:
        if event.device_type != DeviceType.CUDA:
            continue
        has_cuda_event = True
        elapsed_cuda_time = (
            elapsed_cuda_time + event.self_device_time_total
            if hasattr(event, "self_device_time_total")
            else event.self_cuda_time_total
        )
    assert has_cuda_event, "No CUDA events found"
    return elapsed_cuda_time / 1e3

def matmul_fusion(fd: FusionDefinition, inputs: list[torch.Tensor]) -> None:
    a = fd.from_pytorch(inputs[0])
    b = fd.from_pytorch(inputs[1])
    out = fd.ops.matmul(a, b)
    fd.add_output(out)

class MatmulDefinition(FusionDefinition):
    def __init__(self, inputs, config, verbose=False):
        super().__init__()
        self.inputs = inputs
        self.config = config
        self.verbose = verbose

    def definition(self):
        matmul_fusion(self, self.inputs)

    def schedule(self):


        total_in_gbs += _estimate_size(shape, dtype)
    return total_in_gbs

def get_kernel_time(prof_averages: torch.autograd.profiler_util.EventList):
    elapsed_cuda_time = 0
    has_cuda_event = False
    for event in prof_averages:
        if event.device_type != DeviceType.CUDA:
            continue
        has_cuda_event = True
        elapsed_cuda_time = (
            elapsed_cuda_time + event.self_device_time_total
            if hasattr(event, "self_device_time_total")
            else event.self_cuda_time_total
        )
    assert has_cuda_event, "No CUDA events found"
    return elapsed_cuda_time / 1e3

def matmul_fusion(fd: FusionDefinition, inputs: list[torch.Tensor]) -> None:
    a = fd.from_pytorch(inputs[0])
    b = fd.from_pytorch(inputs[1])
    out = fd.ops.matmul(a, b)
    fd.add_output(out)

class MatmulDefinition(FusionDefinition):
    def __init__(self, inputs, config, verbose=False):
        super().__init__()
        self.inputs = inputs
        self.config = config
        self.verbose = verbose

    def definition(self):
        matmul_fusion(self, self.inputs)

    def schedule(self):
        assert self.config is not None
        status, error = self.sched.can_schedule(SchedulerType.matmul)
        assert status, error

        schedule_params = self.sched.compute_matmul_heuristics()

        # Modify original parameters


MatmulDefinition

    for event in prof_averages:
        if event.device_type != DeviceType.CUDA:
            continue
        has_cuda_event = True
        elapsed_cuda_time = (
            elapsed_cuda_time + event.self_device_time_total
            if hasattr(event, "self_device_time_total")
            else event.self_cuda_time_total
        )
    assert has_cuda_event, "No CUDA events found"
    return elapsed_cuda_time / 1e3

def matmul_fusion(fd: FusionDefinition, inputs: list[torch.Tensor]) -> None:
    a = fd.from_pytorch(inputs[0])
    b = fd.from_pytorch(inputs[1])
    out = fd.ops.matmul(a, b)
    fd.add_output(out)

class MatmulDefinition(FusionDefinition):
    def __init__(self, inputs, config, verbose=False):
        super().__init__()
        self.inputs = inputs
        self.config = config
        self.verbose = verbose

    def definition(self):
        matmul_fusion(self, self.inputs)

    def schedule(self):
        assert self.config is not None
        status, error = self.sched.can_schedule(SchedulerType.matmul)
        assert status, error

        schedule_params = self.sched.compute_matmul_heuristics()

        # Modify original parameters
        tile_sizes, macro, tile_order, cluster_dims, stages = self.config
        schedule_params.tile_sizes = tile_sizes
        schedule_params.mma_macro = macro.mma_macro()
        schedule_params.cta_order = tile_order
        schedule_params.cluster_dims = cluster_dims
        schedule_params.circular_buffering_strategy = (
            MatmulCircularBufferingStrategy.warp_specialized
        )
        schedule_params.circular_buffer_options.circular_buffer_smem_write = stages > 1
        schedule_params.circular_buffer_options.smem_circular_buffer_stage = stages
        if self.verbose:
            print(schedule_params)

        # Schedule fusion
        self.sched.schedule()

def test_matmul_nvf(
    problem_config: tuple,
    schedule_config: tuple,
    verbose: bool = False,
    validate: bool = False,
):
    m, n, k, layout = problem_config
    dtype = torch.bfloat16

    # NOTE reduced precision accumulation is not supported in nvFuser
    torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = False
    torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False

    a = torch.randn(m, k, device="cuda", dtype=dtype)
    b = torch.randn(k, n, device="cuda", dtype=dtype)

    if layout == "NT" or layout == "NN":
        a = a.as_strided(size=[m, k], stride=[1, m])


        schedule_params = self.sched.compute_matmul_heuristics()

        # Modify original parameters
        tile_sizes, macro, tile_order, cluster_dims, stages = self.config
        schedule_params.tile_sizes = tile_sizes
        schedule_params.mma_macro = macro.mma_macro()
        schedule_params.cta_order = tile_order
        schedule_params.cluster_dims = cluster_dims
        schedule_params.circular_buffering_strategy = (
            MatmulCircularBufferingStrategy.warp_specialized
        )
        schedule_params.circular_buffer_options.circular_buffer_smem_write = stages > 1
        schedule_params.circular_buffer_options.smem_circular_buffer_stage = stages
        if self.verbose:
            print(schedule_params)

        # Schedule fusion
        self.sched.schedule()

def test_matmul_nvf(
    problem_config: tuple,
    schedule_config: tuple,
    verbose: bool = False,
    validate: bool = False,
):
    m, n, k, layout = problem_config
    dtype = torch.bfloat16

    # NOTE reduced precision accumulation is not supported in nvFuser
    torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = False
    torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False

    a = torch.randn(m, k, device="cuda", dtype=dtype)
    b = torch.randn(k, n, device="cuda", dtype=dtype)

    if layout == "NT" or layout == "NN":
        a = a.as_strided(size=[m, k], stride=[1, m])
    if layout == "TN" or layout == "NN":
        b = b.as_strided(size=[k, n], stride=[1, k])

    scheduled_fd = MatmulDefinition([a, b], schedule_config, verbose)

    try:
        nvf_outputs = scheduled_fd.execute([a, b], profile=True)
    except Exception as e:
        if verbose:
            print(e)
        return -1

    with profile(activities=[ProfilerActivity.CUDA], record_shapes=True) as prof:
        with record_function("matmul"):
            baseline_output = torch.matmul(a, b)
    baseline_time = get_kernel_time(prof.key_averages())

    if validate:
        tolerance = k * 1e-6
        assert torch.allclose(
            nvf_outputs[0], baseline_output, atol=tolerance, rtol=tolerance
        )

    prof = scheduled_fd.profile()
    nvf_time = prof.kernel_profiles[0].time_ms
    return baseline_time, nvf_time

def main():
    import argparse

    parser = argparse.ArgumentParser(
        description="""Run through a combination of matmul parameters and compare relative performance against nvjet for a single problem.""",
        epilog="""How to run script: NVFUSER_ENABLE=fuse_matmul NVFUSER_DISABLE=matmul_expr_eval python single_matmul.py nvjet_pybench.json 1752 4720 584 NN --verbose --validate""",
    )
    parser.add_argument("m", type=int, help="The size of M dimension")
    parser.add_argument("n", type=int, help="The size of N dimension")
    parser.add_argument("k", type=int, help="The size of K dimension")
    parser.add_argument(
        "layout",
        type=str,
        choices=[layout.name for layout in Layout],
        help="The layout for matmul problem.",
    )
    parser.add_argument(
        "--verbose",


main

        if verbose:
            print(e)
        return -1

    with profile(activities=[ProfilerActivity.CUDA], record_shapes=True) as prof:
        with record_function("matmul"):
            baseline_output = torch.matmul(a, b)
    baseline_time = get_kernel_time(prof.key_averages())

    if validate:
        tolerance = k * 1e-6
        assert torch.allclose(
            nvf_outputs[0], baseline_output, atol=tolerance, rtol=tolerance
        )

    prof = scheduled_fd.profile()
    nvf_time = prof.kernel_profiles[0].time_ms
    return baseline_time, nvf_time

def main():
    import argparse

    parser = argparse.ArgumentParser(
        description="""Run through a combination of matmul parameters and compare relative performance against nvjet for a single problem.""",
        epilog="""How to run script: NVFUSER_ENABLE=fuse_matmul NVFUSER_DISABLE=matmul_expr_eval python single_matmul.py nvjet_pybench.json 1752 4720 584 NN --verbose --validate""",
    )
    parser.add_argument("m", type=int, help="The size of M dimension")
    parser.add_argument("n", type=int, help="The size of N dimension")
    parser.add_argument("k", type=int, help="The size of K dimension")
    parser.add_argument(
        "layout",
        type=str,
        choices=[layout.name for layout in Layout],
        help="The layout for matmul problem.",
    )
    parser.add_argument(
        "--verbose",
        action="store_true",
        help="Print matmul parameters and exceptions.",
    )
    parser.add_argument(
        "--validate",
        action="store_true",
        help="Validate nvfuser against pytorch matmul.",
    )
    args = parser.parse_args()

    problem_config = (args.m, args.n, args.k, args.layout)
    device_properties = torch.cuda.get_device_properties(0)
    # short-circuit: problem does not fit on device
    if (
        estimate_matmul_size(problem_config, torch.bfloat16)
        >= device_properties.total_memory
    ):
        assert False

    # These are the parameters we'll optimize
    parameter_configurations = {
        "tile_sizes": [
            MatMulTileOptions(GemmTile(128, 256, 64), GemmTile(64, 256, 64))
        ],
        "mma_macro": [MmaMacroEncode(MmaMacroArch.hopper, 64, 256, 16)],
        "tile_order": [MatmulTileRasterizationOrder.column_major],
        "cluster_dims": [ClusterDims(1, 1, 1)],
        "circular_buffer_stages": [4],
    }

    print(
        f"problem configuration, m: {args.m}, n: {args.n}, k: {args.k}, layout: {args.layout}"
    )
    for idx, scheduler_config in enumerate(
        itertools.product(*parameter_configurations.values())
    ):
        baseline_result, nvf_result = test_matmul_nvf(
            problem_config, scheduler_config, args.verbose, args.validate
        )
        normalized_result = baseline_result / nvf_result
        print(
            f"index: {idx}, baseline(us): {baseline_result: .3e}, "
            f"nvfuser(us): {nvf_result: 3e}, normalized(us): {normalized_result: 2f}"
        )

if __name__ == "__main__":
    main()


    MmaMacroArch,
    MatmulTileRasterizationOrder,
    MatmulCircularBufferingStrategy,
)
import torch
from torch.autograd import DeviceType
from torch.profiler import profile, record_function, ProfilerActivity
import math
import itertools
from enum import IntEnum

class Layout(IntEnum):
    NN = 0
    NT = 1
    TN = 2
    TT = 3

def estimate_matmul_size(config, dtype):
    def _estimate_size(shape, dtype):
        return math.prod(shape) * dtype.itemsize

    m, n, k, layout = config
    total_in_gbs = 0
    for shape in [[m, k], [n, k], [m, n]]:
        total_in_gbs += _estimate_size(shape, dtype)
    return total_in_gbs

def get_kernel_time(prof_averages: torch.autograd.profiler_util.EventList):
    elapsed_cuda_time = 0
    has_cuda_event = False
    for event in prof_averages:
        if event.device_type != DeviceType.CUDA:
            continue
        has_cuda_event = True
        elapsed_cuda_time = (
            elapsed_cuda_time + event.self_device_time_total
            if hasattr(event, "self_device_time_total")
            else event.self_cuda_time_total
        )


        if event.device_type != DeviceType.CUDA:
            continue
        has_cuda_event = True
        elapsed_cuda_time = (
            elapsed_cuda_time + event.self_device_time_total
            if hasattr(event, "self_device_time_total")
            else event.self_cuda_time_total
        )
    assert has_cuda_event, "No CUDA events found"
    return elapsed_cuda_time / 1e3

def matmul_fusion(fd: FusionDefinition, inputs: list[torch.Tensor]) -> None:
    a = fd.from_pytorch(inputs[0])
    b = fd.from_pytorch(inputs[1])
    out = fd.ops.matmul(a, b)
    fd.add_output(out)

class MatmulDefinition(FusionDefinition):
    def __init__(self, inputs, config, verbose=False):
        super().__init__()
        self.inputs = inputs
        self.config = config
        self.verbose = verbose

    def definition(self):
        matmul_fusion(self, self.inputs)

    def schedule(self):
        assert self.config is not None
        status, error = self.sched.can_schedule(SchedulerType.matmul)
        assert status, error

        schedule_params = self.sched.compute_matmul_heuristics()

        # Modify original parameters
        tile_sizes, macro, tile_order, cluster_dims, stages = self.config
        schedule_params.tile_sizes = tile_sizes
        schedule_params.mma_macro = macro.mma_macro()
        schedule_params.cta_order = tile_order
        schedule_params.cluster_dims = cluster_dims
        schedule_params.circular_buffering_strategy = (
            MatmulCircularBufferingStrategy.warp_specialized
        )


definition

            else event.self_cuda_time_total
        )
    assert has_cuda_event, "No CUDA events found"
    return elapsed_cuda_time / 1e3

def matmul_fusion(fd: FusionDefinition, inputs: list[torch.Tensor]) -> None:
    a = fd.from_pytorch(inputs[0])
    b = fd.from_pytorch(inputs[1])
    out = fd.ops.matmul(a, b)
    fd.add_output(out)

class MatmulDefinition(FusionDefinition):
    def __init__(self, inputs, config, verbose=False):
        super().__init__()
        self.inputs = inputs
        self.config = config
        self.verbose = verbose

    def definition(self):
        matmul_fusion(self, self.inputs)

    def schedule(self):
        assert self.config is not None
        status, error = self.sched.can_schedule(SchedulerType.matmul)
        assert status, error

        schedule_params = self.sched.compute_matmul_heuristics()

        # Modify original parameters
        tile_sizes, macro, tile_order, cluster_dims, stages = self.config
        schedule_params.tile_sizes = tile_sizes
        schedule_params.mma_macro = macro.mma_macro()
        schedule_params.cta_order = tile_order
        schedule_params.cluster_dims = cluster_dims
        schedule_params.circular_buffering_strategy = (
            MatmulCircularBufferingStrategy.warp_specialized
        )
        schedule_params.circular_buffer_options.circular_buffer_smem_write = stages > 1
        schedule_params.circular_buffer_options.smem_circular_buffer_stage = stages
        if self.verbose:


schedule

    return elapsed_cuda_time / 1e3

def matmul_fusion(fd: FusionDefinition, inputs: list[torch.Tensor]) -> None:
    a = fd.from_pytorch(inputs[0])
    b = fd.from_pytorch(inputs[1])
    out = fd.ops.matmul(a, b)
    fd.add_output(out)

class MatmulDefinition(FusionDefinition):
    def __init__(self, inputs, config, verbose=False):
        super().__init__()
        self.inputs = inputs
        self.config = config
        self.verbose = verbose

    def definition(self):
        matmul_fusion(self, self.inputs)

    def schedule(self):
        assert self.config is not None
        status, error = self.sched.can_schedule(SchedulerType.matmul)
        assert status, error

        schedule_params = self.sched.compute_matmul_heuristics()

        # Modify original parameters
        tile_sizes, macro, tile_order, cluster_dims, stages = self.config
        schedule_params.tile_sizes = tile_sizes
        schedule_params.mma_macro = macro.mma_macro()
        schedule_params.cta_order = tile_order
        schedule_params.cluster_dims = cluster_dims
        schedule_params.circular_buffering_strategy = (
            MatmulCircularBufferingStrategy.warp_specialized
        )
        schedule_params.circular_buffer_options.circular_buffer_smem_write = stages > 1
        schedule_params.circular_buffer_options.smem_circular_buffer_stage = stages
        if self.verbose:
            print(schedule_params)

        # Schedule fusion
        self.sched.schedule()

def test_matmul_nvf(
    problem_config: tuple,
    schedule_config: tuple,
    verbose: bool = False,
    validate: bool = False,
):
    m, n, k, layout = problem_config
    dtype = torch.bfloat16

    # NOTE reduced precision accumulation is not supported in nvFuser
    torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = False
    torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False

    a = torch.randn(m, k, device="cuda", dtype=dtype)
    b = torch.randn(k, n, device="cuda", dtype=dtype)

    if layout == "NT" or layout == "NN":
        a = a.as_strided(size=[m, k], stride=[1, m])


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import torch
import math
import itertools
from nvfuser import FusionCache, FusionDefinition
from dataclasses import dataclass, astuple

# ================================ Description ================================
# This file contains the utility function for autotuning scripts.
# =============================================================================

# Returns the result of a/b rounded to the nearest integer in the direction of
# positive infinity.
def ceil_div(a, b):
    return int(math.ceil(a / b))

# Returns the result of a/b rounded to the nearest integer in the direction of
# negative infinity.
def floor_div(a, b):
    return int(math.floor(a / b))

@dataclass
class ScriptConfiguration:
    # Settings for input tensor generation
    # number of dimensions in the tensor argument
    num_dimensions: int

    # the data type for the tensor argument
    tensor_datatype: torch.dtype

    # During training, the cartesian product of outer_shapes and inner_shapes
    # is used to define the shape of the input tensor arguments.
    outer_shapes: [int]
    inner_shapes: [int]

    # We profile a range of input shapes with various configurations.
    # This argument determines how much of the profiled data to keep as a test
    # set.
    test_data_percentage: [float]

    # The selected batch size for empirical and nvfuser comparison.
    empirical_batch_size: [int]

    # The range of hidden sizes for empirical and nvfuser comparision.
    empirical_hidden_sizes: [int]

# Converted DataClass to a Tuple. It flattens nested tuples. The function is
# used for compatibility with machine learning model.
def flatten_configuration(scheduler_config):
    new_scheduler_config = []
    for item in astuple(scheduler_config):
        if type(item) is tuple:
            new_scheduler_config.extend(item)
        else:
            new_scheduler_config.append(item)
    return tuple(new_scheduler_config)

# Collect data for machine learning
def collect_data(script_config, autotune_config):
    parameters = []
    performance = []

    for shape in itertools.product(
        script_config.outer_shapes, script_config.inner_shapes
    ):
        print(shape)
        inputs = autotune_config.create_inputs(shape, script_config.tensor_datatype)

        with FusionDefinition() as presched_fd:
            autotune_config.create_fusion_func(inputs)(presched_fd)

        # unroll and vectorization configurations
        for parameter_config in autotune_config.generate_scheduler_configurations(
            shape
        ):
            perf_metric, _ = run_profile(
                autotune_config, presched_fd, inputs, parameter_config
            )
            parameters.append((*shape, *flatten_configuration(parameter_config)))
            performance.append(perf_metric)
    return parameters, performance

# Separate collected data into training and test sets
def separate_data(script_config, parameters, performance):
    import random

    train_inputs = []
    test_inputs = []
    train_perf = []
    test_perf = []
    test_shapes = set()
    all_test_scheduler_config = {}  # key: input_shape, value: (scheduler_config, perf)

    for data, perf in zip(parameters, performance):
        shape = data[: script_config.num_dimensions]
        scheduler_config = data[script_config.num_dimensions :]

        if shape in all_test_scheduler_config:
            all_test_scheduler_config[shape][scheduler_config] = perf
        else:
            all_test_scheduler_config[shape] = {scheduler_config: perf}

        if (
            script_config.test_data_percentage > 0
            and random.random() < script_config.test_data_percentage
        ):
            test_shapes.add(shape)
            test_inputs.append(data)
            test_perf.append(perf)
        else:
            train_inputs.append(data)
            train_perf.append(perf)

    # key: input_shape, value: best_scheduler_config
    best_test_scheduler_config = {
        shape: argmax(all_test_scheduler_config[shape]) for shape in test_shapes
    }

    return (train_inputs, train_perf), (
        test_inputs,
        test_perf,
        test_shapes,
        best_test_scheduler_config,
    )

# Apply schedule decorator, run fusion, and profile performance
def run_profile(autotune_config, presched_fd, inputs, scheduler_config=None):
    scheduled_fd = autotune_config.custom_scheduler(presched_fd, scheduler_config)
    nvf_outputs = scheduled_fd.execute(inputs, profile=True)

    # validate correctness
    assert torch.allclose(
        nvf_outputs[0], autotune_config.eager_reference(inputs), atol=1e-2, rtol=1e-2
    )

    prof = scheduled_fd.profile()
    bandwidth = prof.kernel_profiles[0].effective_bandwidth_gbs
    time = prof.kernel_profiles[0].time_ms
    return bandwidth, time

# Given a map from scheduler configuration to predicted performance, find the
# configuration with the maximum predicted performance
def argmax(map_scheduler_config_to_perf):
    best_perf = -1
    best_scheduler_config = None
    for scheduler_config, perf in map_scheduler_config_to_perf.items():
        if perf > best_perf:
            best_perf = perf
            best_scheduler_config = scheduler_config
    return best_scheduler_config

# Given a prediction model, input_shape, and set of parameter configurations,
# find the best parameters
def find_best_parameters(clf, input_shape, scheduler_configurations):
    map_scheduler_config_to_performance = {
        scheduler_config: clf.predict(
            [[*input_shape, *flatten_configuration(scheduler_config)]]
        )
        for scheduler_config in scheduler_configurations
    }
    return argmax(map_scheduler_config_to_performance)

# Measure model performance with RMSE
def test_model_rmse(clf, script_config, autotune_config, test_data):
    test_inputs, test_perf, test_shapes, best_test_scheduler_config = test_data
    test_pred = clf.predict(test_inputs)

    # Estimate prediction error with RMSE
    import numpy as np

    test_perf = np.array(test_perf)
    print(
        "Test prediction error (RMSE)",
        np.sqrt(np.mean(np.power(test_perf - test_pred, 2))),
    )
    print("Test performance", test_perf)
    print("Test prediction", test_pred)

    print("======================= compare configurations  =======================")
    # Find best configuration for test_shapes
    print("input shape, estimate_config, actual_config, correct")
    correctness_count = 0
    mismatch_configs = []
    for shape in test_shapes:
        estimate_config = find_best_parameters(
            clf, shape, autotune_config.generate_scheduler_configurations(shape)
        )

        match_config = (
            flatten_configuration(estimate_config) == best_test_scheduler_config[shape]
        )
        if not match_config:
            mismatch_configs.append((shape, estimate_config))

        correctness_count += int(match_config)
        print(
            f"{shape}, {estimate_config}, {best_test_scheduler_config[shape]}, {match_config}"
        )
    print(
        "% of predictions match nvfuser parameters",
        correctness_count / len(test_shapes),
    )
    print(correctness_count, "out of", len(test_shapes))

    print("======================= compare performance =========================")

    for shape, estimate_config in mismatch_configs:
        inputs = autotune_config.create_inputs(shape, script_config.tensor_datatype)

        with FusionDefinition() as presched_fd:
            autotune_config.create_fusion_func(inputs)(presched_fd)

        _, est_perf = run_profile(autotune_config, presched_fd, inputs, estimate_config)
        _, nvf_perf = run_profile(autotune_config, presched_fd, inputs)
        est_perf_faster = est_perf < nvf_perf
        print(
            f"{shape} \t estimate_perf: {est_perf: .5f} \t nvfuser_perf: {nvf_perf: .5f} \t is_estimated_config_faster: {est_perf_faster}"
        )
    print("=====================================================================")

# Given a machine learning model, compare the performance of its predicted configuration
# against nvfuser on a given fusion
def test_model(clf, script_config, autotune_config):
    #  For a specific batch size, gather performance across a range of hidden sizes.
    #  Calculate performance for best predicted and nvfuser configurations. Plot a
    #  chart comparing performance using matplotlib.

    # NOTE: The matplotlib experiment plots the kernel runtime, which could be
    # different than the selected performance metric. Currently, the performance
    # metric is effective_bandwidth_gbs.

    import matplotlib.pyplot as plt
    import numpy as np

    FusionCache.reset()
    est_perfs = []
    for hidden_shape in script_config.empirical_hidden_sizes:
        inputs = autotune_config.create_inputs(
            (script_config.empirical_batch_size, hidden_shape),
            script_config.tensor_datatype,
        )

        estimate_config = find_best_parameters(
            clf,
            (script_config.empirical_batch_size, hidden_shape),
            autotune_config.generate_scheduler_configurations(
                (script_config.empirical_batch_size, hidden_shape)
            ),
        )

        with FusionDefinition() as presched_fd:
            autotune_config.create_fusion_func(inputs)(presched_fd)

        _, est_time_ms = run_profile(
            autotune_config, presched_fd, inputs, estimate_config
        )
        est_perfs.append(est_time_ms)
        print(
            f"{script_config.empirical_batch_size}, {hidden_shape}, {estimate_config}, {est_time_ms: .3f}"
        )

    FusionCache.reset()
    nvf_perfs = []
    for hidden_shape in script_config.empirical_hidden_sizes:
        inputs = autotune_config.create_inputs(
            (script_config.empirical_batch_size, hidden_shape),
            script_config.tensor_datatype,
        )

        with FusionDefinition() as presched_fd:
            autotune_config.create_fusion_func(inputs)(presched_fd)

        _, nvf_time_ms = run_profile(autotune_config, presched_fd, inputs)
        nvf_perfs.append(nvf_time_ms)
        print(
            f"{script_config.empirical_batch_size}, {hidden_shape}, {nvf_time_ms: .3f}"
        )

    # Get mean speed-up from nvfuser to empirical configurations across all input shapes.
    # Negative value mean empirical configurations are slower than nvfuser.
    print("Mean speed-up", np.mean(np.array(nvf_perfs) - np.array(est_perfs)))

    np_hidden_size = np.array(script_config.empirical_hidden_sizes)
    plt.plot(np_hidden_size, np.array(est_perfs))
    plt.plot(np_hidden_size, np.array(nvf_perfs))

    plt.xlabel("Hidden Size")
    plt.ylabel("Time(ms)")
    plt.title(
        f"Batch Size = {script_config.empirical_batch_size}, Compare Machine Learning Heuristic vs NvFuser"
    )
    plt.legend(["random_forest", "nvfuser"], loc="lower right")
    plt.savefig(
        f"{autotune_config}_empirical_batch_size_{script_config.empirical_batch_size}.png"
    )
    plt.close("all")


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# Owner(s): ["module: nvfuser"]

import torch
import math
import itertools
from nvfuser import FusionCache, FusionDefinition
from dataclasses import dataclass, astuple

# ================================ Description ================================
# This file contains the utility function for autotuning scripts.
# =============================================================================

# Returns the result of a/b rounded to the nearest integer in the direction of
# positive infinity.
def ceil_div(a, b):
    return int(math.ceil(a / b))

# Returns the result of a/b rounded to the nearest integer in the direction of
# negative infinity.
def floor_div(a, b):
    return int(math.floor(a / b))

@dataclass
class ScriptConfiguration:
    # Settings for input tensor generation
    # number of dimensions in the tensor argument
    num_dimensions: int

    # the data type for the tensor argument
    tensor_datatype: torch.dtype

    # During training, the cartesian product of outer_shapes and inner_shapes
    # is used to define the shape of the input tensor arguments.
    outer_shapes: [int]


import torch
import math
import itertools
from nvfuser import FusionCache, FusionDefinition
from dataclasses import dataclass, astuple

# ================================ Description ================================
# This file contains the utility function for autotuning scripts.
# =============================================================================

# Returns the result of a/b rounded to the nearest integer in the direction of
# positive infinity.
def ceil_div(a, b):
    return int(math.ceil(a / b))

# Returns the result of a/b rounded to the nearest integer in the direction of
# negative infinity.
def floor_div(a, b):
    return int(math.floor(a / b))

@dataclass
class ScriptConfiguration:
    # Settings for input tensor generation
    # number of dimensions in the tensor argument
    num_dimensions: int

    # the data type for the tensor argument
    tensor_datatype: torch.dtype

    # During training, the cartesian product of outer_shapes and inner_shapes
    # is used to define the shape of the input tensor arguments.
    outer_shapes: [int]
    inner_shapes: [int]

    # We profile a range of input shapes with various configurations.
    # This argument determines how much of the profiled data to keep as a test
    # set.
    test_data_percentage: [float]


ScriptConfiguration

from dataclasses import dataclass, astuple

# ================================ Description ================================
# This file contains the utility function for autotuning scripts.
# =============================================================================

# Returns the result of a/b rounded to the nearest integer in the direction of
# positive infinity.
def ceil_div(a, b):
    return int(math.ceil(a / b))

# Returns the result of a/b rounded to the nearest integer in the direction of
# negative infinity.
def floor_div(a, b):
    return int(math.floor(a / b))

@dataclass
class ScriptConfiguration:
    # Settings for input tensor generation
    # number of dimensions in the tensor argument
    num_dimensions: int

    # the data type for the tensor argument
    tensor_datatype: torch.dtype

    # During training, the cartesian product of outer_shapes and inner_shapes
    # is used to define the shape of the input tensor arguments.
    outer_shapes: [int]
    inner_shapes: [int]

    # We profile a range of input shapes with various configurations.
    # This argument determines how much of the profiled data to keep as a test
    # set.
    test_data_percentage: [float]

    # The selected batch size for empirical and nvfuser comparison.
    empirical_batch_size: [int]

    # The range of hidden sizes for empirical and nvfuser comparision.
    empirical_hidden_sizes: [int]

# Converted DataClass to a Tuple. It flattens nested tuples. The function is
# used for compatibility with machine learning model.
def flatten_configuration(scheduler_config):
    new_scheduler_config = []
    for item in astuple(scheduler_config):
        if type(item) is tuple:
            new_scheduler_config.extend(item)
        else:
            new_scheduler_config.append(item)
    return tuple(new_scheduler_config)

# Collect data for machine learning
def collect_data(script_config, autotune_config):
    parameters = []
    performance = []

    for shape in itertools.product(


    # During training, the cartesian product of outer_shapes and inner_shapes
    # is used to define the shape of the input tensor arguments.
    outer_shapes: [int]
    inner_shapes: [int]

    # We profile a range of input shapes with various configurations.
    # This argument determines how much of the profiled data to keep as a test
    # set.
    test_data_percentage: [float]

    # The selected batch size for empirical and nvfuser comparison.
    empirical_batch_size: [int]

    # The range of hidden sizes for empirical and nvfuser comparision.
    empirical_hidden_sizes: [int]

# Converted DataClass to a Tuple. It flattens nested tuples. The function is
# used for compatibility with machine learning model.
def flatten_configuration(scheduler_config):
    new_scheduler_config = []
    for item in astuple(scheduler_config):
        if type(item) is tuple:
            new_scheduler_config.extend(item)
        else:
            new_scheduler_config.append(item)
    return tuple(new_scheduler_config)

# Collect data for machine learning
def collect_data(script_config, autotune_config):
    parameters = []
    performance = []

    for shape in itertools.product(
        script_config.outer_shapes, script_config.inner_shapes
    ):
        print(shape)
        inputs = autotune_config.create_inputs(shape, script_config.tensor_datatype)

        with FusionDefinition() as presched_fd:
            autotune_config.create_fusion_func(inputs)(presched_fd)

        # unroll and vectorization configurations
        for parameter_config in autotune_config.generate_scheduler_configurations(
            shape
        ):


    # The selected batch size for empirical and nvfuser comparison.
    empirical_batch_size: [int]

    # The range of hidden sizes for empirical and nvfuser comparision.
    empirical_hidden_sizes: [int]

# Converted DataClass to a Tuple. It flattens nested tuples. The function is
# used for compatibility with machine learning model.
def flatten_configuration(scheduler_config):
    new_scheduler_config = []
    for item in astuple(scheduler_config):
        if type(item) is tuple:
            new_scheduler_config.extend(item)
        else:
            new_scheduler_config.append(item)
    return tuple(new_scheduler_config)

# Collect data for machine learning
def collect_data(script_config, autotune_config):
    parameters = []
    performance = []

    for shape in itertools.product(
        script_config.outer_shapes, script_config.inner_shapes
    ):
        print(shape)
        inputs = autotune_config.create_inputs(shape, script_config.tensor_datatype)

        with FusionDefinition() as presched_fd:
            autotune_config.create_fusion_func(inputs)(presched_fd)

        # unroll and vectorization configurations
        for parameter_config in autotune_config.generate_scheduler_configurations(
            shape
        ):
            perf_metric, _ = run_profile(
                autotune_config, presched_fd, inputs, parameter_config
            )
            parameters.append((*shape, *flatten_configuration(parameter_config)))
            performance.append(perf_metric)
    return parameters, performance

# Separate collected data into training and test sets
def separate_data(script_config, parameters, performance):
    import random

    train_inputs = []
    test_inputs = []
    train_perf = []
    test_perf = []
    test_shapes = set()
    all_test_scheduler_config = {}  # key: input_shape, value: (scheduler_config, perf)

    for data, perf in zip(parameters, performance):
        shape = data[: script_config.num_dimensions]
        scheduler_config = data[script_config.num_dimensions :]

        if shape in all_test_scheduler_config:
            all_test_scheduler_config[shape][scheduler_config] = perf
        else:


    ):
        print(shape)
        inputs = autotune_config.create_inputs(shape, script_config.tensor_datatype)

        with FusionDefinition() as presched_fd:
            autotune_config.create_fusion_func(inputs)(presched_fd)

        # unroll and vectorization configurations
        for parameter_config in autotune_config.generate_scheduler_configurations(
            shape
        ):
            perf_metric, _ = run_profile(
                autotune_config, presched_fd, inputs, parameter_config
            )
            parameters.append((*shape, *flatten_configuration(parameter_config)))
            performance.append(perf_metric)
    return parameters, performance

# Separate collected data into training and test sets
def separate_data(script_config, parameters, performance):
    import random

    train_inputs = []
    test_inputs = []
    train_perf = []
    test_perf = []
    test_shapes = set()
    all_test_scheduler_config = {}  # key: input_shape, value: (scheduler_config, perf)

    for data, perf in zip(parameters, performance):
        shape = data[: script_config.num_dimensions]
        scheduler_config = data[script_config.num_dimensions :]

        if shape in all_test_scheduler_config:
            all_test_scheduler_config[shape][scheduler_config] = perf
        else:
            all_test_scheduler_config[shape] = {scheduler_config: perf}

        if (
            script_config.test_data_percentage > 0
            and random.random() < script_config.test_data_percentage
        ):
            test_shapes.add(shape)
            test_inputs.append(data)
            test_perf.append(perf)
        else:
            train_inputs.append(data)
            train_perf.append(perf)

    # key: input_shape, value: best_scheduler_config
    best_test_scheduler_config = {
        shape: argmax(all_test_scheduler_config[shape]) for shape in test_shapes
    }

    return (train_inputs, train_perf), (
        test_inputs,
        test_perf,
        test_shapes,
        best_test_scheduler_config,
    )

# Apply schedule decorator, run fusion, and profile performance
def run_profile(autotune_config, presched_fd, inputs, scheduler_config=None):
    scheduled_fd = autotune_config.custom_scheduler(presched_fd, scheduler_config)
    nvf_outputs = scheduled_fd.execute(inputs, profile=True)

    # validate correctness
    assert torch.allclose(
        nvf_outputs[0], autotune_config.eager_reference(inputs), atol=1e-2, rtol=1e-2
    )

    prof = scheduled_fd.profile()
    bandwidth = prof.kernel_profiles[0].effective_bandwidth_gbs
    time = prof.kernel_profiles[0].time_ms
    return bandwidth, time

# Given a map from scheduler configuration to predicted performance, find the
# configuration with the maximum predicted performance


            test_inputs.append(data)
            test_perf.append(perf)
        else:
            train_inputs.append(data)
            train_perf.append(perf)

    # key: input_shape, value: best_scheduler_config
    best_test_scheduler_config = {
        shape: argmax(all_test_scheduler_config[shape]) for shape in test_shapes
    }

    return (train_inputs, train_perf), (
        test_inputs,
        test_perf,
        test_shapes,
        best_test_scheduler_config,
    )

# Apply schedule decorator, run fusion, and profile performance
def run_profile(autotune_config, presched_fd, inputs, scheduler_config=None):
    scheduled_fd = autotune_config.custom_scheduler(presched_fd, scheduler_config)
    nvf_outputs = scheduled_fd.execute(inputs, profile=True)

    # validate correctness
    assert torch.allclose(
        nvf_outputs[0], autotune_config.eager_reference(inputs), atol=1e-2, rtol=1e-2
    )

    prof = scheduled_fd.profile()
    bandwidth = prof.kernel_profiles[0].effective_bandwidth_gbs
    time = prof.kernel_profiles[0].time_ms
    return bandwidth, time

# Given a map from scheduler configuration to predicted performance, find the
# configuration with the maximum predicted performance
def argmax(map_scheduler_config_to_perf):
    best_perf = -1
    best_scheduler_config = None
    for scheduler_config, perf in map_scheduler_config_to_perf.items():
        if perf > best_perf:
            best_perf = perf
            best_scheduler_config = scheduler_config
    return best_scheduler_config

# Given a prediction model, input_shape, and set of parameter configurations,
# find the best parameters
def find_best_parameters(clf, input_shape, scheduler_configurations):
    map_scheduler_config_to_performance = {
        scheduler_config: clf.predict(
            [[*input_shape, *flatten_configuration(scheduler_config)]]