# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
from .core import run_benchmark, clear_dynamo_cache, with_executor, DEFAULT_EXECUTORS
import torch
from .global_params import generate_input_sizes, FLOAT_DTYPES, PROMOTE_DTYPES

def transpose_fusion_input_smem(
    fd: FusionDefinition,
    dtype: DataType,
    is_copy_transpose: bool,
    axes: list,
    rank: int,
):
    """Single input: the transposed input is read through shared memory."""
    shape = [-1] * rank
    contiguity = [True] * rank
    T0 = fd.define_tensor(shape=shape, contiguity=contiguity, dtype=dtype, is_cpu=False)

    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)

    T1 = fd.ops.permute(T0, dims=axes)

    if dtype in PROMOTE_DTYPES:
        T1 = fd.ops.cast(T1, dtype=dtype)

    S2 = fd.define_scalar(0.00000, dtype=DataType.Double)
    T3 = fd.ops.gt(T1, S2)
    T4 = fd.ops.where(T3, T1, S2)
    # add segmenter set to avoid presegment passes setting the output as a
    # view of the input without any data movement. It leads to pointwise
    # instead of transpose scheduler.
    # we can also expose OptimizationPassGuard to python frontend and disable
    # presegmentation passes to enforce output to be contiguous and then
    # transpose scheduler will be used.
    if is_copy_transpose:
        T5 = fd.ops.segment_set(T4)
        fd.add_output(T5)
    else:
        fd.add_output(T4)

def transpose_fusion_output_smem(
    fd: FusionDefinition,
    dtype: DataType,
    is_copy_transpose: bool,
    axes: list,
    rank: int,
):
    """Two inputs: the transposed output is written through shared memory."""
    shape = [-1] * rank
    contiguity = [True] * rank
    T0 = fd.define_tensor(shape=shape, contiguity=contiguity, dtype=dtype, is_cpu=False)
    T1 = fd.define_tensor(shape=shape, contiguity=contiguity, dtype=dtype, is_cpu=False)

    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T1 = fd.ops.cast(T1, dtype=DataType.Float)

    T2 = fd.ops.add(T0, T1)
    T3 = fd.ops.permute(T2, dims=axes)

    if dtype in PROMOTE_DTYPES:
        T3 = fd.ops.cast(T3, dtype=dtype)

    S4 = fd.define_scalar(0.00000, dtype=DataType.Double)
    T5 = fd.ops.gt(T3, S4)
    T6 = fd.ops.where(T5, T3, S4)
    # add segmenter set to avoid presegment passes setting the output as a
    # view of the input without any data movement. It leads to pointwise
    # instead of transpose scheduler.
    # we can also expose OptimizationPassGuard to python frontend and disable
    # presegmentation passes to enforce output to be contiguous and then
    # transpose scheduler will be used.
    if is_copy_transpose:
        T7 = fd.ops.segment_set(T6)
        fd.add_output(T7)
    else:
        fd.add_output(T6)

# Without contiguous, transpose returns a view with swapped strides.
# contiguous() materializes a contiguous copy of the result.
# When compiled with thunder, contiguous version will use nvFuser's transpose scheduler, otherwise it will use the pointwise scheduler.
def transpose_fwd_fn(
    inputs: list,
):  # [input1, input2 (optional), axes, is_copy_transpose]
    is_copy_transpose = inputs[-1]
    axes = inputs[-2]
    if len(inputs) == 4:
        data = inputs[0] + inputs[1]
    else:
        data = inputs[0]
    relu_transpose_result = torch.nn.functional.relu(data.permute(axes))
    if is_copy_transpose:
        return relu_transpose_result.contiguous()
    else:
        return relu_transpose_result

def setup_input_smem(size, dtype, is_copy_transpose, axes, dims):
    """Single input: the transposed input is read through shared memory."""
    input1 = torch.randn(size, device="cuda", dtype=dtype)
    nvfuser_inputs = [input1]

    with FusionDefinition() as fd:
        transpose_fusion_input_smem(
            fd,
            torch_dtype_to_nvfuser_dtype(dtype),
            is_copy_transpose,
            axes,
            rank=dims,
        )

    eager_inputs = [input1, axes, is_copy_transpose]
    return fd, nvfuser_inputs, eager_inputs

def setup_output_smem(size, dtype, is_copy_transpose, axes, dims):
    """Two inputs: the transposed output is written through shared memory."""
    input1 = torch.randn(size, device="cuda", dtype=dtype)
    input2 = torch.randn(size, device="cuda", dtype=dtype)
    nvfuser_inputs = [input1, input2]

    with FusionDefinition() as fd:
        transpose_fusion_output_smem(
            fd,
            torch_dtype_to_nvfuser_dtype(dtype),
            is_copy_transpose,
            axes,
            rank=dims,
        )

    eager_inputs = [input1, input2, axes, is_copy_transpose]
    return fd, nvfuser_inputs, eager_inputs

def _generate_transpose_params():
    params = []
    for dims in (2, 3):
        sizes = generate_input_sizes(dims=dims)
        if dims == 2:
            axes_list = [(1, 0)]
        else:
            axes_list = [(1, 0, 2), (2, 1, 0), (0, 2, 1)]
        for size in sizes:
            for axes in axes_list:
                params.append((size, axes, dims))
    return params

@pytest.mark.parametrize("size,axes,dims", _generate_transpose_params())
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "is_copy_transpose",
    [
        pytest.param(True, marks=pytest.mark.transpose, id="copy"),
        pytest.param(False, marks=pytest.mark.pointwise, id="view"),
    ],
)
@pytest.mark.parametrize(
    "setup_fn",
    [setup_input_smem, setup_output_smem],
    ids=["input_smem", "output_smem"],
)
def test_transpose_nvf_benchmark(
    benchmark,
    size: tuple,
    is_copy_transpose: bool,
    dtype: torch.dtype,
    axes: tuple,
    dims: int,
    setup_fn,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    fd, nvfuser_inputs, eager_inputs = setup_fn(
        size, dtype, is_copy_transpose, axes, dims
    )

    if not disable_validation:
        eager_output = transpose_fwd_fn(eager_inputs)
        fd.validate(nvfuser_inputs, [eager_output])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, nvfuser_inputs)

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size,axes,dims", _generate_transpose_params())
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "is_copy_transpose",
    [True, False],
    ids=["copy", "view"],
)
@pytest.mark.parametrize(
    "setup_fn",
    [setup_input_smem, setup_output_smem],
    ids=["input_smem", "output_smem"],
)
def test_transpose_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    is_copy_transpose: bool,
    axes: tuple,
    dims: int,
    setup_fn,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    _, _, eager_inputs = setup_fn(size, dtype, is_copy_transpose, axes, dims)
    benchmark_fn = with_executor(executor, transpose_fwd_fn)
    run_benchmark(benchmark, benchmark_fn, eager_inputs)


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
from .core import run_benchmark, clear_dynamo_cache, with_executor, DEFAULT_EXECUTORS
import torch
from .global_params import generate_input_sizes, FLOAT_DTYPES, PROMOTE_DTYPES

def transpose_fusion_input_smem(
    fd: FusionDefinition,
    dtype: DataType,
    is_copy_transpose: bool,
    axes: list,
    rank: int,
):
    """Single input: the transposed input is read through shared memory."""
    shape = [-1] * rank
    contiguity = [True] * rank
    T0 = fd.define_tensor(shape=shape, contiguity=contiguity, dtype=dtype, is_cpu=False)

    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)

    T1 = fd.ops.permute(T0, dims=axes)

    if dtype in PROMOTE_DTYPES:
        T1 = fd.ops.cast(T1, dtype=dtype)

    S2 = fd.define_scalar(0.00000, dtype=DataType.Double)
    T3 = fd.ops.gt(T1, S2)
    T4 = fd.ops.where(T3, T1, S2)
    # add segmenter set to avoid presegment passes setting the output as a
    # view of the input without any data movement. It leads to pointwise
    # instead of transpose scheduler.
    # we can also expose OptimizationPassGuard to python frontend and disable
    # presegmentation passes to enforce output to be contiguous and then
    # transpose scheduler will be used.
    if is_copy_transpose:
        T5 = fd.ops.segment_set(T4)
        fd.add_output(T5)
    else:
        fd.add_output(T4)

def transpose_fusion_output_smem(
    fd: FusionDefinition,
    dtype: DataType,
    is_copy_transpose: bool,
    axes: list,
    rank: int,
):
    """Two inputs: the transposed output is written through shared memory."""
    shape = [-1] * rank
    contiguity = [True] * rank
    T0 = fd.define_tensor(shape=shape, contiguity=contiguity, dtype=dtype, is_cpu=False)
    T1 = fd.define_tensor(shape=shape, contiguity=contiguity, dtype=dtype, is_cpu=False)

    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T1 = fd.ops.cast(T1, dtype=DataType.Float)

    T2 = fd.ops.add(T0, T1)


    if dtype in PROMOTE_DTYPES:
        T1 = fd.ops.cast(T1, dtype=dtype)

    S2 = fd.define_scalar(0.00000, dtype=DataType.Double)
    T3 = fd.ops.gt(T1, S2)
    T4 = fd.ops.where(T3, T1, S2)
    # add segmenter set to avoid presegment passes setting the output as a
    # view of the input without any data movement. It leads to pointwise
    # instead of transpose scheduler.
    # we can also expose OptimizationPassGuard to python frontend and disable
    # presegmentation passes to enforce output to be contiguous and then
    # transpose scheduler will be used.
    if is_copy_transpose:
        T5 = fd.ops.segment_set(T4)
        fd.add_output(T5)
    else:
        fd.add_output(T4)

def transpose_fusion_output_smem(
    fd: FusionDefinition,
    dtype: DataType,
    is_copy_transpose: bool,
    axes: list,
    rank: int,
):
    """Two inputs: the transposed output is written through shared memory."""
    shape = [-1] * rank
    contiguity = [True] * rank
    T0 = fd.define_tensor(shape=shape, contiguity=contiguity, dtype=dtype, is_cpu=False)
    T1 = fd.define_tensor(shape=shape, contiguity=contiguity, dtype=dtype, is_cpu=False)

    if dtype in PROMOTE_DTYPES:
        T0 = fd.ops.cast(T0, dtype=DataType.Float)
        T1 = fd.ops.cast(T1, dtype=DataType.Float)

    T2 = fd.ops.add(T0, T1)
    T3 = fd.ops.permute(T2, dims=axes)

    if dtype in PROMOTE_DTYPES:
        T3 = fd.ops.cast(T3, dtype=dtype)

    S4 = fd.define_scalar(0.00000, dtype=DataType.Double)
    T5 = fd.ops.gt(T3, S4)
    T6 = fd.ops.where(T5, T3, S4)
    # add segmenter set to avoid presegment passes setting the output as a
    # view of the input without any data movement. It leads to pointwise
    # instead of transpose scheduler.
    # we can also expose OptimizationPassGuard to python frontend and disable
    # presegmentation passes to enforce output to be contiguous and then
    # transpose scheduler will be used.
    if is_copy_transpose:
        T7 = fd.ops.segment_set(T6)
        fd.add_output(T7)
    else:
        fd.add_output(T6)

# Without contiguous, transpose returns a view with swapped strides.
# contiguous() materializes a contiguous copy of the result.
# When compiled with thunder, contiguous version will use nvFuser's transpose scheduler, otherwise it will use the pointwise scheduler.
def transpose_fwd_fn(
    inputs: list,
):  # [input1, input2 (optional), axes, is_copy_transpose]
    is_copy_transpose = inputs[-1]
    axes = inputs[-2]
    if len(inputs) == 4:
        data = inputs[0] + inputs[1]
    else:
        data = inputs[0]
    relu_transpose_result = torch.nn.functional.relu(data.permute(axes))
    if is_copy_transpose:
        return relu_transpose_result.contiguous()
    else:
        return relu_transpose_result


    S4 = fd.define_scalar(0.00000, dtype=DataType.Double)
    T5 = fd.ops.gt(T3, S4)
    T6 = fd.ops.where(T5, T3, S4)
    # add segmenter set to avoid presegment passes setting the output as a
    # view of the input without any data movement. It leads to pointwise
    # instead of transpose scheduler.
    # we can also expose OptimizationPassGuard to python frontend and disable
    # presegmentation passes to enforce output to be contiguous and then
    # transpose scheduler will be used.
    if is_copy_transpose:
        T7 = fd.ops.segment_set(T6)
        fd.add_output(T7)
    else:
        fd.add_output(T6)

# Without contiguous, transpose returns a view with swapped strides.
# contiguous() materializes a contiguous copy of the result.
# When compiled with thunder, contiguous version will use nvFuser's transpose scheduler, otherwise it will use the pointwise scheduler.
def transpose_fwd_fn(
    inputs: list,
):  # [input1, input2 (optional), axes, is_copy_transpose]
    is_copy_transpose = inputs[-1]
    axes = inputs[-2]
    if len(inputs) == 4:
        data = inputs[0] + inputs[1]
    else:
        data = inputs[0]
    relu_transpose_result = torch.nn.functional.relu(data.permute(axes))
    if is_copy_transpose:
        return relu_transpose_result.contiguous()
    else:
        return relu_transpose_result

def setup_input_smem(size, dtype, is_copy_transpose, axes, dims):
    """Single input: the transposed input is read through shared memory."""
    input1 = torch.randn(size, device="cuda", dtype=dtype)
    nvfuser_inputs = [input1]

    with FusionDefinition() as fd:
        transpose_fusion_input_smem(
            fd,
            torch_dtype_to_nvfuser_dtype(dtype),
            is_copy_transpose,
            axes,
            rank=dims,
        )

    eager_inputs = [input1, axes, is_copy_transpose]
    return fd, nvfuser_inputs, eager_inputs


# Without contiguous, transpose returns a view with swapped strides.
# contiguous() materializes a contiguous copy of the result.
# When compiled with thunder, contiguous version will use nvFuser's transpose scheduler, otherwise it will use the pointwise scheduler.
def transpose_fwd_fn(
    inputs: list,
):  # [input1, input2 (optional), axes, is_copy_transpose]
    is_copy_transpose = inputs[-1]
    axes = inputs[-2]
    if len(inputs) == 4:
        data = inputs[0] + inputs[1]
    else:
        data = inputs[0]
    relu_transpose_result = torch.nn.functional.relu(data.permute(axes))
    if is_copy_transpose:
        return relu_transpose_result.contiguous()
    else:
        return relu_transpose_result

def setup_input_smem(size, dtype, is_copy_transpose, axes, dims):
    """Single input: the transposed input is read through shared memory."""
    input1 = torch.randn(size, device="cuda", dtype=dtype)
    nvfuser_inputs = [input1]

    with FusionDefinition() as fd:
        transpose_fusion_input_smem(
            fd,
            torch_dtype_to_nvfuser_dtype(dtype),
            is_copy_transpose,
            axes,
            rank=dims,
        )

    eager_inputs = [input1, axes, is_copy_transpose]
    return fd, nvfuser_inputs, eager_inputs

def setup_output_smem(size, dtype, is_copy_transpose, axes, dims):
    """Two inputs: the transposed output is written through shared memory."""
    input1 = torch.randn(size, device="cuda", dtype=dtype)
    input2 = torch.randn(size, device="cuda", dtype=dtype)
    nvfuser_inputs = [input1, input2]

    with FusionDefinition() as fd:
        transpose_fusion_output_smem(
            fd,
            torch_dtype_to_nvfuser_dtype(dtype),
            is_copy_transpose,
            axes,
            rank=dims,
        )

    eager_inputs = [input1, input2, axes, is_copy_transpose]
    return fd, nvfuser_inputs, eager_inputs


def setup_input_smem(size, dtype, is_copy_transpose, axes, dims):
    """Single input: the transposed input is read through shared memory."""
    input1 = torch.randn(size, device="cuda", dtype=dtype)
    nvfuser_inputs = [input1]

    with FusionDefinition() as fd:
        transpose_fusion_input_smem(
            fd,
            torch_dtype_to_nvfuser_dtype(dtype),
            is_copy_transpose,
            axes,
            rank=dims,
        )

    eager_inputs = [input1, axes, is_copy_transpose]
    return fd, nvfuser_inputs, eager_inputs

def setup_output_smem(size, dtype, is_copy_transpose, axes, dims):
    """Two inputs: the transposed output is written through shared memory."""
    input1 = torch.randn(size, device="cuda", dtype=dtype)
    input2 = torch.randn(size, device="cuda", dtype=dtype)
    nvfuser_inputs = [input1, input2]

    with FusionDefinition() as fd:
        transpose_fusion_output_smem(
            fd,
            torch_dtype_to_nvfuser_dtype(dtype),
            is_copy_transpose,
            axes,
            rank=dims,
        )

    eager_inputs = [input1, input2, axes, is_copy_transpose]
    return fd, nvfuser_inputs, eager_inputs

def _generate_transpose_params():
    params = []
    for dims in (2, 3):
        sizes = generate_input_sizes(dims=dims)
        if dims == 2:
            axes_list = [(1, 0)]
        else:
            axes_list = [(1, 0, 2), (2, 1, 0), (0, 2, 1)]
        for size in sizes:
            for axes in axes_list:
                params.append((size, axes, dims))
    return params

@pytest.mark.parametrize("size,axes,dims", _generate_transpose_params())
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "is_copy_transpose",


def setup_output_smem(size, dtype, is_copy_transpose, axes, dims):
    """Two inputs: the transposed output is written through shared memory."""
    input1 = torch.randn(size, device="cuda", dtype=dtype)
    input2 = torch.randn(size, device="cuda", dtype=dtype)
    nvfuser_inputs = [input1, input2]

    with FusionDefinition() as fd:
        transpose_fusion_output_smem(
            fd,
            torch_dtype_to_nvfuser_dtype(dtype),
            is_copy_transpose,
            axes,
            rank=dims,
        )

    eager_inputs = [input1, input2, axes, is_copy_transpose]
    return fd, nvfuser_inputs, eager_inputs

def _generate_transpose_params():
    params = []
    for dims in (2, 3):
        sizes = generate_input_sizes(dims=dims)
        if dims == 2:
            axes_list = [(1, 0)]
        else:
            axes_list = [(1, 0, 2), (2, 1, 0), (0, 2, 1)]
        for size in sizes:
            for axes in axes_list:
                params.append((size, axes, dims))
    return params

@pytest.mark.parametrize("size,axes,dims", _generate_transpose_params())
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "is_copy_transpose",
    [
        pytest.param(True, marks=pytest.mark.transpose, id="copy"),
        pytest.param(False, marks=pytest.mark.pointwise, id="view"),
    ],
)
@pytest.mark.parametrize(
    "setup_fn",
    [setup_input_smem, setup_output_smem],
    ids=["input_smem", "output_smem"],
)
def test_transpose_nvf_benchmark(
    benchmark,
    size: tuple,
    is_copy_transpose: bool,


        for size in sizes:
            for axes in axes_list:
                params.append((size, axes, dims))
    return params

@pytest.mark.parametrize("size,axes,dims", _generate_transpose_params())
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "is_copy_transpose",
    [
        pytest.param(True, marks=pytest.mark.transpose, id="copy"),
        pytest.param(False, marks=pytest.mark.pointwise, id="view"),
    ],
)
@pytest.mark.parametrize(
    "setup_fn",
    [setup_input_smem, setup_output_smem],
    ids=["input_smem", "output_smem"],
)
def test_transpose_nvf_benchmark(
    benchmark,
    size: tuple,
    is_copy_transpose: bool,
    dtype: torch.dtype,
    axes: tuple,
    dims: int,
    setup_fn,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    fd, nvfuser_inputs, eager_inputs = setup_fn(
        size, dtype, is_copy_transpose, axes, dims
    )

    if not disable_validation:
        eager_output = transpose_fwd_fn(eager_inputs)
        fd.validate(nvfuser_inputs, [eager_output])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, nvfuser_inputs)

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size,axes,dims", _generate_transpose_params())
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "is_copy_transpose",
    [True, False],
    ids=["copy", "view"],
)
@pytest.mark.parametrize(
    "setup_fn",
    [setup_input_smem, setup_output_smem],
    ids=["input_smem", "output_smem"],
)
def test_transpose_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    is_copy_transpose: bool,


        eager_output = transpose_fwd_fn(eager_inputs)
        fd.validate(nvfuser_inputs, [eager_output])

    if not disable_benchmarking:
        run_benchmark(benchmark, fd.execute, nvfuser_inputs)

@pytest.mark.parametrize("executor", DEFAULT_EXECUTORS)
@pytest.mark.parametrize("size,axes,dims", _generate_transpose_params())
@pytest.mark.parametrize("dtype", FLOAT_DTYPES)
@pytest.mark.parametrize(
    "is_copy_transpose",
    [True, False],
    ids=["copy", "view"],
)
@pytest.mark.parametrize(
    "setup_fn",
    [setup_input_smem, setup_output_smem],
    ids=["input_smem", "output_smem"],
)
def test_transpose_baseline_benchmark(
    benchmark,
    size: tuple,
    dtype: torch.dtype,
    is_copy_transpose: bool,
    axes: tuple,
    dims: int,
    setup_fn,
    executor: str,
):
    if executor == "torchcompile":
        clear_dynamo_cache()
    _, _, eager_inputs = setup_fn(size, dtype, is_copy_transpose, axes, dims)
    benchmark_fn = with_executor(executor, transpose_fwd_fn)
    run_benchmark(benchmark, benchmark_fn, eager_inputs)


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import torch
from typing import Union, List, Tuple
from nvfuser_direct import DataType
from .core import BENCHMARK_CONFIG
from nvfuser_direct.pytorch_utils import DEVICE_PROPERTIES
import itertools
import os
from random import sample

# BENCHMARK_MODE = weekly/nightly.
BENCHMARK_MODE = os.getenv("BENCHMARK_MODE")
if not BENCHMARK_MODE:
    BENCHMARK_MODE = "nightly"

# Datatypes to benchmark
FLOAT_DTYPES = [torch.float32]
# Run only one of float16 / bfloat16.
if DEVICE_PROPERTIES["gpu_compute_capability_major"] >= 8:
    FLOAT_DTYPES.append(torch.bfloat16)
else:
    FLOAT_DTYPES.append(torch.float16)

# Datatypes that will be promoted to Datatype.Float in Fusion Definitions
PROMOTE_DTYPES = [DataType.BFloat16, DataType.Half]

# Model Parameters from LLMs (GPT2/3, PaLM, LLama)

# Embedding size: d_model, d_ff = 4 * d_model
D_MODEL_MIN = 768
D_MODEL_MAX = 18432

# (num_heads, n_embd) configurations seen in models.
LLM_CONFIGS = [
    (12, 768),  # GPT-2 (124M), GPT-3 (125M)
    (16, 1024),  # GPT-2 (350M), GPT-3 (350M)
    (20, 1280),  # GPT-2 (774M)
    (16, 1536),  # GPT-3 (760M)
    (25, 1600),  # GPT-2 (1558M)
    (24, 2048),  # GPT-3 (1.3B)
    (32, 2560),  # GPT-3 (2.7B)
    (16, 4096),  # PaLM (8B)
    (32, 4096),  # LLaMA (7B), GPT-3 (6.7B)
    (40, 5120),  # LLaMA (13B), GPT-3 (13B)
    (52, 6656),  # LLaMA (30B)
    (32, 8192),  # PaLM (63B)
    (64, 8192),  # LLaMA (65B)
    (96, 12288),  # GPT-3 (175B)
    (48, 18432),  # PaLM (540B)
]

# Utility function to generate input sizes for benchmarks
def generate_input_sizes(dims: Union[int, List] = 2) -> List[Tuple]:
    """
    The weekly vs nightly input ranges only differ for 2D inputs currently.
    Nightly input range:
        Batch size: [16, 512, 2048, 8192, 16384] Hidden size: [768, 4*18432] (step size = 256)
    Weekly input range:
        Hidden size: Additonally benchmark hidden sizes at
            [step_size + 2, step_size + 4, step_size + 8, step_size + 16] to check vectorization.
    Note: The hidden size is restricted to 2 * 18432 for the batch size 16384 to avoid OOM.
    """
    inputs = []
    if isinstance(dims, int):
        dims = [dims]

    for dim in dims:
        if dim == 2:
            input_ranges = []

            step_size = 256
            batch_range = [16, 512, 2048, 8192]

            # max_hidden_size = 4 * d_model_max (max hidden size in feedforward layers)
            # NOTE: (This is not applicable to the updated implementation but leaving it here for future updates).
            #    Numpy arrays are not JSON serializable so convert them to enable storing benchmark data.

            hidden_range = []
            for hs in range(
                D_MODEL_MIN, 4 * D_MODEL_MAX + 1, step_size
            ):  # (768, 4*18432)
                hidden_range.append(hs)
                if BENCHMARK_MODE == "weekly":
                    # Additionally benchmark hidden sizes at steps (256 + 2, 256 + 4, 256 + 8, 256 + 16)
                    hidden_range.extend([hs + 2, hs + 4, hs + 8, hs + 16])
            input_ranges.append((batch_range, hidden_range))

            # Reduce max hidden size for largest batch size (16384) to avoid OOM in RMSNorm.
            # Sweeps hidden sizes from # (768, 2*18432) or (768, 2*18432 + 16) for weekly.
            input_ranges.append(
                ([16384], filter(lambda hs: hs <= 2 * D_MODEL_MAX + 16, hidden_range))
            )

            for batch_range, hidden_range in input_ranges:
                inputs.extend(list(itertools.product(batch_range, hidden_range)))

        elif dim == 3:
            dim_range = [2**i for i in range(1, 10)]
            inputs.extend(list(itertools.product(dim_range, repeat=3)))
        elif dim == 4:
            # TODO: Add spatial_dim = 2.
            input_ranges = []

            batch_range = [2**i for i in range(1, 10)]  # {2, 512}
            channel_range = [2**i for i in range(1, 8)]  # {2, 128}
            spatial_range = [2**i for i in range(2, 7)]  # {4, 64}
            input_ranges.append((batch_range, channel_range, spatial_range))

            batch_range = [2**i for i in range(1, 7)]  # {2, 64}
            channel_range = [2**i for i in range(1, 6)]  # {2, 32}
            spatial_range = [128, 256]
            input_ranges.append((batch_range, channel_range, spatial_range))

            # Resnet/ResNext sizes
            batch_range = [2**i for i in range(5, 9)]  # {32, 256}
            channel_range = [2**i for i in range(6, 9)]  # {64, 256}
            spatial_range = [7 * 2**i for i in range(5)]  # {7, 112}
            input_ranges.append((batch_range, channel_range, spatial_range))

            for batch_range, channel_range, spatial_range in input_ranges:
                inputs.extend(
                    [
                        (n, c, hw, hw)
                        for (n, c, hw) in itertools.product(
                            batch_range, channel_range, spatial_range
                        )
                    ]
                )

            inputs.extend(
                [
                    (n, c, hw, hw)
                    for (n, (c, hw)) in itertools.product(
                        [128, 256],
                        [
                            (512, 7),
                            (512, 14),
                            (512, 28),
                            (1024, 7),
                            (1024, 14),
                            (2048, 7),
                        ],
                    )
                ]
            )

        else:
            raise NotImplementedError(
                f"Generating input sizes of dimension {dim} is not implemented"
            )
    if BENCHMARK_CONFIG["num_inputs"] is not None:
        inputs = sample(inputs, BENCHMARK_CONFIG["num_inputs"])

    return inputs

# Utility function to generate input sizes for attention benchmarks.
def generate_attn_inputs():
    batch_range = [16, 32]
    seq_lengths = [2**i for i in range(2, 10)]  # {4, 512}
    inputs = [
        (batch_size, seq_len, nh, n_embd)
        for (batch_size, seq_len, (nh, n_embd)) in itertools.product(
            batch_range, seq_lengths, LLM_CONFIGS
        )
    ]

    if BENCHMARK_CONFIG["num_inputs"] is not None:
        inputs = sample(inputs, BENCHMARK_CONFIG["num_inputs"])

    return inputs


LLM_CONFIGS = [
    (12, 768),  # GPT-2 (124M), GPT-3 (125M)
    (16, 1024),  # GPT-2 (350M), GPT-3 (350M)
    (20, 1280),  # GPT-2 (774M)
    (16, 1536),  # GPT-3 (760M)
    (25, 1600),  # GPT-2 (1558M)
    (24, 2048),  # GPT-3 (1.3B)
    (32, 2560),  # GPT-3 (2.7B)
    (16, 4096),  # PaLM (8B)
    (32, 4096),  # LLaMA (7B), GPT-3 (6.7B)
    (40, 5120),  # LLaMA (13B), GPT-3 (13B)
    (52, 6656),  # LLaMA (30B)
    (32, 8192),  # PaLM (63B)
    (64, 8192),  # LLaMA (65B)
    (96, 12288),  # GPT-3 (175B)
    (48, 18432),  # PaLM (540B)
]

# Utility function to generate input sizes for benchmarks
def generate_input_sizes(dims: Union[int, List] = 2) -> List[Tuple]:
    """
    The weekly vs nightly input ranges only differ for 2D inputs currently.
    Nightly input range:
        Batch size: [16, 512, 2048, 8192, 16384] Hidden size: [768, 4*18432] (step size = 256)
    Weekly input range:
        Hidden size: Additonally benchmark hidden sizes at
            [step_size + 2, step_size + 4, step_size + 8, step_size + 16] to check vectorization.
    Note: The hidden size is restricted to 2 * 18432 for the batch size 16384 to avoid OOM.
    """
    inputs = []
    if isinstance(dims, int):
        dims = [dims]

    for dim in dims:
        if dim == 2:
            input_ranges = []

            step_size = 256
            batch_range = [16, 512, 2048, 8192]

            # max_hidden_size = 4 * d_model_max (max hidden size in feedforward layers)
            # NOTE: (This is not applicable to the updated implementation but leaving it here for future updates).
            #    Numpy arrays are not JSON serializable so convert them to enable storing benchmark data.

            hidden_range = []
            for hs in range(
                D_MODEL_MIN, 4 * D_MODEL_MAX + 1, step_size
            ):  # (768, 4*18432)
                hidden_range.append(hs)
                if BENCHMARK_MODE == "weekly":
                    # Additionally benchmark hidden sizes at steps (256 + 2, 256 + 4, 256 + 8, 256 + 16)
                    hidden_range.extend([hs + 2, hs + 4, hs + 8, hs + 16])
            input_ranges.append((batch_range, hidden_range))

            # Reduce max hidden size for largest batch size (16384) to avoid OOM in RMSNorm.
            # Sweeps hidden sizes from # (768, 2*18432) or (768, 2*18432 + 16) for weekly.
            input_ranges.append(
                ([16384], filter(lambda hs: hs <= 2 * D_MODEL_MAX + 16, hidden_range))
            )

            for batch_range, hidden_range in input_ranges:
                inputs.extend(list(itertools.product(batch_range, hidden_range)))

        elif dim == 3:
            dim_range = [2**i for i in range(1, 10)]
            inputs.extend(list(itertools.product(dim_range, repeat=3)))
        elif dim == 4:
            # TODO: Add spatial_dim = 2.
            input_ranges = []

            batch_range = [2**i for i in range(1, 10)]  # {2, 512}
            channel_range = [2**i for i in range(1, 8)]  # {2, 128}
            spatial_range = [2**i for i in range(2, 7)]  # {4, 64}
            input_ranges.append((batch_range, channel_range, spatial_range))

            batch_range = [2**i for i in range(1, 7)]  # {2, 64}
            channel_range = [2**i for i in range(1, 6)]  # {2, 32}
            spatial_range = [128, 256]
            input_ranges.append((batch_range, channel_range, spatial_range))

            # Resnet/ResNext sizes
            batch_range = [2**i for i in range(5, 9)]  # {32, 256}
            channel_range = [2**i for i in range(6, 9)]  # {64, 256}
            spatial_range = [7 * 2**i for i in range(5)]  # {7, 112}
            input_ranges.append((batch_range, channel_range, spatial_range))

            for batch_range, channel_range, spatial_range in input_ranges:
                inputs.extend(
                    [
                        (n, c, hw, hw)
                        for (n, c, hw) in itertools.product(
                            batch_range, channel_range, spatial_range
                        )
                    ]
                )

            inputs.extend(
                [
                    (n, c, hw, hw)
                    for (n, (c, hw)) in itertools.product(
                        [128, 256],
                        [
                            (512, 7),
                            (512, 14),
                            (512, 28),
                            (1024, 7),
                            (1024, 14),
                            (2048, 7),
                        ],
                    )
                ]
            )

        else:
            raise NotImplementedError(
                f"Generating input sizes of dimension {dim} is not implemented"
            )
    if BENCHMARK_CONFIG["num_inputs"] is not None:
        inputs = sample(inputs, BENCHMARK_CONFIG["num_inputs"])

    return inputs

# Utility function to generate input sizes for attention benchmarks.
def generate_attn_inputs():
    batch_range = [16, 32]
    seq_lengths = [2**i for i in range(2, 10)]  # {4, 512}
    inputs = [
        (batch_size, seq_len, nh, n_embd)
        for (batch_size, seq_len, (nh, n_embd)) in itertools.product(
            batch_range, seq_lengths, LLM_CONFIGS
        )
    ]

    if BENCHMARK_CONFIG["num_inputs"] is not None:
        inputs = sample(inputs, BENCHMARK_CONFIG["num_inputs"])

    return inputs


                            (512, 28),
                            (1024, 7),
                            (1024, 14),
                            (2048, 7),
                        ],
                    )
                ]
            )

        else:
            raise NotImplementedError(
                f"Generating input sizes of dimension {dim} is not implemented"
            )
    if BENCHMARK_CONFIG["num_inputs"] is not None:
        inputs = sample(inputs, BENCHMARK_CONFIG["num_inputs"])

    return inputs

# Utility function to generate input sizes for attention benchmarks.
def generate_attn_inputs():
    batch_range = [16, 32]
    seq_lengths = [2**i for i in range(2, 10)]  # {4, 512}
    inputs = [
        (batch_size, seq_len, nh, n_embd)
        for (batch_size, seq_len, (nh, n_embd)) in itertools.product(
            batch_range, seq_lengths, LLM_CONFIGS
        )
    ]

    if BENCHMARK_CONFIG["num_inputs"] is not None:
        inputs = sample(inputs, BENCHMARK_CONFIG["num_inputs"])

    return inputs


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType, PythonProfiler
from ..core import run_benchmark
import torch

def many_matmul_fusion(fd: FusionDefinition) -> None:
    x = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=DataType.Float, is_cpu=False
    )
    y = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=DataType.Float, is_cpu=False
    )
    a = fd.ops.add(x, y)
    for _ in range(5):
        a_transpose = fd.ops.permute(a, [1, 0])
        matmul_out = fd.ops.matmul(a_transpose, y)
        add_out = fd.ops.add(a_transpose, y)
        a = fd.ops.add(matmul_out, add_out)
    fd.add_output(a)

@pytest.mark.parametrize("host_bench_mode", ["compile", "steady", "dynamic"])
def test_many_segment_benchmark(
    benchmark,
    host_bench_mode: str,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    inputs = [torch.randn(5, 5, device="cuda", dtype=torch.float) for _ in range(2)]

    # Generate multiple inputs to measure dynamic shape overhead.
    if host_bench_mode == "dynamic":
        # Note: Using these sizes to allow kernel reuse in dynamic.
        # Using sizes = [4, 8, 16, 32, 64, 128] led to heuristic mismatch and kernel recompilation.
        input_sizes = [5, 7, 9, 11]
        # Generate matrices of size x size dimensions
        inputs = [
            [
                torch.randn(size, size, device="cuda", dtype=torch.float)
                for _ in range(2)
            ]
            for size in input_sizes
        ]

    with FusionDefinition() as fd:
        many_matmul_fusion(fd)

    def validate(input):
        x, y = input
        eager_output = x + y
        for _ in range(5):
            eager_transpose = eager_output.t()
            matmul_out = torch.matmul(eager_transpose, y)
            add_out = eager_transpose + y
            eager_output = matmul_out + add_out
        fd.validate(input, [eager_output])

        # Validate number of segments
        with PythonProfiler() as prof:
            fd.execute(input)
        num_segments = prof.profile.segments
        expected_segments = 12
        assert (
            num_segments == expected_segments
        ), f"Expected {expected_segments} fusion segments, got {num_segments}."

    if not disable_validation:
        if host_bench_mode == "dynamic":
            # Run validate for all input sizes.
            for input in inputs:
                validate(input)
        else:
            validate(inputs)

    if not disable_benchmarking:
        run_benchmark(
            benchmark,
            None,
            inputs,
            device=f"host:{host_bench_mode}",
            fusion_fn=many_matmul_fusion,
        )


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType, PythonProfiler
from ..core import run_benchmark
import torch

def many_matmul_fusion(fd: FusionDefinition) -> None:
    x = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=DataType.Float, is_cpu=False
    )
    y = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=DataType.Float, is_cpu=False
    )
    a = fd.ops.add(x, y)
    for _ in range(5):
        a_transpose = fd.ops.permute(a, [1, 0])
        matmul_out = fd.ops.matmul(a_transpose, y)
        add_out = fd.ops.add(a_transpose, y)
        a = fd.ops.add(matmul_out, add_out)
    fd.add_output(a)

@pytest.mark.parametrize("host_bench_mode", ["compile", "steady", "dynamic"])
def test_many_segment_benchmark(
    benchmark,
    host_bench_mode: str,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    inputs = [torch.randn(5, 5, device="cuda", dtype=torch.float) for _ in range(2)]

    # Generate multiple inputs to measure dynamic shape overhead.
    if host_bench_mode == "dynamic":
        # Note: Using these sizes to allow kernel reuse in dynamic.
        # Using sizes = [4, 8, 16, 32, 64, 128] led to heuristic mismatch and kernel recompilation.
        input_sizes = [5, 7, 9, 11]
        # Generate matrices of size x size dimensions
        inputs = [
            [
                torch.randn(size, size, device="cuda", dtype=torch.float)


import torch

def many_matmul_fusion(fd: FusionDefinition) -> None:
    x = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=DataType.Float, is_cpu=False
    )
    y = fd.define_tensor(
        shape=[-1, -1], contiguity=[True, True], dtype=DataType.Float, is_cpu=False
    )
    a = fd.ops.add(x, y)
    for _ in range(5):
        a_transpose = fd.ops.permute(a, [1, 0])
        matmul_out = fd.ops.matmul(a_transpose, y)
        add_out = fd.ops.add(a_transpose, y)
        a = fd.ops.add(matmul_out, add_out)
    fd.add_output(a)

@pytest.mark.parametrize("host_bench_mode", ["compile", "steady", "dynamic"])
def test_many_segment_benchmark(
    benchmark,
    host_bench_mode: str,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    inputs = [torch.randn(5, 5, device="cuda", dtype=torch.float) for _ in range(2)]

    # Generate multiple inputs to measure dynamic shape overhead.
    if host_bench_mode == "dynamic":
        # Note: Using these sizes to allow kernel reuse in dynamic.
        # Using sizes = [4, 8, 16, 32, 64, 128] led to heuristic mismatch and kernel recompilation.
        input_sizes = [5, 7, 9, 11]
        # Generate matrices of size x size dimensions
        inputs = [
            [
                torch.randn(size, size, device="cuda", dtype=torch.float)
                for _ in range(2)
            ]
            for size in input_sizes
        ]

    with FusionDefinition() as fd:
        many_matmul_fusion(fd)

    def validate(input):
        x, y = input
        eager_output = x + y
        for _ in range(5):
            eager_transpose = eager_output.t()
            matmul_out = torch.matmul(eager_transpose, y)
            add_out = eager_transpose + y
            eager_output = matmul_out + add_out
        fd.validate(input, [eager_output])

        # Validate number of segments
        with PythonProfiler() as prof:
            fd.execute(input)
        num_segments = prof.profile.segments
        expected_segments = 12
        assert (
            num_segments == expected_segments
        ), f"Expected {expected_segments} fusion segments, got {num_segments}."

    if not disable_validation:
        if host_bench_mode == "dynamic":
            # Run validate for all input sizes.
            for input in inputs:
                validate(input)
        else:
            validate(inputs)

    if not disable_benchmarking:
        run_benchmark(
            benchmark,
            None,
            inputs,
            device=f"host:{host_bench_mode}",
            fusion_fn=many_matmul_fusion,
        )


validate

):
    inputs = [torch.randn(5, 5, device="cuda", dtype=torch.float) for _ in range(2)]

    # Generate multiple inputs to measure dynamic shape overhead.
    if host_bench_mode == "dynamic":
        # Note: Using these sizes to allow kernel reuse in dynamic.
        # Using sizes = [4, 8, 16, 32, 64, 128] led to heuristic mismatch and kernel recompilation.
        input_sizes = [5, 7, 9, 11]
        # Generate matrices of size x size dimensions
        inputs = [
            [
                torch.randn(size, size, device="cuda", dtype=torch.float)
                for _ in range(2)
            ]
            for size in input_sizes
        ]

    with FusionDefinition() as fd:
        many_matmul_fusion(fd)

    def validate(input):
        x, y = input
        eager_output = x + y
        for _ in range(5):
            eager_transpose = eager_output.t()
            matmul_out = torch.matmul(eager_transpose, y)
            add_out = eager_transpose + y
            eager_output = matmul_out + add_out
        fd.validate(input, [eager_output])

        # Validate number of segments
        with PythonProfiler() as prof:
            fd.execute(input)
        num_segments = prof.profile.segments
        expected_segments = 12
        assert (
            num_segments == expected_segments
        ), f"Expected {expected_segments} fusion segments, got {num_segments}."

    if not disable_validation:
        if host_bench_mode == "dynamic":
            # Run validate for all input sizes.
            for input in inputs:
                validate(input)
        else:
            validate(inputs)

    if not disable_benchmarking:
        run_benchmark(
            benchmark,
            None,
            inputs,
            device=f"host:{host_bench_mode}",
            fusion_fn=many_matmul_fusion,
        )


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
from ..core import run_benchmark
import torch
from ..global_params import PROMOTE_DTYPES
from functools import partial

def pointwise_ops_fusion(fd: FusionDefinition, dtype: DataType, num_iters: int):
    x = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)
    y = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)

    if dtype in PROMOTE_DTYPES:
        x = fd.ops.cast(x, dtype=DataType.Float)
        y = fd.ops.cast(y, dtype=DataType.Float)

    a = fd.ops.add(x, y)
    for _ in range(num_iters):
        x = fd.ops.cos(a)
        y = fd.ops.sin(a)
        a = fd.ops.add(x, y)

    if dtype in PROMOTE_DTYPES:
        a = fd.ops.cast(a, dtype=dtype)
    fd.add_output(a)

# TODO: num_iters 32 and 128 are restricted due to issue #5531.
# NOTE: 22 is the largest, runnable num_iters without timeout.
@pytest.mark.parametrize("num_iters", [2, 8, 16, 22])
@pytest.mark.parametrize("host_bench_mode", ["compile", "steady", "dynamic"])
def test_pointwise_ops_benchmark(
    benchmark,
    num_iters: int,
    host_bench_mode: str,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    inputs = [torch.randn(13, device="cuda", dtype=torch.float16) for _ in range(2)]

    # Generate multiple inputs to measure dynamic shape overhead.
    if host_bench_mode == "dynamic":
        input_sizes = [5, 10, 13, 15, 17, 20]
        inputs = [
            [torch.randn(size, device="cuda", dtype=torch.float16) for _ in range(2)]
            for size in input_sizes
        ]

    with FusionDefinition() as fd:
        pointwise_ops_fusion(fd, torch_dtype_to_nvfuser_dtype(torch.float16), num_iters)

    def validate(input):
        eager_output = input[0] + input[1]
        for _ in range(num_iters):
            x = torch.cos(eager_output)
            y = torch.sin(eager_output)
            eager_output = x + y
        fd.validate(input, [eager_output])

    if not disable_validation:
        if host_bench_mode == "dynamic":
            # Run validate for all input sizes.
            for input in inputs:
                validate(input)
        else:
            validate(inputs)

    if not disable_benchmarking:
        run_benchmark(
            benchmark,
            None,
            inputs,
            device=f"host:{host_bench_mode}",
            fusion_fn=partial(
                pointwise_ops_fusion,
                dtype=torch_dtype_to_nvfuser_dtype(torch.float16),
                num_iters=num_iters,
            ),
        )


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType
from nvfuser_direct.pytorch_utils import torch_dtype_to_nvfuser_dtype
from ..core import run_benchmark
import torch
from ..global_params import PROMOTE_DTYPES
from functools import partial

def pointwise_ops_fusion(fd: FusionDefinition, dtype: DataType, num_iters: int):
    x = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)
    y = fd.define_tensor(shape=[-1], contiguity=[True], dtype=dtype, is_cpu=False)

    if dtype in PROMOTE_DTYPES:
        x = fd.ops.cast(x, dtype=DataType.Float)
        y = fd.ops.cast(y, dtype=DataType.Float)

    a = fd.ops.add(x, y)
    for _ in range(num_iters):
        x = fd.ops.cos(a)
        y = fd.ops.sin(a)
        a = fd.ops.add(x, y)

    if dtype in PROMOTE_DTYPES:
        a = fd.ops.cast(a, dtype=dtype)
    fd.add_output(a)

# TODO: num_iters 32 and 128 are restricted due to issue #5531.
# NOTE: 22 is the largest, runnable num_iters without timeout.
@pytest.mark.parametrize("num_iters", [2, 8, 16, 22])
@pytest.mark.parametrize("host_bench_mode", ["compile", "steady", "dynamic"])
def test_pointwise_ops_benchmark(
    benchmark,
    num_iters: int,
    host_bench_mode: str,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    inputs = [torch.randn(13, device="cuda", dtype=torch.float16) for _ in range(2)]

    # Generate multiple inputs to measure dynamic shape overhead.
    if host_bench_mode == "dynamic":
        input_sizes = [5, 10, 13, 15, 17, 20]
        inputs = [
            [torch.randn(size, device="cuda", dtype=torch.float16) for _ in range(2)]


    if dtype in PROMOTE_DTYPES:
        x = fd.ops.cast(x, dtype=DataType.Float)
        y = fd.ops.cast(y, dtype=DataType.Float)

    a = fd.ops.add(x, y)
    for _ in range(num_iters):
        x = fd.ops.cos(a)
        y = fd.ops.sin(a)
        a = fd.ops.add(x, y)

    if dtype in PROMOTE_DTYPES:
        a = fd.ops.cast(a, dtype=dtype)
    fd.add_output(a)

# TODO: num_iters 32 and 128 are restricted due to issue #5531.
# NOTE: 22 is the largest, runnable num_iters without timeout.
@pytest.mark.parametrize("num_iters", [2, 8, 16, 22])
@pytest.mark.parametrize("host_bench_mode", ["compile", "steady", "dynamic"])
def test_pointwise_ops_benchmark(
    benchmark,
    num_iters: int,
    host_bench_mode: str,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    inputs = [torch.randn(13, device="cuda", dtype=torch.float16) for _ in range(2)]

    # Generate multiple inputs to measure dynamic shape overhead.
    if host_bench_mode == "dynamic":
        input_sizes = [5, 10, 13, 15, 17, 20]
        inputs = [
            [torch.randn(size, device="cuda", dtype=torch.float16) for _ in range(2)]
            for size in input_sizes
        ]

    with FusionDefinition() as fd:
        pointwise_ops_fusion(fd, torch_dtype_to_nvfuser_dtype(torch.float16), num_iters)

    def validate(input):
        eager_output = input[0] + input[1]
        for _ in range(num_iters):
            x = torch.cos(eager_output)
            y = torch.sin(eager_output)
            eager_output = x + y
        fd.validate(input, [eager_output])

    if not disable_validation:
        if host_bench_mode == "dynamic":
            # Run validate for all input sizes.
            for input in inputs:
                validate(input)
        else:
            validate(inputs)

    if not disable_benchmarking:
        run_benchmark(
            benchmark,
            None,
            inputs,
            device=f"host:{host_bench_mode}",
            fusion_fn=partial(
                pointwise_ops_fusion,
                dtype=torch_dtype_to_nvfuser_dtype(torch.float16),
                num_iters=num_iters,
            ),
        )


def test_pointwise_ops_benchmark(
    benchmark,
    num_iters: int,
    host_bench_mode: str,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    inputs = [torch.randn(13, device="cuda", dtype=torch.float16) for _ in range(2)]

    # Generate multiple inputs to measure dynamic shape overhead.
    if host_bench_mode == "dynamic":
        input_sizes = [5, 10, 13, 15, 17, 20]
        inputs = [
            [torch.randn(size, device="cuda", dtype=torch.float16) for _ in range(2)]
            for size in input_sizes
        ]

    with FusionDefinition() as fd:
        pointwise_ops_fusion(fd, torch_dtype_to_nvfuser_dtype(torch.float16), num_iters)

    def validate(input):
        eager_output = input[0] + input[1]
        for _ in range(num_iters):
            x = torch.cos(eager_output)
            y = torch.sin(eager_output)
            eager_output = x + y
        fd.validate(input, [eager_output])

    if not disable_validation:
        if host_bench_mode == "dynamic":
            # Run validate for all input sizes.
            for input in inputs:
                validate(input)
        else:
            validate(inputs)

    if not disable_benchmarking:
        run_benchmark(
            benchmark,
            None,
            inputs,
            device=f"host:{host_bench_mode}",
            fusion_fn=partial(
                pointwise_ops_fusion,
                dtype=torch_dtype_to_nvfuser_dtype(torch.float16),
                num_iters=num_iters,
            ),


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType
from ..core import run_benchmark
import torch

def adaptive_layernorm_fwd_fusion(fd: FusionDefinition, eps: float = 1e-6) -> None:
    T0 = fd.define_tensor(
        shape=[-1, -1, -1],
        contiguity=[True, True, True],
        dtype=DataType.Half,
        is_cpu=False,
        stride_order=[2, 1, 0],
    )
    T1 = fd.define_tensor(
        shape=[-1, -1],
        contiguity=[True, True],
        dtype=DataType.Half,
        is_cpu=False,
        stride_order=[1, 0],
    )
    T2 = fd.define_tensor(
        shape=[-1, -1],
        contiguity=[True, True],
        dtype=DataType.Half,
        is_cpu=False,
        stride_order=[1, 0],
    )
    T3 = fd.ops.cast(T0, dtype=DataType.Float)
    T4, T5 = fd.ops.var_mean(T3, dims=[2], correction=0, keepdim=False)
    T10 = fd.ops.broadcast_in_dim(
        T4, shape=[T0.size(0), T0.size(1), 1], broadcast_dims=[0, 1]
    )
    T15 = fd.ops.broadcast_in_dim(
        T5, shape=[T0.size(0), T0.size(1), 1], broadcast_dims=[0, 1]
    )
    S16 = fd.define_scalar(eps, dtype=DataType.Double)
    T17 = fd.ops.add(T10, S16)
    T22 = fd.ops.broadcast_in_dim(T15, shape=T0.shape(), broadcast_dims=[0, 1, 2])
    T23 = fd.ops.rsqrt(T17)
    T24 = fd.ops.sub(T3, T22)
    T29 = fd.ops.broadcast_in_dim(T23, shape=T0.shape(), broadcast_dims=[0, 1, 2])
    T30 = fd.ops.mul(T24, T29)
    T35 = fd.ops.reshape(T1, new_shape=[T1.size(0), 1, T1.size(1)])
    T36 = fd.ops.cast(T35, dtype=DataType.Float)
    S37 = fd.define_scalar(1.00000, dtype=DataType.Double)
    T38 = fd.ops.add(S37, T36)
    T39 = fd.ops.cast(T38, dtype=DataType.Half)
    T44 = fd.ops.broadcast_in_dim(T39, shape=T0.shape(), broadcast_dims=[0, 1, 2])
    T45 = fd.ops.cast(T44, dtype=DataType.Float)
    T46 = fd.ops.mul(T30, T45)
    T51 = fd.ops.reshape(T2, new_shape=[T2.size(0), 1, T2.size(1)])
    T56 = fd.ops.broadcast_in_dim(T51, shape=T0.shape(), broadcast_dims=[0, 1, 2])
    T57 = fd.ops.cast(T56, dtype=DataType.Float)
    T58 = fd.ops.add(T46, T57)
    T59 = fd.ops.cast(T58, dtype=DataType.Half)
    fd.add_output(T5)
    fd.add_output(T23)
    fd.add_output(T59)

# This benchmark is to particularly track nvFuser host overhead for shape
# change (dynamic shape support) in the adapative layernorm case. Running a
# new shape on this fusion without recompiling a new kernel can have significant overhead.
@pytest.mark.parametrize("host_bench_mode", ["compile", "steady", "dynamic"])
def test_adaptive_layernorm_fwd_benchmark(
    benchmark,
    host_bench_mode: str,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    B = 1
    T = 30 * 1024
    D = 1024
    inputs = [
        torch.randn(B, T, D, device="cuda", dtype=torch.float16, requires_grad=True),
        torch.randn(B, D, device="cuda", dtype=torch.float16, requires_grad=True),
        torch.randn(B, D, device="cuda", dtype=torch.float16, requires_grad=True),
    ]

    # Generate multiple inputs to measure dynamic shape overhead.
    if host_bench_mode == "dynamic":
        inputs = []
        for B in range(1, 3, 1):
            for T in range(30 * 1024, 30 * 1024 + 5 * 128, 128):
                inputs.append(
                    [
                        torch.randn(
                            B,
                            T,
                            D,
                            device="cuda",
                            dtype=torch.float16,
                            requires_grad=True,
                        ),
                        torch.randn(
                            B, D, device="cuda", dtype=torch.float16, requires_grad=True
                        ),
                        torch.randn(
                            B, D, device="cuda", dtype=torch.float16, requires_grad=True
                        ),
                    ]
                )

    with FusionDefinition() as fd:
        adaptive_layernorm_fwd_fusion(fd)

    def validate(input):
        eps = 1e-6
        in_tensor, scale, shift = input
        norm_state = torch.nn.LayerNorm(D, elementwise_affine=False, eps=eps)
        norm_out = norm_state(in_tensor)
        mean = in_tensor.to(torch.float).mean(dim=-1)
        variance = in_tensor.to(torch.float).var(dim=-1, unbiased=False)
        invstd = (1.0 / torch.sqrt(variance + eps)).unsqueeze(-1)
        eager_output = norm_out * (1 + scale.view(-1, 1, D)) + shift.view(-1, 1, D)
        fd.validate(input, [mean, invstd, eager_output])

    if not disable_validation:
        if host_bench_mode == "dynamic":
            # Run validate for all input sizes.
            for input in inputs:
                validate(input)
        else:
            validate(inputs)

    if not disable_benchmarking:
        run_benchmark(
            benchmark,
            None,
            inputs,
            device=f"host:{host_bench_mode}",
            fusion_fn=adaptive_layernorm_fwd_fusion,
        )


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import pytest
from nvfuser_direct import FusionDefinition, DataType
from ..core import run_benchmark
import torch

def adaptive_layernorm_fwd_fusion(fd: FusionDefinition, eps: float = 1e-6) -> None:
    T0 = fd.define_tensor(
        shape=[-1, -1, -1],
        contiguity=[True, True, True],
        dtype=DataType.Half,
        is_cpu=False,
        stride_order=[2, 1, 0],
    )
    T1 = fd.define_tensor(
        shape=[-1, -1],
        contiguity=[True, True],
        dtype=DataType.Half,
        is_cpu=False,
        stride_order=[1, 0],
    )
    T2 = fd.define_tensor(
        shape=[-1, -1],
        contiguity=[True, True],
        dtype=DataType.Half,
        is_cpu=False,
        stride_order=[1, 0],
    )
    T3 = fd.ops.cast(T0, dtype=DataType.Float)
    T4, T5 = fd.ops.var_mean(T3, dims=[2], correction=0, keepdim=False)
    T10 = fd.ops.broadcast_in_dim(
        T4, shape=[T0.size(0), T0.size(1), 1], broadcast_dims=[0, 1]
    )
    T15 = fd.ops.broadcast_in_dim(
        T5, shape=[T0.size(0), T0.size(1), 1], broadcast_dims=[0, 1]
    )
    S16 = fd.define_scalar(eps, dtype=DataType.Double)
    T17 = fd.ops.add(T10, S16)
    T22 = fd.ops.broadcast_in_dim(T15, shape=T0.shape(), broadcast_dims=[0, 1, 2])
    T23 = fd.ops.rsqrt(T17)
    T24 = fd.ops.sub(T3, T22)
    T29 = fd.ops.broadcast_in_dim(T23, shape=T0.shape(), broadcast_dims=[0, 1, 2])
    T30 = fd.ops.mul(T24, T29)
    T35 = fd.ops.reshape(T1, new_shape=[T1.size(0), 1, T1.size(1)])
    T36 = fd.ops.cast(T35, dtype=DataType.Float)
    S37 = fd.define_scalar(1.00000, dtype=DataType.Double)
    T38 = fd.ops.add(S37, T36)
    T39 = fd.ops.cast(T38, dtype=DataType.Half)
    T44 = fd.ops.broadcast_in_dim(T39, shape=T0.shape(), broadcast_dims=[0, 1, 2])
    T45 = fd.ops.cast(T44, dtype=DataType.Float)
    T46 = fd.ops.mul(T30, T45)
    T51 = fd.ops.reshape(T2, new_shape=[T2.size(0), 1, T2.size(1)])
    T56 = fd.ops.broadcast_in_dim(T51, shape=T0.shape(), broadcast_dims=[0, 1, 2])
    T57 = fd.ops.cast(T56, dtype=DataType.Float)
    T58 = fd.ops.add(T46, T57)
    T59 = fd.ops.cast(T58, dtype=DataType.Half)
    fd.add_output(T5)
    fd.add_output(T23)
    fd.add_output(T59)

# This benchmark is to particularly track nvFuser host overhead for shape
# change (dynamic shape support) in the adapative layernorm case. Running a
# new shape on this fusion without recompiling a new kernel can have significant overhead.
@pytest.mark.parametrize("host_bench_mode", ["compile", "steady", "dynamic"])
def test_adaptive_layernorm_fwd_benchmark(
    benchmark,
    host_bench_mode: str,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    B = 1
    T = 30 * 1024
    D = 1024
    inputs = [
        torch.randn(B, T, D, device="cuda", dtype=torch.float16, requires_grad=True),
        torch.randn(B, D, device="cuda", dtype=torch.float16, requires_grad=True),
        torch.randn(B, D, device="cuda", dtype=torch.float16, requires_grad=True),
    ]


    S37 = fd.define_scalar(1.00000, dtype=DataType.Double)
    T38 = fd.ops.add(S37, T36)
    T39 = fd.ops.cast(T38, dtype=DataType.Half)
    T44 = fd.ops.broadcast_in_dim(T39, shape=T0.shape(), broadcast_dims=[0, 1, 2])
    T45 = fd.ops.cast(T44, dtype=DataType.Float)
    T46 = fd.ops.mul(T30, T45)
    T51 = fd.ops.reshape(T2, new_shape=[T2.size(0), 1, T2.size(1)])
    T56 = fd.ops.broadcast_in_dim(T51, shape=T0.shape(), broadcast_dims=[0, 1, 2])
    T57 = fd.ops.cast(T56, dtype=DataType.Float)
    T58 = fd.ops.add(T46, T57)
    T59 = fd.ops.cast(T58, dtype=DataType.Half)
    fd.add_output(T5)
    fd.add_output(T23)
    fd.add_output(T59)

# This benchmark is to particularly track nvFuser host overhead for shape
# change (dynamic shape support) in the adapative layernorm case. Running a
# new shape on this fusion without recompiling a new kernel can have significant overhead.
@pytest.mark.parametrize("host_bench_mode", ["compile", "steady", "dynamic"])
def test_adaptive_layernorm_fwd_benchmark(
    benchmark,
    host_bench_mode: str,
    disable_validation: bool,
    disable_benchmarking: bool,
):
    B = 1
    T = 30 * 1024
    D = 1024
    inputs = [
        torch.randn(B, T, D, device="cuda", dtype=torch.float16, requires_grad=True),
        torch.randn(B, D, device="cuda", dtype=torch.float16, requires_grad=True),
        torch.randn(B, D, device="cuda", dtype=torch.float16, requires_grad=True),
    ]

    # Generate multiple inputs to measure dynamic shape overhead.
    if host_bench_mode == "dynamic":
        inputs = []
        for B in range(1, 3, 1):
            for T in range(30 * 1024, 30 * 1024 + 5 * 128, 128):
                inputs.append(
                    [
                        torch.randn(
                            B,
                            T,
                            D,
                            device="cuda",
                            dtype=torch.float16,
                            requires_grad=True,
                        ),
                        torch.randn(
                            B, D, device="cuda", dtype=torch.float16, requires_grad=True
                        ),
                        torch.randn(
                            B, D, device="cuda", dtype=torch.float16, requires_grad=True
                        ),
                    ]
                )

    with FusionDefinition() as fd:
        adaptive_layernorm_fwd_fusion(fd)

    def validate(input):
        eps = 1e-6
        in_tensor, scale, shift = input
        norm_state = torch.nn.LayerNorm(D, elementwise_affine=False, eps=eps)
        norm_out = norm_state(in_tensor)
        mean = in_tensor.to(torch.float).mean(dim=-1)
        variance = in_tensor.to(torch.float).var(dim=-1, unbiased=False)
        invstd = (1.0 / torch.sqrt(variance + eps)).unsqueeze(-1)
        eager_output = norm_out * (1 + scale.view(-1, 1, D)) + shift.view(-1, 1, D)
        fd.validate(input, [mean, invstd, eager_output])

    if not disable_validation:
        if host_bench_mode == "dynamic":
            # Run validate for all input sizes.
            for input in inputs:
                validate(input)
        else:
            validate(inputs)

    if not disable_benchmarking:
        run_benchmark(
            benchmark,
            None,
            inputs,
            device=f"host:{host_bench_mode}",
            fusion_fn=adaptive_layernorm_fwd_fusion,
        )


                        torch.randn(
                            B,
                            T,
                            D,
                            device="cuda",
                            dtype=torch.float16,
                            requires_grad=True,
                        ),
                        torch.randn(
                            B, D, device="cuda", dtype=torch.float16, requires_grad=True
                        ),
                        torch.randn(
                            B, D, device="cuda", dtype=torch.float16, requires_grad=True
                        ),
                    ]
                )

    with FusionDefinition() as fd:
        adaptive_layernorm_fwd_fusion(fd)

    def validate(input):
        eps = 1e-6
        in_tensor, scale, shift = input
        norm_state = torch.nn.LayerNorm(D, elementwise_affine=False, eps=eps)
        norm_out = norm_state(in_tensor)
        mean = in_tensor.to(torch.float).mean(dim=-1)
        variance = in_tensor.to(torch.float).var(dim=-1, unbiased=False)
        invstd = (1.0 / torch.sqrt(variance + eps)).unsqueeze(-1)
        eager_output = norm_out * (1 + scale.view(-1, 1, D)) + shift.view(-1, 1, D)
        fd.validate(input, [mean, invstd, eager_output])

    if not disable_validation:
        if host_bench_mode == "dynamic":
            # Run validate for all input sizes.
            for input in inputs:
                validate(input)
        else:
            validate(inputs)

    if not disable_benchmarking:
        run_benchmark(
            benchmark,
            None,
            inputs,
            device=f"host:{host_bench_mode}",
            fusion_fn=adaptive_layernorm_fwd_fusion,
        )


python.setup

# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
#
# Usage:
# pip install --no-build-isolation -e python -v
#   This build command is equivalent to: python setup.py develop
#   Options:
#     -v: verbose output
#     --no-build-isolation: don't build in a temporary directory
#     -e: install in development mode
#
# Environment variables used during build:
#   MAX_JOBS
#     maximum number of compile jobs we should use to compile your code
#
#   NVFUSER_BUILD_CMAKE_ONLY
#     Only generate ./build directory with cmake setup
#
#   NVFUSER_BUILD_NO_PYTHON
#     Skips python API target `libnvfuser.so`, i.e. `_C.cpython-xxx.so`
#
#   NVFUSER_BUILD_NO_TEST
#     Skips cpp tests `test_nvfuser`
#
#   NVFUSER_BUILD_NO_BENCHMARK
#     Skips benchmark target `nvfuser_bench`
#
#   NVFUSER_BUILD_NO_NINJA
#     In case you want to use make instead of ninja for build
#
#   NVFUSER_BUILD_WITH_UCC
#     Build nvfuser with UCC support. You may need to specify environment variables of UCC_HOME, UCC_DIR, UCX_HOME, UCX_DIR.
#
#   NVFUSER_BUILD_WITHOUT_DISTRIBUTED
#     Build nvfuser without multidevice support
#
#   NVFUSER_BUILD_BUILD_TYPE=Debug
#     Building nvfuser in debug mode
#
#   NVFUSER_BUILD_BUILD_TYPE=RelwithDebInfo
#     Building nvfuser in release mode with debug info, a.k.a. RelwithDebInfo
#
#   NVFUSER_BUILD_DIR=<ABSOLUTE PATH>
#     Specify in which directory to build nvfuser. If not specified, the default build directory is "./build".
#
#   NVFUSER_BUILD_INSTALL_DIR=<ABSOLUTE PATH>
#     Specify in which directory to install nvfuser. If not specified, the default install directory is "./python/nvfuser".
#
#   NVFUSER_BUILD_VERSION_TAG=TAG
#     Specify the tag for build nvfuser version, this is used for pip wheel
#     package nightly where we might want to add a date tag
#     nvfuser-VERSION+TAG+gitSHA1-....-whl
#
#   NVFUSER_BUILD_INSTALL_REQUIRES=pkg0[,pkg1...]
#     this is used for pip wheel build to specify package required for install
#     e.g. NVFUSER_BUILD_INSTALL_REQUIRES=nvidia-cuda-nvrtc-cu12
#
#   NVFUSER_BUILD_NVMMH_INCLUDE_DIR=<ABSOLUTE PATH>
#     Specify the location to find nvMatmulHeuristics.h
#
#   NVFUSER_BUILD_WHEEL_NAME=NAME
#     Specify the wheel name this is used for pip wheel package where we want
#     to identify the cuda toolkit version
#
#   NVFUSER_BUILD_CPP_STANDARD=STANDARD
#     Specify the C++ standard to use for building nvfuser. The default is C++20.
#
#   NVFUSER_BUILD_ENABLE_PCH=1
#     Enable precompiled headers to speed up compilation. Default is OFF.
#

import sys

from utils import (
    BuildConfig,
    run,
    override_build_config_from_env,
)

def version_tag(config):
    from tools.gen_nvfuser_version import get_version

    version = get_version()
    if config.overwrite_version:
        version = version.split("+")[0]
        if len(config.version_tag) != 0:
            # use "." to be pypi friendly
            version = ".".join([version, config.version_tag])
    return version

def main():
    config = BuildConfig()
    # Override build config from environment variables.
    override_build_config_from_env(config)

    if "clean" in sys.argv:
        # only disables BUILD_SETUP, but keep the argument for setuptools
        config.build_setup = False

    if config.cpp_standard < 20:
        raise ValueError("nvfuser requires C++20 standard or higher")

    run(config, version_tag(config), relative_path="..")

if __name__ == "__main__":
    main()


#   NVFUSER_BUILD_WHEEL_NAME=NAME
#     Specify the wheel name this is used for pip wheel package where we want
#     to identify the cuda toolkit version
#
#   NVFUSER_BUILD_CPP_STANDARD=STANDARD
#     Specify the C++ standard to use for building nvfuser. The default is C++20.
#
#   NVFUSER_BUILD_ENABLE_PCH=1
#     Enable precompiled headers to speed up compilation. Default is OFF.
#

import sys

from utils import (
    BuildConfig,
    run,
    override_build_config_from_env,
)

def version_tag(config):
    from tools.gen_nvfuser_version import get_version

    version = get_version()
    if config.overwrite_version:
        version = version.split("+")[0]
        if len(config.version_tag) != 0:
            # use "." to be pypi friendly
            version = ".".join([version, config.version_tag])
    return version

def main():
    config = BuildConfig()
    # Override build config from environment variables.
    override_build_config_from_env(config)

    if "clean" in sys.argv:
        # only disables BUILD_SETUP, but keep the argument for setuptools
        config.build_setup = False

    if config.cpp_standard < 20:
        raise ValueError("nvfuser requires C++20 standard or higher")

    run(config, version_tag(config), relative_path="..")

if __name__ == "__main__":
    main()


main

from utils import (
    BuildConfig,
    run,
    override_build_config_from_env,
)

def version_tag(config):
    from tools.gen_nvfuser_version import get_version

    version = get_version()
    if config.overwrite_version:
        version = version.split("+")[0]
        if len(config.version_tag) != 0:
            # use "." to be pypi friendly
            version = ".".join([version, config.version_tag])
    return version

def main():
    config = BuildConfig()
    # Override build config from environment variables.
    override_build_config_from_env(config)

    if "clean" in sys.argv:
        # only disables BUILD_SETUP, but keep the argument for setuptools
        config.build_setup = False

    if config.cpp_standard < 20:
        raise ValueError("nvfuser requires C++20 standard or higher")

    run(config, version_tag(config), relative_path="..")

if __name__ == "__main__":
    main()


python.utils

# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

import os
import multiprocessing
import subprocess
import sys
import shutil
from dataclasses import dataclass, field
import setuptools.command.build_ext

@dataclass
class BuildConfig:
    cmake_only: bool = False
    build_setup: bool = True
    no_python: bool = False
    no_cutlass: bool = False
    no_test: bool = False
    no_benchmark: bool = False
    no_ninja: bool = False
    build_with_ucc: bool = False
    build_with_asan: bool = False
    build_without_distributed: bool = False
    explicit_error_check: bool = False
    overwrite_version: bool = False
    version_tag: str = None
    build_type: str = "Release"
    wheel_name: str = "nvfuser"
    nvmmh_include_dir: str = ""
    build_dir: str = ""
    install_dir: str = ""
    install_requires: list = field(default_factory=list)
    extras_require: dict = field(default_factory=dict)
    cpp_standard: int = 20
    cutlass_max_jobs: int | None = None
    enable_pch: bool = False

def check_env_flag_bool_default(name: str, default: str = "") -> bool:
    if name not in os.environ:
        return default
    return os.getenv(name).upper() in ["ON", "1", "YES", "TRUE", "Y"]

def get_env_flag_bool(name: str) -> bool:
    assert name in os.environ
    return os.getenv(name).upper() in ["ON", "1", "YES", "TRUE", "Y"]

# Override BuildConfig with environment variables. Only change if variable
# exists. Do not use default to override argparse.
def override_build_config_from_env(config):
    # Command line arguments don't work on PEP517 builds and will be silently ignored,
    # so we need to pass those options as environment variables instead.
    if "NVFUSER_BUILD_CMAKE_ONLY" in os.environ:
        config.cmake_only = get_env_flag_bool("NVFUSER_BUILD_CMAKE_ONLY")
    if "NVFUSER_BUILD_SETUP" in os.environ:
        config.build_setup = get_env_flag_bool("NVFUSER_BUILD_SETUP")
    if "NVFUSER_BUILD_NO_PYTHON" in os.environ:
        config.no_python = get_env_flag_bool("NVFUSER_BUILD_NO_PYTHON")
    if "NVFUSER_BUILD_NO_CUTLASS" in os.environ:
        config.no_cutlass = get_env_flag_bool("NVFUSER_BUILD_NO_CUTLASS")
    if "NVFUSER_BUILD_NO_TEST" in os.environ:
        config.no_test = get_env_flag_bool("NVFUSER_BUILD_NO_TEST")
    if "NVFUSER_BUILD_NO_BENCHMARK" in os.environ:
        config.no_benchmark = get_env_flag_bool("NVFUSER_BUILD_NO_BENCHMARK")
    if "NVFUSER_BUILD_NO_NINJA" in os.environ:
        config.no_ninja = get_env_flag_bool("NVFUSER_BUILD_NO_NINJA")
    if "NVFUSER_BUILD_WITH_UCC" in os.environ:
        config.build_with_ucc = get_env_flag_bool("NVFUSER_BUILD_WITH_UCC")
    if "NVFUSER_BUILD_WITH_ASAN" in os.environ:
        config.build_with_asan = get_env_flag_bool("NVFUSER_BUILD_WITH_ASAN")
    if "NVFUSER_BUILD_WITHOUT_DISTRIBUTED" in os.environ:
        config.build_without_distributed = get_env_flag_bool(
            "NVFUSER_BUILD_WITHOUT_DISTRIBUTED"
        )
    if "NVFUSER_BUILD_EXPLICIT_ERROR_CHECK" in os.environ:
        config.explicit_error_check = get_env_flag_bool(
            "NVFUSER_BUILD_EXPLICIT_ERROR_CHECK"
        )
    if "NVFUSER_BUILD_OVERWRITE_VERSION" in os.environ:
        config.overwrite_version = get_env_flag_bool("NVFUSER_BUILD_OVERWRITE_VERSION")
    if "NVFUSER_BUILD_VERSION_TAG" in os.environ:
        config.version_tag = os.getenv("NVFUSER_BUILD_VERSION_TAG")
    if "NVFUSER_BUILD_BUILD_TYPE" in os.environ:
        config.build_type = os.getenv("NVFUSER_BUILD_BUILD_TYPE")
    if "NVFUSER_BUILD_WHEEL_NAME" in os.environ:
        config.wheel_name = os.getenv("NVFUSER_BUILD_WHEEL_NAME")
    if "NVFUSER_BUILD_DIR" in os.environ:
        config.build_dir = os.getenv("NVFUSER_BUILD_DIR")
    if "NVFUSER_BUILD_INSTALL_DIR" in os.environ:
        config.install_dir = os.getenv("NVFUSER_BUILD_INSTALL_DIR")
    if "NVFUSER_BUILD_INSTALL_REQUIRES" in os.environ:
        config.install_requires = os.getenv("NVFUSER_BUILD_INSTALL_REQUIRES").split(",")
    if "NVFUSER_BUILD_EXTRAS_REQUIRE" in os.environ:
        config.extras_require = eval(os.getenv("NVFUSER_BUILD_EXTRAS_REQUIRE"))
    if "NVFUSER_BUILD_CPP_STANDARD" in os.environ:
        config.cpp_standard = int(os.getenv("NVFUSER_BUILD_CPP_STANDARD"))
    if "NVFUSER_BUILD_VERSION_TAG" in os.environ:
        config.overwrite_version = True
        config.version_tag = os.getenv("NVFUSER_BUILD_VERSION_TAG")
    if "NVFUSER_CUTLASS_MAX_JOBS" in os.environ:
        config.cutlass_max_jobs = int(os.getenv("NVFUSER_CUTLASS_MAX_JOBS"))
    if "NVFUSER_BUILD_NVMMH_INCLUDE_DIR" in os.environ:
        config.nvmmh_include_dir = os.getenv("NVFUSER_BUILD_NVMMH_INCLUDE_DIR")
    if "NVFUSER_BUILD_ENABLE_PCH" in os.environ:
        config.enable_pch = get_env_flag_bool("NVFUSER_BUILD_ENABLE_PCH")

def get_default_install_prefix():
    cwd = os.path.dirname(os.path.abspath(__file__))
    return os.path.join(cwd, "nvfuser_common")

class build_ext(setuptools.command.build_ext.build_ext):
    def __init__(self, *args, **kwargs):
        install_dir = kwargs.pop("install_dir", "")
        self.install_dir = install_dir if install_dir else get_default_install_prefix()
        super().__init__(*args, **kwargs)

    def copy_library(self, ext, library_name):
        # Copy files on necessity.
        filename = self.get_ext_filename(self.get_ext_fullname(ext.name))
        fileext = os.path.splitext(filename)[1]

        libnvfuser_path = os.path.join(
            os.path.join(self.install_dir, "lib"), f"{library_name}{fileext}"
        )
        assert os.path.exists(libnvfuser_path)
        install_dst = os.path.join(self.build_lib, filename)
        if not os.path.exists(os.path.dirname(install_dst)):
            os.makedirs(os.path.dirname(install_dst))
        self.copy_file(libnvfuser_path, install_dst)

    def build_extension(self, ext):
        if ext.name == "nvfuser_direct._C_DIRECT":
            self.copy_library(ext, "libnvfuser_direct")
            self.copy_shared_library("libnvfuser_codegen.so")
        else:
            super().build_extension(ext)

    def copy_shared_library(self, lib_name):
        # Copy shared library to lib/ subdirectory for package data
        src_path = os.path.join(self.install_dir, "lib", lib_name)
        if os.path.exists(src_path):
            dst_dir = os.path.join(self.build_lib, "nvfuser_common", "lib")
            if not os.path.exists(dst_dir):
                os.makedirs(dst_dir)
            dst_path = os.path.join(dst_dir, lib_name)
            self.copy_file(src_path, dst_path)

class concat_third_party_license:
    def __init__(self, directory="third_party"):
        self.license_file = "LICENSE"
        self.directory = directory

    def __enter__(self):
        # read original license file
        with open(self.license_file, "r") as f:
            self.nvfuser_license_txt = f.read()

        licenses = {"LICENSE", "LICENSE.txt", "LICENSE.rst", "COPYING.BSD"}

        # aggregated license, we key on project name
        aggregated_license = {}
        for root, dirs, files in os.walk(self.directory):
            license = list(licenses & set(files))
            if license:
                project_name = root.split("/")[-1]
                # let's worry about multiple license when we see it.
                assert len(license) == 1
                license_entry = os.path.join(root, license[0])
                if project_name in aggregated_license:
                    # Only add it if the license is different
                    aggregated_license[project_name].append(license_entry)
                else:
                    aggregated_license[project_name] = [license_entry]
        return aggregated_license

    def __exit__(self, exception_type, exception_value, traceback):
        # restore original license file
        with open(self.license_file, "w") as f:
            f.write(self.nvfuser_license_txt)

try:
    from wheel.bdist_wheel import bdist_wheel
except ImportError:
    build_whl = None
else:

    class build_whl(bdist_wheel):
        def run(self):
            with concat_third_party_license() as tp_licenses:
                if len(tp_licenses) != 0:
                    with open("LICENSE", "a") as f:
                        f.write("\n\n")
                        f.write(
                            "NVIDIA/fuser depends on libraries with license listed below:"
                        )

                for project_name, license_files in tp_licenses.items():
                    # check all license files are identical
                    with open(license_files[0], "r") as f:
                        license_ref = f.read()

                    def check_file(file_name):
                        with open(file_name, "r") as f:
                            return f.read() == license_ref

                    identical_flag = all(map(check_file, license_files[1:]))
                    if not identical_flag:
                        raise RuntimeError(
                            "inconsistent license found for project: ",
                            project_name,
                            " check its license files under: ",
                            license_files,
                        )

                    with open("LICENSE", "a") as f:
                        f.write("\n\nProject Name: " + project_name)
                        f.write("\nLicense Files:\n")
                        for file_name in license_files:
                            f.write("\t" + file_name)
                        f.write("\n" + license_ref)

                # generate whl before we restore LICENSE
                super().run()

def get_cmake_bin():
    # TODO: double check cmake version here and retrieve later version if necessary
    return "cmake"

def cmake(config, relative_path):
    from tools.memory import get_available_memory_gb

    # make build directories
    cwd = os.path.dirname(os.path.abspath(__file__))
    cmake_build_dir = (
        os.path.join(cwd, "build") if not config.build_dir else config.build_dir
    )
    if not os.path.exists(cmake_build_dir):
        os.makedirs(cmake_build_dir)

    install_prefix = (
        config.install_dir if config.install_dir else get_default_install_prefix()
    )

    from tools.gen_nvfuser_version import (
        get_pytorch_use_distributed,
    )

    pytorch_use_distributed = get_pytorch_use_distributed()

    def on_or_off(flag: bool) -> str:
        return "ON" if flag else "OFF"

    # generate cmake directory
    #
    # cmake options are sticky: when -DFOO=... isn't specified, cmake's option
    # FOO prefers the cached value over the default value. This behavior
    # confused me several times (e.g.
    # https://github.com/NVIDIA/Fuser/pull/4319) when I ran `pip install -e`,
    # so I chose to always pass these options even for default values. Doing so
    # explicitly overrides cached values and ensures consistent behavior across
    # clean and dirty builds.
    cmd_str = [
        get_cmake_bin(),
        f"-DCMAKE_BUILD_TYPE={config.build_type}",
        f"-DCMAKE_INSTALL_PREFIX={install_prefix}",
        f"-DNVFUSER_CPP_STANDARD={config.cpp_standard}",
        f"-DUSE_DISTRIBUTED={pytorch_use_distributed}",
        f"-DNVFUSER_BUILD_WITH_ASAN={on_or_off(config.build_with_asan)}",
        f"-DNVFUSER_STANDALONE_BUILD_WITH_UCC={on_or_off(config.build_with_ucc)}",
        f"-DNVFUSER_EXPLICIT_ERROR_CHECK={on_or_off(config.explicit_error_check)}",
        f"-DBUILD_TEST={on_or_off(not config.no_test)}",
        f"-DBUILD_PYTHON={on_or_off(not config.no_python)}",
        f"-DNVFUSER_DISABLE_CUTLASS={on_or_off(config.no_cutlass)}",
        f"-DPython_EXECUTABLE={sys.executable}",
        f"-DBUILD_NVFUSER_BENCHMARK={on_or_off(not config.no_benchmark)}",
        f"-DNVFUSER_DISTRIBUTED={on_or_off(not config.build_without_distributed)}",
        f"-DNVFUSER_USE_PCH={on_or_off(config.enable_pch)}",
        "-B",
        cmake_build_dir,
    ]
    if config.cutlass_max_jobs:
        cmd_str.append(f"-DCUTLASS_MAX_JOBS={config.cutlass_max_jobs}")
    if config.nvmmh_include_dir:
        cmd_str.append(f"-DNVMMH_INCLUDE_DIR={config.nvmmh_include_dir}")
    if not config.no_ninja:
        cmd_str.append("-G")
        cmd_str.append("Ninja")
    cmd_str.append(relative_path)

    print(f"Configuring CMake with {' '.join(cmd_str)}")
    subprocess.check_call(cmd_str)

    max_jobs = multiprocessing.cpu_count()
    mem_gb_per_task = 3  # Currently compilation of nvFuser souce code takes ~3GB of memory per task, we should adjust this value if it changes in the future.
    available_mem = get_available_memory_gb()
    if available_mem > 0:
        max_jobs_mem = int(available_mem / mem_gb_per_task)
        max_jobs = min(max_jobs, max_jobs_mem)

    if not config.cmake_only:
        # build binary
        max_jobs = os.getenv("MAX_JOBS", str(max_jobs))
        print(f"Using {max_jobs} jobs for compilation")
        cmd_str = [
            get_cmake_bin(),
            "--build",
            cmake_build_dir,
            "--target",
            "install",
            "--",
            "-j",
            max_jobs,
        ]
        subprocess.check_call(cmd_str)

def create_clean(relative_path):
    class clean(setuptools.Command):
        user_options = []

        def initialize_options(self):
            pass

        def finalize_options(self):
            pass

        def run(self):
            import glob

            gitignore_path = os.path.join(relative_path, ".gitignore")
            assert os.path.exists(gitignore_path)
            with open(gitignore_path, "r") as f:
                ignores = f.read()
                for entry in ignores.split("\n"):
                    # ignore comment in .gitignore
                    if len(entry) >= 1 and entry[0] != "#":
                        for filename in glob.glob(entry):
                            print("removing: ", filename)
                            try:
                                os.remove(filename)
                            except OSError:
                                shutil.rmtree(filename, ignore_errors=True)

    return clean

def run(config, version_tag, relative_path):
    from setuptools import Extension, setup, find_packages

    # NOTE(crcrpar): Deliberately build basically two dynamic libraries here so that they can
    # be treated as "nvfuser_package_data".
    if config.build_setup:
        cmake(config, relative_path)
    if not config.cmake_only:
        # NOTE: package include files for cmake
        # TODO(crcrpar): Better avoid hardcoding `libnvfuser_codegen.so`
        # might can be treated by using `exclude_package_data`.
        nvfuser_common_package_data = [
            "lib/libnvfuser_codegen.so",
            "lib/libnvf_cutlass.so",
            "include/nvfuser/*.h",
            "include/nvfuser/struct.inl",
            "include/nvfuser/C++20/type_traits",
            "include/nvfuser/device_lower/*.h",
            "include/nvfuser/device_lower/analysis/*.h",
            "include/nvfuser/device_lower/pass/*.h",
            "include/nvfuser/dynamic_type/*",
            "include/nvfuser/dynamic_type/C++20/*",
            "include/nvfuser/kernel_db/*.h",
            "include/nvfuser/multidevice/*.h",
            "include/nvfuser/ops/*.h",
            "include/nvfuser/ir/*.h",
            "include/nvfuser/scheduler/*.h",
            "include/nvfuser/host_ir/*.h",
            "include/nvfuser/id_model/*.h",
            "share/cmake/nvfuser/NvfuserConfig*",
            # TODO(crcrpar): it'd be better to ship the following two binaries.
            # Would need some change in CMakeLists.txt.
            # "bin/test_nvfuser",
            # "bin/nvfuser_bench"
        ]

        setup(
            name=config.wheel_name,
            version=version_tag,
            url="https://github.com/NVIDIA/Fuser",
            description="A Fusion Code Generator for NVIDIA GPUs (commonly known as 'nvFuser')",
            packages=find_packages(),
            ext_modules=[
                Extension(name="nvfuser_direct._C_DIRECT", sources=[]),
            ],
            license_files=("LICENSE",),
            cmdclass={
                "bdist_wheel": build_whl,
                "build_ext": lambda *args, **kwargs: build_ext(
                    *args, install_dir=config.install_dir, **kwargs
                ),
                "clean": create_clean(relative_path),
            },
            package_data={
                "nvfuser_common": nvfuser_common_package_data,
            },
            install_requires=config.install_requires,
            extras_require={
                "test": ["numpy", "expecttest", "pytest"],
                **config.extras_require,
            },
            license="BSD-3-Clause",
        )


BuildConfig

# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

import os
import multiprocessing
import subprocess
import sys
import shutil
from dataclasses import dataclass, field
import setuptools.command.build_ext

@dataclass
class BuildConfig:
    cmake_only: bool = False
    build_setup: bool = True
    no_python: bool = False
    no_cutlass: bool = False
    no_test: bool = False
    no_benchmark: bool = False
    no_ninja: bool = False
    build_with_ucc: bool = False
    build_with_asan: bool = False
    build_without_distributed: bool = False
    explicit_error_check: bool = False
    overwrite_version: bool = False
    version_tag: str = None
    build_type: str = "Release"
    wheel_name: str = "nvfuser"
    nvmmh_include_dir: str = ""
    build_dir: str = ""
    install_dir: str = ""
    install_requires: list = field(default_factory=list)
    extras_require: dict = field(default_factory=dict)
    cpp_standard: int = 20
    cutlass_max_jobs: int | None = None
    enable_pch: bool = False

def check_env_flag_bool_default(name: str, default: str = "") -> bool:
    if name not in os.environ:
        return default
    return os.getenv(name).upper() in ["ON", "1", "YES", "TRUE", "Y"]

def get_env_flag_bool(name: str) -> bool:
    assert name in os.environ
    return os.getenv(name).upper() in ["ON", "1", "YES", "TRUE", "Y"]

# Override BuildConfig with environment variables. Only change if variable
# exists. Do not use default to override argparse.
def override_build_config_from_env(config):
    # Command line arguments don't work on PEP517 builds and will be silently ignored,
    # so we need to pass those options as environment variables instead.
    if "NVFUSER_BUILD_CMAKE_ONLY" in os.environ:
        config.cmake_only = get_env_flag_bool("NVFUSER_BUILD_CMAKE_ONLY")


    no_benchmark: bool = False
    no_ninja: bool = False
    build_with_ucc: bool = False
    build_with_asan: bool = False
    build_without_distributed: bool = False
    explicit_error_check: bool = False
    overwrite_version: bool = False
    version_tag: str = None
    build_type: str = "Release"
    wheel_name: str = "nvfuser"
    nvmmh_include_dir: str = ""
    build_dir: str = ""
    install_dir: str = ""
    install_requires: list = field(default_factory=list)
    extras_require: dict = field(default_factory=dict)
    cpp_standard: int = 20
    cutlass_max_jobs: int | None = None
    enable_pch: bool = False

def check_env_flag_bool_default(name: str, default: str = "") -> bool:
    if name not in os.environ:
        return default
    return os.getenv(name).upper() in ["ON", "1", "YES", "TRUE", "Y"]

def get_env_flag_bool(name: str) -> bool:
    assert name in os.environ
    return os.getenv(name).upper() in ["ON", "1", "YES", "TRUE", "Y"]

# Override BuildConfig with environment variables. Only change if variable
# exists. Do not use default to override argparse.
def override_build_config_from_env(config):
    # Command line arguments don't work on PEP517 builds and will be silently ignored,
    # so we need to pass those options as environment variables instead.
    if "NVFUSER_BUILD_CMAKE_ONLY" in os.environ:
        config.cmake_only = get_env_flag_bool("NVFUSER_BUILD_CMAKE_ONLY")
    if "NVFUSER_BUILD_SETUP" in os.environ:
        config.build_setup = get_env_flag_bool("NVFUSER_BUILD_SETUP")
    if "NVFUSER_BUILD_NO_PYTHON" in os.environ:
        config.no_python = get_env_flag_bool("NVFUSER_BUILD_NO_PYTHON")
    if "NVFUSER_BUILD_NO_CUTLASS" in os.environ:
        config.no_cutlass = get_env_flag_bool("NVFUSER_BUILD_NO_CUTLASS")


    overwrite_version: bool = False
    version_tag: str = None
    build_type: str = "Release"
    wheel_name: str = "nvfuser"
    nvmmh_include_dir: str = ""
    build_dir: str = ""
    install_dir: str = ""
    install_requires: list = field(default_factory=list)
    extras_require: dict = field(default_factory=dict)
    cpp_standard: int = 20
    cutlass_max_jobs: int | None = None
    enable_pch: bool = False

def check_env_flag_bool_default(name: str, default: str = "") -> bool:
    if name not in os.environ:
        return default
    return os.getenv(name).upper() in ["ON", "1", "YES", "TRUE", "Y"]

def get_env_flag_bool(name: str) -> bool:
    assert name in os.environ
    return os.getenv(name).upper() in ["ON", "1", "YES", "TRUE", "Y"]

# Override BuildConfig with environment variables. Only change if variable
# exists. Do not use default to override argparse.
def override_build_config_from_env(config):
    # Command line arguments don't work on PEP517 builds and will be silently ignored,
    # so we need to pass those options as environment variables instead.
    if "NVFUSER_BUILD_CMAKE_ONLY" in os.environ:
        config.cmake_only = get_env_flag_bool("NVFUSER_BUILD_CMAKE_ONLY")
    if "NVFUSER_BUILD_SETUP" in os.environ:
        config.build_setup = get_env_flag_bool("NVFUSER_BUILD_SETUP")
    if "NVFUSER_BUILD_NO_PYTHON" in os.environ:
        config.no_python = get_env_flag_bool("NVFUSER_BUILD_NO_PYTHON")
    if "NVFUSER_BUILD_NO_CUTLASS" in os.environ:
        config.no_cutlass = get_env_flag_bool("NVFUSER_BUILD_NO_CUTLASS")
    if "NVFUSER_BUILD_NO_TEST" in os.environ:
        config.no_test = get_env_flag_bool("NVFUSER_BUILD_NO_TEST")
    if "NVFUSER_BUILD_NO_BENCHMARK" in os.environ:
        config.no_benchmark = get_env_flag_bool("NVFUSER_BUILD_NO_BENCHMARK")
    if "NVFUSER_BUILD_NO_NINJA" in os.environ:


    install_requires: list = field(default_factory=list)
    extras_require: dict = field(default_factory=dict)
    cpp_standard: int = 20
    cutlass_max_jobs: int | None = None
    enable_pch: bool = False

def check_env_flag_bool_default(name: str, default: str = "") -> bool:
    if name not in os.environ:
        return default
    return os.getenv(name).upper() in ["ON", "1", "YES", "TRUE", "Y"]

def get_env_flag_bool(name: str) -> bool:
    assert name in os.environ
    return os.getenv(name).upper() in ["ON", "1", "YES", "TRUE", "Y"]

# Override BuildConfig with environment variables. Only change if variable
# exists. Do not use default to override argparse.
def override_build_config_from_env(config):
    # Command line arguments don't work on PEP517 builds and will be silently ignored,
    # so we need to pass those options as environment variables instead.
    if "NVFUSER_BUILD_CMAKE_ONLY" in os.environ:
        config.cmake_only = get_env_flag_bool("NVFUSER_BUILD_CMAKE_ONLY")
    if "NVFUSER_BUILD_SETUP" in os.environ:
        config.build_setup = get_env_flag_bool("NVFUSER_BUILD_SETUP")
    if "NVFUSER_BUILD_NO_PYTHON" in os.environ:
        config.no_python = get_env_flag_bool("NVFUSER_BUILD_NO_PYTHON")
    if "NVFUSER_BUILD_NO_CUTLASS" in os.environ:
        config.no_cutlass = get_env_flag_bool("NVFUSER_BUILD_NO_CUTLASS")
    if "NVFUSER_BUILD_NO_TEST" in os.environ:
        config.no_test = get_env_flag_bool("NVFUSER_BUILD_NO_TEST")
    if "NVFUSER_BUILD_NO_BENCHMARK" in os.environ:
        config.no_benchmark = get_env_flag_bool("NVFUSER_BUILD_NO_BENCHMARK")
    if "NVFUSER_BUILD_NO_NINJA" in os.environ:
        config.no_ninja = get_env_flag_bool("NVFUSER_BUILD_NO_NINJA")
    if "NVFUSER_BUILD_WITH_UCC" in os.environ:
        config.build_with_ucc = get_env_flag_bool("NVFUSER_BUILD_WITH_UCC")
    if "NVFUSER_BUILD_WITH_ASAN" in os.environ:
        config.build_with_asan = get_env_flag_bool("NVFUSER_BUILD_WITH_ASAN")
    if "NVFUSER_BUILD_WITHOUT_DISTRIBUTED" in os.environ:
        config.build_without_distributed = get_env_flag_bool(
            "NVFUSER_BUILD_WITHOUT_DISTRIBUTED"
        )
    if "NVFUSER_BUILD_EXPLICIT_ERROR_CHECK" in os.environ:
        config.explicit_error_check = get_env_flag_bool(
            "NVFUSER_BUILD_EXPLICIT_ERROR_CHECK"
        )
    if "NVFUSER_BUILD_OVERWRITE_VERSION" in os.environ:
        config.overwrite_version = get_env_flag_bool("NVFUSER_BUILD_OVERWRITE_VERSION")
    if "NVFUSER_BUILD_VERSION_TAG" in os.environ:
        config.version_tag = os.getenv("NVFUSER_BUILD_VERSION_TAG")
    if "NVFUSER_BUILD_BUILD_TYPE" in os.environ:
        config.build_type = os.getenv("NVFUSER_BUILD_BUILD_TYPE")
    if "NVFUSER_BUILD_WHEEL_NAME" in os.environ:
        config.wheel_name = os.getenv("NVFUSER_BUILD_WHEEL_NAME")
    if "NVFUSER_BUILD_DIR" in os.environ:
        config.build_dir = os.getenv("NVFUSER_BUILD_DIR")
    if "NVFUSER_BUILD_INSTALL_DIR" in os.environ:
        config.install_dir = os.getenv("NVFUSER_BUILD_INSTALL_DIR")
    if "NVFUSER_BUILD_INSTALL_REQUIRES" in os.environ:
        config.install_requires = os.getenv("NVFUSER_BUILD_INSTALL_REQUIRES").split(",")
    if "NVFUSER_BUILD_EXTRAS_REQUIRE" in os.environ:
        config.extras_require = eval(os.getenv("NVFUSER_BUILD_EXTRAS_REQUIRE"))
    if "NVFUSER_BUILD_CPP_STANDARD" in os.environ:
        config.cpp_standard = int(os.getenv("NVFUSER_BUILD_CPP_STANDARD"))
    if "NVFUSER_BUILD_VERSION_TAG" in os.environ:
        config.overwrite_version = True
        config.version_tag = os.getenv("NVFUSER_BUILD_VERSION_TAG")
    if "NVFUSER_CUTLASS_MAX_JOBS" in os.environ:
        config.cutlass_max_jobs = int(os.getenv("NVFUSER_CUTLASS_MAX_JOBS"))
    if "NVFUSER_BUILD_NVMMH_INCLUDE_DIR" in os.environ:
        config.nvmmh_include_dir = os.getenv("NVFUSER_BUILD_NVMMH_INCLUDE_DIR")
    if "NVFUSER_BUILD_ENABLE_PCH" in os.environ:
        config.enable_pch = get_env_flag_bool("NVFUSER_BUILD_ENABLE_PCH")

def get_default_install_prefix():
    cwd = os.path.dirname(os.path.abspath(__file__))
    return os.path.join(cwd, "nvfuser_common")

class build_ext(setuptools.command.build_ext.build_ext):
    def __init__(self, *args, **kwargs):
        install_dir = kwargs.pop("install_dir", "")
        self.install_dir = install_dir if install_dir else get_default_install_prefix()
        super().__init__(*args, **kwargs)

    def copy_library(self, ext, library_name):
        # Copy files on necessity.
        filename = self.get_ext_filename(self.get_ext_fullname(ext.name))
        fileext = os.path.splitext(filename)[1]

        libnvfuser_path = os.path.join(
            os.path.join(self.install_dir, "lib"), f"{library_name}{fileext}"