        torch.randn(8, 16, device="cuda"),
        index := 2,
    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            s1 = fd.define_scalar(dtype=DataType.Int)
            t1 = fd.ops.select(t0, s1, dim)
            fd.add_output(t1)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.select(inputs[0], dim, inputs[1])
        nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

    test_fn(0)
    test_fn(1)

def test_take_along_axis(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
        torch.randn(8, 16, device="cuda"),
        torch.randint(0, 8, (8, 16), device="cuda").to(dtype=torch.long),
    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.from_pytorch(inputs[1])
            t2 = fd.from_pytorch(inputs[2])
            t3 = fd.ops.add(t0, t1)
            t4 = fd.ops.take_along_axis(t3, t2, dim)
            fd.add_output(t4)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.gather(inputs[0] + inputs[1], dim, inputs[2])
        nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

    test_fn(0)
    test_fn(1)

def test_cumsum(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.cumsum(t0, dim)
            fd.add_output(t1)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.cumsum(inputs[0], dim)
        nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

    test_fn(0)
    test_fn(1)


    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.from_pytorch(inputs[1])
            t2 = fd.from_pytorch(inputs[2])
            t3 = fd.ops.add(t0, t1)
            t4 = fd.ops.take_along_axis(t3, t2, dim)
            fd.add_output(t4)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.gather(inputs[0] + inputs[1], dim, inputs[2])
        nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

    test_fn(0)
    test_fn(1)

def test_cumsum(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.cumsum(t0, dim)
            fd.add_output(t1)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.cumsum(inputs[0], dim)
        nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

    test_fn(0)
    test_fn(1)

def test_cumprod(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.cumprod(t0, dim)
            fd.add_output(t1)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.cumprod(inputs[0], dim)
        nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

    test_fn(0)
    test_fn(1)


def test_cumsum(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.cumsum(t0, dim)
            fd.add_output(t1)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.cumsum(inputs[0], dim)
        nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

    test_fn(0)
    test_fn(1)

def test_cumprod(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.cumprod(t0, dim)
            fd.add_output(t1)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.cumprod(inputs[0], dim)
        nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

    test_fn(0)
    test_fn(1)

def test_cummin(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.cummin(t0, dim)
            fd.add_output(t1)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.cummin(inputs[0], dim)
        nvfuser_direct_test.assertEqual(eager_out[0], nvf_out[0])

    test_fn(0)
    test_fn(1)


def test_cumprod(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.cumprod(t0, dim)
            fd.add_output(t1)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.cumprod(inputs[0], dim)
        nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

    test_fn(0)
    test_fn(1)

def test_cummin(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.cummin(t0, dim)
            fd.add_output(t1)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.cummin(inputs[0], dim)
        nvfuser_direct_test.assertEqual(eager_out[0], nvf_out[0])

    test_fn(0)
    test_fn(1)

def test_cummax(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.cummax(t0, dim)
            fd.add_output(t1)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.cummax(inputs[0], dim)
        nvfuser_direct_test.assertEqual(eager_out[0], nvf_out[0])

    test_fn(0)
    test_fn(1)


def test_cummin(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.cummin(t0, dim)
            fd.add_output(t1)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.cummin(inputs[0], dim)
        nvfuser_direct_test.assertEqual(eager_out[0], nvf_out[0])

    test_fn(0)
    test_fn(1)

def test_cummax(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.cummax(t0, dim)
            fd.add_output(t1)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.cummax(inputs[0], dim)
        nvfuser_direct_test.assertEqual(eager_out[0], nvf_out[0])

    test_fn(0)
    test_fn(1)

def test_where(nvfuser_direct_test):
    # nvfuser_where is a decorator function. It takes the input arguments
    # and creates a function that builds a FusionDefinition.
    def nvfuser_where(pred, a, b):
        def fusion_func(fd: FusionDefinition):
            nv_pred = fd.define_tensor(
                sizes=pred.shape, strides=pred.stride(), dtype=DataType.Bool
            )
            nv_a = fd.define_tensor(
                sizes=a.shape,
                strides=a.stride(),
                dtype=torch_dtype_to_nvfuser_dtype(a.dtype),
            )
            nv_b = fd.define_tensor(
                sizes=b.shape,
                strides=b.stride(),
                dtype=torch_dtype_to_nvfuser_dtype(b.dtype),
            )


def test_cummax(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
    ]

    def test_fn(dim):
        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.cummax(t0, dim)
            fd.add_output(t1)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.cummax(inputs[0], dim)
        nvfuser_direct_test.assertEqual(eager_out[0], nvf_out[0])

    test_fn(0)
    test_fn(1)

def test_where(nvfuser_direct_test):
    # nvfuser_where is a decorator function. It takes the input arguments
    # and creates a function that builds a FusionDefinition.
    def nvfuser_where(pred, a, b):
        def fusion_func(fd: FusionDefinition):
            nv_pred = fd.define_tensor(
                sizes=pred.shape, strides=pred.stride(), dtype=DataType.Bool
            )
            nv_a = fd.define_tensor(
                sizes=a.shape,
                strides=a.stride(),
                dtype=torch_dtype_to_nvfuser_dtype(a.dtype),
            )
            nv_b = fd.define_tensor(
                sizes=b.shape,
                strides=b.stride(),
                dtype=torch_dtype_to_nvfuser_dtype(b.dtype),
            )
            result = fd.ops.where(nv_pred, nv_a, nv_b)
            fd.add_output(result)

        return fusion_func

    # get list of dtypes to test with
    list_of_dtype = [torch.float16, torch.float32]
    if not is_pre_ampere():
        list_of_dtype.append(torch.bfloat16)

    pred = torch.testing.make_tensor((5,), device="cuda", dtype=torch.bool)
    for atype, btype in itertools.product(list_of_dtype, list_of_dtype):
        a = torch.randn((5,), device="cuda", dtype=atype)
        b = torch.randn((5,), device="cuda", dtype=btype)
        fusion_func = nvfuser_where(pred, a, b)
        nv_result, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, [pred, a, b])
        torch_result = torch.where(pred, a, b)
        nvfuser_direct_test.assertEqual(nv_result[0], torch_result)

def test_where_dtypes(nvfuser_direct_test):
    inputs = [
        torch.arange(2, device="cuda").type(torch.bool),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])

        c0 = fd.define_scalar(3.0)
        c1 = fd.define_scalar(5.0)
        t1 = fd.ops.where(t0, c0, c1)  # DataType.Double
        fd.add_output(t1)

        c0f = fd.define_scalar(3.0, DataType.Float)
        c1f = fd.define_scalar(5.0, DataType.Float)
        t1f = fd.ops.where(t0, c0f, c1f)  # DataType.Float
        fd.add_output(t1f)


            result = fd.ops.where(nv_pred, nv_a, nv_b)
            fd.add_output(result)

        return fusion_func

    # get list of dtypes to test with
    list_of_dtype = [torch.float16, torch.float32]
    if not is_pre_ampere():
        list_of_dtype.append(torch.bfloat16)

    pred = torch.testing.make_tensor((5,), device="cuda", dtype=torch.bool)
    for atype, btype in itertools.product(list_of_dtype, list_of_dtype):
        a = torch.randn((5,), device="cuda", dtype=atype)
        b = torch.randn((5,), device="cuda", dtype=btype)
        fusion_func = nvfuser_where(pred, a, b)
        nv_result, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, [pred, a, b])
        torch_result = torch.where(pred, a, b)
        nvfuser_direct_test.assertEqual(nv_result[0], torch_result)

def test_where_dtypes(nvfuser_direct_test):
    inputs = [
        torch.arange(2, device="cuda").type(torch.bool),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])

        c0 = fd.define_scalar(3.0)
        c1 = fd.define_scalar(5.0)
        t1 = fd.ops.where(t0, c0, c1)  # DataType.Double
        fd.add_output(t1)

        c0f = fd.define_scalar(3.0, DataType.Float)
        c1f = fd.define_scalar(5.0, DataType.Float)
        t1f = fd.ops.where(t0, c0f, c1f)  # DataType.Float
        fd.add_output(t1f)

        c0d = fd.define_scalar(3.0, DataType.Double)
        c1d = fd.define_scalar(5.0, DataType.Double)
        t1d = fd.ops.where(t0, c0d, c1d)  # DataType.Double
        fd.add_output(t1d)

        c0i = fd.define_scalar(3, DataType.Int32)
        c1i = fd.define_scalar(5, DataType.Int32)
        t1i = fd.ops.where(t0, c0i, c1i)  # DataType.Int32
        fd.add_output(t1i)

        c0l = fd.define_scalar(3)
        c1l = fd.define_scalar(5)
        t1l = fd.ops.where(t0, c0l, c1l)  # DataType.Int
        fd.add_output(t1l)

        c0c = fd.define_scalar(complex(3.0))
        c1c = fd.define_scalar(complex(5.0))
        t1c = fd.ops.where(t0, c0c, c1c)  # DataType.ComplexDouble
        fd.add_output(t1c)

        c0cf = fd.define_scalar(3.0 + 0j, DataType.ComplexFloat)
        c1cf = fd.define_scalar(5.0 + 0j, DataType.ComplexFloat)
        t1cf = fd.ops.where(t0, c0cf, c1cf)  # DataType.ComplexFloat
        fd.add_output(t1cf)

        c0cd = fd.define_scalar(3.0 + 0j, DataType.ComplexDouble)
        c1cd = fd.define_scalar(5.0 + 0j, DataType.ComplexDouble)
        t1cd = fd.ops.where(t0, c0cd, c1cd)  # DataType.ComplexDouble
        fd.add_output(t1cd)

        c0b = fd.define_scalar(True, DataType.Bool)
        c1b = fd.define_scalar(False, DataType.Bool)
        t1b = fd.ops.where(t0, c0b, c1b)  # DataType.Bool
        fd.add_output(t1b)

    (
        n,
        nf,
        nd,
        ni,
        nl,
        nc,
        ncf,
        ncd,
        nb,
    ), _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    eager_out = torch.where(inputs[0], 3.0, 5.0)

    # explicit Float dtype matches torch.where behavior
    nvfuser_direct_test.assertEqual(eager_out, nf)

    assert n.dtype == torch.float64
    assert nf.dtype == torch.float32
    assert nd.dtype == torch.float64
    assert ni.dtype == torch.int32
    assert nl.dtype == torch.int64
    assert nc.dtype == torch.complex128
    assert ncf.dtype == torch.complex64
    assert ncd.dtype == torch.complex128
    assert nb.dtype == torch.bool

def test_addcmul(nvfuser_direct_test):
    inputs = [
        torch.randn(4, device="cuda", dtype=torch.float32),
        torch.randn(4, device="cuda", dtype=torch.float32),
        torch.randn(4, device="cuda", dtype=torch.float32),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.from_pytorch(inputs[2])
        c0 = fd.define_scalar(0.1)

        t3 = fd.ops.addcmul(t0, t1, t2, c0)

        fd.add_output(t3)

    nvfout, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)


        ncd,
        nb,
    ), _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    eager_out = torch.where(inputs[0], 3.0, 5.0)

    # explicit Float dtype matches torch.where behavior
    nvfuser_direct_test.assertEqual(eager_out, nf)

    assert n.dtype == torch.float64
    assert nf.dtype == torch.float32
    assert nd.dtype == torch.float64
    assert ni.dtype == torch.int32
    assert nl.dtype == torch.int64
    assert nc.dtype == torch.complex128
    assert ncf.dtype == torch.complex64
    assert ncd.dtype == torch.complex128
    assert nb.dtype == torch.bool

def test_addcmul(nvfuser_direct_test):
    inputs = [
        torch.randn(4, device="cuda", dtype=torch.float32),
        torch.randn(4, device="cuda", dtype=torch.float32),
        torch.randn(4, device="cuda", dtype=torch.float32),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.from_pytorch(inputs[2])
        c0 = fd.define_scalar(0.1)

        t3 = fd.ops.addcmul(t0, t1, t2, c0)

        fd.add_output(t3)

    nvfout, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    torch_out = torch.addcmul(*inputs, value=0.1)

    nvfuser_direct_test.assertEqual(nvfout[0], torch_out)

def test_slice(nvfuser_direct_test):
    x = torch.randn((2, 5, 10), dtype=torch.float32, device="cuda:0")

    offset = (0, 1, 2)

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[-1, -1, -1],
            contiguity=[True, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T1 = fd.ops.slice(
            T0, start_indices=offset, end_indices=(2, 5, 10), strides=(1, 1, 1)
        )
        fd.add_output(T1)
        V_start = list(offset)


        torch.randn(4, device="cuda", dtype=torch.float32),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.from_pytorch(inputs[2])
        c0 = fd.define_scalar(0.1)

        t3 = fd.ops.addcmul(t0, t1, t2, c0)

        fd.add_output(t3)

    nvfout, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    torch_out = torch.addcmul(*inputs, value=0.1)

    nvfuser_direct_test.assertEqual(nvfout[0], torch_out)

def test_slice(nvfuser_direct_test):
    x = torch.randn((2, 5, 10), dtype=torch.float32, device="cuda:0")

    offset = (0, 1, 2)

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[-1, -1, -1],
            contiguity=[True, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T1 = fd.ops.slice(
            T0, start_indices=offset, end_indices=(2, 5, 10), strides=(1, 1, 1)
        )
        fd.add_output(T1)
        V_start = list(offset)
        V_end = T0.shape()
        T2 = fd.ops.slice(T0, V_start, V_end)
        fd.add_output(T2)
        dynamic_start = fd.define_vector(3)
        dynamic_end = fd.define_vector(3)
        T3 = fd.ops.slice(T0, dynamic_start, dynamic_end)
        fd.add_output(T3)

    inputs = [x, *offset, *x.shape]

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    for out in nvf_out:
        torch.testing.assert_close(out, x[:, 1:, 2:])

def test_iota(nvfuser_direct_test):
    inputs = [
        (2, 0, 2, DataType.Int),
        (3, 100, 1, DataType.Int32),
    ]

    def fusion_func(fd: FusionDefinition):
        for input in inputs:
            c0 = fd.define_scalar(input[0])
            c1 = None if input[1] is None else fd.define_scalar(input[1])
            c2 = None if input[2] is None else fd.define_scalar(input[2])
            dt = input[3]
            t3 = fd.ops.iota(c0, c1, c2, dt)
            fd.add_output(t3)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, [])

    eager_out1 = torch.tensor([0, 2], dtype=torch.long, device="cuda")


        T1 = fd.ops.slice(
            T0, start_indices=offset, end_indices=(2, 5, 10), strides=(1, 1, 1)
        )
        fd.add_output(T1)
        V_start = list(offset)
        V_end = T0.shape()
        T2 = fd.ops.slice(T0, V_start, V_end)
        fd.add_output(T2)
        dynamic_start = fd.define_vector(3)
        dynamic_end = fd.define_vector(3)
        T3 = fd.ops.slice(T0, dynamic_start, dynamic_end)
        fd.add_output(T3)

    inputs = [x, *offset, *x.shape]

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    for out in nvf_out:
        torch.testing.assert_close(out, x[:, 1:, 2:])

def test_iota(nvfuser_direct_test):
    inputs = [
        (2, 0, 2, DataType.Int),
        (3, 100, 1, DataType.Int32),
    ]

    def fusion_func(fd: FusionDefinition):
        for input in inputs:
            c0 = fd.define_scalar(input[0])
            c1 = None if input[1] is None else fd.define_scalar(input[1])
            c2 = None if input[2] is None else fd.define_scalar(input[2])
            dt = input[3]
            t3 = fd.ops.iota(c0, c1, c2, dt)
            fd.add_output(t3)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, [])

    eager_out1 = torch.tensor([0, 2], dtype=torch.long, device="cuda")
    eager_out2 = torch.tensor([100, 101, 102], dtype=torch.int, device="cuda")
    nvfuser_direct_test.assertEqual(eager_out1, nvf_out[0])
    nvfuser_direct_test.assertEqual(eager_out2, nvf_out[1])

def test_scalar_only_inputs(nvfuser_direct_test):
    # We don't allow scalar outputs, currently,
    # so a tensor has to be returned
    def fusion_func(fd: FusionDefinition):
        s0 = fd.define_scalar()
        s1 = fd.define_scalar()
        s2 = fd.ops.add(s0, s1)
        c0 = fd.define_scalar(1.0, DataType.Float)
        t3 = fd.ops.full(shape=[2, 2], fill_value=c0, dtype=DataType.Float)
        t4 = fd.ops.mul(t3, s2)
        fd.add_output(t4)

    with FusionDefinition() as fd:
        fusion_func(fd)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, [2.0, 3.0])
    eager_out = torch.full([2, 2], 1.0) * 5.0
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])


        (3, 100, 1, DataType.Int32),
    ]

    def fusion_func(fd: FusionDefinition):
        for input in inputs:
            c0 = fd.define_scalar(input[0])
            c1 = None if input[1] is None else fd.define_scalar(input[1])
            c2 = None if input[2] is None else fd.define_scalar(input[2])
            dt = input[3]
            t3 = fd.ops.iota(c0, c1, c2, dt)
            fd.add_output(t3)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, [])

    eager_out1 = torch.tensor([0, 2], dtype=torch.long, device="cuda")
    eager_out2 = torch.tensor([100, 101, 102], dtype=torch.int, device="cuda")
    nvfuser_direct_test.assertEqual(eager_out1, nvf_out[0])
    nvfuser_direct_test.assertEqual(eager_out2, nvf_out[1])

def test_scalar_only_inputs(nvfuser_direct_test):
    # We don't allow scalar outputs, currently,
    # so a tensor has to be returned
    def fusion_func(fd: FusionDefinition):
        s0 = fd.define_scalar()
        s1 = fd.define_scalar()
        s2 = fd.ops.add(s0, s1)
        c0 = fd.define_scalar(1.0, DataType.Float)
        t3 = fd.ops.full(shape=[2, 2], fill_value=c0, dtype=DataType.Float)
        t4 = fd.ops.mul(t3, s2)
        fd.add_output(t4)

    with FusionDefinition() as fd:
        fusion_func(fd)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, [2.0, 3.0])
    eager_out = torch.full([2, 2], 1.0) * 5.0
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_alias_output_to_input(nvfuser_direct_test):
    in_tensors = [
        torch.ones(4, 4, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(in_tensors[0])  # = 1.0
        one = fd.define_scalar(1.0)
        two = fd.define_scalar(2.0)
        t1 = fd.ops.add(t0, one)  # = t0 + 1.0 = 2.0
        t2 = fd.ops.add(t1, two)  # = t1 + 2.0 = 4.0
        fd.add_output(t1, alias_input=t0)
        fd.add_output(t2)

    out_tensors, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, in_tensors)

    # t1 is an alias and therefore is hidden.
    nvfuser_direct_test.assertEqual(len(out_tensors), 1)


def test_scalar_only_inputs(nvfuser_direct_test):
    # We don't allow scalar outputs, currently,
    # so a tensor has to be returned
    def fusion_func(fd: FusionDefinition):
        s0 = fd.define_scalar()
        s1 = fd.define_scalar()
        s2 = fd.ops.add(s0, s1)
        c0 = fd.define_scalar(1.0, DataType.Float)
        t3 = fd.ops.full(shape=[2, 2], fill_value=c0, dtype=DataType.Float)
        t4 = fd.ops.mul(t3, s2)
        fd.add_output(t4)

    with FusionDefinition() as fd:
        fusion_func(fd)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, [2.0, 3.0])
    eager_out = torch.full([2, 2], 1.0) * 5.0
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_alias_output_to_input(nvfuser_direct_test):
    in_tensors = [
        torch.ones(4, 4, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(in_tensors[0])  # = 1.0
        one = fd.define_scalar(1.0)
        two = fd.define_scalar(2.0)
        t1 = fd.ops.add(t0, one)  # = t0 + 1.0 = 2.0
        t2 = fd.ops.add(t1, two)  # = t1 + 2.0 = 4.0
        fd.add_output(t1, alias_input=t0)
        fd.add_output(t2)

    out_tensors, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, in_tensors)

    # t1 is an alias and therefore is hidden.
    nvfuser_direct_test.assertEqual(len(out_tensors), 1)
    nvfuser_direct_test.assertEqual(
        out_tensors[0], torch.full((4, 4), 4.0, device="cuda")
    )
    nvfuser_direct_test.assertEqual(
        in_tensors[0], torch.full((4, 4), 2.0, device="cuda")
    )

def test_returning_aliased_outputs(nvfuser_direct_test):
    inputs = [torch.randn((1, 2, 3, 4), dtype=torch.float32, device="cuda:0")]

    def fusion_func(fd: FusionDefinition):
        T0 = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=[True, True, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        S1 = fd.define_scalar(0.00000, dtype=DataType.Double)
        T2 = fd.ops.gt(T0, S1)
        S3 = fd.define_scalar(0.00000, dtype=DataType.Double)
        T4 = fd.ops.where(T2, T0, S3)
        fd.add_output(T4)
        fd.add_output(T4, T0)
        fd.add_output(T4)


        t0 = fd.from_pytorch(in_tensors[0])  # = 1.0
        one = fd.define_scalar(1.0)
        two = fd.define_scalar(2.0)
        t1 = fd.ops.add(t0, one)  # = t0 + 1.0 = 2.0
        t2 = fd.ops.add(t1, two)  # = t1 + 2.0 = 4.0
        fd.add_output(t1, alias_input=t0)
        fd.add_output(t2)

    out_tensors, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, in_tensors)

    # t1 is an alias and therefore is hidden.
    nvfuser_direct_test.assertEqual(len(out_tensors), 1)
    nvfuser_direct_test.assertEqual(
        out_tensors[0], torch.full((4, 4), 4.0, device="cuda")
    )
    nvfuser_direct_test.assertEqual(
        in_tensors[0], torch.full((4, 4), 2.0, device="cuda")
    )

def test_returning_aliased_outputs(nvfuser_direct_test):
    inputs = [torch.randn((1, 2, 3, 4), dtype=torch.float32, device="cuda:0")]

    def fusion_func(fd: FusionDefinition):
        T0 = fd.define_tensor(
            shape=[-1, -1, -1, -1],
            contiguity=[True, True, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        S1 = fd.define_scalar(0.00000, dtype=DataType.Double)
        T2 = fd.ops.gt(T0, S1)
        S3 = fd.define_scalar(0.00000, dtype=DataType.Double)
        T4 = fd.ops.where(T2, T0, S3)
        fd.add_output(T4)
        fd.add_output(T4, T0)
        fd.add_output(T4)
        fd.add_output(T0)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    num_out = len(nvf_out)
    nvfuser_direct_test.assertEqual(num_out, 3)
    for i in range(num_out):
        nvfuser_direct_test.assertEqual(nvf_out[i].data_ptr(), inputs[0].data_ptr())

def test_welford(nvfuser_direct_test):
    inputs = [torch.randn(2, 2, device="cuda")]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        mean, var_sum, n = fd.ops.welford(t0, [-1])
        var = fd.ops.div(var_sum, n)
        fd.add_output(var)
        fd.add_output(mean)

    fuser_result, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    torch_result = torch.var_mean(inputs[0], [-1], correction=0)
    nvfuser_direct_test.assertEqual(fuser_result, torch_result)

def test_gather(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),


            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        S1 = fd.define_scalar(0.00000, dtype=DataType.Double)
        T2 = fd.ops.gt(T0, S1)
        S3 = fd.define_scalar(0.00000, dtype=DataType.Double)
        T4 = fd.ops.where(T2, T0, S3)
        fd.add_output(T4)
        fd.add_output(T4, T0)
        fd.add_output(T4)
        fd.add_output(T0)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    num_out = len(nvf_out)
    nvfuser_direct_test.assertEqual(num_out, 3)
    for i in range(num_out):
        nvfuser_direct_test.assertEqual(nvf_out[i].data_ptr(), inputs[0].data_ptr())

def test_welford(nvfuser_direct_test):
    inputs = [torch.randn(2, 2, device="cuda")]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        mean, var_sum, n = fd.ops.welford(t0, [-1])
        var = fd.ops.div(var_sum, n)
        fd.add_output(var)
        fd.add_output(mean)

    fuser_result, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    torch_result = torch.var_mean(inputs[0], [-1], correction=0)
    nvfuser_direct_test.assertEqual(fuser_result, torch_result)

def test_gather(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
        torch.randn(8, 16, device="cuda"),
        torch.randint(0, 8, (4, 4), device="cuda").to(dtype=torch.long),
    ]

    for dim in [0, 1]:

        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.from_pytorch(inputs[1])
            t2 = fd.from_pytorch(inputs[2])
            t3 = fd.ops.add(t0, t1)
            t4 = fd.ops.gather(t3, t2, dim)
            fd.add_output(t4)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)


    nvfuser_direct_test.assertEqual(num_out, 3)
    for i in range(num_out):
        nvfuser_direct_test.assertEqual(nvf_out[i].data_ptr(), inputs[0].data_ptr())

def test_welford(nvfuser_direct_test):
    inputs = [torch.randn(2, 2, device="cuda")]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        mean, var_sum, n = fd.ops.welford(t0, [-1])
        var = fd.ops.div(var_sum, n)
        fd.add_output(var)
        fd.add_output(mean)

    fuser_result, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    torch_result = torch.var_mean(inputs[0], [-1], correction=0)
    nvfuser_direct_test.assertEqual(fuser_result, torch_result)

def test_gather(nvfuser_direct_test):
    inputs = [
        torch.randn(8, 16, device="cuda"),
        torch.randn(8, 16, device="cuda"),
        torch.randint(0, 8, (4, 4), device="cuda").to(dtype=torch.long),
    ]

    for dim in [0, 1]:

        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.from_pytorch(inputs[1])
            t2 = fd.from_pytorch(inputs[2])
            t3 = fd.ops.add(t0, t1)
            t4 = fd.ops.gather(t3, t2, dim)
            fd.add_output(t4)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.gather(inputs[0] + inputs[1], dim, inputs[2])
        nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_pad(nvfuser_direct_test):
    inputs = [
        torch.testing.make_tensor((1, 2, 3), dtype=torch.float32, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])

        t1 = fd.ops.pad(t0, [1, 1, 1, 1])
        fd.add_output(t1)

        # zero padding in some dims
        t2 = fd.ops.pad(t0, [0, 0, 2, 3])
        fd.add_output(t2)

        # zero padding in all dims
        t3 = fd.ops.pad(t0, [0, 0, 0, 0])
        fd.add_output(t3)


        torch.randn(8, 16, device="cuda"),
        torch.randint(0, 8, (4, 4), device="cuda").to(dtype=torch.long),
    ]

    for dim in [0, 1]:

        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.from_pytorch(inputs[1])
            t2 = fd.from_pytorch(inputs[2])
            t3 = fd.ops.add(t0, t1)
            t4 = fd.ops.gather(t3, t2, dim)
            fd.add_output(t4)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        eager_out = torch.gather(inputs[0] + inputs[1], dim, inputs[2])
        nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_pad(nvfuser_direct_test):
    inputs = [
        torch.testing.make_tensor((1, 2, 3), dtype=torch.float32, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])

        t1 = fd.ops.pad(t0, [1, 1, 1, 1])
        fd.add_output(t1)

        # zero padding in some dims
        t2 = fd.ops.pad(t0, [0, 0, 2, 3])
        fd.add_output(t2)

        # zero padding in all dims
        t3 = fd.ops.pad(t0, [0, 0, 0, 0])
        fd.add_output(t3)

        # no padding provided in first dim
        t4 = fd.ops.pad(t0, [2, 3])
        fd.add_output(t4)

        # test padding with a value other than 0
        fill_val = fd.define_scalar(2.0)
        t5 = fd.ops.pad(t0, [2, 3], fill_val)
        fd.add_output(t5)

        # pad a broadcast dimension with a value other than 0
        t6 = fd.ops.pad(t0, [2, 3, 0, 0, 0, 0])
        fd.add_output(t6)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    nvfuser_direct_test.assertEqual(
        torch.nn.functional.pad(inputs[0], [1, 1, 1, 1]), nvf_out[0]
    )
    nvfuser_direct_test.assertEqual(
        torch.nn.functional.pad(inputs[0], [0, 0, 2, 3]), nvf_out[1]
    )
    nvfuser_direct_test.assertEqual(
        torch.nn.functional.pad(inputs[0], [0, 0, 0, 0]), nvf_out[2]
    )
    nvfuser_direct_test.assertEqual(
        torch.nn.functional.pad(inputs[0], [2, 3]), nvf_out[3]
    )
    nvfuser_direct_test.assertEqual(
        torch.nn.functional.pad(inputs[0], [2, 3], "constant", 2.0), nvf_out[4]
    )
    nvfuser_direct_test.assertEqual(
        torch.nn.functional.pad(inputs[0], [2, 3, 0, 0, 0, 0]), nvf_out[5]
    )

def test_pad_dynamic(nvfuser_direct_test):
    inputs = [
        torch.testing.make_tensor((1, 2, 3), dtype=torch.float32, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])

        S10 = fd.define_scalar(2.5, dtype=DataType.Float)
        S13 = fd.define_scalar(7, dtype=DataType.Int)
        S15 = fd.ops.mul(S10, S13)
        S16 = fd.ops.cast(S15, dtype=DataType.Int)
        t1 = fd.ops.pad(t0, [S16, S16, S16, S16])
        fd.add_output(t1)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    nvfuser_direct_test.assertEqual(


    nvfuser_direct_test.assertEqual(
        torch.nn.functional.pad(inputs[0], [1, 1, 1, 1]), nvf_out[0]
    )
    nvfuser_direct_test.assertEqual(
        torch.nn.functional.pad(inputs[0], [0, 0, 2, 3]), nvf_out[1]
    )
    nvfuser_direct_test.assertEqual(
        torch.nn.functional.pad(inputs[0], [0, 0, 0, 0]), nvf_out[2]
    )
    nvfuser_direct_test.assertEqual(
        torch.nn.functional.pad(inputs[0], [2, 3]), nvf_out[3]
    )
    nvfuser_direct_test.assertEqual(
        torch.nn.functional.pad(inputs[0], [2, 3], "constant", 2.0), nvf_out[4]
    )
    nvfuser_direct_test.assertEqual(
        torch.nn.functional.pad(inputs[0], [2, 3, 0, 0, 0, 0]), nvf_out[5]
    )

def test_pad_dynamic(nvfuser_direct_test):
    inputs = [
        torch.testing.make_tensor((1, 2, 3), dtype=torch.float32, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])

        S10 = fd.define_scalar(2.5, dtype=DataType.Float)
        S13 = fd.define_scalar(7, dtype=DataType.Int)
        S15 = fd.ops.mul(S10, S13)
        S16 = fd.ops.cast(S15, dtype=DataType.Int)
        t1 = fd.ops.pad(t0, [S16, S16, S16, S16])
        fd.add_output(t1)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    nvfuser_direct_test.assertEqual(
        torch.nn.functional.pad(inputs[0], [17, 17, 17, 17]), nvf_out[0]
    )

def test_cat(nvfuser_direct_test):
    inputs = [
        torch.randn(2, 4, device="cuda"),
        torch.randn(2, 3, device="cuda"),
        torch.randn(4, 4, device="cuda"),
        torch.randn(0, 4, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.from_pytorch(inputs[2])
        t3 = fd.from_pytorch(inputs[3])

        t3 = fd.ops.cat([t0, t1], 1)
        fd.add_output(t3)

        t4 = fd.ops.cat([t0, t2], 0)


        torch.testing.make_tensor((1, 2, 3), dtype=torch.float32, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])

        S10 = fd.define_scalar(2.5, dtype=DataType.Float)
        S13 = fd.define_scalar(7, dtype=DataType.Int)
        S15 = fd.ops.mul(S10, S13)
        S16 = fd.ops.cast(S15, dtype=DataType.Int)
        t1 = fd.ops.pad(t0, [S16, S16, S16, S16])
        fd.add_output(t1)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    nvfuser_direct_test.assertEqual(
        torch.nn.functional.pad(inputs[0], [17, 17, 17, 17]), nvf_out[0]
    )

def test_cat(nvfuser_direct_test):
    inputs = [
        torch.randn(2, 4, device="cuda"),
        torch.randn(2, 3, device="cuda"),
        torch.randn(4, 4, device="cuda"),
        torch.randn(0, 4, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.from_pytorch(inputs[2])
        t3 = fd.from_pytorch(inputs[3])

        t3 = fd.ops.cat([t0, t1], 1)
        fd.add_output(t3)

        t4 = fd.ops.cat([t0, t2], 0)
        fd.add_output(t4)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    nvfuser_direct_test.assertEqual(
        torch.cat([inputs[0], inputs[1]], dim=1), nvf_out[0]
    )
    nvfuser_direct_test.assertEqual(
        torch.cat([inputs[0], inputs[2]], dim=0), nvf_out[1]
    )

def test_normal(nvfuser_direct_test):
    input_size = [64, 128, 1024]
    dtype = torch.float32
    device = "cuda"
    inputs = [
        torch.randn(*input_size, device=device, dtype=dtype),
    ]
    mean = 3.7
    std = 2.5

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        s_mean = fd.define_scalar(mean)
        s_std = fd.define_scalar(std)
        t1 = fd.ops.normal(s_mean, s_std, t0.shape(), dtype=DataType.Double)
        fd.add_output(t1)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)


        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.from_pytorch(inputs[2])
        t3 = fd.from_pytorch(inputs[3])

        t3 = fd.ops.cat([t0, t1], 1)
        fd.add_output(t3)

        t4 = fd.ops.cat([t0, t2], 0)
        fd.add_output(t4)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    nvfuser_direct_test.assertEqual(
        torch.cat([inputs[0], inputs[1]], dim=1), nvf_out[0]
    )
    nvfuser_direct_test.assertEqual(
        torch.cat([inputs[0], inputs[2]], dim=0), nvf_out[1]
    )

def test_normal(nvfuser_direct_test):
    input_size = [64, 128, 1024]
    dtype = torch.float32
    device = "cuda"
    inputs = [
        torch.randn(*input_size, device=device, dtype=dtype),
    ]
    mean = 3.7
    std = 2.5

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        s_mean = fd.define_scalar(mean)
        s_std = fd.define_scalar(std)
        t1 = fd.ops.normal(s_mean, s_std, t0.shape(), dtype=DataType.Double)
        fd.add_output(t1)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    # Is there a better way to test distribution?!
    nvfuser_direct_test.assertTrue(
        nvf_out[0]
        .mean()
        .cpu()
        .float()
        .isclose(torch.tensor(mean), rtol=1e-2, atol=1e-2)
        .item()
    )
    nvfuser_direct_test.assertTrue(
        nvf_out[0]
        .std()
        .cpu()
        .float()
        .isclose(torch.tensor(std), rtol=1e-2, atol=1e-2)
        .item()
    )

def test_uniform(nvfuser_direct_test):
    input_size = [64, 128, 1024]
    dtype = torch.float32
    device = "cuda"
    inputs = [
        torch.randn(*input_size, device=device, dtype=dtype),
    ]
    lo = 1.8
    hi = 1223.5

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        s_lo = fd.define_scalar(lo)
        s_hi = fd.define_scalar(hi)
        t1 = fd.ops.uniform(s_lo, s_hi, t0.shape(), dtype=DataType.Double)
        fd.add_output(t1)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)


    # Is there a better way to test distribution?!
    nvfuser_direct_test.assertTrue(
        nvf_out[0]
        .mean()
        .cpu()
        .float()
        .isclose(torch.tensor(mean), rtol=1e-2, atol=1e-2)
        .item()
    )
    nvfuser_direct_test.assertTrue(
        nvf_out[0]
        .std()
        .cpu()
        .float()
        .isclose(torch.tensor(std), rtol=1e-2, atol=1e-2)
        .item()
    )

def test_uniform(nvfuser_direct_test):
    input_size = [64, 128, 1024]
    dtype = torch.float32
    device = "cuda"
    inputs = [
        torch.randn(*input_size, device=device, dtype=dtype),
    ]
    lo = 1.8
    hi = 1223.5

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        s_lo = fd.define_scalar(lo)
        s_hi = fd.define_scalar(hi)
        t1 = fd.ops.uniform(s_lo, s_hi, t0.shape(), dtype=DataType.Double)
        fd.add_output(t1)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    # Is there a better way to test distribution?!
    nvfuser_direct_test.assertTrue(
        nvf_out[0]
        .mean()
        .cpu()
        .float()
        .isclose(torch.tensor((hi - lo) / 2.0), rtol=1e-2, atol=1e-2)
        .item()
    )
    nvfuser_direct_test.assertTrue(
        nvf_out[0]
        .min()
        .cpu()
        .float()
        .isclose(torch.tensor(lo), rtol=1e-2, atol=1e-2)
        .item()
    )
    nvfuser_direct_test.assertTrue(
        nvf_out[0]
        .max()
        .cpu()
        .float()
        .isclose(torch.tensor(hi), rtol=1e-2, atol=1e-2)
        .item()
    )

def test_random_distinct_values(nvfuser_direct_test):
    dtypes = [DataType.Double, DataType.Float, DataType.Half, DataType.BFloat16]
    for dtype, rand_op_name in itertools.product(dtypes, ["uniform", "normal"]):

        def fusion_fn(fd: FusionDefinition):
            # generate 4 values and check that they are all distinct
            rand_op = getattr(fd.ops, rand_op_name)
            S0 = fd.define_scalar(0.00000, dtype=DataType.Double)
            S1 = fd.define_scalar(1.00000, dtype=DataType.Double)
            output = rand_op(S0, S1, shape=[2, 2], dtype=dtype)
            fd.add_output(output)

        with FusionDefinition() as fd:
            fusion_fn(fd)

        for i in range(100):
            output = fd.execute([])[0]


        .item()
    )
    nvfuser_direct_test.assertTrue(
        nvf_out[0]
        .min()
        .cpu()
        .float()
        .isclose(torch.tensor(lo), rtol=1e-2, atol=1e-2)
        .item()
    )
    nvfuser_direct_test.assertTrue(
        nvf_out[0]
        .max()
        .cpu()
        .float()
        .isclose(torch.tensor(hi), rtol=1e-2, atol=1e-2)
        .item()
    )

def test_random_distinct_values(nvfuser_direct_test):
    dtypes = [DataType.Double, DataType.Float, DataType.Half, DataType.BFloat16]
    for dtype, rand_op_name in itertools.product(dtypes, ["uniform", "normal"]):

        def fusion_fn(fd: FusionDefinition):
            # generate 4 values and check that they are all distinct
            rand_op = getattr(fd.ops, rand_op_name)
            S0 = fd.define_scalar(0.00000, dtype=DataType.Double)
            S1 = fd.define_scalar(1.00000, dtype=DataType.Double)
            output = rand_op(S0, S1, shape=[2, 2], dtype=dtype)
            fd.add_output(output)

        with FusionDefinition() as fd:
            fusion_fn(fd)

        for i in range(100):
            output = fd.execute([])[0]

            # Rarely we might have a pair of matching lower precision
            # samples. However, it is extremely rare that we would have a
            # set of three matching elements in only 100 repeats unless we
            # have a bug.

            match = output.flatten().unsqueeze(0) == output.flatten().unsqueeze(1)
            match_pairs = (
                match ^ torch.eye(4, dtype=torch.bool, device="cuda")
            ).sum() // 2

            assert match_pairs.item() < 3, f"At least three entries match in {output}"

@pytest.mark.parametrize("padding_idx", [None, -2])
@pytest.mark.parametrize("max_norm", [None, 1e-5])
@pytest.mark.parametrize("norm_type", [None, 1.0])
@pytest.mark.parametrize("scale_grad_by_freq", [None, True])
@pytest.mark.parametrize("sparse", [None, True])
def test_embedding(
    padding_idx: None | int,
    max_norm: None | float,
    norm_type: None | float,
    scale_grad_by_freq: None | bool,
    sparse: None | bool,
):
    def fusion_func(
        fd: FusionDefinition,
        has_optional_inputs: list[bool],
        optional_inputs_dtypes: list[DataType],
    ):
        input = fd.define_tensor(


            output = fd.execute([])[0]

            # Rarely we might have a pair of matching lower precision
            # samples. However, it is extremely rare that we would have a
            # set of three matching elements in only 100 repeats unless we
            # have a bug.

            match = output.flatten().unsqueeze(0) == output.flatten().unsqueeze(1)
            match_pairs = (
                match ^ torch.eye(4, dtype=torch.bool, device="cuda")
            ).sum() // 2

            assert match_pairs.item() < 3, f"At least three entries match in {output}"

@pytest.mark.parametrize("padding_idx", [None, -2])
@pytest.mark.parametrize("max_norm", [None, 1e-5])
@pytest.mark.parametrize("norm_type", [None, 1.0])
@pytest.mark.parametrize("scale_grad_by_freq", [None, True])
@pytest.mark.parametrize("sparse", [None, True])
def test_embedding(
    padding_idx: None | int,
    max_norm: None | float,
    norm_type: None | float,
    scale_grad_by_freq: None | bool,
    sparse: None | bool,
):
    def fusion_func(
        fd: FusionDefinition,
        has_optional_inputs: list[bool],
        optional_inputs_dtypes: list[DataType],
    ):
        input = fd.define_tensor(
            shape=[-1],
            contiguity=[True],
            dtype=DataType.Int,
            is_cpu=False,
        )
        weight = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        # padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse
        optional_inputs = [None] * 5
        for idx in range(len(optional_inputs)):
            if has_optional_inputs[idx]:
                optional_inputs[idx] = fd.define_scalar(
                    value=None, dtype=optional_inputs_dtypes[idx]
                )
        out = fd.ops.embedding_fwd(input, weight, *optional_inputs)
        fd.add_output(out)

    N, S = 10, 3
    input = torch.randint(
        N, (S,), dtype=torch.int64, device="cuda", requires_grad=False
    )
    weight = torch.randn(N, S, dtype=torch.bfloat16, device="cuda", requires_grad=True)
    optional_inputs_dtypes = [
        DataType.Int,
        DataType.Float,
        DataType.Float,
        DataType.Bool,
        DataType.Bool,
    ]

    # This is not in pytest_ops.py since the torch API does not accept None values for some arguments.
    # Different inputs for nvfuser and torch API cannot be handled within OpInfo
    optional_inputs = [padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse]
    has_optional_inputs = [None] * 5
    inputs = [input, weight]
    for idx, param in enumerate(optional_inputs):
        if param is not None:
            has_optional_inputs[idx] = True
            inputs.append(param)

    with FusionDefinition() as fd:
        fusion_func(
            fd,
            has_optional_inputs=has_optional_inputs,
            optional_inputs_dtypes=optional_inputs_dtypes,
        )
    nvf_out = fd.execute(inputs)

    norm_type = 2.0 if norm_type is None else norm_type
    scale_grad_by_freq = False if scale_grad_by_freq is None else scale_grad_by_freq
    sparse = False if sparse is None else sparse
    ref_out = torch.nn.functional.embedding(
        input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse
    )
    torch.testing.assert_close(nvf_out[0], ref_out)

def test_stride_order_with_explicit_broadcast(nvfuser_direct_test):
    inputs = [
        torch.randn(3, device="cuda").unsqueeze(-1),
        torch.randn(2, 3, device="cuda").unsqueeze(-1).expand(2, 3, 4).transpose(2, 0),
        torch.randn(5 * 960, device="cuda").as_strided(
            (5, 4, 1, 5, 16), (960, 48, 16, 192, 1)
        ),
        torch.randn(6, device="cuda").as_strided((2, 16, 3), (3, 0, 1)),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.from_pytorch(inputs[2])
        t3 = fd.define_tensor(
            shape=[-1, 16, 3],
            contiguity=[None, True, True],
            dtype=DataType.Float,


            has_optional_inputs[idx] = True
            inputs.append(param)

    with FusionDefinition() as fd:
        fusion_func(
            fd,
            has_optional_inputs=has_optional_inputs,
            optional_inputs_dtypes=optional_inputs_dtypes,
        )
    nvf_out = fd.execute(inputs)

    norm_type = 2.0 if norm_type is None else norm_type
    scale_grad_by_freq = False if scale_grad_by_freq is None else scale_grad_by_freq
    sparse = False if sparse is None else sparse
    ref_out = torch.nn.functional.embedding(
        input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse
    )
    torch.testing.assert_close(nvf_out[0], ref_out)

def test_stride_order_with_explicit_broadcast(nvfuser_direct_test):
    inputs = [
        torch.randn(3, device="cuda").unsqueeze(-1),
        torch.randn(2, 3, device="cuda").unsqueeze(-1).expand(2, 3, 4).transpose(2, 0),
        torch.randn(5 * 960, device="cuda").as_strided(
            (5, 4, 1, 5, 16), (960, 48, 16, 192, 1)
        ),
        torch.randn(6, device="cuda").as_strided((2, 16, 3), (3, 0, 1)),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.from_pytorch(inputs[2])
        t3 = fd.define_tensor(
            shape=[-1, 16, 3],
            contiguity=[None, True, True],
            dtype=DataType.Float,
            stride_order=[1, 2, 0],
            is_cpu=False,
        )

        t0_b = fd.ops.broadcast(t0, [True, False, False])
        t4 = fd.ops.add(t0_b, t1)
        c0 = fd.define_scalar(3.0)
        t5 = fd.ops.add(t2, c0)
        t6 = fd.ops.mul(t3, c0)

        fd.add_output(t4)
        fd.add_output(t5)
        fd.add_output(t6)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out = inputs[0] + inputs[1]
    nvfuser_direct_test.assertEqual(nvf_out[0], inputs[0] + inputs[1])
    nvfuser_direct_test.assertEqual(nvf_out[1], inputs[2] + 3.0)
    nvfuser_direct_test.assertEqual(nvf_out[2], inputs[3] * 3.0)

def test_output_stride_order(nvfuser_direct_test):
    inputs = [
        torch.arange(0, 24).reshape(2, 3, 4).cuda().float(),
    ]
    eager_out = inputs[0] + 3.0

    for perm in itertools.permutations(range(3), 3):
        # testing stride_order in set
        def fusion_set_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            c0 = fd.define_scalar(3.0)
            t1 = fd.ops.add(t0, c0)
            t2 = fd.ops.stride_order(t1, perm)
            fd.add_output(t2)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_set_func, inputs)
        nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])


            is_cpu=False,
        )

        t0_b = fd.ops.broadcast(t0, [True, False, False])
        t4 = fd.ops.add(t0_b, t1)
        c0 = fd.define_scalar(3.0)
        t5 = fd.ops.add(t2, c0)
        t6 = fd.ops.mul(t3, c0)

        fd.add_output(t4)
        fd.add_output(t5)
        fd.add_output(t6)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out = inputs[0] + inputs[1]
    nvfuser_direct_test.assertEqual(nvf_out[0], inputs[0] + inputs[1])
    nvfuser_direct_test.assertEqual(nvf_out[1], inputs[2] + 3.0)
    nvfuser_direct_test.assertEqual(nvf_out[2], inputs[3] * 3.0)

def test_output_stride_order(nvfuser_direct_test):
    inputs = [
        torch.arange(0, 24).reshape(2, 3, 4).cuda().float(),
    ]
    eager_out = inputs[0] + 3.0

    for perm in itertools.permutations(range(3), 3):
        # testing stride_order in set
        def fusion_set_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            c0 = fd.define_scalar(3.0)
            t1 = fd.ops.add(t0, c0)
            t2 = fd.ops.stride_order(t1, perm)
            fd.add_output(t2)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_set_func, inputs)
        nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

        nvf_stride = nvf_out[0].stride()
        verify_stride_order(nvf_stride, perm)

def test_output_stride_order_with_reduction(nvfuser_direct_test):
    inputs = [torch.randn(2, 3, 4, 5, device="cuda", dtype=torch.float)]

    for stride_order in itertools.permutations(range(3), 3):

        def fusion_stride_order_op(fd: FusionDefinition) -> None:
            T0 = fd.from_pytorch(inputs[0])
            T1 = fd.ops.sum(T0, dims=[2])
            T2 = fd.ops.stride_order(T1, stride_order)
            fd.add_output(T2)

        with FusionDefinition() as fd:
            fusion_stride_order_op(fd)

        out = fd.execute(inputs)[0]
        verify_stride_order(out.stride(), stride_order)


        torch.arange(0, 24).reshape(2, 3, 4).cuda().float(),
    ]
    eager_out = inputs[0] + 3.0

    for perm in itertools.permutations(range(3), 3):
        # testing stride_order in set
        def fusion_set_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            c0 = fd.define_scalar(3.0)
            t1 = fd.ops.add(t0, c0)
            t2 = fd.ops.stride_order(t1, perm)
            fd.add_output(t2)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_set_func, inputs)
        nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

        nvf_stride = nvf_out[0].stride()
        verify_stride_order(nvf_stride, perm)

def test_output_stride_order_with_reduction(nvfuser_direct_test):
    inputs = [torch.randn(2, 3, 4, 5, device="cuda", dtype=torch.float)]

    for stride_order in itertools.permutations(range(3), 3):

        def fusion_stride_order_op(fd: FusionDefinition) -> None:
            T0 = fd.from_pytorch(inputs[0])
            T1 = fd.ops.sum(T0, dims=[2])
            T2 = fd.ops.stride_order(T1, stride_order)
            fd.add_output(T2)

        with FusionDefinition() as fd:
            fusion_stride_order_op(fd)

        out = fd.execute(inputs)[0]
        verify_stride_order(out.stride(), stride_order)

def test_triu(nvfuser_direct_test):
    inputs = [
        torch.randn(4, 16, device="cuda", dtype=torch.float16),
    ]

    def fusion_func(fd: FusionDefinition) -> None:
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.ops.triu(t0, -1)
        fd.add_output(t1)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out0 = torch.triu(inputs[0], -1)
    nvfuser_direct_test.assertEqual(eager_out0, nvf_out[0])

def test_scatter_output_intermediate(nvfuser_direct_test):
    bsz = 128
    hidden = 1024


def test_output_stride_order_with_reduction(nvfuser_direct_test):
    inputs = [torch.randn(2, 3, 4, 5, device="cuda", dtype=torch.float)]

    for stride_order in itertools.permutations(range(3), 3):

        def fusion_stride_order_op(fd: FusionDefinition) -> None:
            T0 = fd.from_pytorch(inputs[0])
            T1 = fd.ops.sum(T0, dims=[2])
            T2 = fd.ops.stride_order(T1, stride_order)
            fd.add_output(T2)

        with FusionDefinition() as fd:
            fusion_stride_order_op(fd)

        out = fd.execute(inputs)[0]
        verify_stride_order(out.stride(), stride_order)

def test_triu(nvfuser_direct_test):
    inputs = [
        torch.randn(4, 16, device="cuda", dtype=torch.float16),
    ]

    def fusion_func(fd: FusionDefinition) -> None:
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.ops.triu(t0, -1)
        fd.add_output(t1)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out0 = torch.triu(inputs[0], -1)
    nvfuser_direct_test.assertEqual(eager_out0, nvf_out[0])

def test_scatter_output_intermediate(nvfuser_direct_test):
    bsz = 128
    hidden = 1024
    scatter_size = 64
    scatter_dim = 0

    x = torch.randn([bsz, hidden], device="cuda")
    _, ind = torch.topk(x, k=scatter_size, dim=scatter_dim)
    src = torch.randn(scatter_size, hidden, device="cuda")
    inputs = [x, ind, src]

    def fusion_func(fd: FusionDefinition):
        T0 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[1, 0],


        out = fd.execute(inputs)[0]
        verify_stride_order(out.stride(), stride_order)

def test_triu(nvfuser_direct_test):
    inputs = [
        torch.randn(4, 16, device="cuda", dtype=torch.float16),
    ]

    def fusion_func(fd: FusionDefinition) -> None:
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.ops.triu(t0, -1)
        fd.add_output(t1)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out0 = torch.triu(inputs[0], -1)
    nvfuser_direct_test.assertEqual(eager_out0, nvf_out[0])

def test_scatter_output_intermediate(nvfuser_direct_test):
    bsz = 128
    hidden = 1024
    scatter_size = 64
    scatter_dim = 0

    x = torch.randn([bsz, hidden], device="cuda")
    _, ind = torch.topk(x, k=scatter_size, dim=scatter_dim)
    src = torch.randn(scatter_size, hidden, device="cuda")
    inputs = [x, ind, src]

    def fusion_func(fd: FusionDefinition):
        T0 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[1, 0],
        )
        T1 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.Int,
            is_cpu=False,
            stride_order=[1, 0],
        )
        T2 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[1, 0],
        )
        T3 = fd.ops.scatter(T0, T1, T2, scatter_dim)
        T4 = fd.ops.sigmoid(T3)
        fd.add_output(T4)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out = refs.sigmoid(torch.scatter(x, scatter_dim, ind, src))
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_scatter_scalar_src(nvfuser_direct_test):
    bsz = 128
    hidden = 1024
    scatter_size = 64
    scatter_dim = 0

    x = torch.randn([bsz, hidden], device="cuda")
    _, ind = torch.topk(x, k=scatter_size, dim=scatter_dim)
    src = 1.5
    inputs = [x, ind, src]

    def fusion_func(fd: FusionDefinition):
        T0 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[1, 0],


            dtype=DataType.Int,
            is_cpu=False,
            stride_order=[1, 0],
        )
        T2 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[1, 0],
        )
        T3 = fd.ops.scatter(T0, T1, T2, scatter_dim)
        T4 = fd.ops.sigmoid(T3)
        fd.add_output(T4)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out = refs.sigmoid(torch.scatter(x, scatter_dim, ind, src))
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_scatter_scalar_src(nvfuser_direct_test):
    bsz = 128
    hidden = 1024
    scatter_size = 64
    scatter_dim = 0

    x = torch.randn([bsz, hidden], device="cuda")
    _, ind = torch.topk(x, k=scatter_size, dim=scatter_dim)
    src = 1.5
    inputs = [x, ind, src]

    def fusion_func(fd: FusionDefinition):
        T0 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[1, 0],
        )
        T1 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.Int,
            is_cpu=False,
            stride_order=[1, 0],
        )
        S2 = fd.define_scalar(None, dtype=DataType.Double)
        T3 = fd.ops.scatter(T0, T1, S2, scatter_dim)
        fd.add_output(T3)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out = torch.scatter(x, scatter_dim, ind, src)
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_compute_tensor_descriptor(nvfuser_direct_test):
    configs = (
        (
            # size
            [2, 1, 3, 1, 4, 3],
            # stride
            [12, 4, 4, 4, 1, 0],
            # expected contiguity
            [True, None, True, None, True, None],
            # expected stride_order
            [5, 4, 3, 2, 1, 0],
        ),
        (
            [2, 3, 1, 5, 4],
            [28, 4, 14, 0, 1],
            [False, None, True, None, True],
            [4, 2, 3, 1, 0],
        ),


            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[1, 0],
        )
        T1 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.Int,
            is_cpu=False,
            stride_order=[1, 0],
        )
        S2 = fd.define_scalar(None, dtype=DataType.Double)
        T3 = fd.ops.scatter(T0, T1, S2, scatter_dim)
        fd.add_output(T3)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out = torch.scatter(x, scatter_dim, ind, src)
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_compute_tensor_descriptor(nvfuser_direct_test):
    configs = (
        (
            # size
            [2, 1, 3, 1, 4, 3],
            # stride
            [12, 4, 4, 4, 1, 0],
            # expected contiguity
            [True, None, True, None, True, None],
            # expected stride_order
            [5, 4, 3, 2, 1, 0],
        ),
        (
            [2, 3, 1, 5, 4],
            [28, 4, 14, 0, 1],
            [False, None, True, None, True],
            [4, 2, 3, 1, 0],
        ),
        (
            [2, 2, 1, 1, 2, 2, 2],
            [8, 4, 3, 9, 2, 0, 1],
            [None, True, True, None, True, None, True],
            [5, 4, 3, 6, 2, 1, 0],
        ),
        (
            [2, 2, 1, 2, 4, 2],
            [2, 32, 1, 8, 0, 4],
            [False, True, True, False, None, None],
            [2, 5, 0, 4, 1, 3],
        ),
        (
            [2, 2, 2, 2],
            [8, 4, 2, 1],
            [True, True, True, True],
            [3, 2, 1, 0],
        ),
        (
            [2, 1, 3, 1, 4],
            [24, 4, 8, 4, 2],
            [True, True, None, None, False],
            [4, 2, 3, 1, 0],
        ),
        (
            [2, 2, 2, 2],
            [8, 4, 0, 2],
            [True, True, None, False],
            [3, 2, 1, 0],
        ),
    )

    for sizes, strides, contiguity, stride_order in configs:
        computed_contiguity, computed_stride_order = compute_tensor_descriptor(
            sizes, strides
        )
        nvfuser_direct_test.assertEqual(computed_contiguity, contiguity)
        nvfuser_direct_test.assertEqual(computed_stride_order, stride_order)

def test_complex_constants(nvfuser_direct_test):
    inputs = [
        torch.arange(2, device="cuda").type(torch.complex64),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        c0 = fd.define_scalar(complex(3.0, 0.5))
        t1 = fd.ops.mul(t0, c0)
        fd.add_output(t1)

    (n,), _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    eager_out = inputs[0] * (3.0 + 0.5j)

    nvfuser_direct_test.assertEqual(eager_out, n)
    assert n.dtype == torch.complex64


            [24, 4, 8, 4, 2],
            [True, True, None, None, False],
            [4, 2, 3, 1, 0],
        ),
        (
            [2, 2, 2, 2],
            [8, 4, 0, 2],
            [True, True, None, False],
            [3, 2, 1, 0],
        ),
    )

    for sizes, strides, contiguity, stride_order in configs:
        computed_contiguity, computed_stride_order = compute_tensor_descriptor(
            sizes, strides
        )
        nvfuser_direct_test.assertEqual(computed_contiguity, contiguity)
        nvfuser_direct_test.assertEqual(computed_stride_order, stride_order)

def test_complex_constants(nvfuser_direct_test):
    inputs = [
        torch.arange(2, device="cuda").type(torch.complex64),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        c0 = fd.define_scalar(complex(3.0, 0.5))
        t1 = fd.ops.mul(t0, c0)
        fd.add_output(t1)

    (n,), _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    eager_out = inputs[0] * (3.0 + 0.5j)

    nvfuser_direct_test.assertEqual(eager_out, n)
    assert n.dtype == torch.complex64

def test_complex_rsqrt(nvfuser_direct_test):
    inputs = [
        torch.randn(4, device="cuda", dtype=torch.complex64),
        torch.randn(4, device="cuda", dtype=torch.complex128),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.ops.rsqrt(t0)
        fd.add_output(t2)
        t3 = fd.ops.rsqrt(t1)
        fd.add_output(t3)

    (rfloat, rdouble), _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    at_rfloat = inputs[0].rsqrt()
    at_rdouble = inputs[1].rsqrt()


def test_complex_constants(nvfuser_direct_test):
    inputs = [
        torch.arange(2, device="cuda").type(torch.complex64),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        c0 = fd.define_scalar(complex(3.0, 0.5))
        t1 = fd.ops.mul(t0, c0)
        fd.add_output(t1)

    (n,), _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    eager_out = inputs[0] * (3.0 + 0.5j)

    nvfuser_direct_test.assertEqual(eager_out, n)
    assert n.dtype == torch.complex64

def test_complex_rsqrt(nvfuser_direct_test):
    inputs = [
        torch.randn(4, device="cuda", dtype=torch.complex64),
        torch.randn(4, device="cuda", dtype=torch.complex128),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.ops.rsqrt(t0)
        fd.add_output(t2)
        t3 = fd.ops.rsqrt(t1)
        fd.add_output(t3)

    (rfloat, rdouble), _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    at_rfloat = inputs[0].rsqrt()
    at_rdouble = inputs[1].rsqrt()

    nvfuser_direct_test.assertEqual(at_rfloat, rfloat)
    nvfuser_direct_test.assertEqual(at_rdouble, rdouble)

def test_constant_nans(nvfuser_direct_test):
    inputs = [
        torch.randn(4, 4, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition) -> None:
        t0 = fd.from_pytorch(inputs[0])
        c0 = fd.define_scalar(float("nan"))
        t1 = fd.ops.add(t0, c0)
        fd.add_output(t1)

    eager_out = inputs[0] + float("nan")

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_gcd(nvfuser_direct_test):


        torch.randn(4, device="cuda", dtype=torch.complex128),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.ops.rsqrt(t0)
        fd.add_output(t2)
        t3 = fd.ops.rsqrt(t1)
        fd.add_output(t3)

    (rfloat, rdouble), _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    at_rfloat = inputs[0].rsqrt()
    at_rdouble = inputs[1].rsqrt()

    nvfuser_direct_test.assertEqual(at_rfloat, rfloat)
    nvfuser_direct_test.assertEqual(at_rdouble, rdouble)

def test_constant_nans(nvfuser_direct_test):
    inputs = [
        torch.randn(4, 4, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition) -> None:
        t0 = fd.from_pytorch(inputs[0])
        c0 = fd.define_scalar(float("nan"))
        t1 = fd.ops.add(t0, c0)
        fd.add_output(t1)

    eager_out = inputs[0] + float("nan")

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_gcd(nvfuser_direct_test):
    inputs = [
        torch.testing.make_tensor(1024, device="cuda", dtype=torch.long),
        torch.testing.make_tensor(1024, device="cuda", dtype=torch.long),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.ops.gcd(t0, t1)
        fd.add_output(t2)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    nvfuser_direct_test.assertEqual(nvf_out[0], torch.gcd(inputs[0], inputs[1]))

def test_input_scalar(nvfuser_direct_test):
    inputs = [


    nvfuser_direct_test.assertEqual(at_rdouble, rdouble)

def test_constant_nans(nvfuser_direct_test):
    inputs = [
        torch.randn(4, 4, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition) -> None:
        t0 = fd.from_pytorch(inputs[0])
        c0 = fd.define_scalar(float("nan"))
        t1 = fd.ops.add(t0, c0)
        fd.add_output(t1)

    eager_out = inputs[0] + float("nan")

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_gcd(nvfuser_direct_test):
    inputs = [
        torch.testing.make_tensor(1024, device="cuda", dtype=torch.long),
        torch.testing.make_tensor(1024, device="cuda", dtype=torch.long),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.ops.gcd(t0, t1)
        fd.add_output(t2)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    nvfuser_direct_test.assertEqual(nvf_out[0], torch.gcd(inputs[0], inputs[1]))

def test_input_scalar(nvfuser_direct_test):
    inputs = [
        torch.randn((3,), dtype=torch.float32, device="cuda:0"),
        0.1,
    ]

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.from_pytorch(inputs[0])
        S1 = fd.define_scalar()
        T1 = fd.ops.mul(T0, S1)
        fd.add_output(T1)

    # Just test that this executes, not that it's correct
    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

def test_integer_division(nvfuser_direct_test):
    inputs = [


    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_gcd(nvfuser_direct_test):
    inputs = [
        torch.testing.make_tensor(1024, device="cuda", dtype=torch.long),
        torch.testing.make_tensor(1024, device="cuda", dtype=torch.long),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.ops.gcd(t0, t1)
        fd.add_output(t2)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    nvfuser_direct_test.assertEqual(nvf_out[0], torch.gcd(inputs[0], inputs[1]))

def test_input_scalar(nvfuser_direct_test):
    inputs = [
        torch.randn((3,), dtype=torch.float32, device="cuda:0"),
        0.1,
    ]

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.from_pytorch(inputs[0])
        S1 = fd.define_scalar()
        T1 = fd.ops.mul(T0, S1)
        fd.add_output(T1)

    # Just test that this executes, not that it's correct
    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

def test_integer_division(nvfuser_direct_test):
    inputs = [
        torch.testing.make_tensor(1024, device="cuda", dtype=torch.long),
        torch.testing.make_tensor(1024, device="cuda", dtype=torch.long),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.ops.div(t0, t1)
        t3 = fd.ops.truediv(t0, t1)
        fd.add_output(t2)
        fd.add_output(t3)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    nvfuser_direct_test.assertEqual(
        nvf_out[0], torch.div(inputs[0], inputs[1], rounding_mode="trunc")
    )


    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    nvfuser_direct_test.assertEqual(nvf_out[0], torch.gcd(inputs[0], inputs[1]))

def test_input_scalar(nvfuser_direct_test):
    inputs = [
        torch.randn((3,), dtype=torch.float32, device="cuda:0"),
        0.1,
    ]

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.from_pytorch(inputs[0])
        S1 = fd.define_scalar()
        T1 = fd.ops.mul(T0, S1)
        fd.add_output(T1)

    # Just test that this executes, not that it's correct
    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

def test_integer_division(nvfuser_direct_test):
    inputs = [
        torch.testing.make_tensor(1024, device="cuda", dtype=torch.long),
        torch.testing.make_tensor(1024, device="cuda", dtype=torch.long),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.ops.div(t0, t1)
        t3 = fd.ops.truediv(t0, t1)
        fd.add_output(t2)
        fd.add_output(t3)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    nvfuser_direct_test.assertEqual(
        nvf_out[0], torch.div(inputs[0], inputs[1], rounding_mode="trunc")
    )
    nvfuser_direct_test.assertEqual(nvf_out[1], torch.true_divide(inputs[0], inputs[1]))

def test_mark_alias_pass(nvfuser_direct_test):
    def reshape(fd: FusionDefinition) -> None:
        x = fd.define_tensor(
            [2, 3, 4], contiguity=[True, True, True], dtype=DataType.Float
        )
        y = fd.ops.reshape(x, [2, 12])
        fd.add_output(y)

    x = torch.rand(2, 3, 4, device="cuda")
    ys, _ = nvfuser_direct_test.exec_nvfuser(reshape, [x])
    nvfuser_direct_test.assertEqual(len(ys), 1)
    y = ys[0]

    nvfuser_direct_test.assertEqual(y.data_ptr(), x.data_ptr())

def test_misaligned_add(nvfuser_direct_test):
    inputs = [


    inputs = [
        torch.testing.make_tensor(1024, device="cuda", dtype=torch.long),
        torch.testing.make_tensor(1024, device="cuda", dtype=torch.long),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.ops.div(t0, t1)
        t3 = fd.ops.truediv(t0, t1)
        fd.add_output(t2)
        fd.add_output(t3)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    nvfuser_direct_test.assertEqual(
        nvf_out[0], torch.div(inputs[0], inputs[1], rounding_mode="trunc")
    )
    nvfuser_direct_test.assertEqual(nvf_out[1], torch.true_divide(inputs[0], inputs[1]))

def test_mark_alias_pass(nvfuser_direct_test):
    def reshape(fd: FusionDefinition) -> None:
        x = fd.define_tensor(
            [2, 3, 4], contiguity=[True, True, True], dtype=DataType.Float
        )
        y = fd.ops.reshape(x, [2, 12])
        fd.add_output(y)

    x = torch.rand(2, 3, 4, device="cuda")
    ys, _ = nvfuser_direct_test.exec_nvfuser(reshape, [x])
    nvfuser_direct_test.assertEqual(len(ys), 1)
    y = ys[0]

    nvfuser_direct_test.assertEqual(y.data_ptr(), x.data_ptr())

def test_misaligned_add(nvfuser_direct_test):
    inputs = [
        torch.ones(2**20 + 1, device="cuda")[1:],  # cannot vectorize
        torch.ones(2**20, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        c0 = fd.define_scalar(3.0)

        t2 = fd.ops.add(t0, t1)

        fd.add_output(t2)

    # Fails because vectorization 4 is set but only 1 supported
    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)


    )
    nvfuser_direct_test.assertEqual(nvf_out[1], torch.true_divide(inputs[0], inputs[1]))

def test_mark_alias_pass(nvfuser_direct_test):
    def reshape(fd: FusionDefinition) -> None:
        x = fd.define_tensor(
            [2, 3, 4], contiguity=[True, True, True], dtype=DataType.Float
        )
        y = fd.ops.reshape(x, [2, 12])
        fd.add_output(y)

    x = torch.rand(2, 3, 4, device="cuda")
    ys, _ = nvfuser_direct_test.exec_nvfuser(reshape, [x])
    nvfuser_direct_test.assertEqual(len(ys), 1)
    y = ys[0]

    nvfuser_direct_test.assertEqual(y.data_ptr(), x.data_ptr())

def test_misaligned_add(nvfuser_direct_test):
    inputs = [
        torch.ones(2**20 + 1, device="cuda")[1:],  # cannot vectorize
        torch.ones(2**20, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        c0 = fd.define_scalar(3.0)

        t2 = fd.ops.add(t0, t1)

        fd.add_output(t2)

    # Fails because vectorization 4 is set but only 1 supported
    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

def test_nextafter(nvfuser_direct_test):
    inputs = [
        # torch.nextafter is only defined for float{32,64} tensor inputs
        torch.testing.make_tensor(4, device="cuda", dtype=torch.float32),
        torch.testing.make_tensor(4, device="cuda", dtype=torch.float64),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])

        s0 = fd.define_scalar(1.0, dtype=DataType.Float)
        s1 = fd.define_scalar(-1.0, dtype=DataType.Double)

        for a, b in itertools.product(
            [t0, t1, s0, s1],
            [t0, t1, s0, s1],
        ):


def test_misaligned_add(nvfuser_direct_test):
    inputs = [
        torch.ones(2**20 + 1, device="cuda")[1:],  # cannot vectorize
        torch.ones(2**20, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        c0 = fd.define_scalar(3.0)

        t2 = fd.ops.add(t0, t1)

        fd.add_output(t2)

    # Fails because vectorization 4 is set but only 1 supported
    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

def test_nextafter(nvfuser_direct_test):
    inputs = [
        # torch.nextafter is only defined for float{32,64} tensor inputs
        torch.testing.make_tensor(4, device="cuda", dtype=torch.float32),
        torch.testing.make_tensor(4, device="cuda", dtype=torch.float64),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])

        s0 = fd.define_scalar(1.0, dtype=DataType.Float)
        s1 = fd.define_scalar(-1.0, dtype=DataType.Double)

        for a, b in itertools.product(
            [t0, t1, s0, s1],
            [t0, t1, s0, s1],
        ):
            # always enter the fusion...
            t = fd.ops.nextafter(a, b)
            if t.is_tensor():
                # ...but skip outputting scalars, which we don't support
                fd.add_output(t)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    ab = [inputs[0], inputs[1], 1.0, -1.0]
    i = 0
    for a, b in itertools.product(ab, ab):
        if not (isinstance(a, torch.Tensor) or isinstance(b, torch.Tensor)):
            continue
        n = nvf_out[i]
        i += 1
        torch_out = torch.nextafter(
            torch.as_tensor(a, device="cuda"), torch.as_tensor(b, device="cuda")
        )
        nvfuser_direct_test.assertEqual(n, torch_out)

def test_prod(nvfuser_direct_test):
    inputs = [
        torch.ones(2, 4, 8, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])

        t1 = fd.ops.prod(t0, DataType.Float)
        t2 = fd.ops.prod(t0, 1, False, DataType.Float)
        t3 = fd.ops.prod(t0, 1, True, DataType.Float)
        t4 = fd.ops.prod(t0, [-1], False, DataType.Float)

        fd.add_output(t1)
        fd.add_output(t2)
        fd.add_output(t3)
        fd.add_output(t4)


            t = fd.ops.nextafter(a, b)
            if t.is_tensor():
                # ...but skip outputting scalars, which we don't support
                fd.add_output(t)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    ab = [inputs[0], inputs[1], 1.0, -1.0]
    i = 0
    for a, b in itertools.product(ab, ab):
        if not (isinstance(a, torch.Tensor) or isinstance(b, torch.Tensor)):
            continue
        n = nvf_out[i]
        i += 1
        torch_out = torch.nextafter(
            torch.as_tensor(a, device="cuda"), torch.as_tensor(b, device="cuda")
        )
        nvfuser_direct_test.assertEqual(n, torch_out)

def test_prod(nvfuser_direct_test):
    inputs = [
        torch.ones(2, 4, 8, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])

        t1 = fd.ops.prod(t0, DataType.Float)
        t2 = fd.ops.prod(t0, 1, False, DataType.Float)
        t3 = fd.ops.prod(t0, 1, True, DataType.Float)
        t4 = fd.ops.prod(t0, [-1], False, DataType.Float)

        fd.add_output(t1)
        fd.add_output(t2)
        fd.add_output(t3)
        fd.add_output(t4)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    eager_outs = [
        torch.prod(inputs[0], dtype=torch.float32),
        torch.prod(inputs[0], 1, False, dtype=torch.float32),
        torch.prod(inputs[0], 1, True, dtype=torch.float32),
        torch.prod(inputs[0], -1, False, dtype=torch.float32),
    ]
    assert len(nvf_out) == len(eager_outs)

    for n, e in zip(nvf_out, eager_outs):
        nvfuser_direct_test.assertEqual(n, e)

def test_real_imag(nvfuser_direct_test):
    for dtype in [torch.complex128, torch.complex64]:
        inputs = [
            torch.randn(5, dtype=dtype, device="cuda"),
        ]

        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            fd.add_output(fd.ops.real(t0))
            fd.add_output(fd.ops.imag(t0))

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        nvfuser_direct_test.assertEqual(torch.real(inputs[0]), nvf_out[0])
        nvfuser_direct_test.assertEqual(torch.imag(inputs[0]), nvf_out[1])

def test_reduction_complex_number(nvfuser_direct_test):


        fd.add_output(t1)
        fd.add_output(t2)
        fd.add_output(t3)
        fd.add_output(t4)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

    eager_outs = [
        torch.prod(inputs[0], dtype=torch.float32),
        torch.prod(inputs[0], 1, False, dtype=torch.float32),
        torch.prod(inputs[0], 1, True, dtype=torch.float32),
        torch.prod(inputs[0], -1, False, dtype=torch.float32),
    ]
    assert len(nvf_out) == len(eager_outs)

    for n, e in zip(nvf_out, eager_outs):
        nvfuser_direct_test.assertEqual(n, e)

def test_real_imag(nvfuser_direct_test):
    for dtype in [torch.complex128, torch.complex64]:
        inputs = [
            torch.randn(5, dtype=dtype, device="cuda"),
        ]

        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            fd.add_output(fd.ops.real(t0))
            fd.add_output(fd.ops.imag(t0))

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        nvfuser_direct_test.assertEqual(torch.real(inputs[0]), nvf_out[0])
        nvfuser_direct_test.assertEqual(torch.imag(inputs[0]), nvf_out[1])

def test_reduction_complex_number(nvfuser_direct_test):
    def test_dtype(torch_dtype):
        inputs = [torch.randn(2, 32, device="cuda", dtype=torch_dtype)]

        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.sum(t0, [-1], False, torch_dtype_to_nvfuser_dtype(torch_dtype))
            fd.add_output(t1)

        nvf_out1, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
        eager_out = torch.sum(inputs[0], dim=-1)
        nvfuser_direct_test.assertEqual(eager_out, nvf_out1[0])

    list_of_dtype = [torch.complex64, torch.complex128]
    for torch_dtype in list_of_dtype:
        test_dtype(torch_dtype)


        nvfuser_direct_test.assertEqual(n, e)

def test_real_imag(nvfuser_direct_test):
    for dtype in [torch.complex128, torch.complex64]:
        inputs = [
            torch.randn(5, dtype=dtype, device="cuda"),
        ]

        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            fd.add_output(fd.ops.real(t0))
            fd.add_output(fd.ops.imag(t0))

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

        nvfuser_direct_test.assertEqual(torch.real(inputs[0]), nvf_out[0])
        nvfuser_direct_test.assertEqual(torch.imag(inputs[0]), nvf_out[1])

def test_reduction_complex_number(nvfuser_direct_test):
    def test_dtype(torch_dtype):
        inputs = [torch.randn(2, 32, device="cuda", dtype=torch_dtype)]

        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.sum(t0, [-1], False, torch_dtype_to_nvfuser_dtype(torch_dtype))
            fd.add_output(t1)

        nvf_out1, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
        eager_out = torch.sum(inputs[0], dim=-1)
        nvfuser_direct_test.assertEqual(eager_out, nvf_out1[0])

    list_of_dtype = [torch.complex64, torch.complex128]
    for torch_dtype in list_of_dtype:
        test_dtype(torch_dtype)

def test_right_shift_arithmetic(nvfuser_direct_test):
    inputs = [torch.tensor([-2147483648, 1073741824], dtype=torch.int32, device="cuda")]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        c0 = fd.define_scalar(3)
        t1 = fd.ops.bitwise_right_shift(t0, c0)
        fd.add_output(t1)

    nvf_out1, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out1 = torch.bitwise_right_shift(inputs[0], 3)
    nvfuser_direct_test.assertEqual(eager_out1, nvf_out1[0])

def test_segment_set(nvfuser_direct_test):
    inputs = [
        torch.randn(5, 5, 5, device="cuda"),
    ]


def test_reduction_complex_number(nvfuser_direct_test):
    def test_dtype(torch_dtype):
        inputs = [torch.randn(2, 32, device="cuda", dtype=torch_dtype)]

        def fusion_func(fd: FusionDefinition):
            t0 = fd.from_pytorch(inputs[0])
            t1 = fd.ops.sum(t0, [-1], False, torch_dtype_to_nvfuser_dtype(torch_dtype))
            fd.add_output(t1)

        nvf_out1, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
        eager_out = torch.sum(inputs[0], dim=-1)
        nvfuser_direct_test.assertEqual(eager_out, nvf_out1[0])

    list_of_dtype = [torch.complex64, torch.complex128]
    for torch_dtype in list_of_dtype:
        test_dtype(torch_dtype)

def test_right_shift_arithmetic(nvfuser_direct_test):
    inputs = [torch.tensor([-2147483648, 1073741824], dtype=torch.int32, device="cuda")]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        c0 = fd.define_scalar(3)
        t1 = fd.ops.bitwise_right_shift(t0, c0)
        fd.add_output(t1)

    nvf_out1, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out1 = torch.bitwise_right_shift(inputs[0], 3)
    nvfuser_direct_test.assertEqual(eager_out1, nvf_out1[0])

def test_segment_set(nvfuser_direct_test):
    inputs = [
        torch.randn(5, 5, 5, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.from_pytorch(inputs[0])
        T1 = fd.ops.neg(T0)
        T2 = fd.ops.segment_set(T1)
        T3 = fd.ops.relu(T2)
        fd.add_output(T3)

    eager_out = inputs[0].neg().relu()

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])


    list_of_dtype = [torch.complex64, torch.complex128]
    for torch_dtype in list_of_dtype:
        test_dtype(torch_dtype)

def test_right_shift_arithmetic(nvfuser_direct_test):
    inputs = [torch.tensor([-2147483648, 1073741824], dtype=torch.int32, device="cuda")]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        c0 = fd.define_scalar(3)
        t1 = fd.ops.bitwise_right_shift(t0, c0)
        fd.add_output(t1)

    nvf_out1, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out1 = torch.bitwise_right_shift(inputs[0], 3)
    nvfuser_direct_test.assertEqual(eager_out1, nvf_out1[0])

def test_segment_set(nvfuser_direct_test):
    inputs = [
        torch.randn(5, 5, 5, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.from_pytorch(inputs[0])
        T1 = fd.ops.neg(T0)
        T2 = fd.ops.segment_set(T1)
        T3 = fd.ops.relu(T2)
        fd.add_output(T3)

    eager_out = inputs[0].neg().relu()

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_signbit(nvfuser_direct_test):
    inputs = [
        torch.randn(3, 4, 5, device="cuda", dtype=torch.float32),
        torch.randn(3, 4, 5, device="cuda", dtype=torch.float32),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.ops.where(
            fd.ops.signbit(t0), fd.ops.neg(fd.ops.abs(t1)), fd.ops.abs(t1)
        )
        fd.add_output(t2)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    at_out = torch.where(
        torch.signbit(inputs[0]), -torch.abs(inputs[1]), torch.abs(inputs[1])
    )


def test_segment_set(nvfuser_direct_test):
    inputs = [
        torch.randn(5, 5, 5, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.from_pytorch(inputs[0])
        T1 = fd.ops.neg(T0)
        T2 = fd.ops.segment_set(T1)
        T3 = fd.ops.relu(T2)
        fd.add_output(T3)

    eager_out = inputs[0].neg().relu()

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_signbit(nvfuser_direct_test):
    inputs = [
        torch.randn(3, 4, 5, device="cuda", dtype=torch.float32),
        torch.randn(3, 4, 5, device="cuda", dtype=torch.float32),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.ops.where(
            fd.ops.signbit(t0), fd.ops.neg(fd.ops.abs(t1)), fd.ops.abs(t1)
        )
        fd.add_output(t2)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    at_out = torch.where(
        torch.signbit(inputs[0]), -torch.abs(inputs[1]), torch.abs(inputs[1])
    )
    nvfuser_direct_test.assertEqual(at_out, nvf_out[0])

def test_tensor_shape(nvfuser_direct_test):
    inputs = [
        torch.randn(2, 3, 4, device="cuda"),
        torch.randn(4, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])

        t1_b = fd.ops.broadcast_in_dim(t1, t0.shape(), [2])
        t2 = fd.ops.sub(t0, t1_b)

        fd.add_output(t2)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out = refs.sub(
        inputs[0], prims.broadcast_in_dim(inputs[1], inputs[0].size(), [2])


    inputs = [
        torch.randn(3, 4, 5, device="cuda", dtype=torch.float32),
        torch.randn(3, 4, 5, device="cuda", dtype=torch.float32),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])
        t2 = fd.ops.where(
            fd.ops.signbit(t0), fd.ops.neg(fd.ops.abs(t1)), fd.ops.abs(t1)
        )
        fd.add_output(t2)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    at_out = torch.where(
        torch.signbit(inputs[0]), -torch.abs(inputs[1]), torch.abs(inputs[1])
    )
    nvfuser_direct_test.assertEqual(at_out, nvf_out[0])

def test_tensor_shape(nvfuser_direct_test):
    inputs = [
        torch.randn(2, 3, 4, device="cuda"),
        torch.randn(4, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])

        t1_b = fd.ops.broadcast_in_dim(t1, t0.shape(), [2])
        t2 = fd.ops.sub(t0, t1_b)

        fd.add_output(t2)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out = refs.sub(
        inputs[0], prims.broadcast_in_dim(inputs[1], inputs[0].size(), [2])
    )
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_tensor_shape_expand_bcast(nvfuser_direct_test):
    def fusion_func(fd: FusionDefinition):
        t0 = fd.define_tensor(shape=[-1, -1, -1], contiguity=[True, True, True])
        t1 = fd.define_tensor(shape=[-1, 1, -1], contiguity=[True, None, True])
        t2 = fd.define_tensor(shape=[-1, 1, -1], contiguity=[True, None, True])

        t1_b = fd.ops.broadcast_in_dim(t1, t0.shape(), [0, 1, 2])
        t2_b = fd.ops.broadcast_in_dim(t2, t1_b.shape(), [0, 1, 2])

        fd.add_output(t2_b)

    inputs = [
        torch.randn(2, 3, 4, device="cuda"),
        torch.randn(2, 1, 4, device="cuda"),
        torch.randn(2, 1, 4, device="cuda"),
    ]

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)


        torch.randn(2, 3, 4, device="cuda"),
        torch.randn(4, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])

        t1_b = fd.ops.broadcast_in_dim(t1, t0.shape(), [2])
        t2 = fd.ops.sub(t0, t1_b)

        fd.add_output(t2)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out = refs.sub(
        inputs[0], prims.broadcast_in_dim(inputs[1], inputs[0].size(), [2])
    )
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_tensor_shape_expand_bcast(nvfuser_direct_test):
    def fusion_func(fd: FusionDefinition):
        t0 = fd.define_tensor(shape=[-1, -1, -1], contiguity=[True, True, True])
        t1 = fd.define_tensor(shape=[-1, 1, -1], contiguity=[True, None, True])
        t2 = fd.define_tensor(shape=[-1, 1, -1], contiguity=[True, None, True])

        t1_b = fd.ops.broadcast_in_dim(t1, t0.shape(), [0, 1, 2])
        t2_b = fd.ops.broadcast_in_dim(t2, t1_b.shape(), [0, 1, 2])

        fd.add_output(t2_b)

    inputs = [
        torch.randn(2, 3, 4, device="cuda"),
        torch.randn(2, 1, 4, device="cuda"),
        torch.randn(2, 1, 4, device="cuda"),
    ]

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out1 = prims.broadcast_in_dim(inputs[1], inputs[0].size(), [0, 1, 2])
    eager_out2 = prims.broadcast_in_dim(inputs[2], eager_out1.size(), [0, 1, 2])
    nvfuser_direct_test.assertEqual(eager_out2, nvf_out[0])

def test_tensor_shape_nobcast(nvfuser_direct_test):
    inputs = [
        torch.randn(2, 3, device="cuda"),
        torch.randn(2, 3, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])

        t1_b = fd.ops.broadcast_in_dim(t1, t0.shape(), [0, 1])
        t2 = fd.ops.add(t0, t1_b)

        fd.add_output(t2)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out = refs.add(
        inputs[0], prims.broadcast_in_dim(inputs[1], inputs[0].size(), [0, 1])


        t1 = fd.define_tensor(shape=[-1, 1, -1], contiguity=[True, None, True])
        t2 = fd.define_tensor(shape=[-1, 1, -1], contiguity=[True, None, True])

        t1_b = fd.ops.broadcast_in_dim(t1, t0.shape(), [0, 1, 2])
        t2_b = fd.ops.broadcast_in_dim(t2, t1_b.shape(), [0, 1, 2])

        fd.add_output(t2_b)

    inputs = [
        torch.randn(2, 3, 4, device="cuda"),
        torch.randn(2, 1, 4, device="cuda"),
        torch.randn(2, 1, 4, device="cuda"),
    ]

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out1 = prims.broadcast_in_dim(inputs[1], inputs[0].size(), [0, 1, 2])
    eager_out2 = prims.broadcast_in_dim(inputs[2], eager_out1.size(), [0, 1, 2])
    nvfuser_direct_test.assertEqual(eager_out2, nvf_out[0])

def test_tensor_shape_nobcast(nvfuser_direct_test):
    inputs = [
        torch.randn(2, 3, device="cuda"),
        torch.randn(2, 3, device="cuda"),
    ]

    def fusion_func(fd: FusionDefinition):
        t0 = fd.from_pytorch(inputs[0])
        t1 = fd.from_pytorch(inputs[1])

        t1_b = fd.ops.broadcast_in_dim(t1, t0.shape(), [0, 1])
        t2 = fd.ops.add(t0, t1_b)

        fd.add_output(t2)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
    eager_out = refs.add(
        inputs[0], prims.broadcast_in_dim(inputs[1], inputs[0].size(), [0, 1])
    )
    nvfuser_direct_test.assertEqual(eager_out, nvf_out[0])

def test_tensor_shape_with_output_bcast(nvfuser_direct_test):
    def fusion_func(fd: FusionDefinition):
        t0 = fd.define_tensor(shape=[-1, -1, -1], contiguity=[True, True, True])

        t1 = fd.ops.sum(t0, dims=[2])
        t1_b = fd.ops.broadcast_in_dim(t1, t0.shape(), [0, 1])

        fd.add_output(t1_b)

    inputs_1 = [
        torch.randn(2, 3, 4, device="cuda"),
    ]

    inputs_2 = [
        torch.randn(4, 5, 32, device="cuda"),
    ]

    inputs = inputs_1