            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (2560,),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 16384, 2560),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
    ]
    nvfuser_direct_test.exec_nvfuser(fusion_func, inputs, validate_results=True)

def test_loop_promotion_cyclic_war(nvfuser_direct_test):
    """
    Test for loop promotion with cyclic WAR (Write-After-Read) dependencies.

    This test verifies complex tensor operations with BFloat16 and Float data types,
    including reshape, cast, broadcast, and mathematical operations with slice operations.
    """

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[4096, 128],
            contiguity=[True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[1, 0],
        )
        T1 = fd.define_tensor(
            shape=[4096, 128],
            contiguity=[True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[1, 0],
        )
        T2 = fd.define_tensor(
            shape=[1, 4096, 5120],
            contiguity=[None, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T3 = fd.define_tensor(
            shape=[1, 4096, 640],
            contiguity=[None, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T4 = fd.define_tensor(
            shape=[1, 4096, 640],
            contiguity=[None, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T5 = fd.define_tensor(
            shape=[1, 4096, 16640],
            contiguity=[None, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T6 = fd.define_tensor(
            shape=[1, 4096, 16640],
            contiguity=[None, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T7 = fd.define_tensor(
            shape=[1, 4096, 5120],
            contiguity=[None, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T8 = fd.define_tensor(
            shape=[1, 4096, 640],
            contiguity=[None, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T9 = fd.define_tensor(
            shape=[1, 4096, 640],
            contiguity=[None, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T10 = fd.define_tensor(
            shape=[1, 4096, 16640],
            contiguity=[None, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T11 = fd.define_tensor(
            shape=[1, 4096, 16640],
            contiguity=[None, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T16 = fd.ops.reshape(T0, new_shape=[1, 4096, 128])
        T22 = fd.ops.broadcast_in_dim(
            T16, shape=[1, 1, 4096, 128], broadcast_dims=[0, 2, 3]
        )
        T27 = fd.ops.reshape(T1, new_shape=[1, 4096, 128])
        T33 = fd.ops.broadcast_in_dim(
            T27, shape=[1, 1, 4096, 128], broadcast_dims=[0, 2, 3]
        )
        T39 = fd.ops.broadcast_in_dim(
            T22, shape=[1, 40, 4096, 128], broadcast_dims=[0, 1, 2, 3]
        )
        T40 = fd.ops.cast(T39, dtype=DataType.Float)
        T46 = fd.ops.broadcast_in_dim(
            T33, shape=[1, 40, 4096, 128], broadcast_dims=[0, 1, 2, 3]
        )
        T47 = fd.ops.cast(T46, dtype=DataType.Float)
        T48 = fd.ops.cast(T2, dtype=DataType.Float)
        T49 = fd.ops.cast(T3, dtype=DataType.Float)
        T50 = fd.ops.cast(T4, dtype=DataType.Float)
        T51 = fd.ops.cast(T5, dtype=DataType.Float)
        T52 = fd.ops.cast(T6, dtype=DataType.Float)
        S53 = fd.define_scalar(2.00000, dtype=DataType.Double)
        T54 = fd.ops.mul(T7, S53)
        S55 = fd.define_scalar(2.00000, dtype=DataType.Double)
        T56 = fd.ops.mul(T8, S55)
        S57 = fd.define_scalar(2.00000, dtype=DataType.Double)
        T58 = fd.ops.mul(T9, S57)
        S59 = fd.define_scalar(2.00000, dtype=DataType.Double)
        T60 = fd.ops.mul(T10, S59)
        S61 = fd.define_scalar(2.00000, dtype=DataType.Double)
        T62 = fd.ops.mul(T11, S61)
        T63 = fd.ops.add(T48, T54)
        T64 = fd.ops.add(T49, T56)
        T65 = fd.ops.add(T50, T58)
        T66 = fd.ops.add(T51, T60)
        T67 = fd.ops.add(T52, T62)
        T68 = fd.ops.cast(T63, dtype=DataType.BFloat16)
        T74 = fd.ops.reshape(T68, new_shape=[1, 4096, 40, 128])
        T75 = fd.ops.cast(T64, dtype=DataType.BFloat16)
        T81 = fd.ops.reshape(T75, new_shape=[1, 4096, 5, 128])
        T82 = fd.ops.cast(T65, dtype=DataType.BFloat16)
        T88 = fd.ops.reshape(T82, new_shape=[1, 4096, 5, 128])
        T89 = fd.ops.cast(T66, dtype=DataType.BFloat16)
        T90 = fd.ops.neg(T66)
        T91 = fd.ops.cast(T67, dtype=DataType.BFloat16)
        T92 = fd.ops.permute(T74, dims=[0, 2, 1, 3])
        T93 = fd.ops.permute(T81, dims=[0, 2, 1, 3])
        T94 = fd.ops.permute(T88, dims=[0, 2, 1, 3])
        T95 = fd.ops.exp(T90)
        T105 = fd.ops.broadcast_in_dim(
            T93, shape=[1, 1, 8, 5, 1, 4096, 1, 128], broadcast_dims=[1, 3, 5, 7]
        )
        T111 = fd.ops.reshape(T105, new_shape=[1, 40, 4096, 128])
        T121 = fd.ops.broadcast_in_dim(
            T94, shape=[1, 1, 8, 5, 1, 4096, 1, 128], broadcast_dims=[1, 3, 5, 7]
        )
        T127 = fd.ops.reshape(T121, new_shape=[1, 40, 4096, 128])
        T128 = fd.ops.cast(T92, dtype=DataType.Float)
        T144 = fd.ops.slice(
            T92,
            start_indices=[0, 0, 0, 0],
            end_indices=[1, 40, 4096, 64],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T160 = fd.ops.slice(
            T92,
            start_indices=[0, 0, 0, 64],
            end_indices=[1, 40, 4096, 128],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T161 = fd.ops.cast(T160, dtype=DataType.Float)
        T162 = fd.ops.neg(T161)
        T163 = fd.ops.cast(T162, dtype=DataType.BFloat16)
        T164 = fd.ops.cast(T111, dtype=DataType.Float)
        T180 = fd.ops.slice(
            T111,
            start_indices=[0, 0, 0, 0],
            end_indices=[1, 40, 4096, 64],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T196 = fd.ops.slice(
            T111,
            start_indices=[0, 0, 0, 64],
            end_indices=[1, 40, 4096, 128],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T197 = fd.ops.cast(T196, dtype=DataType.Float)
        T198 = fd.ops.neg(T197)
        T199 = fd.ops.cast(T198, dtype=DataType.BFloat16)
        S200 = fd.define_scalar(1.00000, dtype=DataType.Double)
        T201 = fd.ops.add(S200, T95)
        T202 = fd.ops.mul(T128, T40)
        T203 = fd.ops.cat([T163, T144], dim=-1, manual_padding=0)
        T204 = fd.ops.mul(T164, T40)
        T205 = fd.ops.cat([T199, T180], dim=-1, manual_padding=0)
        T206 = fd.ops.reciprocal(T201)
        T207 = fd.ops.cast(T203, dtype=DataType.Float)
        T208 = fd.ops.cast(T205, dtype=DataType.Float)
        T209 = fd.ops.mul(T207, T47)
        T210 = fd.ops.mul(T208, T47)
        T211 = fd.ops.mul(T66, T206)
        T212 = fd.ops.add(T202, T209)
        T213 = fd.ops.add(T204, T210)
        T214 = fd.ops.mul(T211, T67)
        T215 = fd.ops.cast(T212, dtype=DataType.BFloat16)
        T216 = fd.ops.cast(T213, dtype=DataType.BFloat16)
        T217 = fd.ops.cast(T214, dtype=DataType.BFloat16)
        fd.add_output(T89)
        fd.add_output(T91)
        fd.add_output(T127)
        fd.add_output(T215)
        fd.add_output(T216)
        fd.add_output(T217)
        fd.add_output(T214)

    with FusionDefinition() as fd:
        fusion_func(fd)

    inputs = [
        torch.testing.make_tensor(
            (4096, 128),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (4096, 128),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 5120),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 640),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 640),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 16640),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 16640),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 5120),
            dtype=torch.float32,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 640),
            dtype=torch.float32,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 640),
            dtype=torch.float32,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 16640),
            dtype=torch.float32,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 16640),
            dtype=torch.float32,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
    ]
    nvfuser_direct_test.exec_nvfuser(fusion_func, inputs, validate_results=True)

def test_reshape_cancellation(nvfuser_direct_test):
    """
    Test for reshape cancellation operations with complex tensor manipulations.

    This test verifies complex tensor operations with BFloat16 and Float data types,
    including slice operations, concatenation, reshape, cast, broadcast, and mathematical operations.
    """

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[1, 2048, 24, 32],
            contiguity=[None, True, True, False],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        T1 = fd.define_tensor(
            shape=[1, 2048, 24, 32],


            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 16640),
            dtype=torch.float32,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 16640),
            dtype=torch.float32,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
    ]
    nvfuser_direct_test.exec_nvfuser(fusion_func, inputs, validate_results=True)

def test_reshape_cancellation(nvfuser_direct_test):
    """
    Test for reshape cancellation operations with complex tensor manipulations.

    This test verifies complex tensor operations with BFloat16 and Float data types,
    including slice operations, concatenation, reshape, cast, broadcast, and mathematical operations.
    """

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[1, 2048, 24, 32],
            contiguity=[None, True, True, False],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        T1 = fd.define_tensor(
            shape=[1, 2048, 24, 32],
            contiguity=[None, True, True, False],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        T2 = fd.define_tensor(
            shape=[1, 2048, 24, 32],
            contiguity=[None, True, True, False],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        T3 = fd.define_tensor(
            shape=[1, 2048, 4, 4608],
            contiguity=[None, True, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        T4 = fd.define_tensor(
            shape=[1, 2048, 24, 32],
            contiguity=[None, True, True, False],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        T5 = fd.define_tensor(
            shape=[1, 2048, 24, 64],
            contiguity=[None, True, None, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        T6 = fd.define_tensor(
            shape=[1, 2048, 24, 64],
            contiguity=[None, True, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        T7 = fd.define_tensor(
            shape=[1, 2048, 24, 64],
            contiguity=[None, True, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        T8 = fd.ops.cast(T0, dtype=DataType.Float)
        T9 = fd.ops.neg(T8)
        T10 = fd.ops.cast(T9, dtype=DataType.BFloat16)
        T17 = fd.ops.broadcast_in_dim(
            T1, shape=[1, 2048, 24, 32, 1], broadcast_dims=[0, 1, 2, 3]
        )
        T24 = fd.ops.broadcast_in_dim(
            T10, shape=[1, 2048, 24, 32, 1], broadcast_dims=[0, 1, 2, 3]
        )
        T25 = fd.ops.cast(T2, dtype=DataType.Float)
        T41 = fd.ops.slice(
            T3,
            start_indices=[0, 0, 0, 3072],
            end_indices=[1, 2048, 4, 4608],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T42 = fd.ops.cat([T24, T17], dim=-1, manual_padding=0)
        T43 = fd.ops.neg(T25)
        T50 = fd.ops.reshape(T41, new_shape=[1, 2048, 4, 6, 256])
        T56 = fd.ops.reshape(T42, new_shape=[1, 2048, 24, 64])
        T57 = fd.ops.cast(T43, dtype=DataType.BFloat16)
        T63 = fd.ops.reshape(T50, new_shape=[1, 2048, 24, 256])
        T64 = fd.ops.cast(T56, dtype=DataType.Float)
        T71 = fd.ops.broadcast_in_dim(
            T4, shape=[1, 2048, 24, 32, 1], broadcast_dims=[0, 1, 2, 3]
        )
        T78 = fd.ops.broadcast_in_dim(
            T57, shape=[1, 2048, 24, 32, 1], broadcast_dims=[0, 1, 2, 3]
        )
        T94 = fd.ops.slice(
            T63,
            start_indices=[0, 0, 0, 64],
            end_indices=[1, 2048, 24, 256],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T95 = fd.ops.mul(T64, T5)
        T111 = fd.ops.slice(
            T3,
            start_indices=[0, 0, 0, 0],
            end_indices=[1, 2048, 4, 1536],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T112 = fd.ops.cat([T78, T71], dim=-1, manual_padding=0)
        T113 = fd.ops.cast(T94, dtype=DataType.Float)
        T114 = fd.ops.add(T6, T95)
        T121 = fd.ops.reshape(T111, new_shape=[1, 2048, 4, 6, 256])
        T127 = fd.ops.reshape(T112, new_shape=[1, 2048, 24, 64])
        T128 = fd.ops.cat([T114, T113], dim=-1, manual_padding=0)
        T134 = fd.ops.reshape(T121, new_shape=[1, 2048, 24, 256])
        T135 = fd.ops.cast(T127, dtype=DataType.Float)
        T136 = fd.ops.permute(T128, dims=[0, 2, 1, 3])
        T152 = fd.ops.slice(
            T134,
            start_indices=[0, 0, 0, 64],
            end_indices=[1, 2048, 24, 256],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T153 = fd.ops.mul(T135, T5)
        T154 = fd.ops.cast(T136, dtype=DataType.BFloat16)
        T155 = fd.ops.cast(T152, dtype=DataType.Float)
        T156 = fd.ops.add(T7, T153)
        T157 = fd.ops.cat([T156, T155], dim=-1, manual_padding=0)
        T158 = fd.ops.permute(T136, dims=[0, 1, 3, 2])
        T159 = fd.ops.permute(T157, dims=[0, 2, 1, 3])
        fd.add_output(T159)
        fd.add_output(T154)
        fd.add_output(T158)

    with FusionDefinition() as fd:
        fusion_func(fd)

    inputs = [
        torch.randn(3145727, dtype=torch.bfloat16, device="cuda:0").as_strided(
            (1, 2048, 24, 32), (3145728, 1536, 64, 2)
        ),
        torch.randn(3145727, dtype=torch.bfloat16, device="cuda:0").as_strided(
            (1, 2048, 24, 32), (3145728, 1536, 64, 2)
        ),
        torch.randn(3145727, dtype=torch.bfloat16, device="cuda:0").as_strided(
            (1, 2048, 24, 32), (3145728, 1536, 64, 2)
        ),
        torch.testing.make_tensor(
            (1, 2048, 4, 4608),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.randn(3145727, dtype=torch.bfloat16, device="cuda:0").as_strided(
            (1, 2048, 24, 32), (3145728, 1536, 64, 2)
        ),
        torch.randn(131072, dtype=torch.float32, device="cuda:0").as_strided(
            (1, 2048, 24, 64), (131072, 64, 0, 1)
        ),
        torch.testing.make_tensor(
            (1, 2048, 24, 64),
            dtype=torch.float32,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 2048, 24, 64),
            dtype=torch.float32,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
    ]
    nvfuser_direct_test.exec_nvfuser(fusion_func, inputs, validate_results=True)

def test_reduction_reference_missing_input_ids(nvfuser_direct_test):
    """
    Test for issue 4840 - reduction reference missing input IDs.

    This test verifies complex tensor operations with Half and Float data types,
    including slice operations, concatenation, reshape, cast, broadcast, and mathematical operations.
    """

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[1, 16, 4096, 128],
            contiguity=[None, True, True, True],
            dtype=DataType.Half,
            is_cpu=False,
            stride_order=[3, 1, 2, 0],
        )
        T1 = fd.define_tensor(
            shape=[1, 4096, 16, 128],


            (1, 2048, 24, 64), (131072, 64, 0, 1)
        ),
        torch.testing.make_tensor(
            (1, 2048, 24, 64),
            dtype=torch.float32,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 2048, 24, 64),
            dtype=torch.float32,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
    ]
    nvfuser_direct_test.exec_nvfuser(fusion_func, inputs, validate_results=True)

def test_reduction_reference_missing_input_ids(nvfuser_direct_test):
    """
    Test for issue 4840 - reduction reference missing input IDs.

    This test verifies complex tensor operations with Half and Float data types,
    including slice operations, concatenation, reshape, cast, broadcast, and mathematical operations.
    """

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[1, 16, 4096, 128],
            contiguity=[None, True, True, True],
            dtype=DataType.Half,
            is_cpu=False,
            stride_order=[3, 1, 2, 0],
        )
        T1 = fd.define_tensor(
            shape=[1, 4096, 16, 128],
            contiguity=[None, True, True, True],
            dtype=DataType.Half,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        T2 = fd.define_tensor(
            shape=[1, 16, 4096, 128],
            contiguity=[None, True, True, True],
            dtype=DataType.Half,
            is_cpu=False,
            stride_order=[3, 1, 2, 0],
        )
        T3 = fd.define_tensor(
            shape=[1, 4096, 16, 128],
            contiguity=[None, True, None, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        T4 = fd.define_tensor(
            shape=[1, 4096, 6144],
            contiguity=[None, True, True],
            dtype=DataType.Half,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T5 = fd.define_tensor(
            shape=[1, 16, 4096, 128],
            contiguity=[None, True, True, True],
            dtype=DataType.Half,
            is_cpu=False,
            stride_order=[3, 1, 2, 0],
        )
        T6 = fd.define_tensor(
            shape=[1, 4096, 16, 128],
            contiguity=[None, True, None, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        T7 = fd.define_tensor(
            shape=[1, 4096, 16, 128],
            contiguity=[None, True, True, True],
            dtype=DataType.Half,
            is_cpu=False,
            stride_order=[3, 2, 1, 0],
        )
        T8 = fd.ops.permute(T0, dims=[0, 2, 1, 3])
        T9 = fd.ops.cast(T8, dtype=DataType.Float)
        T10 = fd.ops.cast(T1, dtype=DataType.Float)
        T11 = fd.ops.add(T10, T9)
        T12 = fd.ops.permute(T2, dims=[0, 2, 1, 3])
        T13 = fd.ops.cast(T12, dtype=DataType.Float)
        T29 = fd.ops.slice(
            T11,
            start_indices=[0, 0, 0, 0],
            end_indices=[1, 4096, 16, 128],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T45 = fd.ops.slice(
            T13,
            start_indices=[0, 0, 0, 0],
            end_indices=[1, 4096, 16, 128],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T46 = fd.ops.mul(T3, T29)
        T47 = fd.ops.mul(T3, T45)
        T63 = fd.ops.slice(
            T46,
            start_indices=[0, 0, 0, 0],
            end_indices=[1, 4096, 16, 64],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T79 = fd.ops.slice(
            T47,
            start_indices=[0, 0, 0, 0],
            end_indices=[1, 4096, 16, 64],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T95 = fd.ops.slice(
            T46,
            start_indices=[0, 0, 0, 64],
            end_indices=[1, 4096, 16, 128],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T96 = fd.ops.neg(T63)
        T112 = fd.ops.slice(
            T47,
            start_indices=[0, 0, 0, 64],
            end_indices=[1, 4096, 16, 128],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T113 = fd.ops.neg(T79)
        T120 = fd.ops.broadcast_in_dim(
            T95, shape=[1, 4096, 16, 1, 64], broadcast_dims=[0, 1, 2, 4]
        )
        T127 = fd.ops.broadcast_in_dim(
            T96, shape=[1, 4096, 16, 1, 64], broadcast_dims=[0, 1, 2, 4]
        )
        T134 = fd.ops.broadcast_in_dim(
            T112, shape=[1, 4096, 16, 1, 64], broadcast_dims=[0, 1, 2, 4]
        )
        T141 = fd.ops.broadcast_in_dim(
            T113, shape=[1, 4096, 16, 1, 64], broadcast_dims=[0, 1, 2, 4]
        )
        S142 = fd.define_scalar(0.00000, dtype=DataType.Double)
        T154 = fd.ops.pad(T120, [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], S142)
        S155 = fd.define_scalar(0.00000, dtype=DataType.Double)
        T167 = fd.ops.pad(T127, [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], S155)
        S168 = fd.define_scalar(0.00000, dtype=DataType.Double)
        T180 = fd.ops.pad(T134, [0, 0, 0, 1, 0, 0, 0, 0, 0, 0], S168)
        S181 = fd.define_scalar(0.00000, dtype=DataType.Double)
        T193 = fd.ops.pad(T141, [0, 0, 1, 0, 0, 0, 0, 0, 0, 0], S181)
        T206 = fd.ops.slice(
            T4,
            start_indices=[0, 0, 0],
            end_indices=[1, 4096, 2048],
            strides=[1, 1, 1],
            manual_normalization=0,
        )
        T219 = fd.ops.slice(
            T4,
            start_indices=[0, 0, 2048],
            end_indices=[1, 4096, 4096],
            strides=[1, 1, 1],
            manual_normalization=0,
        )
        T220 = fd.ops.add(T167, T154)
        T221 = fd.ops.add(T193, T180)
        T227 = fd.ops.reshape(T206, new_shape=[1, 4096, 16, 128])
        T233 = fd.ops.reshape(T219, new_shape=[1, 4096, 16, 128])
        T234 = fd.ops.permute(T5, dims=[0, 2, 1, 3])
        S235 = fd.define_scalar(0, dtype=DataType.Int)
        T241 = fd.ops.full(
            shape=[1, 4096, 16, 0], fill_value=S235, dtype=DataType.Float
        )
        T242 = fd.ops.mul(T6, T29)
        T248 = fd.ops.reshape(T220, new_shape=[1, 4096, 16, 128])
        T249 = fd.ops.mul(T6, T45)
        T255 = fd.ops.reshape(T221, new_shape=[1, 4096, 16, 128])
        T256 = fd.ops.cast(T227, dtype=DataType.Float)
        T257 = fd.ops.cast(T233, dtype=DataType.Float)
        T258 = fd.ops.cast(T234, dtype=DataType.Float)
        T259 = fd.ops.cast(T7, dtype=DataType.Float)
        S260 = fd.define_scalar(0.00000, dtype=DataType.Double)
        T270 = fd.ops.pad(T241, [0, 128, 0, 0, 0, 0, 0, 0], S260)
        T271 = fd.ops.add(T248, T242)
        T272 = fd.ops.add(T255, T249)
        T279 = fd.ops.reshape(T256, new_shape=[1, 4096, 16, 2, 64])
        T286 = fd.ops.reshape(T257, new_shape=[1, 4096, 16, 2, 64])
        T287 = fd.ops.add(T259, T258)
        T288 = fd.ops.add(T271, T270)
        T289 = fd.ops.add(T272, T270)
        T308 = fd.ops.slice(
            T279,
            start_indices=[0, 0, 0, 1, 0],
            end_indices=[1, 4096, 16, 2, 64],
            strides=[1, 1, 1, 1, 1],
            manual_normalization=0,
        )
        T327 = fd.ops.slice(
            T286,
            start_indices=[0, 0, 0, 1, 0],
            end_indices=[1, 4096, 16, 2, 64],
            strides=[1, 1, 1, 1, 1],
            manual_normalization=0,
        )
        T328 = fd.ops.cast(T287, dtype=DataType.Half)
        T329 = fd.ops.cast(T288, dtype=DataType.Half)
        T330 = fd.ops.cast(T289, dtype=DataType.Half)
        T349 = fd.ops.slice(
            T279,
            start_indices=[0, 0, 0, 0, 0],
            end_indices=[1, 4096, 16, 1, 64],
            strides=[1, 1, 1, 1, 1],
            manual_normalization=0,
        )
        T350 = fd.ops.squeeze(T308, dims=[3], squeeze_expanded=False)
        T369 = fd.ops.slice(
            T286,
            start_indices=[0, 0, 0, 0, 0],
            end_indices=[1, 4096, 16, 1, 64],
            strides=[1, 1, 1, 1, 1],
            manual_normalization=0,
        )
        T370 = fd.ops.squeeze(T327, dims=[3], squeeze_expanded=False)
        T375 = fd.ops.reshape(T328, new_shape=[1, 4096, 2048])
        T380 = fd.ops.reshape(T329, new_shape=[1, 4096, 2048])
        T385 = fd.ops.reshape(T330, new_shape=[1, 4096, 2048])
        T386 = fd.ops.squeeze(T349, dims=[3], squeeze_expanded=False)
        T387 = fd.ops.neg(T350)
        T388 = fd.ops.squeeze(T369, dims=[3], squeeze_expanded=False)
        T389 = fd.ops.neg(T370)
        T390 = fd.ops.cat([T385, T380, T375], dim=2, manual_padding=0)
        T391 = fd.ops.cat([T387, T386], dim=-1, manual_padding=0)
        T392 = fd.ops.cat([T389, T388], dim=-1, manual_padding=0)
        T393 = fd.ops.cast(T390, dtype=DataType.Float)
        T394 = fd.ops.mul(T256, T45)
        T395 = fd.ops.mul(T257, T29)
        T396 = fd.ops.mul(T391, T45)
        T397 = fd.ops.mul(T392, T29)
        S398 = fd.define_scalar(2.00000, dtype=DataType.Double)
        T399 = fd.ops.mul(S398, T393)
        T400 = fd.ops.sum(T394, dims=[0, 2], keepdim=False, dtype=DataType.Null)
        T401 = fd.ops.sum(T395, dims=[0, 2], keepdim=False, dtype=DataType.Null)
        T402 = fd.ops.sum(T396, dims=[0, 2], keepdim=False, dtype=DataType.Null)
        T403 = fd.ops.sum(T397, dims=[0, 2], keepdim=False, dtype=DataType.Null)
        T407 = fd.ops.reshape(T399, new_shape=[4096, 6144])
        T413 = fd.ops.broadcast_in_dim(
            T400, shape=[1, 4096, 1, 128], broadcast_dims=[1, 3]
        )
        T419 = fd.ops.broadcast_in_dim(
            T401, shape=[1, 4096, 1, 128], broadcast_dims=[1, 3]
        )
        T425 = fd.ops.broadcast_in_dim(
            T402, shape=[1, 4096, 1, 128], broadcast_dims=[1, 3]
        )
        T431 = fd.ops.broadcast_in_dim(
            T403, shape=[1, 4096, 1, 128], broadcast_dims=[1, 3]
        )
        T432 = fd.ops.permute(T407, dims=[1, 0])
        T436 = fd.ops.reshape(T390, new_shape=[4096, 6144])
        T437 = fd.ops.add(T419, T413)
        T438 = fd.ops.add(T431, T425)
        fd.add_output(T432)
        fd.add_output(T407)
        fd.add_output(T436)
        fd.add_output(T437)
        fd.add_output(T438)

    with FusionDefinition() as fd:
        fusion_func(fd)

    # input range is revised to [-1, 1] to avoid small validation errors
    # which highly likely caused by the strict tolerance values in
    # ValidationConstants.
    inputs = [
        torch.testing.make_tensor(
            (8388608,), dtype=torch.float16, device="cuda:0", low=-1, high=1
        ).as_strided((1, 16, 4096, 128), (8388608, 128, 2048, 1)),
        torch.testing.make_tensor(
            (1, 4096, 16, 128), dtype=torch.float16, device="cuda:0", low=-1, high=1
        ),
        torch.testing.make_tensor(
            (8388608,), dtype=torch.float16, device="cuda:0", low=-1, high=1
        ).as_strided((1, 16, 4096, 128), (8388608, 128, 2048, 1)),
        torch.testing.make_tensor(
            (524288,), dtype=torch.float32, device="cuda:0", low=-1, high=1
        ).as_strided((1, 4096, 16, 128), (1048576, 128, 0, 1)),
        torch.testing.make_tensor(
            (1, 4096, 6144), dtype=torch.float16, device="cuda:0", low=-1, high=1
        ),
        torch.testing.make_tensor(
            (8388608,), dtype=torch.float16, device="cuda:0", low=-1, high=1
        ).as_strided((1, 16, 4096, 128), (8388608, 128, 2048, 1)),
        torch.testing.make_tensor(
            (524288,), dtype=torch.float32, device="cuda:0", low=-1, high=1
        ).as_strided((1, 4096, 16, 128), (1048576, 128, 0, 1)),
        torch.testing.make_tensor(
            (1, 4096, 16, 128), dtype=torch.float16, device="cuda:0", low=-1, high=1
        ),
    ]
    nvfuser_direct_test.exec_nvfuser(fusion_func, inputs, validate_results=True)

def test_ws_tma_normalization6(nvfuser_direct_test):
    """
    Test for scalar input handling in TMA normalization.

    This test verifies complex tensor operations with BFloat16 and Float data types,
    including scalar tensor operations, reshape, cast, broadcast, and mathematical operations.
    """
    skip_if_global_memory_below_gb(32)

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[3072],
            contiguity=[True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[0],
        )
        T1 = fd.define_tensor(


        ).as_strided((1, 16, 4096, 128), (8388608, 128, 2048, 1)),
        torch.testing.make_tensor(
            (524288,), dtype=torch.float32, device="cuda:0", low=-1, high=1
        ).as_strided((1, 4096, 16, 128), (1048576, 128, 0, 1)),
        torch.testing.make_tensor(
            (1, 4096, 6144), dtype=torch.float16, device="cuda:0", low=-1, high=1
        ),
        torch.testing.make_tensor(
            (8388608,), dtype=torch.float16, device="cuda:0", low=-1, high=1
        ).as_strided((1, 16, 4096, 128), (8388608, 128, 2048, 1)),
        torch.testing.make_tensor(
            (524288,), dtype=torch.float32, device="cuda:0", low=-1, high=1
        ).as_strided((1, 4096, 16, 128), (1048576, 128, 0, 1)),
        torch.testing.make_tensor(
            (1, 4096, 16, 128), dtype=torch.float16, device="cuda:0", low=-1, high=1
        ),
    ]
    nvfuser_direct_test.exec_nvfuser(fusion_func, inputs, validate_results=True)

def test_ws_tma_normalization6(nvfuser_direct_test):
    """
    Test for scalar input handling in TMA normalization.

    This test verifies complex tensor operations with BFloat16 and Float data types,
    including scalar tensor operations, reshape, cast, broadcast, and mathematical operations.
    """
    skip_if_global_memory_below_gb(32)

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[3072],
            contiguity=[True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[0],
        )
        T1 = fd.define_tensor(
            shape=[1, 4096, 3072],
            contiguity=[None, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T2 = fd.define_tensor(
            shape=[1, 4096, 3072],
            contiguity=[None, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T3 = fd.define_tensor(
            shape=[1, 4096, 1],
            contiguity=[None, True, None],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T4 = fd.define_tensor(
            shape=[1, 4096, 3072],
            contiguity=[None, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T5 = fd.define_tensor(
            shape=[], contiguity=[], dtype=DataType.Float, is_cpu=False
        )
        T6 = fd.ops.cast(T0, dtype=DataType.Float)
        S7 = fd.define_scalar(1.00000, dtype=DataType.Double)
        T8 = fd.ops.add(S7, T6)
        T9 = fd.ops.cast(T1, dtype=DataType.Float)
        T14 = fd.ops.broadcast_in_dim(T8, shape=[1, 4096, 3072], broadcast_dims=[2])
        T15 = fd.ops.mul(T14, T9)
        T16 = fd.ops.cast(T2, dtype=DataType.Float)
        T17 = fd.ops.mul(T16, T15)
        T18 = fd.ops.sum(T17, dims=[0, 2], keepdim=False, dtype=DataType.Null)
        T23 = fd.ops.broadcast_in_dim(T18, shape=[1, 4096, 1], broadcast_dims=[1])
        S24 = fd.define_scalar(3.00000, dtype=DataType.Double)
        T25 = fd.ops.pow(T3, S24)
        S26 = fd.define_scalar(-0.500000, dtype=DataType.Double)
        T27 = fd.ops.mul(S26, T23)
        T28 = fd.ops.mul(T27, T25)
        S29 = fd.define_scalar(3072.00, dtype=DataType.Double)
        S30 = fd.ops.reciprocal(S29)
        T31 = fd.ops.mul(T28, S30)
        T32 = fd.ops.sum(T31, dims=[0, 2], keepdim=False, dtype=DataType.Null)
        T36 = fd.ops.broadcast_in_dim(T32, shape=[1, 4096], broadcast_dims=[1])
        T41 = fd.ops.broadcast_in_dim(T36, shape=[1, 4096, 1], broadcast_dims=[0, 1])
        T46 = fd.ops.broadcast_in_dim(
            T41, shape=[1, 4096, 3072], broadcast_dims=[0, 1, 2]
        )
        T51 = fd.ops.broadcast_in_dim(
            T3, shape=[1, 4096, 3072], broadcast_dims=[0, 1, 2]
        )
        T52 = fd.ops.mul(T16, T46)
        T53 = fd.ops.mul(T51, T15)
        T54 = fd.ops.add(T53, T52)
        T55 = fd.ops.add(T54, T52)
        T56 = fd.ops.cast(T4, dtype=DataType.Float)
        T57 = fd.ops.mul(T16, T51)
        T58 = fd.ops.add(T56, T55)
        T59 = fd.ops.mul(T57, T9)
        T60 = fd.ops.sum(T59, dims=[0, 1], keepdim=False, dtype=DataType.Null)
        T61 = fd.ops.cast(T60, dtype=DataType.BFloat16)
        T62 = fd.ops.mul(T5, T58)
        T63 = fd.ops.cast(T62, dtype=DataType.BFloat16)
        fd.add_output(T61)
        fd.add_output(T63)

    with FusionDefinition() as fd:
        fusion_func(fd)

    inputs = [
        torch.testing.make_tensor(
            (3072,),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 3072),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 3072),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 1),
            dtype=torch.float32,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 3072),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (),
            dtype=torch.float32,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
    ]
    nvfuser_direct_test.exec_nvfuser(fusion_func, inputs, validate_results=True)

def test_domain_map_hang(nvfuser_direct_test):
    """
    Test for issue 4960 - domain map hang in complex tensor operations.

    This test verifies complex tensor operations with Float and BFloat16 data types,
    including iota operations, broadcast, mathematical operations, index_select, reshape, cast, slice operations, and concatenation.
    """

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[16],
            contiguity=[True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[0],
        )
        T1 = fd.define_tensor(
            shape=[16],


            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (1, 4096, 3072),
            dtype=torch.bfloat16,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
        torch.testing.make_tensor(
            (),
            dtype=torch.float32,
            device="cuda:0",
            low=LOW_VAL,
            high=HIGH_VAL,
        ),
    ]
    nvfuser_direct_test.exec_nvfuser(fusion_func, inputs, validate_results=True)

def test_domain_map_hang(nvfuser_direct_test):
    """
    Test for issue 4960 - domain map hang in complex tensor operations.

    This test verifies complex tensor operations with Float and BFloat16 data types,
    including iota operations, broadcast, mathematical operations, index_select, reshape, cast, slice operations, and concatenation.
    """

    def fusion_func(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[16],
            contiguity=[True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[0],
        )
        T1 = fd.define_tensor(
            shape=[16],
            contiguity=[True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[0],
        )
        T2 = fd.define_tensor(
            shape=[4096, 4096],
            contiguity=[True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[1, 0],
        )
        T3 = fd.define_tensor(
            shape=[1, 4096, 2560],
            contiguity=[None, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T4 = fd.define_tensor(
            shape=[1, 4096, 2560],
            contiguity=[None, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T5 = fd.define_tensor(
            shape=[1, 4096, 2560],
            contiguity=[None, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T6 = fd.define_tensor(
            shape=[1, 4096, 10240],
            contiguity=[None, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T7 = fd.define_tensor(
            shape=[1, 4096, 2560],
            contiguity=[None, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T8 = fd.define_tensor(
            shape=[1, 4096, 2560],
            contiguity=[None, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T9 = fd.define_tensor(
            shape=[1, 4096, 2560],
            contiguity=[None, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        T10 = fd.define_tensor(
            shape=[1, 4096, 10240],
            contiguity=[None, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        S11 = fd.define_scalar(4096, dtype=DataType.Int)
        S12 = fd.define_scalar(0, dtype=DataType.Int)
        S13 = fd.define_scalar(1, dtype=DataType.Int)
        T14 = fd.ops.iota(S11, S12, S13, dtype=DataType.Int)
        T18 = fd.ops.broadcast_in_dim(T14, shape=[1, 4096], broadcast_dims=[1])
        T19 = fd.ops.cast(T14, dtype=DataType.Float)
        T23 = fd.ops.broadcast_in_dim(T19, shape=[4096, 1], broadcast_dims=[0])
        T27 = fd.ops.broadcast_in_dim(T0, shape=[1, 16], broadcast_dims=[1])
        T31 = fd.ops.broadcast_in_dim(T23, shape=[4096, 16], broadcast_dims=[0, 1])
        T35 = fd.ops.broadcast_in_dim(T27, shape=[4096, 16], broadcast_dims=[0, 1])
        T39 = fd.ops.broadcast_in_dim(T1, shape=[1, 16], broadcast_dims=[1])
        T43 = fd.ops.broadcast_in_dim(T39, shape=[4096, 16], broadcast_dims=[0, 1])
        S44 = fd.define_scalar(0, dtype=DataType.Int)
        T45 = fd.ops.lt(T18, S44)
        T46 = fd.ops.mul(T31, T35)
        T47 = fd.ops.mul(T31, T43)
        S48 = fd.define_scalar(4096, dtype=DataType.Int)
        S49 = fd.define_scalar(0, dtype=DataType.Int)
        T50 = fd.ops.where(T45, S48, S49)
        T51 = fd.ops.cast(T50, dtype=DataType.Int)
        T52 = fd.ops.cat([T46, T46], dim=-1, manual_padding=0)
        T53 = fd.ops.cat([T47, T47], dim=-1, manual_padding=0)
        T54 = fd.ops.add(T18, T51)
        T55 = fd.ops.cos(T52)
        T56 = fd.ops.sin(T52)
        T57 = fd.ops.cos(T53)
        T58 = fd.ops.sin(T53)
        T59 = fd.ops.cast(T55, dtype=DataType.BFloat16)
        T60 = fd.ops.cast(T56, dtype=DataType.BFloat16)
        T63 = fd.ops.reshape(T54, new_shape=[4096])
        T64 = fd.ops.cast(T57, dtype=DataType.BFloat16)
        T65 = fd.ops.cast(T58, dtype=DataType.BFloat16)
        T66 = fd.ops.index_select(T59, T63, dim=0)
        T67 = fd.ops.index_select(T60, T63, dim=0)
        T68 = fd.ops.index_select(T64, T63, dim=0)
        T69 = fd.ops.index_select(T65, T63, dim=0)
        T74 = fd.ops.reshape(T66, new_shape=[1, 4096, 32])
        T80 = fd.ops.broadcast_in_dim(
            T74, shape=[1, 1, 4096, 32], broadcast_dims=[0, 2, 3]
        )
        T85 = fd.ops.reshape(T67, new_shape=[1, 4096, 32])
        T91 = fd.ops.broadcast_in_dim(
            T85, shape=[1, 1, 4096, 32], broadcast_dims=[0, 2, 3]
        )
        T97 = fd.ops.broadcast_in_dim(
            T80, shape=[1, 32, 4096, 32], broadcast_dims=[0, 1, 2, 3]
        )
        T98 = fd.ops.cast(T97, dtype=DataType.Float)
        T104 = fd.ops.broadcast_in_dim(
            T91, shape=[1, 32, 4096, 32], broadcast_dims=[0, 1, 2, 3]
        )
        T105 = fd.ops.cast(T104, dtype=DataType.Float)
        T110 = fd.ops.reshape(T68, new_shape=[1, 4096, 32])
        T116 = fd.ops.broadcast_in_dim(
            T110, shape=[1, 1, 4096, 32], broadcast_dims=[0, 2, 3]
        )
        T121 = fd.ops.reshape(T69, new_shape=[1, 4096, 32])
        T127 = fd.ops.broadcast_in_dim(
            T121, shape=[1, 1, 4096, 32], broadcast_dims=[0, 2, 3]
        )
        T133 = fd.ops.broadcast_in_dim(
            T116, shape=[1, 32, 4096, 32], broadcast_dims=[0, 1, 2, 3]
        )
        T134 = fd.ops.cast(T133, dtype=DataType.Float)
        T140 = fd.ops.broadcast_in_dim(
            T127, shape=[1, 32, 4096, 32], broadcast_dims=[0, 1, 2, 3]
        )
        T141 = fd.ops.cast(T140, dtype=DataType.Float)
        T142 = fd.ops.cast(T2, dtype=DataType.BFloat16)
        T143 = fd.ops.cast(T3, dtype=DataType.Float)
        T144 = fd.ops.cast(T4, dtype=DataType.Float)
        T145 = fd.ops.cast(T5, dtype=DataType.Float)
        T146 = fd.ops.cast(T6, dtype=DataType.Float)
        S147 = fd.define_scalar(2.00000, dtype=DataType.Double)
        T148 = fd.ops.mul(T7, S147)
        S149 = fd.define_scalar(2.00000, dtype=DataType.Double)
        T150 = fd.ops.mul(T8, S149)
        S151 = fd.define_scalar(2.00000, dtype=DataType.Double)
        T152 = fd.ops.mul(T9, S151)
        S153 = fd.define_scalar(2.00000, dtype=DataType.Double)
        T154 = fd.ops.mul(T10, S153)
        T155 = fd.ops.add(T143, T148)
        T156 = fd.ops.add(T144, T150)
        T157 = fd.ops.add(T145, T152)
        T158 = fd.ops.add(T146, T154)
        T159 = fd.ops.cast(T155, dtype=DataType.BFloat16)
        T160 = fd.ops.cast(T156, dtype=DataType.BFloat16)
        T161 = fd.ops.cast(T157, dtype=DataType.BFloat16)
        T167 = fd.ops.reshape(T159, new_shape=[1, 4096, 32, 80])
        T173 = fd.ops.reshape(T160, new_shape=[1, 4096, 32, 80])
        T179 = fd.ops.reshape(T161, new_shape=[1, 4096, 32, 80])
        T180 = fd.ops.cast(T158, dtype=DataType.BFloat16)
        T181 = fd.ops.permute(T167, dims=[0, 2, 1, 3])
        T182 = fd.ops.permute(T173, dims=[0, 2, 1, 3])
        T183 = fd.ops.permute(T179, dims=[0, 2, 1, 3])
        S184 = fd.define_scalar(0.500000, dtype=DataType.Double)
        T185 = fd.ops.mul(S184, T158)
        S186 = fd.define_scalar(3.00000, dtype=DataType.Double)
        T187 = fd.ops.pow(T158, S186)
        T203 = fd.ops.slice(
            T181,
            start_indices=[0, 0, 0, 0],
            end_indices=[1, 32, 4096, 32],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T219 = fd.ops.slice(
            T181,
            start_indices=[0, 0, 0, 32],
            end_indices=[1, 32, 4096, 80],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T235 = fd.ops.slice(
            T182,
            start_indices=[0, 0, 0, 0],
            end_indices=[1, 32, 4096, 32],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T251 = fd.ops.slice(
            T182,
            start_indices=[0, 0, 0, 32],
            end_indices=[1, 32, 4096, 80],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T252 = fd.ops.cast(T203, dtype=DataType.Float)
        T268 = fd.ops.slice(
            T203,
            start_indices=[0, 0, 0, 0],
            end_indices=[1, 32, 4096, 16],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T284 = fd.ops.slice(
            T203,
            start_indices=[0, 0, 0, 16],
            end_indices=[1, 32, 4096, 32],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T285 = fd.ops.cast(T284, dtype=DataType.Float)
        T286 = fd.ops.neg(T285)
        T287 = fd.ops.cast(T286, dtype=DataType.BFloat16)
        T288 = fd.ops.cast(T235, dtype=DataType.Float)
        T304 = fd.ops.slice(
            T235,
            start_indices=[0, 0, 0, 0],
            end_indices=[1, 32, 4096, 16],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T320 = fd.ops.slice(
            T235,
            start_indices=[0, 0, 0, 16],
            end_indices=[1, 32, 4096, 32],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T321 = fd.ops.cast(T320, dtype=DataType.Float)
        T322 = fd.ops.neg(T321)
        T323 = fd.ops.cast(T322, dtype=DataType.BFloat16)
        T324 = fd.ops.mul(T252, T98)
        T325 = fd.ops.cat([T287, T268], dim=-1, manual_padding=0)
        T326 = fd.ops.mul(T288, T98)
        T327 = fd.ops.cat([T323, T304], dim=-1, manual_padding=0)
        S328 = fd.define_scalar(0.0447150, dtype=DataType.Double)
        T329 = fd.ops.mul(S328, T187)
        T330 = fd.ops.cast(T325, dtype=DataType.Float)
        T331 = fd.ops.cast(T327, dtype=DataType.Float)
        T332 = fd.ops.mul(T330, T105)
        T333 = fd.ops.mul(T331, T105)
        T334 = fd.ops.add(T158, T329)
        T335 = fd.ops.add(T324, T332)
        T336 = fd.ops.add(T326, T333)
        S337 = fd.define_scalar(0.797885, dtype=DataType.Double)
        T338 = fd.ops.mul(S337, T334)
        T339 = fd.ops.cast(T335, dtype=DataType.BFloat16)
        T340 = fd.ops.cast(T336, dtype=DataType.BFloat16)
        T341 = fd.ops.cat([T339, T219], dim=-1, manual_padding=0)
        T342 = fd.ops.cat([T340, T251], dim=-1, manual_padding=0)
        T343 = fd.ops.tanh(T338)
        T344 = fd.ops.cast(T341, dtype=DataType.Float)
        T345 = fd.ops.cast(T342, dtype=DataType.Float)
        T346 = fd.ops.permute(T345, dims=[0, 1, 3, 2])
        S347 = fd.define_scalar(1.00000, dtype=DataType.Double)
        T348 = fd.ops.add(S347, T343)
        T349 = fd.ops.mul(T185, T348)
        T350 = fd.ops.cast(T349, dtype=DataType.BFloat16)
        fd.add_output(T59)
        fd.add_output(T60)
        fd.add_output(T64)
        fd.add_output(T65)
        fd.add_output(T80)
        fd.add_output(T91)
        fd.add_output(T134)
        fd.add_output(T141)
        fd.add_output(T142)
        fd.add_output(T180)
        fd.add_output(T183)
        fd.add_output(T342)
        fd.add_output(T344)
        fd.add_output(T346)
        fd.add_output(T350)
        fd.add_output(T349)

    with FusionDefinition() as fd:
        fusion_func(fd)

    inputs = [
        torch.testing.make_tensor((16,), dtype=torch.float32, device="cuda:0"),
        torch.testing.make_tensor((16,), dtype=torch.float32, device="cuda:0"),
        torch.testing.make_tensor((4096, 4096), dtype=torch.float32, device="cuda:0"),
        torch.testing.make_tensor(
            (1, 4096, 2560), dtype=torch.bfloat16, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (1, 4096, 2560), dtype=torch.bfloat16, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (1, 4096, 2560), dtype=torch.bfloat16, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (1, 4096, 10240), dtype=torch.bfloat16, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (1, 4096, 2560), dtype=torch.float32, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (1, 4096, 2560), dtype=torch.float32, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (1, 4096, 2560), dtype=torch.float32, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (1, 4096, 10240), dtype=torch.float32, device="cuda:0"
        ),
    ]
    nvfuser_direct_test.exec_nvfuser(fusion_func, inputs, validate_results=True)

def test_shared_memory_usage(nvfuser_direct_test):
    # repro of benchmarks.python.test_dropout_rmsnorm_bwd.test_dropout_rmsnorm_bwd_nvf_benchmark[dtype=torch.bfloat16-size=[16384_24578]]
    # avoid overflow of shared memory usage since previous version didn't consider static smem size
    def nvfuser_fusion_id0(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        T1 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        T2 = fd.define_tensor(
            shape=[-1, -1], contiguity=[True, True], dtype=DataType.Bool, is_cpu=False


        ),
        torch.testing.make_tensor(
            (1, 4096, 10240), dtype=torch.bfloat16, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (1, 4096, 2560), dtype=torch.float32, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (1, 4096, 2560), dtype=torch.float32, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (1, 4096, 2560), dtype=torch.float32, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (1, 4096, 10240), dtype=torch.float32, device="cuda:0"
        ),
    ]
    nvfuser_direct_test.exec_nvfuser(fusion_func, inputs, validate_results=True)

def test_shared_memory_usage(nvfuser_direct_test):
    # repro of benchmarks.python.test_dropout_rmsnorm_bwd.test_dropout_rmsnorm_bwd_nvf_benchmark[dtype=torch.bfloat16-size=[16384_24578]]
    # avoid overflow of shared memory usage since previous version didn't consider static smem size
    def nvfuser_fusion_id0(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        T1 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        T2 = fd.define_tensor(
            shape=[-1, -1], contiguity=[True, True], dtype=DataType.Bool, is_cpu=False
        )
        T3 = fd.define_tensor(
            shape=[-1, 1], contiguity=[True, None], dtype=DataType.Float, is_cpu=False
        )
        T4 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        T5 = fd.define_tensor(
            shape=[-1], contiguity=[True], dtype=DataType.BFloat16, is_cpu=False
        )
        T6 = fd.ops.cast(T1, dtype=DataType.Float)
        T7 = fd.ops.cast(T0, dtype=DataType.Float)
        T8 = fd.ops.cast(T2, dtype=DataType.Float)
        T9 = fd.ops.cast(T4, dtype=DataType.Float)
        T10 = fd.ops.cast(T5, dtype=DataType.Float)
        T11 = fd.ops.mul(T7, T8)
        S12 = fd.define_scalar(1.25000, dtype=DataType.Double)
        T13 = fd.ops.mul(T11, S12)
        T14 = fd.ops.add(T6, T13)
        V15 = fd.ops.shape(T7)
        T16 = fd.ops.broadcast_in_dim(T3, shape=V15, broadcast_dims=[0, 1])
        T17 = fd.ops.reciprocal(T16)
        T18 = fd.ops.mul(T14, T17)
        T19 = fd.ops.broadcast_in_dim(T10, shape=V15, broadcast_dims=[1])
        T20 = fd.ops.mul(T9, T18)
        T21 = fd.ops.mul(T9, T19)
        T22 = fd.ops.sum(T20, dims=[0], keepdim=False, dtype=DataType.Null)
        T23 = fd.ops.mul(T21, T17)
        T24 = fd.ops.neg(T21)
        T25 = fd.ops.mul(T24, T14)
        S26 = fd.define_scalar(2.00000, dtype=DataType.Double)
        T27 = fd.ops.pow(T16, S26)
        T28 = fd.ops.reciprocal(T27)
        T29 = fd.ops.mul(T25, T28)
        T30 = fd.ops.sum(T29, dims=[1], keepdim=False, dtype=DataType.Null)
        S31 = fd.ops.size(T7, dim=0)
        T34 = fd.ops.broadcast_in_dim(T30, shape=[S31, 1], broadcast_dims=[0])
        T35 = fd.ops.mul(S26, T3)
        T36 = fd.ops.reciprocal(T35)
        T37 = fd.ops.mul(T34, T36)
        S38 = fd.ops.size(T7, dim=1)
        S39 = fd.ops.reciprocal(S38)
        T40 = fd.ops.mul(T37, S39)
        T41 = fd.ops.sum(T40, dims=[1], keepdim=False, dtype=DataType.Null)
        T42 = fd.ops.broadcast_in_dim(T41, shape=[S31, 1], broadcast_dims=[0])
        T43 = fd.ops.broadcast_in_dim(T42, shape=V15, broadcast_dims=[0, 1])
        T44 = fd.ops.mul(T43, S26)
        T45 = fd.ops.mul(T44, T14)
        T46 = fd.ops.add(T23, T45)
        T47 = fd.ops.mul(T46, S12)
        T48 = fd.ops.mul(T47, T8)
        T49 = fd.ops.cast(T46, dtype=DataType.BFloat16)
        T50 = fd.ops.cast(T48, dtype=DataType.BFloat16)
        T51 = fd.ops.cast(T22, dtype=DataType.BFloat16)
        fd.add_output(T50)
        fd.add_output(T49)
        fd.add_output(T51)

    inputs = [
        torch.testing.make_tensor((16, 24578), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((16, 24578), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((16, 24578), dtype=torch.bool, device="cuda:0"),
        torch.testing.make_tensor((16, 1), dtype=torch.float32, device="cuda:0"),
        torch.testing.make_tensor((16, 24578), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((24578,), dtype=torch.bfloat16, device="cuda:0"),
    ]
    nvfuser_direct_test.exec_nvfuser(nvfuser_fusion_id0, inputs, validate_results=True)

def test_ca_map_concrete_loop_id(nvfuser_direct_test):
    def nvfuser_fusion_id10(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[16, 1, 1],
            contiguity=[True, None, None],
            dtype=DataType.Float,
            is_cpu=False,
        )
        T1 = fd.define_tensor(
            shape=[1, 1, 1024],
            contiguity=[None, None, True],
            dtype=DataType.Float,
            is_cpu=False,
        )
        T2 = fd.ops.sub(T0, T0)
        T3 = fd.ops.exp(T2)
        T4 = fd.ops.reciprocal(T3)
        T5 = fd.ops.mul(T3, T4)


        T47 = fd.ops.mul(T46, S12)
        T48 = fd.ops.mul(T47, T8)
        T49 = fd.ops.cast(T46, dtype=DataType.BFloat16)
        T50 = fd.ops.cast(T48, dtype=DataType.BFloat16)
        T51 = fd.ops.cast(T22, dtype=DataType.BFloat16)
        fd.add_output(T50)
        fd.add_output(T49)
        fd.add_output(T51)

    inputs = [
        torch.testing.make_tensor((16, 24578), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((16, 24578), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((16, 24578), dtype=torch.bool, device="cuda:0"),
        torch.testing.make_tensor((16, 1), dtype=torch.float32, device="cuda:0"),
        torch.testing.make_tensor((16, 24578), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((24578,), dtype=torch.bfloat16, device="cuda:0"),
    ]
    nvfuser_direct_test.exec_nvfuser(nvfuser_fusion_id0, inputs, validate_results=True)

def test_ca_map_concrete_loop_id(nvfuser_direct_test):
    def nvfuser_fusion_id10(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[16, 1, 1],
            contiguity=[True, None, None],
            dtype=DataType.Float,
            is_cpu=False,
        )
        T1 = fd.define_tensor(
            shape=[1, 1, 1024],
            contiguity=[None, None, True],
            dtype=DataType.Float,
            is_cpu=False,
        )
        T2 = fd.ops.sub(T0, T0)
        T3 = fd.ops.exp(T2)
        T4 = fd.ops.reciprocal(T3)
        T5 = fd.ops.mul(T3, T4)
        T6 = fd.ops.broadcast(T5, is_broadcast_dim=[False, False, True, False])
        S7 = fd.ops.size(T5, dim=1)
        S8 = fd.define_scalar(16, dtype=DataType.Int)
        S9 = fd.define_scalar(64, dtype=DataType.Int)
        T11 = fd.ops.reshape(T1, new_shape=[S7, S7, S8, S9])
        T12 = fd.ops.permute(T11, dims=[0, 2, 1, 3])
        T13 = fd.ops.squeeze(T12, dims=[0], squeeze_expanded=True)
        T14 = fd.ops.permute(T13, dims=[0, 2, 1])
        T15 = fd.ops.broadcast(T14, is_broadcast_dim=[False, True, False, False])
        T16 = fd.ops.mul(T6, T15)
        T17 = fd.ops.squeeze(T16, dims=[3], squeeze_expanded=True)
        T18 = fd.ops.broadcast(T17, is_broadcast_dim=[True, False, False, False])
        T19 = fd.ops.permute(T18, dims=[0, 2, 1, 3])
        S20 = fd.define_scalar(1024, dtype=DataType.Int)
        T22 = fd.ops.reshape(T19, new_shape=[S7, S7, S20])
        fd.add_output(T5)
        fd.add_output(T13)
        fd.add_output(T22)

    inputs = [
        torch.testing.make_tensor((16, 1, 1), dtype=torch.float32, device="cuda:0"),
        torch.testing.make_tensor((1, 1, 1024), dtype=torch.float32, device="cuda:0"),
    ]
    nvfuser_direct_test.exec_nvfuser(nvfuser_fusion_id10, inputs, validate_results=True)

def test_issue5377(nvfuser_direct_test):
    """
    Repro for issue 5377, where a traversal for vectorize validation
    failed to find an allocation ID that corresponds to a vectorized
    ID.
    """

    def nvfuser_fusion(fd: FusionDefinition) -> None:
        tv0 = fd.define_tensor(
            shape=[1, 16],
            contiguity=[None, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        tv1 = fd.define_tensor(
            shape=[1, 16],
            contiguity=[None, True],
            dtype=DataType.BFloat16,


        T13 = fd.ops.squeeze(T12, dims=[0], squeeze_expanded=True)
        T14 = fd.ops.permute(T13, dims=[0, 2, 1])
        T15 = fd.ops.broadcast(T14, is_broadcast_dim=[False, True, False, False])
        T16 = fd.ops.mul(T6, T15)
        T17 = fd.ops.squeeze(T16, dims=[3], squeeze_expanded=True)
        T18 = fd.ops.broadcast(T17, is_broadcast_dim=[True, False, False, False])
        T19 = fd.ops.permute(T18, dims=[0, 2, 1, 3])
        S20 = fd.define_scalar(1024, dtype=DataType.Int)
        T22 = fd.ops.reshape(T19, new_shape=[S7, S7, S20])
        fd.add_output(T5)
        fd.add_output(T13)
        fd.add_output(T22)

    inputs = [
        torch.testing.make_tensor((16, 1, 1), dtype=torch.float32, device="cuda:0"),
        torch.testing.make_tensor((1, 1, 1024), dtype=torch.float32, device="cuda:0"),
    ]
    nvfuser_direct_test.exec_nvfuser(nvfuser_fusion_id10, inputs, validate_results=True)

def test_issue5377(nvfuser_direct_test):
    """
    Repro for issue 5377, where a traversal for vectorize validation
    failed to find an allocation ID that corresponds to a vectorized
    ID.
    """

    def nvfuser_fusion(fd: FusionDefinition) -> None:
        tv0 = fd.define_tensor(
            shape=[1, 16],
            contiguity=[None, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        tv1 = fd.define_tensor(
            shape=[1, 16],
            contiguity=[None, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        tv2 = fd.define_tensor(
            shape=[1, 5120],
            contiguity=[None, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        tv3 = fd.define_tensor(
            shape=[16, 5120, 16384],
            contiguity=[True, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        tv4 = fd.define_tensor(
            shape=[16, 8192, 5120],
            contiguity=[True, True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        tv5 = fd.define_tensor(
            shape=[8192, 5120],
            contiguity=[True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        tv6 = fd.define_tensor(
            shape=[8192, 5120],
            contiguity=[True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        tv7 = fd.define_tensor(
            shape=[5120, 8192],
            contiguity=[True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        tv41 = fd.ops.linear(tv2, tv5)
        tv42 = fd.ops.cast(tv41, dtype=DataType.Float)
        tv43 = fd.ops.neg(tv42)
        tv44 = fd.ops.exp(tv43)
        tv45 = fd.ops.add(1.00000, tv44)
        tv46 = fd.ops.reciprocal(tv45)
        tv47 = fd.ops.mul(tv42, tv46)
        tv48 = fd.ops.linear(tv2, tv6)
        tv49 = fd.ops.cast(tv48, dtype=DataType.Float)
        tv50 = fd.ops.mul(tv47, tv49)
        tv51 = fd.ops.cast(tv50, dtype=DataType.BFloat16)
        tv52 = fd.ops.linear(tv51, tv7)
        tv15 = fd.ops.broadcast(tv2, is_broadcast_dim=[True, False, True, False])
        tv16 = fd.ops.expand(tv15, shape=[16, 1, 1, 5120])
        tv17 = fd.ops.squeeze(tv16, dims=[1, 2])
        tv22 = fd.ops.cast(tv17, dtype=DataType.Float)
        tv8 = fd.ops.cast(tv0, dtype=DataType.BFloat16)
        tv9 = fd.ops.cast(tv8, dtype=DataType.Float)
        tv10 = fd.ops.neg(tv9)
        tv11 = fd.ops.exp(tv10)
        tv12 = fd.ops.add(1.00000, tv11)
        tv13 = fd.ops.reciprocal(tv12)
        tv14 = fd.ops.cast(tv13, dtype=DataType.BFloat16)
        tv18 = fd.ops.squeeze(tv14, dims=[0])
        tv19 = fd.ops.broadcast(tv18, is_broadcast_dim=[False, True])
        tv20 = fd.ops.cast(tv19, dtype=DataType.BFloat16)
        tv21 = fd.ops.expand(tv20, shape=[16, 5120])
        tv23 = fd.ops.cast(tv21, dtype=DataType.Float)
        tv24 = fd.ops.mul(tv22, tv23)
        tv25 = fd.ops.cast(tv24, dtype=DataType.BFloat16)
        tv26 = fd.ops.broadcast(tv25, is_broadcast_dim=[False, True, False])
        tv27 = fd.ops.matmul(tv26, tv3)
        tv29 = fd.ops.slice(
            tv27,
            start_indices=[0, 0, 8192],
            end_indices=[16, 1, 16384],
            strides=[1, 1, 1],
            manual_normalization=True,
        )
        tv36 = fd.ops.cast(tv29, dtype=DataType.Float)
        tv28 = fd.ops.slice(
            tv27,
            start_indices=[0, 0, 0],
            end_indices=[16, 1, 8192],
            strides=[1, 1, 1],
            manual_normalization=True,
        )
        tv30 = fd.ops.cast(tv28, dtype=DataType.Float)
        tv31 = fd.ops.neg(tv30)
        tv32 = fd.ops.exp(tv31)
        tv33 = fd.ops.add(1.00000, tv32)
        tv34 = fd.ops.reciprocal(tv33)
        tv35 = fd.ops.mul(tv30, tv34)
        tv37 = fd.ops.mul(tv36, tv35)
        tv38 = fd.ops.cast(tv37, dtype=DataType.BFloat16)
        tv39 = fd.ops.matmul(tv38, tv4)
        tv40 = fd.ops.squeeze(tv39, dims=[1])
        tv53 = fd.ops.broadcast(tv40, is_broadcast_dim=[False, True, False])
        tv54 = fd.ops.cast(tv53, dtype=DataType.Float)
        tv55 = fd.ops.sum(tv54, dims=[0], dtype=DataType.Float)
        tv56 = fd.ops.cast(tv55, dtype=DataType.BFloat16)
        fd.add_output(tv8, tv1)
        fd.add_output(tv52)
        fd.add_output(tv56)

    inputs = [
        torch.testing.make_tensor((1, 16), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((1, 16), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((1, 5120), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor(
            (16, 5120, 16384), dtype=torch.bfloat16, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (16, 8192, 5120), dtype=torch.bfloat16, device="cuda:0"
        ),
        torch.testing.make_tensor((8192, 5120), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((8192, 5120), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((5120, 8192), dtype=torch.bfloat16, device="cuda:0"),
    ]

    nvfuser_direct_test.exec_nvfuser(nvfuser_fusion, inputs, validate_results=True)

    # https://github.com/NVIDIA/Fuser/issues/5346
    def test_register_aliasing_bug(nvfuser_direct_test):
        def nvfuser_fusion(fd: FusionDefinition, dim0, dim1) -> None:
            T0 = fd.define_tensor(
                shape=[dim1],
                contiguity=[True],
                dtype=DataType.Float,
                is_cpu=False,
                stride_order=[0],
            )
            T1 = fd.define_tensor(
                shape=[1, dim0, dim1],
                contiguity=[None, True, True],
                dtype=DataType.Float,
                is_cpu=False,
                stride_order=[2, 1, 0],
            )
            T2 = fd.define_tensor(
                shape=[1, dim0, dim1],
                contiguity=[None, True, True],
                dtype=DataType.Float,
                is_cpu=False,
                stride_order=[2, 1, 0],
            )
            T3 = fd.define_tensor(
                shape=[1, dim0, 1],
                contiguity=[None, True, None],
                dtype=DataType.Float,
                is_cpu=False,
                stride_order=[2, 1, 0],
            )
            T8 = fd.ops.broadcast_in_dim(T0, shape=[1, dim0, dim1], broadcast_dims=[2])
            T9 = fd.ops.mul(T8, T1)
            T10 = fd.ops.mul(T2, T9)
            T11 = fd.ops.sum(T10, dims=[0, 2], keepdim=False, dtype=DataType.Null)
            T16 = fd.ops.broadcast_in_dim(T11, shape=[1, dim0, 1], broadcast_dims=[1])
            S17 = fd.define_scalar(3.00000, dtype=DataType.Double)
            T18 = fd.ops.pow(T3, S17)
            S19 = fd.define_scalar(-0.500000, dtype=DataType.Double)
            T20 = fd.ops.mul(S19, T16)
            T21 = fd.ops.mul(T20, T18)
            S22 = fd.define_scalar(dim1, dtype=DataType.Double)
            S23 = fd.ops.reciprocal(S22)
            T24 = fd.ops.mul(T21, S23)
            T25 = fd.ops.sum(T24, dims=[0, 2], keepdim=False, dtype=DataType.Null)
            T29 = fd.ops.broadcast_in_dim(T25, shape=[1, dim0], broadcast_dims=[1])
            T34 = fd.ops.broadcast_in_dim(
                T29, shape=[1, dim0, 1], broadcast_dims=[0, 1]
            )
            T39 = fd.ops.broadcast_in_dim(
                T3, shape=[1, dim0, dim1], broadcast_dims=[0, 1, 2]
            )
            T44 = fd.ops.broadcast_in_dim(
                T34, shape=[1, dim0, dim1], broadcast_dims=[0, 1, 2]
            )
            T45 = fd.ops.mul(T2, T39)
            T46 = fd.ops.mul(T2, T44)
            T47 = fd.ops.mul(T39, T9)
            T48 = fd.ops.mul(T45, T1)
            T49 = fd.ops.add(T47, T46)
            T50 = fd.ops.add(T49, T46)
            T51 = fd.ops.sum(T48, dims=[0, 1], keepdim=False, dtype=DataType.Null)
            T55 = fd.ops.reshape(T50, new_shape=[dim0, dim1])
            fd.add_output(T50)
            fd.add_output(T55)
            fd.add_output(T51)

        dim0 = 2048
        dim1 = 32
        with FusionDefinition() as fd:
            nvfuser_fusion(fd, dim0, dim1)

        inputs = [
            torch.testing.make_tensor(
                (dim1,),
                dtype=torch.float32,
                device="cuda:0",
                low=LOW_VAL,
                high=HIGH_VAL,
            ),
            torch.testing.make_tensor(
                (1, dim0, dim1),
                dtype=torch.float32,
                device="cuda:0",
                low=LOW_VAL,
                high=HIGH_VAL,
            ),
            torch.testing.make_tensor(
                (1, dim0, dim1),
                dtype=torch.float32,
                device="cuda:0",
                low=LOW_VAL,
                high=HIGH_VAL,
            ),
            torch.testing.make_tensor(
                (1, dim0, 1),
                dtype=torch.float32,
                device="cuda:0",
                low=LOW_VAL,
                high=HIGH_VAL,
            ),
        ]
        nvfuser_direct_test.exec_nvfuser(nvfuser_fusion, inputs, validate_results=True)


def test_issue1246(nvfuser_direct_test):
    """
    Test for issue 1246 - tests concatenation with empty tensors and strided tensors.

    This test verifies that concatenation operations work correctly with:
    - Strided tensors with non-standard memory layouts
    - Empty tensors (zero-sized dimensions)
    - Both with and without additional operations after concatenation
    """
    inputs = [
        torch.randn((8388608,), dtype=torch.float32, device="cuda:0").as_strided(
            (1, 32, 2048, 128), (8388608, 262144, 128, 1)
        ),
        torch.randn((0,), dtype=torch.float32, device="cuda:0").as_strided(
            (1, 32, 2048, 0), (8388608, 262144, 128, 1)
        ),
    ]

    for final_mul in [False, True]:

        def fusion_func(fd: FusionDefinition) -> None:
            T0 = fd.define_tensor(
                shape=[1, -1, -1, -1],
                contiguity=[None, True, True, True],
                dtype=DataType.Float,
                is_cpu=False,
            )
            T1 = fd.define_tensor(
                shape=[1, -1, -1, -1],
                contiguity=[None, True, False, True],
                dtype=DataType.Float,
                is_cpu=False,
            )
            S2 = fd.define_scalar(2.00000, dtype=DataType.Double)
            T3 = fd.ops.mul(T0, S2)
            T4 = fd.ops.cat([T3, T1], dim=-1)
            if final_mul:
                # NOTE: original repro does not have this final op
                S3 = fd.define_scalar(1.00000, dtype=DataType.Double)
                T5 = fd.ops.mul(T4, S3)
                fd.add_output(T5)
            else:
                fd.add_output(T4)

        nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)
        torch_ref = torch.cat([2.0 * inputs[0], inputs[1]], dim=-1)
        nvfuser_direct_test.assertEqual(nvf_out[0], torch_ref)

def test_issue1270(nvfuser_direct_test):
    """
    Test for issue 1270 - tests operations with empty tensors and dead code removal.

    This test verifies that operations work correctly with:
    - Empty tensors (zero-sized dimensions)
    - Dead code removal during fusion optimization
    - Complex operations involving casting, multiplication, and reduction
    - Proper handling of empty tensor operations that should not cause problems
    """
    inputs = [
        torch.randn(0, device="cuda", dtype=torch.bfloat16).as_strided((5, 0), (1, 0)),
        torch.randn(0, device="cuda", dtype=torch.bfloat16).as_strided((5, 0), (0, 1)),
    ]


    nvfuser_direct_test.exec_nvfuser(fusion_func, inputs, validate_results=True)

def test_issue2395(nvfuser_direct_test):
    """
    Test for issue 2395 - tests complex operations with strided tensors and multiple data types.

    Migrated from `test_issue_2395` in legacy test_pointwise.py

    This test verifies that complex operations work correctly with:
    - Large strided tensors with complex memory layouts
    - Multiple data types (Float32 and BFloat16)
    - Complex tensor operations (permute, reshape, slice, sum, mul, neg, add)
    - Broadcasting operations with different shapes
    - Padding operations with scalar values
    - Multiple output tensors from a single fusion
    - Proper handling of tensor contiguity and stride order
    """

    def create_fusion(fd: FusionDefinition) -> None:
        cond0 = fd.define_tensor(
            shape=[-1, -1, -1],
            contiguity=[True, True, None],
            dtype=DataType.Bool,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        values = fd.define_tensor(
            shape=[-1, -1, -1],
            contiguity=[True, True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[2, 1, 0],
        )
        cond1 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.Bool,
            is_cpu=False,
            stride_order=[1, 0],
        )
        cond1 = fd.ops.broadcast_in_dim(
            cond1, shape=[16, 16, 32], broadcast_dims=[0, 1]
        )
        sliced = fd.ops.slice(
            values,
            start_indices=[0, 0, 16],
            end_indices=[16, 16, 32],
            strides=[1, 1, 1],
        )
        zero = fd.define_scalar(0.00000, dtype=DataType.Double)
        masked = fd.ops.where(cond1, zero, values)
        masked = fd.ops.where(cond0, zero, masked)
        fd.add_output(sliced)
        fd.add_output(masked)

    ins = [
        torch.randint(0, 2, (256,), dtype=torch.bool, device="cuda:0").as_strided(
            (16, 16, 32), (16, 1, 0)
        ),
        torch.randn((8192,), dtype=torch.float32, device="cuda:0").as_strided(
            (16, 16, 32), (512, 32, 1)
        ),
        torch.randint(0, 2, (256,), dtype=torch.bool, device="cuda:0").as_strided(
            (16, 16), (16, 1)
        ),
    ]
    outs, _ = nvfuser_direct_test.exec_nvfuser(create_fusion, ins)

    nvfuser_direct_test.assertEqual(outs[0], ins[1][:, :, 16:], rtol=0, atol=0)
    nvfuser_direct_test.assertEqual(
        outs[1],
        torch.where(
            torch.logical_or(ins[0] == 1, ins[2].unsqueeze(-1) == 1), 0, ins[1]
        ),


        ),
        torch.testing.make_tensor(
            (1, 4096, 2560), dtype=torch.float32, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (1, 4096, 2560), dtype=torch.float32, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (1, 4096, 2560), dtype=torch.float32, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (1, 4096, 10240), dtype=torch.float32, device="cuda:0"
        ),
    ]
    nvfuser_direct_test.exec_nvfuser(fusion_func, inputs, validate_results=True)

def test_shared_memory_usage(nvfuser_direct_test):
    # repro of benchmarks.python.test_dropout_rmsnorm_bwd.test_dropout_rmsnorm_bwd_nvf_benchmark[dtype=torch.bfloat16-size=[16384_24578]]
    # avoid overflow of shared memory usage since previous version didn't consider static smem size
    def nvfuser_fusion_id0(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        T1 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        T2 = fd.define_tensor(
            shape=[-1, -1], contiguity=[True, True], dtype=DataType.Bool, is_cpu=False
        )
        T3 = fd.define_tensor(
            shape=[-1, 1], contiguity=[True, None], dtype=DataType.Float, is_cpu=False
        )
        T4 = fd.define_tensor(
            shape=[-1, -1],
            contiguity=[True, True],
            dtype=DataType.BFloat16,
            is_cpu=False,
        )
        T5 = fd.define_tensor(
            shape=[-1], contiguity=[True], dtype=DataType.BFloat16, is_cpu=False
        )
        T6 = fd.ops.cast(T1, dtype=DataType.Float)
        T7 = fd.ops.cast(T0, dtype=DataType.Float)
        T8 = fd.ops.cast(T2, dtype=DataType.Float)
        T9 = fd.ops.cast(T4, dtype=DataType.Float)
        T10 = fd.ops.cast(T5, dtype=DataType.Float)
        T11 = fd.ops.mul(T7, T8)
        S12 = fd.define_scalar(1.25000, dtype=DataType.Double)
        T13 = fd.ops.mul(T11, S12)
        T14 = fd.ops.add(T6, T13)
        V15 = fd.ops.shape(T7)
        T16 = fd.ops.broadcast_in_dim(T3, shape=V15, broadcast_dims=[0, 1])
        T17 = fd.ops.reciprocal(T16)
        T18 = fd.ops.mul(T14, T17)
        T19 = fd.ops.broadcast_in_dim(T10, shape=V15, broadcast_dims=[1])
        T20 = fd.ops.mul(T9, T18)
        T21 = fd.ops.mul(T9, T19)
        T22 = fd.ops.sum(T20, dims=[0], keepdim=False, dtype=DataType.Null)
        T23 = fd.ops.mul(T21, T17)
        T24 = fd.ops.neg(T21)
        T25 = fd.ops.mul(T24, T14)
        S26 = fd.define_scalar(2.00000, dtype=DataType.Double)
        T27 = fd.ops.pow(T16, S26)
        T28 = fd.ops.reciprocal(T27)
        T29 = fd.ops.mul(T25, T28)
        T30 = fd.ops.sum(T29, dims=[1], keepdim=False, dtype=DataType.Null)
        S31 = fd.ops.size(T7, dim=0)
        T34 = fd.ops.broadcast_in_dim(T30, shape=[S31, 1], broadcast_dims=[0])
        T35 = fd.ops.mul(S26, T3)
        T36 = fd.ops.reciprocal(T35)
        T37 = fd.ops.mul(T34, T36)
        S38 = fd.ops.size(T7, dim=1)
        S39 = fd.ops.reciprocal(S38)
        T40 = fd.ops.mul(T37, S39)
        T41 = fd.ops.sum(T40, dims=[1], keepdim=False, dtype=DataType.Null)
        T42 = fd.ops.broadcast_in_dim(T41, shape=[S31, 1], broadcast_dims=[0])
        T43 = fd.ops.broadcast_in_dim(T42, shape=V15, broadcast_dims=[0, 1])
        T44 = fd.ops.mul(T43, S26)
        T45 = fd.ops.mul(T44, T14)
        T46 = fd.ops.add(T23, T45)
        T47 = fd.ops.mul(T46, S12)
        T48 = fd.ops.mul(T47, T8)
        T49 = fd.ops.cast(T46, dtype=DataType.BFloat16)
        T50 = fd.ops.cast(T48, dtype=DataType.BFloat16)
        T51 = fd.ops.cast(T22, dtype=DataType.BFloat16)
        fd.add_output(T50)
        fd.add_output(T49)
        fd.add_output(T51)

    inputs = [
        torch.testing.make_tensor((16, 24578), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((16, 24578), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((16, 24578), dtype=torch.bool, device="cuda:0"),
        torch.testing.make_tensor((16, 1), dtype=torch.float32, device="cuda:0"),
        torch.testing.make_tensor((16, 24578), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((24578,), dtype=torch.bfloat16, device="cuda:0"),
    ]
    nvfuser_direct_test.exec_nvfuser(nvfuser_fusion_id0, inputs, validate_results=True)

def test_ca_map_concrete_loop_id(nvfuser_direct_test):
    def nvfuser_fusion_id10(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[16, 1, 1],
            contiguity=[True, None, None],
            dtype=DataType.Float,
            is_cpu=False,
        )


    inputs = [
        torch.testing.make_tensor((1, 16), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((1, 16), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((1, 5120), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor(
            (16, 5120, 16384), dtype=torch.bfloat16, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (16, 8192, 5120), dtype=torch.bfloat16, device="cuda:0"
        ),
        torch.testing.make_tensor((8192, 5120), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((8192, 5120), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((5120, 8192), dtype=torch.bfloat16, device="cuda:0"),
    ]

    nvfuser_direct_test.exec_nvfuser(nvfuser_fusion, inputs, validate_results=True)

    # https://github.com/NVIDIA/Fuser/issues/5346
    def test_register_aliasing_bug(nvfuser_direct_test):
        def nvfuser_fusion(fd: FusionDefinition, dim0, dim1) -> None:
            T0 = fd.define_tensor(
                shape=[dim1],
                contiguity=[True],
                dtype=DataType.Float,
                is_cpu=False,
                stride_order=[0],
            )
            T1 = fd.define_tensor(
                shape=[1, dim0, dim1],
                contiguity=[None, True, True],
                dtype=DataType.Float,
                is_cpu=False,
                stride_order=[2, 1, 0],
            )
            T2 = fd.define_tensor(
                shape=[1, dim0, dim1],
                contiguity=[None, True, True],
                dtype=DataType.Float,
                is_cpu=False,
                stride_order=[2, 1, 0],
            )
            T3 = fd.define_tensor(
                shape=[1, dim0, 1],
                contiguity=[None, True, None],
                dtype=DataType.Float,
                is_cpu=False,
                stride_order=[2, 1, 0],
            )
            T8 = fd.ops.broadcast_in_dim(T0, shape=[1, dim0, dim1], broadcast_dims=[2])
            T9 = fd.ops.mul(T8, T1)
            T10 = fd.ops.mul(T2, T9)
            T11 = fd.ops.sum(T10, dims=[0, 2], keepdim=False, dtype=DataType.Null)
            T16 = fd.ops.broadcast_in_dim(T11, shape=[1, dim0, 1], broadcast_dims=[1])
            S17 = fd.define_scalar(3.00000, dtype=DataType.Double)
            T18 = fd.ops.pow(T3, S17)
            S19 = fd.define_scalar(-0.500000, dtype=DataType.Double)
            T20 = fd.ops.mul(S19, T16)
            T21 = fd.ops.mul(T20, T18)
            S22 = fd.define_scalar(dim1, dtype=DataType.Double)
            S23 = fd.ops.reciprocal(S22)
            T24 = fd.ops.mul(T21, S23)
            T25 = fd.ops.sum(T24, dims=[0, 2], keepdim=False, dtype=DataType.Null)
            T29 = fd.ops.broadcast_in_dim(T25, shape=[1, dim0], broadcast_dims=[1])
            T34 = fd.ops.broadcast_in_dim(
                T29, shape=[1, dim0, 1], broadcast_dims=[0, 1]
            )
            T39 = fd.ops.broadcast_in_dim(
                T3, shape=[1, dim0, dim1], broadcast_dims=[0, 1, 2]
            )
            T44 = fd.ops.broadcast_in_dim(
                T34, shape=[1, dim0, dim1], broadcast_dims=[0, 1, 2]
            )
            T45 = fd.ops.mul(T2, T39)
            T46 = fd.ops.mul(T2, T44)
            T47 = fd.ops.mul(T39, T9)
            T48 = fd.ops.mul(T45, T1)
            T49 = fd.ops.add(T47, T46)
            T50 = fd.ops.add(T49, T46)
            T51 = fd.ops.sum(T48, dims=[0, 1], keepdim=False, dtype=DataType.Null)
            T55 = fd.ops.reshape(T50, new_shape=[dim0, dim1])
            fd.add_output(T50)
            fd.add_output(T55)
            fd.add_output(T51)

        dim0 = 2048
        dim1 = 32
        with FusionDefinition() as fd:
            nvfuser_fusion(fd, dim0, dim1)

        inputs = [
            torch.testing.make_tensor(
                (dim1,),
                dtype=torch.float32,
                device="cuda:0",
                low=LOW_VAL,
                high=HIGH_VAL,
            ),
            torch.testing.make_tensor(
                (1, dim0, dim1),
                dtype=torch.float32,
                device="cuda:0",
                low=LOW_VAL,
                high=HIGH_VAL,


        T222 = fd.ops.slice(
            T108,
            start_indices=[0, 0, 0, 0],
            end_indices=[5, 7, 5, 0],
            strides=[1, 1, 1, 1],
            manual_normalization=0,
        )
        T223 = fd.ops.cat([T169, T222], dim=-1, manual_padding=0)
        fd.add_output(T223)

    nvf_out, _ = nvfuser_direct_test.exec_nvfuser(fusion_func, inputs)

def test_issue3369(nvfuser_direct_test):
    """
    Test for issue 3369 - tests square linear operations.

    This test verifies that square linear operations work correctly.
    """

    def nvfuser_fusion_id28(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[5, 5],
            contiguity=[True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[1, 0],
        )
        T1 = fd.define_tensor(
            shape=[5, 5],
            contiguity=[True, True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[1, 0],
        )
        T2 = fd.define_tensor(
            shape=[5],
            contiguity=[True],
            dtype=DataType.Float,
            is_cpu=False,
            stride_order=[0],
        )
        T3 = fd.ops.linear(T0, T1, T2)
        fd.add_output(T3)

    inputs = [
        torch.testing.make_tensor((5, 5), dtype=torch.float32, device="cuda:0"),
        torch.testing.make_tensor((5, 5), dtype=torch.float32, device="cuda:0"),
        torch.testing.make_tensor((5,), dtype=torch.float32, device="cuda:0"),
    ]
    nvfuser_direct_test.exec_nvfuser(nvfuser_fusion_id28, inputs, validate_results=True)

def test_issue4444(nvfuser_direct_test):
    """
    Test for issue 4444 - complex tensor operations with multiple slice operations,
    manual normalization, padding, and reshaping operations.

    This test validates:
    - Multiple slice operations with manual normalization
    - Complex tensor reshaping and permutation operations
    - Manual padding operations with scalar values
    - Broadcast operations with specific dimensions
    - Cast operations between different data types


        T48 = fd.ops.mul(T47, T8)
        T49 = fd.ops.cast(T46, dtype=DataType.BFloat16)
        T50 = fd.ops.cast(T48, dtype=DataType.BFloat16)
        T51 = fd.ops.cast(T22, dtype=DataType.BFloat16)
        fd.add_output(T50)
        fd.add_output(T49)
        fd.add_output(T51)

    inputs = [
        torch.testing.make_tensor((16, 24578), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((16, 24578), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((16, 24578), dtype=torch.bool, device="cuda:0"),
        torch.testing.make_tensor((16, 1), dtype=torch.float32, device="cuda:0"),
        torch.testing.make_tensor((16, 24578), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((24578,), dtype=torch.bfloat16, device="cuda:0"),
    ]
    nvfuser_direct_test.exec_nvfuser(nvfuser_fusion_id0, inputs, validate_results=True)

def test_ca_map_concrete_loop_id(nvfuser_direct_test):
    def nvfuser_fusion_id10(fd: FusionDefinition) -> None:
        T0 = fd.define_tensor(
            shape=[16, 1, 1],
            contiguity=[True, None, None],
            dtype=DataType.Float,
            is_cpu=False,
        )
        T1 = fd.define_tensor(
            shape=[1, 1, 1024],
            contiguity=[None, None, True],
            dtype=DataType.Float,
            is_cpu=False,
        )
        T2 = fd.ops.sub(T0, T0)
        T3 = fd.ops.exp(T2)
        T4 = fd.ops.reciprocal(T3)
        T5 = fd.ops.mul(T3, T4)
        T6 = fd.ops.broadcast(T5, is_broadcast_dim=[False, False, True, False])
        S7 = fd.ops.size(T5, dim=1)
        S8 = fd.define_scalar(16, dtype=DataType.Int)
        S9 = fd.define_scalar(64, dtype=DataType.Int)
        T11 = fd.ops.reshape(T1, new_shape=[S7, S7, S8, S9])
        T12 = fd.ops.permute(T11, dims=[0, 2, 1, 3])
        T13 = fd.ops.squeeze(T12, dims=[0], squeeze_expanded=True)
        T14 = fd.ops.permute(T13, dims=[0, 2, 1])
        T15 = fd.ops.broadcast(T14, is_broadcast_dim=[False, True, False, False])
        T16 = fd.ops.mul(T6, T15)
        T17 = fd.ops.squeeze(T16, dims=[3], squeeze_expanded=True)
        T18 = fd.ops.broadcast(T17, is_broadcast_dim=[True, False, False, False])
        T19 = fd.ops.permute(T18, dims=[0, 2, 1, 3])
        S20 = fd.define_scalar(1024, dtype=DataType.Int)
        T22 = fd.ops.reshape(T19, new_shape=[S7, S7, S20])
        fd.add_output(T5)
        fd.add_output(T13)
        fd.add_output(T22)

    inputs = [
        torch.testing.make_tensor((16, 1, 1), dtype=torch.float32, device="cuda:0"),
        torch.testing.make_tensor((1, 1, 1024), dtype=torch.float32, device="cuda:0"),
    ]
    nvfuser_direct_test.exec_nvfuser(nvfuser_fusion_id10, inputs, validate_results=True)

def test_issue5377(nvfuser_direct_test):
    """
    Repro for issue 5377, where a traversal for vectorize validation
    failed to find an allocation ID that corresponds to a vectorized
    ID.
    """

    def nvfuser_fusion(fd: FusionDefinition) -> None:
        tv0 = fd.define_tensor(
            shape=[1, 16],
            contiguity=[None, True],
            dtype=DataType.BFloat16,


        fd.add_output(tv56)

    inputs = [
        torch.testing.make_tensor((1, 16), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((1, 16), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((1, 5120), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor(
            (16, 5120, 16384), dtype=torch.bfloat16, device="cuda:0"
        ),
        torch.testing.make_tensor(
            (16, 8192, 5120), dtype=torch.bfloat16, device="cuda:0"
        ),
        torch.testing.make_tensor((8192, 5120), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((8192, 5120), dtype=torch.bfloat16, device="cuda:0"),
        torch.testing.make_tensor((5120, 8192), dtype=torch.bfloat16, device="cuda:0"),
    ]

    nvfuser_direct_test.exec_nvfuser(nvfuser_fusion, inputs, validate_results=True)

    # https://github.com/NVIDIA/Fuser/issues/5346
    def test_register_aliasing_bug(nvfuser_direct_test):
        def nvfuser_fusion(fd: FusionDefinition, dim0, dim1) -> None:
            T0 = fd.define_tensor(
                shape=[dim1],
                contiguity=[True],
                dtype=DataType.Float,
                is_cpu=False,
                stride_order=[0],
            )
            T1 = fd.define_tensor(
                shape=[1, dim0, dim1],
                contiguity=[None, True, True],
                dtype=DataType.Float,
                is_cpu=False,
                stride_order=[2, 1, 0],
            )
            T2 = fd.define_tensor(
                shape=[1, dim0, dim1],
                contiguity=[None, True, True],
                dtype=DataType.Float,
                is_cpu=False,
                stride_order=[2, 1, 0],
            )
            T3 = fd.define_tensor(
                shape=[1, dim0, 1],
                contiguity=[None, True, None],
                dtype=DataType.Float,
                is_cpu=False,
                stride_order=[2, 1, 0],
            )
            T8 = fd.ops.broadcast_in_dim(T0, shape=[1, dim0, dim1], broadcast_dims=[2])
            T9 = fd.ops.mul(T8, T1)
            T10 = fd.ops.mul(T2, T9)
            T11 = fd.ops.sum(T10, dims=[0, 2], keepdim=False, dtype=DataType.Null)
            T16 = fd.ops.broadcast_in_dim(T11, shape=[1, dim0, 1], broadcast_dims=[1])
            S17 = fd.define_scalar(3.00000, dtype=DataType.Double)
            T18 = fd.ops.pow(T3, S17)
            S19 = fd.define_scalar(-0.500000, dtype=DataType.Double)
            T20 = fd.ops.mul(S19, T16)
            T21 = fd.ops.mul(T20, T18)
            S22 = fd.define_scalar(dim1, dtype=DataType.Double)
            S23 = fd.ops.reciprocal(S22)
            T24 = fd.ops.mul(T21, S23)
            T25 = fd.ops.sum(T24, dims=[0, 2], keepdim=False, dtype=DataType.Null)
            T29 = fd.ops.broadcast_in_dim(T25, shape=[1, dim0], broadcast_dims=[1])
            T34 = fd.ops.broadcast_in_dim(
                T29, shape=[1, dim0, 1], broadcast_dims=[0, 1]
            )
            T39 = fd.ops.broadcast_in_dim(
                T3, shape=[1, dim0, dim1], broadcast_dims=[0, 1, 2]
            )
            T44 = fd.ops.broadcast_in_dim(
                T34, shape=[1, dim0, dim1], broadcast_dims=[0, 1, 2]
            )
            T45 = fd.ops.mul(T2, T39)
            T46 = fd.ops.mul(T2, T44)
            T47 = fd.ops.mul(T39, T9)
            T48 = fd.ops.mul(T45, T1)
            T49 = fd.ops.add(T47, T46)
            T50 = fd.ops.add(T49, T46)
            T51 = fd.ops.sum(T48, dims=[0, 1], keepdim=False, dtype=DataType.Null)
            T55 = fd.ops.reshape(T50, new_shape=[dim0, dim1])
            fd.add_output(T50)
            fd.add_output(T55)
            fd.add_output(T51)

        dim0 = 2048
        dim1 = 32
        with FusionDefinition() as fd:
            nvfuser_fusion(fd, dim0, dim1)

        inputs = [
            torch.testing.make_tensor(
                (dim1,),
                dtype=torch.float32,
                device="cuda:0",
                low=LOW_VAL,
                high=HIGH_VAL,
            ),
            torch.testing.make_tensor(
                (1, dim0, dim1),
                dtype=torch.float32,
                device="cuda:0",
                low=LOW_VAL,
                high=HIGH_VAL,
            ),
            torch.testing.make_tensor(
                (1, dim0, dim1),
                dtype=torch.float32,
                device="cuda:0",
                low=LOW_VAL,
                high=HIGH_VAL,
            ),
            torch.testing.make_tensor(
                (1, dim0, 1),
                dtype=torch.float32,
                device="cuda:0",
                low=LOW_VAL,
                high=HIGH_VAL,
            ),
        ]
        nvfuser_direct_test.exec_nvfuser(nvfuser_fusion, inputs, validate_results=True)