        )
        grad_x, grad_w = op([input, weight, grad_output])
        return (grad_x, grad_w)

class TensorParallelLinear(torch.nn.Linear):
    def __init__(
        self,
        in_features: int,
        out_features: int,
        in_placements: Iterable[Placement] = [],
    ):
        # Unlike normal layers whose `__init__` allocates the parameters,
        # `TensorParallelLinear` is expected to be created from a
        # non-distributed Linear layer via the `distribute` method. Therefore,
        # here, we construct super() with no memory allocated for weights. The
        # weights will be derived by the `distribute` method.
        super().__init__(in_features, out_features, bias=False, device="meta")
        self.in_placements = in_placements

    def __repr__(self):
        base_repr = super().__repr__()
        return f"{base_repr[:-1]}, in_placements={self.in_placements}, weight_placements={self.weight.data.placements})"

    @classmethod
    def distribute(
        cls,
        linear: torch.nn.Linear,
        mesh: dist.device_mesh.DeviceMesh,
        in_placements: Iterable[Placement] = [],
        weight_placements: Iterable[Placement] = [],
    ):
        tp_linear = cls(linear.in_features, linear.out_features, in_placements)
        tp_linear.weight = torch.nn.Parameter(
            dist.tensor.distribute_tensor(linear.weight, mesh, weight_placements)
        )
        return tp_linear

    def forward(self, input: torch.Tensor) -> torch.Tensor:
        in_dtensor = DTensor.from_local(
            input, self.weight.device_mesh, self.in_placements
        )
        out_dtensor = LinearFunction.apply(in_dtensor, self.weight)


distribute

class TensorParallelLinear(torch.nn.Linear):
    def __init__(
        self,
        in_features: int,
        out_features: int,
        in_placements: Iterable[Placement] = [],
    ):
        # Unlike normal layers whose `__init__` allocates the parameters,
        # `TensorParallelLinear` is expected to be created from a
        # non-distributed Linear layer via the `distribute` method. Therefore,
        # here, we construct super() with no memory allocated for weights. The
        # weights will be derived by the `distribute` method.
        super().__init__(in_features, out_features, bias=False, device="meta")
        self.in_placements = in_placements

    def __repr__(self):
        base_repr = super().__repr__()
        return f"{base_repr[:-1]}, in_placements={self.in_placements}, weight_placements={self.weight.data.placements})"

    @classmethod
    def distribute(
        cls,
        linear: torch.nn.Linear,
        mesh: dist.device_mesh.DeviceMesh,
        in_placements: Iterable[Placement] = [],
        weight_placements: Iterable[Placement] = [],
    ):
        tp_linear = cls(linear.in_features, linear.out_features, in_placements)
        tp_linear.weight = torch.nn.Parameter(
            dist.tensor.distribute_tensor(linear.weight, mesh, weight_placements)
        )
        return tp_linear

    def forward(self, input: torch.Tensor) -> torch.Tensor:
        in_dtensor = DTensor.from_local(
            input, self.weight.device_mesh, self.in_placements
        )
        out_dtensor = LinearFunction.apply(in_dtensor, self.weight)
        return out_dtensor.to_local()


# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

# Run command:
# mpirun -np 1 pytest tests/python/multidevice/test_deepseek_v3.py --only-mpi -s

from contextlib import contextmanager
from enum import Enum, auto
from functools import wraps
from typing import Optional

import pytest

import torch
import torch.distributed as dist
from torch.distributed.tensor import DTensor
from torch.distributed.tensor.parallel import (
    parallelize_module,
    ParallelStyle,
    RowwiseParallel,
    ColwiseParallel,
)
from torch.distributed.tensor.placement_types import Shard

import transformers

from .benchmark_utils import get_benchmark_fns
from .linear import TensorParallelLinear

@contextmanager
def default_tensor_type(dtype=torch.float32, device="cpu"):
    # Save
    prev_dtype = torch.get_default_dtype()
    prev_device = torch.get_default_device()

    # Set
    torch.set_default_dtype(dtype)
    torch.set_default_device(device)

    yield

    # Restore
    torch.set_default_dtype(prev_dtype)
    torch.set_default_device(prev_device)

# This decorator ensures that the model/config is downloaded only once by rank
# 0. Other ranks will load from the cache that's stored on the same machine.
def download_once(fn):
    @wraps(fn)
    def wrapper(*args, **kwargs):
        rank = dist.get_rank()
        if rank == 0:
            # Download only once.
            result = fn(*args, **kwargs)

        dist.barrier()

        if rank != 0:
            # Other ranks load from cache.
            result = fn(*args, **kwargs)

        return result

    return wrapper

@download_once
def load_config(model_name: str) -> transformers.PretrainedConfig:
    return transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=True)

@download_once
def load_model(config: transformers.PretrainedConfig) -> transformers.PreTrainedModel:
    return transformers.AutoModel.from_config(config, trust_remote_code=True)

class Executor(Enum):
    # https://docs.pytorch.org/docs/stable/distributed.tensor.parallel.html
    TORCH_TP = auto()
    NVFUSER = auto()

def parallelize_linear_with_nvfuser(
    linear: torch.nn.Linear,
    mesh: dist.device_mesh.DeviceMesh,
    parallel_style: ParallelStyle,
) -> torch.nn.Linear:
    assert isinstance(linear, torch.nn.Linear), f"Unsupported layer: {linear}"

    assert len(parallel_style.input_layouts) == 1, "Expect 1D mesh"
    input_layout = parallel_style.input_layouts[0]

    assert len(parallel_style.output_layouts) == 1, "Expect 1D mesh"
    output_layout = parallel_style.output_layouts[0]

    if isinstance(parallel_style, RowwiseParallel):
        # We only support TP at this moment. A row-wise parallel linear is
        # expected to have the input sharded on the contracting dimension and
        # the output replicated.
        assert input_layout.is_shard(-1), f"Unsupported layout: {input_layout}"
        assert output_layout.is_replicate(), f"Unsupported layout: {output_layout}"
        return TensorParallelLinear.distribute(
            linear, mesh, in_placements=[input_layout], weight_placements=[Shard(-1)]
        )

    if isinstance(parallel_style, ColwiseParallel):
        # We only support TP at this moment. A column-wise parallel linear is
        # expected to have the input replicated and the output sharded on the
        # feature dimension.
        assert input_layout.is_replicate(), f"Unsupported layout: {input_layout}"
        assert output_layout.is_shard(-1), f"Unsupported layout: {output_layout}"
        return TensorParallelLinear.distribute(
            linear, mesh, in_placements=[input_layout], weight_placements=[Shard(0)]
        )

    assert False, f"Unsupported parallel style: {parallel_style}"

# Recursively finds all linear modules and replaces them with tensor-parallel
# nvFuser definitions if a parallel plan is found.
def parallelize_module_with_nvfuser(
    module: torch.nn.Module,
    mesh: dist.device_mesh.DeviceMesh,
    parallel_plan: dict[str, ParallelStyle],
    fqn: str,  # stands for fully qualified name
    parent_module: Optional[torch.nn.Module] = None,
):
    for child_module_name, child_module in module.named_children():
        if fqn:
            child_fqn = f"{fqn}.{child_module_name}"
        else:
            child_fqn = child_module_name

        parallelize_module_with_nvfuser(
            child_module, mesh, parallel_plan, child_fqn, module
        )

    if (parallel_style := parallel_plan.get(fqn)) is None:
        return

    new_module = parallelize_linear_with_nvfuser(module, mesh, parallel_style)
    assert parent_module is not None
    module_name = fqn.split(".")[-1]
    setattr(parent_module, module_name, new_module)

@pytest.mark.skip(
    reason="Flaky. The test occasionally failed for too many download requests. The test itself downloads the HuggingFace model only once. However, with CI runs triggered by other PRs and for other GPU architectures, it can easily exceed the limit."
)
@pytest.mark.mpi
@pytest.mark.parametrize(
    "executor",
    [Executor.TORCH_TP, Executor.NVFUSER],
    ids=lambda e: e.name,
)
def test_transformer_layer(setup_default_process_group, benchmark, executor: Executor):
    config = load_config("deepseek-ai/deepseek-v3")
    # Create only one layer which is sufficient for the test.
    config.num_hidden_layers = 1
    # Without this, the first and only layer will have a dense MLP instead of MoE.
    config.first_k_dense_replace = 0
    # Disable quantization so the test can run on A100 and is made easier for nvFuser.
    delattr(config, "quantization_config")

    # This ensures the input tokens are identically replicated on all ranks.
    # Otherwise, some ranks may skip an expert because they have no tokens to
    # send, while other ranks don't. This will cause a deadlock because a NCCL
    # collective is expected to be called by all ranks in the process group.
    torch.manual_seed(0)

    d = dist.get_world_size()
    mesh = dist.device_mesh.init_device_mesh("cuda", [d])

    with default_tensor_type(dtype=config.torch_dtype, device="cuda"):
        # Loading the model under `device="cuda"` makes weight initialization
        # much faster but requires full GPU memory allocation.
        #
        # Alternatively, I think the following may work but haven't tried it:
        # 1. Load the model under torch.nn.utils.init_empty_weights. This skips weight initialization and allocates full weights on CPU not GPU.
        # 2. parallelize_module
        # 3. Load pre-trained parameters.
        # 4. Move the model to CUDA.
        model = load_model(config)
        # Training is unavailable (cf. https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/modeling_deepseek.py#L439)
        model.eval()

        transformer_layer = model.layers[0]

        # By default, RowwiseParallel and ColwiseParallel output a local tensor
        # and therefore num_heads needs to be adjusted to accomodate the local
        # size. Alternatively, I could RowwiseParallel(use_local_output=False)
        # so the linear layer outputs a DTensor, which can be viewed using the
        # original num_heads. This requires all activations, parameters, and
        # buffers to be DTensor; otherwise aten ops would complain "got mixed
        # torch.Tensor and DTensor". Doing so is challenging because
        # DeepseekV3RotaryEmbedding creates cos_cached and sin_cached during
        # the first forward call (cf.
        # https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/modeling_deepseek.py#L143-L144).
        transformer_layer.self_attn.num_heads //= d

        # Create the parallel plan
        parallel_plan = {
            "self_attn.q_b_proj": ColwiseParallel(),
            "self_attn.kv_b_proj": ColwiseParallel(),
            "self_attn.o_proj": RowwiseParallel(),
        }

        for expert in range(config.n_routed_experts):
            parallel_plan[f"mlp.experts.{expert}.gate_proj"] = ColwiseParallel()
            parallel_plan[f"mlp.experts.{expert}.up_proj"] = ColwiseParallel()
            parallel_plan[f"mlp.experts.{expert}.down_proj"] = RowwiseParallel()

        parallel_plan["mlp.shared_experts.gate_proj"] = ColwiseParallel()
        parallel_plan["mlp.shared_experts.up_proj"] = ColwiseParallel()
        parallel_plan["mlp.shared_experts.down_proj"] = RowwiseParallel()

        match executor:
            case Executor.TORCH_TP:
                transformer_layer = parallelize_module(
                    transformer_layer,
                    mesh,
                    parallel_plan,
                )

                # Sanity-check parameters are indeed distributed
                distributed_params: list[str] = [
                    name
                    for name, parameter in transformer_layer.named_parameters()
                    if isinstance(parameter.data, DTensor)
                ]
                assert len(distributed_params) == 3 + (config.n_routed_experts + 1) * 3
            case Executor.NVFUSER:
                parallelize_module_with_nvfuser(
                    transformer_layer, mesh, parallel_plan, fqn=""
                )

        batch_size = 1
        seq_len = 1024
        inp = torch.randn(batch_size, seq_len, config.hidden_size)
        mask = transformers.modeling_attn_mask_utils._prepare_4d_causal_attention_mask(
            None, [batch_size, seq_len], inp, past_key_values_length=0
        )
        warmup_fn, benchmark_fn = get_benchmark_fns(
            lambda: transformer_layer(inp, attention_mask=mask)
        )

        (out,) = warmup_fn()
        assert out.size() == (batch_size, seq_len, config.hidden_size)
        assert out.dtype == config.torch_dtype
        assert out.is_cuda

        benchmark.pedantic(benchmark_fn, rounds=5)


import pytest

import torch
import torch.distributed as dist
from torch.distributed.tensor import DTensor
from torch.distributed.tensor.parallel import (
    parallelize_module,
    ParallelStyle,
    RowwiseParallel,
    ColwiseParallel,
)
from torch.distributed.tensor.placement_types import Shard

import transformers

from .benchmark_utils import get_benchmark_fns
from .linear import TensorParallelLinear

@contextmanager
def default_tensor_type(dtype=torch.float32, device="cpu"):
    # Save
    prev_dtype = torch.get_default_dtype()
    prev_device = torch.get_default_device()

    # Set
    torch.set_default_dtype(dtype)
    torch.set_default_device(device)

    yield

    # Restore
    torch.set_default_dtype(prev_dtype)
    torch.set_default_device(prev_device)

# This decorator ensures that the model/config is downloaded only once by rank
# 0. Other ranks will load from the cache that's stored on the same machine.
def download_once(fn):
    @wraps(fn)
    def wrapper(*args, **kwargs):
        rank = dist.get_rank()
        if rank == 0:
            # Download only once.
            result = fn(*args, **kwargs)

        dist.barrier()

        if rank != 0:
            # Other ranks load from cache.
            result = fn(*args, **kwargs)

        return result


@contextmanager
def default_tensor_type(dtype=torch.float32, device="cpu"):
    # Save
    prev_dtype = torch.get_default_dtype()
    prev_device = torch.get_default_device()

    # Set
    torch.set_default_dtype(dtype)
    torch.set_default_device(device)

    yield

    # Restore
    torch.set_default_dtype(prev_dtype)
    torch.set_default_device(prev_device)

# This decorator ensures that the model/config is downloaded only once by rank
# 0. Other ranks will load from the cache that's stored on the same machine.
def download_once(fn):
    @wraps(fn)
    def wrapper(*args, **kwargs):
        rank = dist.get_rank()
        if rank == 0:
            # Download only once.
            result = fn(*args, **kwargs)

        dist.barrier()

        if rank != 0:
            # Other ranks load from cache.
            result = fn(*args, **kwargs)

        return result

    return wrapper

@download_once
def load_config(model_name: str) -> transformers.PretrainedConfig:
    return transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=True)

@download_once
def load_model(config: transformers.PretrainedConfig) -> transformers.PreTrainedModel:
    return transformers.AutoModel.from_config(config, trust_remote_code=True)

class Executor(Enum):
    # https://docs.pytorch.org/docs/stable/distributed.tensor.parallel.html
    TORCH_TP = auto()
    NVFUSER = auto()

def parallelize_linear_with_nvfuser(
    linear: torch.nn.Linear,


def download_once(fn):
    @wraps(fn)
    def wrapper(*args, **kwargs):
        rank = dist.get_rank()
        if rank == 0:
            # Download only once.
            result = fn(*args, **kwargs)

        dist.barrier()

        if rank != 0:
            # Other ranks load from cache.
            result = fn(*args, **kwargs)

        return result

    return wrapper

@download_once
def load_config(model_name: str) -> transformers.PretrainedConfig:
    return transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=True)

@download_once
def load_model(config: transformers.PretrainedConfig) -> transformers.PreTrainedModel:
    return transformers.AutoModel.from_config(config, trust_remote_code=True)

class Executor(Enum):
    # https://docs.pytorch.org/docs/stable/distributed.tensor.parallel.html
    TORCH_TP = auto()
    NVFUSER = auto()

def parallelize_linear_with_nvfuser(
    linear: torch.nn.Linear,
    mesh: dist.device_mesh.DeviceMesh,
    parallel_style: ParallelStyle,
) -> torch.nn.Linear:
    assert isinstance(linear, torch.nn.Linear), f"Unsupported layer: {linear}"


            # Download only once.
            result = fn(*args, **kwargs)

        dist.barrier()

        if rank != 0:
            # Other ranks load from cache.
            result = fn(*args, **kwargs)

        return result

    return wrapper

@download_once
def load_config(model_name: str) -> transformers.PretrainedConfig:
    return transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=True)

@download_once
def load_model(config: transformers.PretrainedConfig) -> transformers.PreTrainedModel:
    return transformers.AutoModel.from_config(config, trust_remote_code=True)

class Executor(Enum):
    # https://docs.pytorch.org/docs/stable/distributed.tensor.parallel.html
    TORCH_TP = auto()
    NVFUSER = auto()

def parallelize_linear_with_nvfuser(
    linear: torch.nn.Linear,
    mesh: dist.device_mesh.DeviceMesh,
    parallel_style: ParallelStyle,
) -> torch.nn.Linear:
    assert isinstance(linear, torch.nn.Linear), f"Unsupported layer: {linear}"

    assert len(parallel_style.input_layouts) == 1, "Expect 1D mesh"
    input_layout = parallel_style.input_layouts[0]

    assert len(parallel_style.output_layouts) == 1, "Expect 1D mesh"
    output_layout = parallel_style.output_layouts[0]


Executor

        if rank != 0:
            # Other ranks load from cache.
            result = fn(*args, **kwargs)

        return result

    return wrapper

@download_once
def load_config(model_name: str) -> transformers.PretrainedConfig:
    return transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=True)

@download_once
def load_model(config: transformers.PretrainedConfig) -> transformers.PreTrainedModel:
    return transformers.AutoModel.from_config(config, trust_remote_code=True)

class Executor(Enum):
    # https://docs.pytorch.org/docs/stable/distributed.tensor.parallel.html
    TORCH_TP = auto()
    NVFUSER = auto()

def parallelize_linear_with_nvfuser(
    linear: torch.nn.Linear,
    mesh: dist.device_mesh.DeviceMesh,
    parallel_style: ParallelStyle,
) -> torch.nn.Linear:
    assert isinstance(linear, torch.nn.Linear), f"Unsupported layer: {linear}"

    assert len(parallel_style.input_layouts) == 1, "Expect 1D mesh"
    input_layout = parallel_style.input_layouts[0]

    assert len(parallel_style.output_layouts) == 1, "Expect 1D mesh"
    output_layout = parallel_style.output_layouts[0]

    if isinstance(parallel_style, RowwiseParallel):
        # We only support TP at this moment. A row-wise parallel linear is
        # expected to have the input sharded on the contracting dimension and
        # the output replicated.
        assert input_layout.is_shard(-1), f"Unsupported layout: {input_layout}"


    return wrapper

@download_once
def load_config(model_name: str) -> transformers.PretrainedConfig:
    return transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=True)

@download_once
def load_model(config: transformers.PretrainedConfig) -> transformers.PreTrainedModel:
    return transformers.AutoModel.from_config(config, trust_remote_code=True)

class Executor(Enum):
    # https://docs.pytorch.org/docs/stable/distributed.tensor.parallel.html
    TORCH_TP = auto()
    NVFUSER = auto()

def parallelize_linear_with_nvfuser(
    linear: torch.nn.Linear,
    mesh: dist.device_mesh.DeviceMesh,
    parallel_style: ParallelStyle,
) -> torch.nn.Linear:
    assert isinstance(linear, torch.nn.Linear), f"Unsupported layer: {linear}"

    assert len(parallel_style.input_layouts) == 1, "Expect 1D mesh"
    input_layout = parallel_style.input_layouts[0]

    assert len(parallel_style.output_layouts) == 1, "Expect 1D mesh"
    output_layout = parallel_style.output_layouts[0]

    if isinstance(parallel_style, RowwiseParallel):
        # We only support TP at this moment. A row-wise parallel linear is
        # expected to have the input sharded on the contracting dimension and
        # the output replicated.
        assert input_layout.is_shard(-1), f"Unsupported layout: {input_layout}"
        assert output_layout.is_replicate(), f"Unsupported layout: {output_layout}"
        return TensorParallelLinear.distribute(
            linear, mesh, in_placements=[input_layout], weight_placements=[Shard(-1)]
        )

    if isinstance(parallel_style, ColwiseParallel):
        # We only support TP at this moment. A column-wise parallel linear is
        # expected to have the input replicated and the output sharded on the
        # feature dimension.
        assert input_layout.is_replicate(), f"Unsupported layout: {input_layout}"
        assert output_layout.is_shard(-1), f"Unsupported layout: {output_layout}"
        return TensorParallelLinear.distribute(
            linear, mesh, in_placements=[input_layout], weight_placements=[Shard(0)]
        )

    assert False, f"Unsupported parallel style: {parallel_style}"

# Recursively finds all linear modules and replaces them with tensor-parallel
# nvFuser definitions if a parallel plan is found.
def parallelize_module_with_nvfuser(
    module: torch.nn.Module,
    mesh: dist.device_mesh.DeviceMesh,
    parallel_plan: dict[str, ParallelStyle],
    fqn: str,  # stands for fully qualified name
    parent_module: Optional[torch.nn.Module] = None,
):
    for child_module_name, child_module in module.named_children():
        if fqn:
            child_fqn = f"{fqn}.{child_module_name}"
        else:
            child_fqn = child_module_name

        parallelize_module_with_nvfuser(
            child_module, mesh, parallel_plan, child_fqn, module
        )


        assert output_layout.is_replicate(), f"Unsupported layout: {output_layout}"
        return TensorParallelLinear.distribute(
            linear, mesh, in_placements=[input_layout], weight_placements=[Shard(-1)]
        )

    if isinstance(parallel_style, ColwiseParallel):
        # We only support TP at this moment. A column-wise parallel linear is
        # expected to have the input replicated and the output sharded on the
        # feature dimension.
        assert input_layout.is_replicate(), f"Unsupported layout: {input_layout}"
        assert output_layout.is_shard(-1), f"Unsupported layout: {output_layout}"
        return TensorParallelLinear.distribute(
            linear, mesh, in_placements=[input_layout], weight_placements=[Shard(0)]
        )

    assert False, f"Unsupported parallel style: {parallel_style}"

# Recursively finds all linear modules and replaces them with tensor-parallel
# nvFuser definitions if a parallel plan is found.
def parallelize_module_with_nvfuser(
    module: torch.nn.Module,
    mesh: dist.device_mesh.DeviceMesh,
    parallel_plan: dict[str, ParallelStyle],
    fqn: str,  # stands for fully qualified name
    parent_module: Optional[torch.nn.Module] = None,
):
    for child_module_name, child_module in module.named_children():
        if fqn:
            child_fqn = f"{fqn}.{child_module_name}"
        else:
            child_fqn = child_module_name

        parallelize_module_with_nvfuser(
            child_module, mesh, parallel_plan, child_fqn, module
        )

    if (parallel_style := parallel_plan.get(fqn)) is None:
        return

    new_module = parallelize_linear_with_nvfuser(module, mesh, parallel_style)
    assert parent_module is not None
    module_name = fqn.split(".")[-1]
    setattr(parent_module, module_name, new_module)

@pytest.mark.skip(
    reason="Flaky. The test occasionally failed for too many download requests. The test itself downloads the HuggingFace model only once. However, with CI runs triggered by other PRs and for other GPU architectures, it can easily exceed the limit."
)
@pytest.mark.mpi
@pytest.mark.parametrize(
    "executor",
    [Executor.TORCH_TP, Executor.NVFUSER],
    ids=lambda e: e.name,
)
def test_transformer_layer(setup_default_process_group, benchmark, executor: Executor):
    config = load_config("deepseek-ai/deepseek-v3")
    # Create only one layer which is sufficient for the test.
    config.num_hidden_layers = 1
    # Without this, the first and only layer will have a dense MLP instead of MoE.
    config.first_k_dense_replace = 0
    # Disable quantization so the test can run on A100 and is made easier for nvFuser.
    delattr(config, "quantization_config")


        )

    if (parallel_style := parallel_plan.get(fqn)) is None:
        return

    new_module = parallelize_linear_with_nvfuser(module, mesh, parallel_style)
    assert parent_module is not None
    module_name = fqn.split(".")[-1]
    setattr(parent_module, module_name, new_module)

@pytest.mark.skip(
    reason="Flaky. The test occasionally failed for too many download requests. The test itself downloads the HuggingFace model only once. However, with CI runs triggered by other PRs and for other GPU architectures, it can easily exceed the limit."
)
@pytest.mark.mpi
@pytest.mark.parametrize(
    "executor",
    [Executor.TORCH_TP, Executor.NVFUSER],
    ids=lambda e: e.name,
)
def test_transformer_layer(setup_default_process_group, benchmark, executor: Executor):
    config = load_config("deepseek-ai/deepseek-v3")
    # Create only one layer which is sufficient for the test.
    config.num_hidden_layers = 1
    # Without this, the first and only layer will have a dense MLP instead of MoE.
    config.first_k_dense_replace = 0
    # Disable quantization so the test can run on A100 and is made easier for nvFuser.
    delattr(config, "quantization_config")

    # This ensures the input tokens are identically replicated on all ranks.
    # Otherwise, some ranks may skip an expert because they have no tokens to
    # send, while other ranks don't. This will cause a deadlock because a NCCL
    # collective is expected to be called by all ranks in the process group.
    torch.manual_seed(0)

    d = dist.get_world_size()
    mesh = dist.device_mesh.init_device_mesh("cuda", [d])

    with default_tensor_type(dtype=config.torch_dtype, device="cuda"):
        # Loading the model under `device="cuda"` makes weight initialization
        # much faster but requires full GPU memory allocation.
        #
        # Alternatively, I think the following may work but haven't tried it:
        # 1. Load the model under torch.nn.utils.init_empty_weights. This skips weight initialization and allocates full weights on CPU not GPU.
        # 2. parallelize_module
        # 3. Load pre-trained parameters.
        # 4. Move the model to CUDA.
        model = load_model(config)
        # Training is unavailable (cf. https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/modeling_deepseek.py#L439)
        model.eval()

        transformer_layer = model.layers[0]

        # By default, RowwiseParallel and ColwiseParallel output a local tensor
        # and therefore num_heads needs to be adjusted to accomodate the local
        # size. Alternatively, I could RowwiseParallel(use_local_output=False)
        # so the linear layer outputs a DTensor, which can be viewed using the
        # original num_heads. This requires all activations, parameters, and
        # buffers to be DTensor; otherwise aten ops would complain "got mixed
        # torch.Tensor and DTensor". Doing so is challenging because
        # DeepseekV3RotaryEmbedding creates cos_cached and sin_cached during
        # the first forward call (cf.
        # https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/modeling_deepseek.py#L143-L144).
        transformer_layer.self_attn.num_heads //= d

        # Create the parallel plan
        parallel_plan = {
            "self_attn.q_b_proj": ColwiseParallel(),
            "self_attn.kv_b_proj": ColwiseParallel(),
            "self_attn.o_proj": RowwiseParallel(),
        }

        for expert in range(config.n_routed_experts):
            parallel_plan[f"mlp.experts.{expert}.gate_proj"] = ColwiseParallel()
            parallel_plan[f"mlp.experts.{expert}.up_proj"] = ColwiseParallel()
            parallel_plan[f"mlp.experts.{expert}.down_proj"] = RowwiseParallel()

        parallel_plan["mlp.shared_experts.gate_proj"] = ColwiseParallel()
        parallel_plan["mlp.shared_experts.up_proj"] = ColwiseParallel()
        parallel_plan["mlp.shared_experts.down_proj"] = RowwiseParallel()

        match executor:
            case Executor.TORCH_TP:
                transformer_layer = parallelize_module(
                    transformer_layer,
                    mesh,
                    parallel_plan,
                )

                # Sanity-check parameters are indeed distributed
                distributed_params: list[str] = [
                    name
                    for name, parameter in transformer_layer.named_parameters()
                    if isinstance(parameter.data, DTensor)
                ]
                assert len(distributed_params) == 3 + (config.n_routed_experts + 1) * 3
            case Executor.NVFUSER:
                parallelize_module_with_nvfuser(
                    transformer_layer, mesh, parallel_plan, fqn=""
                )

        batch_size = 1
        seq_len = 1024
        inp = torch.randn(batch_size, seq_len, config.hidden_size)
        mask = transformers.modeling_attn_mask_utils._prepare_4d_causal_attention_mask(
            None, [batch_size, seq_len], inp, past_key_values_length=0
        )
        warmup_fn, benchmark_fn = get_benchmark_fns(
            lambda: transformer_layer(inp, attention_mask=mask)
        )

        (out,) = warmup_fn()
        assert out.size() == (batch_size, seq_len, config.hidden_size)
        assert out.dtype == config.torch_dtype
        assert out.is_cuda

        benchmark.pedantic(benchmark_fn, rounds=5)


wrapper

def default_tensor_type(dtype=torch.float32, device="cpu"):
    # Save
    prev_dtype = torch.get_default_dtype()
    prev_device = torch.get_default_device()

    # Set
    torch.set_default_dtype(dtype)
    torch.set_default_device(device)

    yield

    # Restore
    torch.set_default_dtype(prev_dtype)
    torch.set_default_device(prev_device)

# This decorator ensures that the model/config is downloaded only once by rank
# 0. Other ranks will load from the cache that's stored on the same machine.
def download_once(fn):
    @wraps(fn)
    def wrapper(*args, **kwargs):
        rank = dist.get_rank()
        if rank == 0:
            # Download only once.
            result = fn(*args, **kwargs)

        dist.barrier()

        if rank != 0:
            # Other ranks load from cache.
            result = fn(*args, **kwargs)

        return result

    return wrapper

@download_once
def load_config(model_name: str) -> transformers.PretrainedConfig:
    return transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=True)

@download_once
def load_model(config: transformers.PretrainedConfig) -> transformers.PreTrainedModel:
    return transformers.AutoModel.from_config(config, trust_remote_code=True)

class Executor(Enum):
    # https://docs.pytorch.org/docs/stable/distributed.tensor.parallel.html
    TORCH_TP = auto()
    NVFUSER = auto()


# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

# This file contains certain building blocks of the AlphaFold3 model.

import pytest
import torch
from dataclasses import dataclass
from enum import Enum, auto

import nvfuser_direct as nvfuser
from nvfuser_direct import FusionDefinition, DataType, TensorView

@dataclass
class ModelConfig:
    c_z: int = 128
    c_hidden: int = 32
    n_heads: int = 4

_DEFAULT_CONFIG = ModelConfig()

class Direction(Enum):
    INCOMING = auto()  # aka ending node
    OUTGOING = auto()  # aka starting node

def layer_norm(
    fd: FusionDefinition, x: TensorView, w: TensorView, b: TensorView
) -> TensorView:
    io_dtype = x.dtype()
    x = fd.ops.cast(x, dtype=DataType.Float)
    var, mean = fd.ops.var_mean(x, dims=[-1], correction=0, keepdim=True)
    y = fd.ops.sub(x, mean)
    var = fd.ops.add(var, fd.define_scalar(1e-5))
    y = fd.ops.mul(y, fd.ops.rsqrt(var))
    shape = fd.ops.shape(x)
    w = fd.ops.broadcast_in_dim(w, shape=shape, broadcast_dims=[-1])
    y = fd.ops.mul(y, w)
    b = fd.ops.broadcast_in_dim(b, shape=shape, broadcast_dims=[-1])
    y = fd.ops.add(y, b)
    y = fd.ops.cast(y, dtype=io_dtype)
    return y

def gating(
    fd: FusionDefinition,
    z: TensorView,
    w_p: TensorView,
    z_in: TensorView,
    w_g: TensorView,
) -> TensorView:
    io_dtype = z.dtype()
    p = fd.ops.linear(z, w_p)
    g = fd.ops.linear(z_in, w_g)
    g = fd.ops.sigmoid(g)
    z = fd.ops.mul(p, g)
    return fd.ops.cast(z, dtype=io_dtype)

# https://elanapearl.github.io/blog/2024/the-illustrated-alphafold/#triangle-updates
#
# Jumper, J., Evans, R., Pritzel, A. et al. Highly accurate protein structure
# prediction with AlphaFold. Nature 596, 583–589 (2021).
# https://doi.org/10.1038/s41586-021-03819-2
# (see Supplementary Methods 1.6.5 for details)
@pytest.mark.mpi
@pytest.mark.parametrize(
    "direction", [Direction.OUTGOING, Direction.INCOMING], ids=lambda d: d.name.lower()
)
def test_triangle_updates(direction, multidevice_test):
    d = multidevice_test.size
    cp_size = 2
    if d % (cp_size * cp_size) != 0:
        pytest.skip(
            f"We only support even split, so {d} has to be divisible by {cp_size * cp_size} for {cp_size=}."
        )
    dp_size = d // (cp_size * cp_size)

    c_z = _DEFAULT_CONFIG.c_z

    with FusionDefinition() as fd:
        z_in_tv = fd.define_tensor(
            shape=[-1, -1, -1, c_z],
            dtype=DataType.BFloat16,
            contiguity=True,
        )  # [b, i, j, c_z]
        w_norm_in = fd.define_tensor(
            shape=[c_z], dtype=DataType.BFloat16, contiguity=True
        )
        b_norm_in = fd.define_tensor(
            shape=[c_z], dtype=DataType.BFloat16, contiguity=True
        )
        w_p_in = fd.define_tensor(
            shape=[c_z * 2, c_z], dtype=DataType.BFloat16, contiguity=True
        )
        w_g_in = fd.define_tensor(
            shape=[c_z * 2, c_z], dtype=DataType.BFloat16, contiguity=True
        )
        w_norm_out = fd.define_tensor(
            shape=[c_z], dtype=DataType.BFloat16, contiguity=True
        )
        b_norm_out = fd.define_tensor(
            shape=[c_z], dtype=DataType.BFloat16, contiguity=True
        )
        w_p_out = fd.define_tensor(
            shape=[c_z, c_z], dtype=DataType.BFloat16, contiguity=True
        )
        w_g_out = fd.define_tensor(
            shape=[c_z, c_z], dtype=DataType.BFloat16, contiguity=True
        )
        # Masking is used in an internal implementation: http://nv/e-4
        mask_tv = fd.define_tensor(
            shape=[-1, -1, -1], dtype=DataType.Bool, contiguity=True
        )  # [b, i, j]

        batch_size = fd.ops.size(z_in_tv, 0)
        n_tokens = fd.ops.size(z_in_tv, 1)

        z_in = layer_norm(fd, z_in_tv, w_norm_in, b_norm_in)
        z = gating(fd, z_in, w_p_in, z_in, w_g_in)
        mask = fd.ops.broadcast_in_dim(
            mask_tv,
            shape=[batch_size, n_tokens, n_tokens, c_z],
            broadcast_dims=[0, 1, 2],
        )
        z = fd.ops.where(mask, z, 0.0)
        a = fd.ops.slice(z, [0, 0, 0, 0], [batch_size, n_tokens, n_tokens, c_z])
        b = fd.ops.slice(z, [0, 0, 0, c_z], [batch_size, n_tokens, n_tokens, c_z * 2])

        match direction:
            case Direction.OUTGOING:
                # z_out = einsum("bikc,bjkc->bijc", a, b)
                a = fd.ops.permute(a, [0, 3, 1, 2])  # [b, c, i, k]
                b = fd.ops.permute(b, [0, 3, 2, 1])  # [b, c, k, j]
            case Direction.INCOMING:
                # z_out = einsum("bkic,bkjc->bijc", a, b)
                a = fd.ops.permute(a, [0, 3, 2, 1])  # [b, c, i, k]
                b = fd.ops.permute(b, [0, 3, 1, 2])  # [b, c, k, j]
        z = fd.ops.matmul(a, b)  # [b, c, i, j]
        matmul_out = z
        z = fd.ops.permute(z, [0, 2, 3, 1])  # [b, i, j, c]

        z = layer_norm(fd, z, w_norm_out, b_norm_out)
        z = gating(fd, z, w_p_out, z_in, w_g_out)
        fd.add_output(z)

        mesh = nvfuser.multidevice.DeviceMesh(
            torch.arange(d).reshape(dp_size, cp_size, cp_size)
        )
        for tv in [
            z_in_tv,
            w_norm_in,
            b_norm_in,
            w_p_in,
            w_g_in,
            w_norm_out,
            b_norm_out,
            w_p_out,
            w_g_out,
            mask_tv,
            matmul_out,
        ]:
            tv.set_device_mesh(mesh)

        for tv in [z_in_tv, mask_tv]:
            tv.outer_split(2, cp_size)
            tv.axis(2).parallelize(nvfuser.ParallelType.mesh_x)
            tv.outer_split(1, cp_size)
            tv.axis(1).parallelize(nvfuser.ParallelType.mesh_y)
            tv.outer_split(0, dp_size)
            tv.axis(0).parallelize(nvfuser.ParallelType.mesh_z)

        # TODO(#5901): this can be avoided with a better sharding propagation.
        #
        # matmul_out is of shape [b, c, i, j]. We shard `b` by `DIDz`, `i` by
        # `DIDy`, and `j` by `DIDx`.
        matmul_out.outer_split(-1, cp_size)
        match direction:
            case Direction.OUTGOING:
                matmul_out.axis(-2).parallelize(nvfuser.ParallelType.mesh_x)
            case Direction.INCOMING:
                matmul_out.axis(-2).parallelize(nvfuser.ParallelType.mesh_y)
        matmul_out.outer_split(3, cp_size)
        matmul_out.axis(3).parallelize(nvfuser.ParallelType.mesh_x)
        matmul_out.outer_split(2, cp_size)
        matmul_out.axis(2).parallelize(nvfuser.ParallelType.mesh_y)
        matmul_out.outer_split(0, dp_size)
        matmul_out.axis(0).parallelize(nvfuser.ParallelType.mesh_z)

    batch_per_rank = 3
    n_tokens_per_rank = 5
    z_in_ref = torch.testing.make_tensor(
        batch_per_rank * dp_size,
        n_tokens_per_rank * cp_size,
        n_tokens_per_rank * cp_size,
        c_z,
        dtype=torch.bfloat16,
        device="cpu",
    )
    mask_ref = torch.testing.make_tensor(
        batch_per_rank * dp_size,
        n_tokens_per_rank * cp_size,
        n_tokens_per_rank * cp_size,
        dtype=torch.bool,
        device="cpu",
    )

    z_in = multidevice_test.shard_tensor(z_in_ref, z_in_tv)
    w_norm_in = torch.testing.make_tensor(c_z, dtype=torch.bfloat16, device="cuda")
    b_norm_in = torch.testing.make_tensor(c_z, dtype=torch.bfloat16, device="cuda")
    w_p_in = torch.testing.make_tensor(
        c_z * 2, c_z, dtype=torch.bfloat16, device="cuda"
    )
    w_g_in = torch.testing.make_tensor(
        c_z * 2, c_z, dtype=torch.bfloat16, device="cuda"
    )
    w_norm_out = torch.testing.make_tensor(c_z, dtype=torch.bfloat16, device="cuda")
    b_norm_out = torch.testing.make_tensor(c_z, dtype=torch.bfloat16, device="cuda")
    w_p_out = torch.testing.make_tensor(c_z, c_z, dtype=torch.bfloat16, device="cuda")
    w_g_out = torch.testing.make_tensor(c_z, c_z, dtype=torch.bfloat16, device="cuda")
    mask = multidevice_test.shard_tensor(mask_ref, mask_tv)
    (z_out,) = fd.execute(
        [
            z_in,
            w_norm_in,
            b_norm_in,
            w_p_in,
            w_g_in,
            w_norm_out,
            b_norm_out,
            w_p_out,
            w_g_out,
            mask,
        ]
    )
    assert z_out.shape == (batch_per_rank, n_tokens_per_rank, n_tokens_per_rank, c_z)


ModelConfig

# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

# This file contains certain building blocks of the AlphaFold3 model.

import pytest
import torch
from dataclasses import dataclass
from enum import Enum, auto

import nvfuser_direct as nvfuser
from nvfuser_direct import FusionDefinition, DataType, TensorView

@dataclass
class ModelConfig:
    c_z: int = 128
    c_hidden: int = 32
    n_heads: int = 4

_DEFAULT_CONFIG = ModelConfig()

class Direction(Enum):
    INCOMING = auto()  # aka ending node
    OUTGOING = auto()  # aka starting node

def layer_norm(
    fd: FusionDefinition, x: TensorView, w: TensorView, b: TensorView
) -> TensorView:
    io_dtype = x.dtype()
    x = fd.ops.cast(x, dtype=DataType.Float)
    var, mean = fd.ops.var_mean(x, dims=[-1], correction=0, keepdim=True)
    y = fd.ops.sub(x, mean)
    var = fd.ops.add(var, fd.define_scalar(1e-5))
    y = fd.ops.mul(y, fd.ops.rsqrt(var))
    shape = fd.ops.shape(x)


Direction

import pytest
import torch
from dataclasses import dataclass
from enum import Enum, auto

import nvfuser_direct as nvfuser
from nvfuser_direct import FusionDefinition, DataType, TensorView

@dataclass
class ModelConfig:
    c_z: int = 128
    c_hidden: int = 32
    n_heads: int = 4

_DEFAULT_CONFIG = ModelConfig()

class Direction(Enum):
    INCOMING = auto()  # aka ending node
    OUTGOING = auto()  # aka starting node

def layer_norm(
    fd: FusionDefinition, x: TensorView, w: TensorView, b: TensorView
) -> TensorView:
    io_dtype = x.dtype()
    x = fd.ops.cast(x, dtype=DataType.Float)
    var, mean = fd.ops.var_mean(x, dims=[-1], correction=0, keepdim=True)
    y = fd.ops.sub(x, mean)
    var = fd.ops.add(var, fd.define_scalar(1e-5))
    y = fd.ops.mul(y, fd.ops.rsqrt(var))
    shape = fd.ops.shape(x)
    w = fd.ops.broadcast_in_dim(w, shape=shape, broadcast_dims=[-1])
    y = fd.ops.mul(y, w)
    b = fd.ops.broadcast_in_dim(b, shape=shape, broadcast_dims=[-1])
    y = fd.ops.add(y, b)
    y = fd.ops.cast(y, dtype=io_dtype)
    return y


import nvfuser_direct as nvfuser
from nvfuser_direct import FusionDefinition, DataType, TensorView

@dataclass
class ModelConfig:
    c_z: int = 128
    c_hidden: int = 32
    n_heads: int = 4

_DEFAULT_CONFIG = ModelConfig()

class Direction(Enum):
    INCOMING = auto()  # aka ending node
    OUTGOING = auto()  # aka starting node

def layer_norm(
    fd: FusionDefinition, x: TensorView, w: TensorView, b: TensorView
) -> TensorView:
    io_dtype = x.dtype()
    x = fd.ops.cast(x, dtype=DataType.Float)
    var, mean = fd.ops.var_mean(x, dims=[-1], correction=0, keepdim=True)
    y = fd.ops.sub(x, mean)
    var = fd.ops.add(var, fd.define_scalar(1e-5))
    y = fd.ops.mul(y, fd.ops.rsqrt(var))
    shape = fd.ops.shape(x)
    w = fd.ops.broadcast_in_dim(w, shape=shape, broadcast_dims=[-1])
    y = fd.ops.mul(y, w)
    b = fd.ops.broadcast_in_dim(b, shape=shape, broadcast_dims=[-1])
    y = fd.ops.add(y, b)
    y = fd.ops.cast(y, dtype=io_dtype)
    return y

def gating(
    fd: FusionDefinition,
    z: TensorView,
    w_p: TensorView,
    z_in: TensorView,
    w_g: TensorView,
) -> TensorView:
    io_dtype = z.dtype()
    p = fd.ops.linear(z, w_p)
    g = fd.ops.linear(z_in, w_g)
    g = fd.ops.sigmoid(g)
    z = fd.ops.mul(p, g)
    return fd.ops.cast(z, dtype=io_dtype)

# https://elanapearl.github.io/blog/2024/the-illustrated-alphafold/#triangle-updates
#
# Jumper, J., Evans, R., Pritzel, A. et al. Highly accurate protein structure


gating

def layer_norm(
    fd: FusionDefinition, x: TensorView, w: TensorView, b: TensorView
) -> TensorView:
    io_dtype = x.dtype()
    x = fd.ops.cast(x, dtype=DataType.Float)
    var, mean = fd.ops.var_mean(x, dims=[-1], correction=0, keepdim=True)
    y = fd.ops.sub(x, mean)
    var = fd.ops.add(var, fd.define_scalar(1e-5))
    y = fd.ops.mul(y, fd.ops.rsqrt(var))
    shape = fd.ops.shape(x)
    w = fd.ops.broadcast_in_dim(w, shape=shape, broadcast_dims=[-1])
    y = fd.ops.mul(y, w)
    b = fd.ops.broadcast_in_dim(b, shape=shape, broadcast_dims=[-1])
    y = fd.ops.add(y, b)
    y = fd.ops.cast(y, dtype=io_dtype)
    return y

def gating(
    fd: FusionDefinition,
    z: TensorView,
    w_p: TensorView,
    z_in: TensorView,
    w_g: TensorView,
) -> TensorView:
    io_dtype = z.dtype()
    p = fd.ops.linear(z, w_p)
    g = fd.ops.linear(z_in, w_g)
    g = fd.ops.sigmoid(g)
    z = fd.ops.mul(p, g)
    return fd.ops.cast(z, dtype=io_dtype)

# https://elanapearl.github.io/blog/2024/the-illustrated-alphafold/#triangle-updates
#
# Jumper, J., Evans, R., Pritzel, A. et al. Highly accurate protein structure
# prediction with AlphaFold. Nature 596, 583–589 (2021).
# https://doi.org/10.1038/s41586-021-03819-2
# (see Supplementary Methods 1.6.5 for details)
@pytest.mark.mpi
@pytest.mark.parametrize(
    "direction", [Direction.OUTGOING, Direction.INCOMING], ids=lambda d: d.name.lower()
)
def test_triangle_updates(direction, multidevice_test):
    d = multidevice_test.size
    cp_size = 2
    if d % (cp_size * cp_size) != 0:
        pytest.skip(
            f"We only support even split, so {d} has to be divisible by {cp_size * cp_size} for {cp_size=}."
        )
    dp_size = d // (cp_size * cp_size)


    w_g: TensorView,
) -> TensorView:
    io_dtype = z.dtype()
    p = fd.ops.linear(z, w_p)
    g = fd.ops.linear(z_in, w_g)
    g = fd.ops.sigmoid(g)
    z = fd.ops.mul(p, g)
    return fd.ops.cast(z, dtype=io_dtype)

# https://elanapearl.github.io/blog/2024/the-illustrated-alphafold/#triangle-updates
#
# Jumper, J., Evans, R., Pritzel, A. et al. Highly accurate protein structure
# prediction with AlphaFold. Nature 596, 583–589 (2021).
# https://doi.org/10.1038/s41586-021-03819-2
# (see Supplementary Methods 1.6.5 for details)
@pytest.mark.mpi
@pytest.mark.parametrize(
    "direction", [Direction.OUTGOING, Direction.INCOMING], ids=lambda d: d.name.lower()
)
def test_triangle_updates(direction, multidevice_test):
    d = multidevice_test.size
    cp_size = 2
    if d % (cp_size * cp_size) != 0:
        pytest.skip(
            f"We only support even split, so {d} has to be divisible by {cp_size * cp_size} for {cp_size=}."
        )
    dp_size = d // (cp_size * cp_size)

    c_z = _DEFAULT_CONFIG.c_z

    with FusionDefinition() as fd:
        z_in_tv = fd.define_tensor(
            shape=[-1, -1, -1, c_z],
            dtype=DataType.BFloat16,
            contiguity=True,
        )  # [b, i, j, c_z]
        w_norm_in = fd.define_tensor(
            shape=[c_z], dtype=DataType.BFloat16, contiguity=True
        )
        b_norm_in = fd.define_tensor(
            shape=[c_z], dtype=DataType.BFloat16, contiguity=True
        )
        w_p_in = fd.define_tensor(
            shape=[c_z * 2, c_z], dtype=DataType.BFloat16, contiguity=True
        )
        w_g_in = fd.define_tensor(
            shape=[c_z * 2, c_z], dtype=DataType.BFloat16, contiguity=True
        )
        w_norm_out = fd.define_tensor(
            shape=[c_z], dtype=DataType.BFloat16, contiguity=True
        )
        b_norm_out = fd.define_tensor(
            shape=[c_z], dtype=DataType.BFloat16, contiguity=True
        )
        w_p_out = fd.define_tensor(
            shape=[c_z, c_z], dtype=DataType.BFloat16, contiguity=True
        )
        w_g_out = fd.define_tensor(
            shape=[c_z, c_z], dtype=DataType.BFloat16, contiguity=True
        )
        # Masking is used in an internal implementation: http://nv/e-4
        mask_tv = fd.define_tensor(
            shape=[-1, -1, -1], dtype=DataType.Bool, contiguity=True
        )  # [b, i, j]

        batch_size = fd.ops.size(z_in_tv, 0)
        n_tokens = fd.ops.size(z_in_tv, 1)

        z_in = layer_norm(fd, z_in_tv, w_norm_in, b_norm_in)
        z = gating(fd, z_in, w_p_in, z_in, w_g_in)
        mask = fd.ops.broadcast_in_dim(
            mask_tv,
            shape=[batch_size, n_tokens, n_tokens, c_z],
            broadcast_dims=[0, 1, 2],
        )
        z = fd.ops.where(mask, z, 0.0)
        a = fd.ops.slice(z, [0, 0, 0, 0], [batch_size, n_tokens, n_tokens, c_z])
        b = fd.ops.slice(z, [0, 0, 0, c_z], [batch_size, n_tokens, n_tokens, c_z * 2])

        match direction:
            case Direction.OUTGOING:
                # z_out = einsum("bikc,bjkc->bijc", a, b)
                a = fd.ops.permute(a, [0, 3, 1, 2])  # [b, c, i, k]
                b = fd.ops.permute(b, [0, 3, 2, 1])  # [b, c, k, j]
            case Direction.INCOMING:
                # z_out = einsum("bkic,bkjc->bijc", a, b)
                a = fd.ops.permute(a, [0, 3, 2, 1])  # [b, c, i, k]
                b = fd.ops.permute(b, [0, 3, 1, 2])  # [b, c, k, j]
        z = fd.ops.matmul(a, b)  # [b, c, i, j]
        matmul_out = z
        z = fd.ops.permute(z, [0, 2, 3, 1])  # [b, i, j, c]

        z = layer_norm(fd, z, w_norm_out, b_norm_out)
        z = gating(fd, z, w_p_out, z_in, w_g_out)
        fd.add_output(z)

        mesh = nvfuser.multidevice.DeviceMesh(
            torch.arange(d).reshape(dp_size, cp_size, cp_size)
        )
        for tv in [
            z_in_tv,
            w_norm_in,
            b_norm_in,
            w_p_in,
            w_g_in,
            w_norm_out,
            b_norm_out,
            w_p_out,
            w_g_out,
            mask_tv,
            matmul_out,
        ]:
            tv.set_device_mesh(mesh)

        for tv in [z_in_tv, mask_tv]:
            tv.outer_split(2, cp_size)
            tv.axis(2).parallelize(nvfuser.ParallelType.mesh_x)
            tv.outer_split(1, cp_size)
            tv.axis(1).parallelize(nvfuser.ParallelType.mesh_y)
            tv.outer_split(0, dp_size)
            tv.axis(0).parallelize(nvfuser.ParallelType.mesh_z)

        # TODO(#5901): this can be avoided with a better sharding propagation.
        #
        # matmul_out is of shape [b, c, i, j]. We shard `b` by `DIDz`, `i` by
        # `DIDy`, and `j` by `DIDx`.
        matmul_out.outer_split(-1, cp_size)
        match direction:
            case Direction.OUTGOING:
                matmul_out.axis(-2).parallelize(nvfuser.ParallelType.mesh_x)
            case Direction.INCOMING:
                matmul_out.axis(-2).parallelize(nvfuser.ParallelType.mesh_y)
        matmul_out.outer_split(3, cp_size)
        matmul_out.axis(3).parallelize(nvfuser.ParallelType.mesh_x)
        matmul_out.outer_split(2, cp_size)
        matmul_out.axis(2).parallelize(nvfuser.ParallelType.mesh_y)
        matmul_out.outer_split(0, dp_size)
        matmul_out.axis(0).parallelize(nvfuser.ParallelType.mesh_z)

    batch_per_rank = 3
    n_tokens_per_rank = 5
    z_in_ref = torch.testing.make_tensor(
        batch_per_rank * dp_size,
        n_tokens_per_rank * cp_size,
        n_tokens_per_rank * cp_size,
        c_z,
        dtype=torch.bfloat16,
        device="cpu",
    )
    mask_ref = torch.testing.make_tensor(
        batch_per_rank * dp_size,
        n_tokens_per_rank * cp_size,
        n_tokens_per_rank * cp_size,
        dtype=torch.bool,
        device="cpu",
    )

    z_in = multidevice_test.shard_tensor(z_in_ref, z_in_tv)
    w_norm_in = torch.testing.make_tensor(c_z, dtype=torch.bfloat16, device="cuda")
    b_norm_in = torch.testing.make_tensor(c_z, dtype=torch.bfloat16, device="cuda")
    w_p_in = torch.testing.make_tensor(
        c_z * 2, c_z, dtype=torch.bfloat16, device="cuda"
    )
    w_g_in = torch.testing.make_tensor(
        c_z * 2, c_z, dtype=torch.bfloat16, device="cuda"
    )
    w_norm_out = torch.testing.make_tensor(c_z, dtype=torch.bfloat16, device="cuda")
    b_norm_out = torch.testing.make_tensor(c_z, dtype=torch.bfloat16, device="cuda")
    w_p_out = torch.testing.make_tensor(c_z, c_z, dtype=torch.bfloat16, device="cuda")
    w_g_out = torch.testing.make_tensor(c_z, c_z, dtype=torch.bfloat16, device="cuda")
    mask = multidevice_test.shard_tensor(mask_ref, mask_tv)
    (z_out,) = fd.execute(
        [
            z_in,
            w_norm_in,
            b_norm_in,
            w_p_in,
            w_g_in,
            w_norm_out,
            b_norm_out,
            w_p_out,
            w_g_out,
            mask,
        ]
    )
    assert z_out.shape == (batch_per_rank, n_tokens_per_rank, n_tokens_per_rank, c_z)


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

import pytest
import torch

import nvfuser_direct as nvfuser
from nvfuser_direct import DataType, FusionDefinition

@pytest.mark.mpi
def test_allgather(multidevice_test):
    d = multidevice_test.size
    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((d * 4,), contiguity=True, dtype=DataType.Float)
        out = fd.ops.set(inp_tv)
        fd.add_output(out)

        inp_tv.set_device_mesh(mesh)
        inp_tv.outer_split(0, d)
        inp_tv.axis(0).parallelize(nvfuser.ParallelType.mesh_x)

        out.set_device_mesh(mesh)

    inp_ref = torch.randn(d * 4)
    out_ref = inp_ref

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out.cpu(), out_ref)

@pytest.mark.mpi
def test_allgather_2d(multidevice_test):
    d = multidevice_test.size
    tp_size = 2
    if d % tp_size != 0:
        pytest.skip(f"Number of devices ({d=}) must be divisible by {tp_size=}")
    dp_size = d // tp_size
    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d).reshape(dp_size, tp_size))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        out_tv = fd.ops.set(inp_tv)
        fd.add_output(out_tv)

        for tv in [inp_tv, out_tv]:
            tv.set_device_mesh(mesh)
            tv.outer_split(0, dp_size)
            tv.axis(0).parallelize(nvfuser.ParallelType.mesh_y)
        inp_tv.outer_split(2, tp_size)
        inp_tv.axis(2).parallelize(nvfuser.ParallelType.mesh_x)

    rows_per_rank, cols_per_rank = 2, 3
    rows, cols = dp_size * rows_per_rank, tp_size * cols_per_rank
    inp_ref = torch.randn(rows, cols)
    out_ref = inp_ref

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out, multidevice_test.shard_tensor(out_ref, out_tv))

@pytest.mark.mpi
def test_allgather_expanded_broadcast(multidevice_test):
    d = multidevice_test.size
    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor([d], contiguity=True, dtype=DataType.Float)
        expanded = fd.ops.broadcast_in_dim(inp_tv, [d, 3], [0])
        out = fd.ops.set(expanded)
        fd.add_output(out)

        for tv in [inp_tv, expanded, out]:
            tv.set_device_mesh(mesh)
        inp_tv.axis(0).parallelize(nvfuser.ParallelType.mesh_x)
        expanded.axis(0).parallelize(nvfuser.ParallelType.mesh_x)

    inp_ref = torch.randn(d)
    out_ref = inp_ref.unsqueeze(-1).expand(-1, 3)

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out.cpu(), out_ref)

@pytest.mark.mpi
def test_allreduce(multidevice_test):
    d = multidevice_test.size
    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((-1, -1, -1), contiguity=True, dtype=DataType.Float)
        out = fd.ops.sum(inp_tv, [1])
        fd.add_output(out)

        inp_tv.set_device_mesh(mesh)
        inp_tv.outer_split(1, d)
        inp_tv.axis(1).parallelize(nvfuser.ParallelType.mesh_x)

    m = 2
    k = d * 3
    n = 5
    inp_ref = torch.randn(m, k, n)
    out_ref = inp_ref.sum(1)

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])

    torch.testing.assert_close(out.cpu(), out_ref)

@pytest.mark.mpi
def test_reduce_scatter(multidevice_test):
    d = multidevice_test.size

    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((d, d * 4), contiguity=True, dtype=DataType.Float)
        out_tv = fd.ops.sum(inp_tv, [0])
        fd.add_output(out_tv)

        inp_tv.set_device_mesh(mesh)
        inp_tv.axis(0).parallelize(nvfuser.ParallelType.mesh_x)

        out_tv.set_device_mesh(mesh)
        out_tv.outer_split(-1, d)
        out_tv.axis(-2).parallelize(nvfuser.ParallelType.mesh_x)

    inp_ref = torch.randn(d, d * 4)
    out_ref = inp_ref.sum(0)

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out, multidevice_test.shard_tensor(out_ref, out_tv))

@pytest.mark.mpi
def test_reduce_scatter_noncontiguous(multidevice_test):
    d = multidevice_test.size

    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((d, 3, d * 4), contiguity=True, dtype=DataType.Float)
        out_tv = fd.ops.sum(inp_tv, [0])
        fd.add_output(out_tv)

        # inp: [iDID{d}, i{3}, i{d*4}]
        # out: [r{d}, i{3}, i{d*4}]
        #                   /    \
        #               iDID{d}  i{4}
        #
        # Unlike test_reduce_scatter, this leads to extra data copy because
        # the scattered axis is not outermost in allocation.
        inp_tv.set_device_mesh(mesh)
        inp_tv.axis(0).parallelize(nvfuser.ParallelType.mesh_x)

        out_tv.set_device_mesh(mesh)
        out_tv.outer_split(-1, d)
        out_tv.axis(-2).parallelize(nvfuser.ParallelType.mesh_x)

    inp_ref = torch.randn(d, 3, d * 4)
    out_ref = inp_ref.sum(0)

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out, multidevice_test.shard_tensor(out_ref, out_tv))

# AllToAll patterns seen in expert parallelism
@pytest.mark.mpi
@pytest.mark.parametrize(
    "inp_axis,out_axis", [(0, 1), (1, 0)], ids=["dispatch", "combine"]
)
def test_alltoall(multidevice_test, inp_axis, out_axis):
    d = multidevice_test.size
    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))
    n = 3

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((d, d * n), contiguity=True, dtype=DataType.Half)
        out_tv = fd.ops.set(inp_tv)
        fd.add_output(out_tv)

        inp_tv.set_device_mesh(mesh)
        inp_tv.outer_split(inp_axis, d)
        inp_tv.axis(inp_axis).parallelize(nvfuser.ParallelType.mesh_x)

        out_tv.set_device_mesh(mesh)
        out_tv.outer_split(out_axis, d)
        out_tv.axis(out_axis).parallelize(nvfuser.ParallelType.mesh_x)

    in_ref = torch.randn(d, d * n, dtype=torch.float16)
    out_ref = in_ref

    inp = multidevice_test.shard_tensor(in_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out, multidevice_test.shard_tensor(out_ref, out_tv))


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

import pytest
import torch

import nvfuser_direct as nvfuser
from nvfuser_direct import DataType, FusionDefinition

@pytest.mark.mpi
def test_allgather(multidevice_test):
    d = multidevice_test.size
    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((d * 4,), contiguity=True, dtype=DataType.Float)
        out = fd.ops.set(inp_tv)
        fd.add_output(out)

        inp_tv.set_device_mesh(mesh)
        inp_tv.outer_split(0, d)
        inp_tv.axis(0).parallelize(nvfuser.ParallelType.mesh_x)

        out.set_device_mesh(mesh)

    inp_ref = torch.randn(d * 4)
    out_ref = inp_ref

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out.cpu(), out_ref)

@pytest.mark.mpi
def test_allgather_2d(multidevice_test):
    d = multidevice_test.size
    tp_size = 2
    if d % tp_size != 0:
        pytest.skip(f"Number of devices ({d=}) must be divisible by {tp_size=}")
    dp_size = d // tp_size
    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d).reshape(dp_size, tp_size))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        out_tv = fd.ops.set(inp_tv)
        fd.add_output(out_tv)

        for tv in [inp_tv, out_tv]:
            tv.set_device_mesh(mesh)
            tv.outer_split(0, dp_size)
            tv.axis(0).parallelize(nvfuser.ParallelType.mesh_y)


    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((d * 4,), contiguity=True, dtype=DataType.Float)
        out = fd.ops.set(inp_tv)
        fd.add_output(out)

        inp_tv.set_device_mesh(mesh)
        inp_tv.outer_split(0, d)
        inp_tv.axis(0).parallelize(nvfuser.ParallelType.mesh_x)

        out.set_device_mesh(mesh)

    inp_ref = torch.randn(d * 4)
    out_ref = inp_ref

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out.cpu(), out_ref)

@pytest.mark.mpi
def test_allgather_2d(multidevice_test):
    d = multidevice_test.size
    tp_size = 2
    if d % tp_size != 0:
        pytest.skip(f"Number of devices ({d=}) must be divisible by {tp_size=}")
    dp_size = d // tp_size
    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d).reshape(dp_size, tp_size))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor([-1, -1], contiguity=True, dtype=DataType.Float)
        out_tv = fd.ops.set(inp_tv)
        fd.add_output(out_tv)

        for tv in [inp_tv, out_tv]:
            tv.set_device_mesh(mesh)
            tv.outer_split(0, dp_size)
            tv.axis(0).parallelize(nvfuser.ParallelType.mesh_y)
        inp_tv.outer_split(2, tp_size)
        inp_tv.axis(2).parallelize(nvfuser.ParallelType.mesh_x)

    rows_per_rank, cols_per_rank = 2, 3
    rows, cols = dp_size * rows_per_rank, tp_size * cols_per_rank
    inp_ref = torch.randn(rows, cols)
    out_ref = inp_ref

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out, multidevice_test.shard_tensor(out_ref, out_tv))

@pytest.mark.mpi
def test_allgather_expanded_broadcast(multidevice_test):
    d = multidevice_test.size
    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor([d], contiguity=True, dtype=DataType.Float)
        expanded = fd.ops.broadcast_in_dim(inp_tv, [d, 3], [0])
        out = fd.ops.set(expanded)
        fd.add_output(out)

        for tv in [inp_tv, expanded, out]:
            tv.set_device_mesh(mesh)
        inp_tv.axis(0).parallelize(nvfuser.ParallelType.mesh_x)
        expanded.axis(0).parallelize(nvfuser.ParallelType.mesh_x)

    inp_ref = torch.randn(d)
    out_ref = inp_ref.unsqueeze(-1).expand(-1, 3)


        fd.add_output(out_tv)

        for tv in [inp_tv, out_tv]:
            tv.set_device_mesh(mesh)
            tv.outer_split(0, dp_size)
            tv.axis(0).parallelize(nvfuser.ParallelType.mesh_y)
        inp_tv.outer_split(2, tp_size)
        inp_tv.axis(2).parallelize(nvfuser.ParallelType.mesh_x)

    rows_per_rank, cols_per_rank = 2, 3
    rows, cols = dp_size * rows_per_rank, tp_size * cols_per_rank
    inp_ref = torch.randn(rows, cols)
    out_ref = inp_ref

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out, multidevice_test.shard_tensor(out_ref, out_tv))

@pytest.mark.mpi
def test_allgather_expanded_broadcast(multidevice_test):
    d = multidevice_test.size
    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor([d], contiguity=True, dtype=DataType.Float)
        expanded = fd.ops.broadcast_in_dim(inp_tv, [d, 3], [0])
        out = fd.ops.set(expanded)
        fd.add_output(out)

        for tv in [inp_tv, expanded, out]:
            tv.set_device_mesh(mesh)
        inp_tv.axis(0).parallelize(nvfuser.ParallelType.mesh_x)
        expanded.axis(0).parallelize(nvfuser.ParallelType.mesh_x)

    inp_ref = torch.randn(d)
    out_ref = inp_ref.unsqueeze(-1).expand(-1, 3)

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out.cpu(), out_ref)

@pytest.mark.mpi
def test_allreduce(multidevice_test):
    d = multidevice_test.size
    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((-1, -1, -1), contiguity=True, dtype=DataType.Float)
        out = fd.ops.sum(inp_tv, [1])
        fd.add_output(out)

        inp_tv.set_device_mesh(mesh)
        inp_tv.outer_split(1, d)
        inp_tv.axis(1).parallelize(nvfuser.ParallelType.mesh_x)

    m = 2
    k = d * 3
    n = 5
    inp_ref = torch.randn(m, k, n)


    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor([d], contiguity=True, dtype=DataType.Float)
        expanded = fd.ops.broadcast_in_dim(inp_tv, [d, 3], [0])
        out = fd.ops.set(expanded)
        fd.add_output(out)

        for tv in [inp_tv, expanded, out]:
            tv.set_device_mesh(mesh)
        inp_tv.axis(0).parallelize(nvfuser.ParallelType.mesh_x)
        expanded.axis(0).parallelize(nvfuser.ParallelType.mesh_x)

    inp_ref = torch.randn(d)
    out_ref = inp_ref.unsqueeze(-1).expand(-1, 3)

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out.cpu(), out_ref)

@pytest.mark.mpi
def test_allreduce(multidevice_test):
    d = multidevice_test.size
    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((-1, -1, -1), contiguity=True, dtype=DataType.Float)
        out = fd.ops.sum(inp_tv, [1])
        fd.add_output(out)

        inp_tv.set_device_mesh(mesh)
        inp_tv.outer_split(1, d)
        inp_tv.axis(1).parallelize(nvfuser.ParallelType.mesh_x)

    m = 2
    k = d * 3
    n = 5
    inp_ref = torch.randn(m, k, n)
    out_ref = inp_ref.sum(1)

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])

    torch.testing.assert_close(out.cpu(), out_ref)

@pytest.mark.mpi
def test_reduce_scatter(multidevice_test):
    d = multidevice_test.size

    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((d, d * 4), contiguity=True, dtype=DataType.Float)
        out_tv = fd.ops.sum(inp_tv, [0])
        fd.add_output(out_tv)

        inp_tv.set_device_mesh(mesh)
        inp_tv.axis(0).parallelize(nvfuser.ParallelType.mesh_x)

        out_tv.set_device_mesh(mesh)
        out_tv.outer_split(-1, d)
        out_tv.axis(-2).parallelize(nvfuser.ParallelType.mesh_x)


        out = fd.ops.sum(inp_tv, [1])
        fd.add_output(out)

        inp_tv.set_device_mesh(mesh)
        inp_tv.outer_split(1, d)
        inp_tv.axis(1).parallelize(nvfuser.ParallelType.mesh_x)

    m = 2
    k = d * 3
    n = 5
    inp_ref = torch.randn(m, k, n)
    out_ref = inp_ref.sum(1)

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])

    torch.testing.assert_close(out.cpu(), out_ref)

@pytest.mark.mpi
def test_reduce_scatter(multidevice_test):
    d = multidevice_test.size

    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((d, d * 4), contiguity=True, dtype=DataType.Float)
        out_tv = fd.ops.sum(inp_tv, [0])
        fd.add_output(out_tv)

        inp_tv.set_device_mesh(mesh)
        inp_tv.axis(0).parallelize(nvfuser.ParallelType.mesh_x)

        out_tv.set_device_mesh(mesh)
        out_tv.outer_split(-1, d)
        out_tv.axis(-2).parallelize(nvfuser.ParallelType.mesh_x)

    inp_ref = torch.randn(d, d * 4)
    out_ref = inp_ref.sum(0)

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out, multidevice_test.shard_tensor(out_ref, out_tv))

@pytest.mark.mpi
def test_reduce_scatter_noncontiguous(multidevice_test):
    d = multidevice_test.size

    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((d, 3, d * 4), contiguity=True, dtype=DataType.Float)
        out_tv = fd.ops.sum(inp_tv, [0])
        fd.add_output(out_tv)

        # inp: [iDID{d}, i{3}, i{d*4}]
        # out: [r{d}, i{3}, i{d*4}]
        #                   /    \
        #               iDID{d}  i{4}
        #
        # Unlike test_reduce_scatter, this leads to extra data copy because
        # the scattered axis is not outermost in allocation.


        inp_tv = fd.define_tensor((d, d * 4), contiguity=True, dtype=DataType.Float)
        out_tv = fd.ops.sum(inp_tv, [0])
        fd.add_output(out_tv)

        inp_tv.set_device_mesh(mesh)
        inp_tv.axis(0).parallelize(nvfuser.ParallelType.mesh_x)

        out_tv.set_device_mesh(mesh)
        out_tv.outer_split(-1, d)
        out_tv.axis(-2).parallelize(nvfuser.ParallelType.mesh_x)

    inp_ref = torch.randn(d, d * 4)
    out_ref = inp_ref.sum(0)

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out, multidevice_test.shard_tensor(out_ref, out_tv))

@pytest.mark.mpi
def test_reduce_scatter_noncontiguous(multidevice_test):
    d = multidevice_test.size

    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((d, 3, d * 4), contiguity=True, dtype=DataType.Float)
        out_tv = fd.ops.sum(inp_tv, [0])
        fd.add_output(out_tv)

        # inp: [iDID{d}, i{3}, i{d*4}]
        # out: [r{d}, i{3}, i{d*4}]
        #                   /    \
        #               iDID{d}  i{4}
        #
        # Unlike test_reduce_scatter, this leads to extra data copy because
        # the scattered axis is not outermost in allocation.
        inp_tv.set_device_mesh(mesh)
        inp_tv.axis(0).parallelize(nvfuser.ParallelType.mesh_x)

        out_tv.set_device_mesh(mesh)
        out_tv.outer_split(-1, d)
        out_tv.axis(-2).parallelize(nvfuser.ParallelType.mesh_x)

    inp_ref = torch.randn(d, 3, d * 4)
    out_ref = inp_ref.sum(0)

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out, multidevice_test.shard_tensor(out_ref, out_tv))

# AllToAll patterns seen in expert parallelism
@pytest.mark.mpi
@pytest.mark.parametrize(
    "inp_axis,out_axis", [(0, 1), (1, 0)], ids=["dispatch", "combine"]
)
def test_alltoall(multidevice_test, inp_axis, out_axis):
    d = multidevice_test.size
    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))
    n = 3

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((d, d * n), contiguity=True, dtype=DataType.Half)
        out_tv = fd.ops.set(inp_tv)
        fd.add_output(out_tv)

        inp_tv.set_device_mesh(mesh)
        inp_tv.outer_split(inp_axis, d)
        inp_tv.axis(inp_axis).parallelize(nvfuser.ParallelType.mesh_x)


        inp_tv.set_device_mesh(mesh)
        inp_tv.axis(0).parallelize(nvfuser.ParallelType.mesh_x)

        out_tv.set_device_mesh(mesh)
        out_tv.outer_split(-1, d)
        out_tv.axis(-2).parallelize(nvfuser.ParallelType.mesh_x)

    inp_ref = torch.randn(d, 3, d * 4)
    out_ref = inp_ref.sum(0)

    inp = multidevice_test.shard_tensor(inp_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out, multidevice_test.shard_tensor(out_ref, out_tv))

# AllToAll patterns seen in expert parallelism
@pytest.mark.mpi
@pytest.mark.parametrize(
    "inp_axis,out_axis", [(0, 1), (1, 0)], ids=["dispatch", "combine"]
)
def test_alltoall(multidevice_test, inp_axis, out_axis):
    d = multidevice_test.size
    mesh = nvfuser.multidevice.DeviceMesh(torch.arange(d))
    n = 3

    with FusionDefinition() as fd:
        inp_tv = fd.define_tensor((d, d * n), contiguity=True, dtype=DataType.Half)
        out_tv = fd.ops.set(inp_tv)
        fd.add_output(out_tv)

        inp_tv.set_device_mesh(mesh)
        inp_tv.outer_split(inp_axis, d)
        inp_tv.axis(inp_axis).parallelize(nvfuser.ParallelType.mesh_x)

        out_tv.set_device_mesh(mesh)
        out_tv.outer_split(out_axis, d)
        out_tv.axis(out_axis).parallelize(nvfuser.ParallelType.mesh_x)

    in_ref = torch.randn(d, d * n, dtype=torch.float16)
    out_ref = in_ref

    inp = multidevice_test.shard_tensor(in_ref, inp_tv)
    (out,) = fd.execute([inp])
    torch.testing.assert_close(out, multidevice_test.shard_tensor(out_ref, out_tv))


# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

import pytest
import torch
import torch.distributed as dist

# Partitions n into k integers whose sum is n.
def random_partition(n: int, k: int) -> torch.Tensor:
    offsets = torch.randint(0, n + 1, [k - 1]).tolist()
    # The last partition ends at index n.
    offsets.append(n)
    offsets.sort()

    sizes = []
    prev = 0
    for offset in offsets:
        sizes.append(offset - prev)
        prev = offset
    assert sum(sizes) == n
    assert len(sizes) == k
    return torch.tensor(sizes, device="cuda")

def list_to_matrix(l: list, rows: int, cols: int) -> list[list]:
    return [l[row * cols : (row + 1) * cols] for row in range(rows)]

def matrix_to_list(m: list[list]) -> list:
    return [item for row in m for item in row]

# This can be fused into one kernel.
def rank_first_to_expert_first(
    tokens: torch.Tensor, n_tokens_for_expert_from_rank: torch.Tensor
) -> torch.Tensor:
    d, num_experts_per_rank = n_tokens_for_expert_from_rank.size()
    # Rank first, expert second
    tokens_for_expert_from_rank = tokens.split(
        n_tokens_for_expert_from_rank.flatten().tolist()
    )
    tokens_for_expert_from_rank = list_to_matrix(
        tokens_for_expert_from_rank, d, num_experts_per_rank
    )
    # list(zip(*matrix)) transposes the matrix.
    # Expert first, rank second
    tokens_for_expert_from_rank = list(zip(*tokens_for_expert_from_rank))
    tokens_for_expert_from_rank = matrix_to_list(tokens_for_expert_from_rank)
    tokens = torch.cat(tokens_for_expert_from_rank, dim=0)
    return tokens

# This can be fused into one kernel as well.
def expert_first_to_rank_first(
    tokens: torch.Tensor, n_tokens_for_expert_from_rank: torch.Tensor
) -> torch.Tensor:
    d, num_experts_per_rank = n_tokens_for_expert_from_rank.size()
    # Expert first, rank second
    tokens_for_expert_from_rank = tokens.split(
        n_tokens_for_expert_from_rank.t().flatten().tolist()
    )
    tokens_for_expert_from_rank = list_to_matrix(
        tokens_for_expert_from_rank, num_experts_per_rank, d
    )
    # Rank first, expert second
    tokens_for_expert_from_rank = list(zip(*tokens_for_expert_from_rank))
    tokens_for_expert_from_rank = matrix_to_list(tokens_for_expert_from_rank)
    tokens = torch.cat(tokens_for_expert_from_rank, dim=0)
    return tokens

# This test serves as the reference implementation for the expert parallelism
# dispatch and combine logic. The test is written in a way that is easy to
# understand, but not efficient.
@pytest.mark.mpi
def test_dispatch_and_combine(setup_default_process_group):
    rank = dist.get_rank()
    # So RNG generates different numbers for each rank.
    torch.manual_seed(rank)

    d = dist.get_world_size()
    num_experts_per_rank = 3
    n_experts = d * num_experts_per_rank

    # Input tokens are distributed among ranks, so the effective sequence length is `n_tokens * d`.
    n_tokens = 12
    # GPU 0: [n_tokens_for_expert_0_from_rank_0, n_tokens_for_expert_1_from_rank_0, ..., n_tokens_for_expert_5_from_rank_0]
    # GPU 1: [n_tokens_for_expert_0_from_rank_1, n_tokens_for_expert_1_from_rank_1, ..., n_tokens_for_expert_5_from_rank_1]
    # Both sum to `n_tokens`.
    n_tokens_for_expert = random_partition(n_tokens, n_experts)

    # For illustration, input tokens are complex numbers of value `token_id + expert_id * j`.
    expert_ids = []
    for expert, n in enumerate(n_tokens_for_expert.tolist()):
        expert_ids.extend([expert] * n)
    tokens = (
        torch.arange(
            rank * n_tokens, (rank + 1) * n_tokens, dtype=torch.float, device="cuda"
        )
        + torch.tensor(expert_ids, dtype=torch.float, device="cuda") * 1j
    )

    # --------------------------------------------------------------------------
    # An expert-parallel MoE layer dispatches, processes, and combines tokens
    # in the following steps:
    #
    # Step 1: Tokens are dispatched to the rank hosting the corresponding expert.
    # --------------------------------------------------------------------------
    n_tokens_for_expert_from_rank = torch.empty(
        d, num_experts_per_rank, dtype=torch.int64, device="cuda"
    )
    # The following all_to_all_single communicates the number of tokens each
    # rank sends to each expert. This way, we can compute `output_split_sizes`
    # of the next all_to_all_single. In a more efficient implementation (e.g.
    # https://www.perplexity.ai/hub/blog/efficient-and-portable-mixture-of-experts-communication),
    # the two `all_to_all_single`s can be fused into one kernel.
    #
    # GPU 0: [[n_tokens_for_expert_0_from_rank_0, n_tokens_for_expert_1_from_rank_0, n_tokens_for_expert_2_from_rank_0],
    #         [n_tokens_for_expert_0_from_rank_1, n_tokens_for_expert_1_from_rank_1, n_tokens_for_expert_2_from_rank_1]]
    # GPU 1: [[n_tokens_for_expert_3_from_rank_0, n_tokens_for_expert_4_from_rank_0, n_tokens_for_expert_5_from_rank_0],
    #         [n_tokens_for_expert_3_from_rank_1, n_tokens_for_expert_4_from_rank_1, n_tokens_for_expert_5_from_rank_1]]
    dist.all_to_all_single(n_tokens_for_expert_from_rank, n_tokens_for_expert)

    # Number of tokens to receive from each rank
    # GPU 0: [n_tokens_for_expert_012_from_rank_0, n_tokens_for_expert_012_from_rank_1]
    # GPU 1: [n_tokens_for_expert_345_from_rank_0, n_tokens_for_expert_345_from_rank_1]
    n_tokens_from_rank = n_tokens_for_expert_from_rank.sum(-1)

    # Number of tokens to send to each rank
    # GPU 0: [n_tokens_for_expert_012_from_rank_0, n_tokens_for_expert_345_from_rank_0]
    # GPU 1: [n_tokens_for_expert_012_from_rank_1, n_tokens_for_expert_345_from_rank_1]
    n_tokens_to_rank = n_tokens_for_expert.view(d, -1).sum(-1)

    tokens_by_rank = torch.empty(
        n_tokens_from_rank.sum(), dtype=torch.complex64, device="cuda"
    )
    # GPU 0: tokens_for_expert_0_from_rank_0 || tokens_for_expert_1_from_rank_0 || tokens_for_expert_2_from_rank_0 || tokens_for_expert_0_from_rank_1 || tokens_for_expert_1_from_rank_1 || tokens_for_expert_2_from_rank_1
    # GPU 1: tokens_for_expert_3_from_rank_0 || tokens_for_expert_4_from_rank_0 || tokens_for_expert_5_from_rank_0 || tokens_for_expert_3_from_rank_1 || tokens_for_expert_4_from_rank_1 || tokens_for_expert_5_from_rank_1
    dist.all_to_all_single(
        tokens_by_rank, tokens, n_tokens_from_rank.tolist(), n_tokens_to_rank.tolist()
    )

    # --------------------------------------------------------------------------
    # Step 2: Each rank sorts the received tokens by expert ID.
    # --------------------------------------------------------------------------
    # GPU 0: tokens_for_expert_0_from_rank_0 || tokens_for_expert_0_from_rank_1 || tokens_for_expert_1_from_rank_0 || tokens_for_expert_1_from_rank_1 || tokens_for_expert_2_from_rank_0 || tokens_for_expert_2_from_rank_1
    # GPU 1: tokens_for_expert_3_from_rank_0 || tokens_for_expert_3_from_rank_1 || tokens_for_expert_4_from_rank_0 || tokens_for_expert_4_from_rank_1 || tokens_for_expert_5_from_rank_0 || tokens_for_expert_5_from_rank_1
    tokens_by_expert = rank_first_to_expert_first(
        tokens_by_rank, n_tokens_for_expert_from_rank
    )

    imag = torch.imag(tokens_by_expert)
    assert (imag >= num_experts_per_rank * rank).all()
    assert (imag < num_experts_per_rank * (rank + 1)).all()
    assert (imag[:-1] <= imag[1:]).all()

    # --------------------------------------------------------------------------
    # Step 3: Each rank processes the received tokens with their corresponding
    # experts. This is skipped by the test for simplicity.
    # --------------------------------------------------------------------------
    processed_tokens_by_expert = tokens_by_expert

    # --------------------------------------------------------------------------
    # Step 4: Each rank sorts the processed tokens by rank ID.
    # --------------------------------------------------------------------------
    # GPU 0: tokens_for_expert_0_from_rank_0 || tokens_for_expert_1_from_rank_0 || tokens_for_expert_2_from_rank_0 || tokens_for_expert_0_from_rank_1 || tokens_for_expert_1_from_rank_1 || tokens_for_expert_2_from_rank_1
    # GPU 1: tokens_for_expert_3_from_rank_0 || tokens_for_expert_4_from_rank_0 || tokens_for_expert_5_from_rank_0 || tokens_for_expert_3_from_rank_1 || tokens_for_expert_4_from_rank_1 || tokens_for_expert_5_from_rank_1
    processed_tokens_by_rank = expert_first_to_rank_first(
        processed_tokens_by_expert, n_tokens_for_expert_from_rank
    )

    # --------------------------------------------------------------------------
    # Step 5: Processed tokens are sent back to the original ranks.
    # --------------------------------------------------------------------------
    processed_tokens = torch.empty(n_tokens, dtype=torch.complex64, device="cuda")
    # GPU 0: tokens_for_expert_0_from_rank_0 || tokens_for_expert_1_from_rank_0 || tokens_for_expert_2_from_rank_0 || tokens_for_expert_3_from_rank_0 || tokens_for_expert_4_from_rank_0 || tokens_for_expert_5_from_rank_0
    # GPU 1: tokens_for_expert_0_from_rank_1 || tokens_for_expert_1_from_rank_1 || tokens_for_expert_2_from_rank_1 || tokens_for_expert_3_from_rank_1 || tokens_for_expert_4_from_rank_1 || tokens_for_expert_5_from_rank_1
    dist.all_to_all_single(
        processed_tokens,
        processed_tokens_by_rank,
        n_tokens_to_rank.tolist(),
        n_tokens_from_rank.tolist(),
    )

    # Because we skipped expert processing, the processed tokens should be the
    # same as the original tokens.
    torch.testing.assert_close(processed_tokens, tokens, rtol=0, atol=0)


# SPDX-FileCopyrightText: Copyright (c) 2025-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

import pytest
import torch
import torch.distributed as dist

# Partitions n into k integers whose sum is n.
def random_partition(n: int, k: int) -> torch.Tensor:
    offsets = torch.randint(0, n + 1, [k - 1]).tolist()
    # The last partition ends at index n.
    offsets.append(n)
    offsets.sort()

    sizes = []
    prev = 0
    for offset in offsets:
        sizes.append(offset - prev)
        prev = offset
    assert sum(sizes) == n
    assert len(sizes) == k
    return torch.tensor(sizes, device="cuda")

def list_to_matrix(l: list, rows: int, cols: int) -> list[list]:
    return [l[row * cols : (row + 1) * cols] for row in range(rows)]

def matrix_to_list(m: list[list]) -> list:
    return [item for row in m for item in row]

# This can be fused into one kernel.
def rank_first_to_expert_first(
    tokens: torch.Tensor, n_tokens_for_expert_from_rank: torch.Tensor
) -> torch.Tensor:
    d, num_experts_per_rank = n_tokens_for_expert_from_rank.size()
    # Rank first, expert second
    tokens_for_expert_from_rank = tokens.split(
        n_tokens_for_expert_from_rank.flatten().tolist()
    )
    tokens_for_expert_from_rank = list_to_matrix(


import torch.distributed as dist

# Partitions n into k integers whose sum is n.
def random_partition(n: int, k: int) -> torch.Tensor:
    offsets = torch.randint(0, n + 1, [k - 1]).tolist()
    # The last partition ends at index n.
    offsets.append(n)
    offsets.sort()

    sizes = []
    prev = 0
    for offset in offsets:
        sizes.append(offset - prev)
        prev = offset
    assert sum(sizes) == n
    assert len(sizes) == k
    return torch.tensor(sizes, device="cuda")

def list_to_matrix(l: list, rows: int, cols: int) -> list[list]:
    return [l[row * cols : (row + 1) * cols] for row in range(rows)]

def matrix_to_list(m: list[list]) -> list:
    return [item for row in m for item in row]

# This can be fused into one kernel.
def rank_first_to_expert_first(
    tokens: torch.Tensor, n_tokens_for_expert_from_rank: torch.Tensor
) -> torch.Tensor:
    d, num_experts_per_rank = n_tokens_for_expert_from_rank.size()
    # Rank first, expert second
    tokens_for_expert_from_rank = tokens.split(
        n_tokens_for_expert_from_rank.flatten().tolist()
    )
    tokens_for_expert_from_rank = list_to_matrix(
        tokens_for_expert_from_rank, d, num_experts_per_rank
    )
    # list(zip(*matrix)) transposes the matrix.
    # Expert first, rank second


def random_partition(n: int, k: int) -> torch.Tensor:
    offsets = torch.randint(0, n + 1, [k - 1]).tolist()
    # The last partition ends at index n.
    offsets.append(n)
    offsets.sort()

    sizes = []
    prev = 0
    for offset in offsets:
        sizes.append(offset - prev)
        prev = offset
    assert sum(sizes) == n
    assert len(sizes) == k
    return torch.tensor(sizes, device="cuda")

def list_to_matrix(l: list, rows: int, cols: int) -> list[list]:
    return [l[row * cols : (row + 1) * cols] for row in range(rows)]

def matrix_to_list(m: list[list]) -> list:
    return [item for row in m for item in row]

# This can be fused into one kernel.
def rank_first_to_expert_first(
    tokens: torch.Tensor, n_tokens_for_expert_from_rank: torch.Tensor
) -> torch.Tensor:
    d, num_experts_per_rank = n_tokens_for_expert_from_rank.size()
    # Rank first, expert second
    tokens_for_expert_from_rank = tokens.split(
        n_tokens_for_expert_from_rank.flatten().tolist()
    )
    tokens_for_expert_from_rank = list_to_matrix(
        tokens_for_expert_from_rank, d, num_experts_per_rank
    )
    # list(zip(*matrix)) transposes the matrix.
    # Expert first, rank second
    tokens_for_expert_from_rank = list(zip(*tokens_for_expert_from_rank))
    tokens_for_expert_from_rank = matrix_to_list(tokens_for_expert_from_rank)
    tokens = torch.cat(tokens_for_expert_from_rank, dim=0)
    return tokens


    sizes = []
    prev = 0
    for offset in offsets:
        sizes.append(offset - prev)
        prev = offset
    assert sum(sizes) == n
    assert len(sizes) == k
    return torch.tensor(sizes, device="cuda")

def list_to_matrix(l: list, rows: int, cols: int) -> list[list]:
    return [l[row * cols : (row + 1) * cols] for row in range(rows)]

def matrix_to_list(m: list[list]) -> list:
    return [item for row in m for item in row]

# This can be fused into one kernel.
def rank_first_to_expert_first(
    tokens: torch.Tensor, n_tokens_for_expert_from_rank: torch.Tensor
) -> torch.Tensor:
    d, num_experts_per_rank = n_tokens_for_expert_from_rank.size()
    # Rank first, expert second
    tokens_for_expert_from_rank = tokens.split(
        n_tokens_for_expert_from_rank.flatten().tolist()
    )
    tokens_for_expert_from_rank = list_to_matrix(
        tokens_for_expert_from_rank, d, num_experts_per_rank
    )
    # list(zip(*matrix)) transposes the matrix.
    # Expert first, rank second
    tokens_for_expert_from_rank = list(zip(*tokens_for_expert_from_rank))
    tokens_for_expert_from_rank = matrix_to_list(tokens_for_expert_from_rank)
    tokens = torch.cat(tokens_for_expert_from_rank, dim=0)
    return tokens

# This can be fused into one kernel as well.
def expert_first_to_rank_first(
    tokens: torch.Tensor, n_tokens_for_expert_from_rank: torch.Tensor
) -> torch.Tensor:
    d, num_experts_per_rank = n_tokens_for_expert_from_rank.size()
    # Expert first, rank second
    tokens_for_expert_from_rank = tokens.split(
        n_tokens_for_expert_from_rank.t().flatten().tolist()
    )
    tokens_for_expert_from_rank = list_to_matrix(
        tokens_for_expert_from_rank, num_experts_per_rank, d
    )
    # Rank first, expert second
    tokens_for_expert_from_rank = list(zip(*tokens_for_expert_from_rank))
    tokens_for_expert_from_rank = matrix_to_list(tokens_for_expert_from_rank)
    tokens = torch.cat(tokens_for_expert_from_rank, dim=0)
    return tokens


def rank_first_to_expert_first(
    tokens: torch.Tensor, n_tokens_for_expert_from_rank: torch.Tensor
) -> torch.Tensor:
    d, num_experts_per_rank = n_tokens_for_expert_from_rank.size()
    # Rank first, expert second
    tokens_for_expert_from_rank = tokens.split(
        n_tokens_for_expert_from_rank.flatten().tolist()
    )
    tokens_for_expert_from_rank = list_to_matrix(
        tokens_for_expert_from_rank, d, num_experts_per_rank
    )
    # list(zip(*matrix)) transposes the matrix.
    # Expert first, rank second
    tokens_for_expert_from_rank = list(zip(*tokens_for_expert_from_rank))
    tokens_for_expert_from_rank = matrix_to_list(tokens_for_expert_from_rank)
    tokens = torch.cat(tokens_for_expert_from_rank, dim=0)
    return tokens

# This can be fused into one kernel as well.
def expert_first_to_rank_first(
    tokens: torch.Tensor, n_tokens_for_expert_from_rank: torch.Tensor
) -> torch.Tensor:
    d, num_experts_per_rank = n_tokens_for_expert_from_rank.size()
    # Expert first, rank second
    tokens_for_expert_from_rank = tokens.split(
        n_tokens_for_expert_from_rank.t().flatten().tolist()
    )
    tokens_for_expert_from_rank = list_to_matrix(
        tokens_for_expert_from_rank, num_experts_per_rank, d
    )
    # Rank first, expert second
    tokens_for_expert_from_rank = list(zip(*tokens_for_expert_from_rank))
    tokens_for_expert_from_rank = matrix_to_list(tokens_for_expert_from_rank)
    tokens = torch.cat(tokens_for_expert_from_rank, dim=0)
    return tokens

# This test serves as the reference implementation for the expert parallelism
# dispatch and combine logic. The test is written in a way that is easy to
# understand, but not efficient.
@pytest.mark.mpi
def test_dispatch_and_combine(setup_default_process_group):
    rank = dist.get_rank()
    # So RNG generates different numbers for each rank.
    torch.manual_seed(rank)

    d = dist.get_world_size()
    num_experts_per_rank = 3
    n_experts = d * num_experts_per_rank

    # Input tokens are distributed among ranks, so the effective sequence length is `n_tokens * d`.
    n_tokens = 12
    # GPU 0: [n_tokens_for_expert_0_from_rank_0, n_tokens_for_expert_1_from_rank_0, ..., n_tokens_for_expert_5_from_rank_0]
    # GPU 1: [n_tokens_for_expert_0_from_rank_1, n_tokens_for_expert_1_from_rank_1, ..., n_tokens_for_expert_5_from_rank_1]
    # Both sum to `n_tokens`.


) -> torch.Tensor:
    d, num_experts_per_rank = n_tokens_for_expert_from_rank.size()
    # Expert first, rank second
    tokens_for_expert_from_rank = tokens.split(
        n_tokens_for_expert_from_rank.t().flatten().tolist()
    )
    tokens_for_expert_from_rank = list_to_matrix(
        tokens_for_expert_from_rank, num_experts_per_rank, d
    )
    # Rank first, expert second
    tokens_for_expert_from_rank = list(zip(*tokens_for_expert_from_rank))
    tokens_for_expert_from_rank = matrix_to_list(tokens_for_expert_from_rank)
    tokens = torch.cat(tokens_for_expert_from_rank, dim=0)
    return tokens

# This test serves as the reference implementation for the expert parallelism
# dispatch and combine logic. The test is written in a way that is easy to
# understand, but not efficient.
@pytest.mark.mpi
def test_dispatch_and_combine(setup_default_process_group):
    rank = dist.get_rank()
    # So RNG generates different numbers for each rank.
    torch.manual_seed(rank)

    d = dist.get_world_size()
    num_experts_per_rank = 3
    n_experts = d * num_experts_per_rank

    # Input tokens are distributed among ranks, so the effective sequence length is `n_tokens * d`.
    n_tokens = 12
    # GPU 0: [n_tokens_for_expert_0_from_rank_0, n_tokens_for_expert_1_from_rank_0, ..., n_tokens_for_expert_5_from_rank_0]
    # GPU 1: [n_tokens_for_expert_0_from_rank_1, n_tokens_for_expert_1_from_rank_1, ..., n_tokens_for_expert_5_from_rank_1]
    # Both sum to `n_tokens`.
    n_tokens_for_expert = random_partition(n_tokens, n_experts)

    # For illustration, input tokens are complex numbers of value `token_id + expert_id * j`.
    expert_ids = []
    for expert, n in enumerate(n_tokens_for_expert.tolist()):
        expert_ids.extend([expert] * n)
    tokens = (
        torch.arange(
            rank * n_tokens, (rank + 1) * n_tokens, dtype=torch.float, device="cuda"
        )
        + torch.tensor(expert_ids, dtype=torch.float, device="cuda") * 1j
    )

    # --------------------------------------------------------------------------
    # An expert-parallel MoE layer dispatches, processes, and combines tokens
    # in the following steps:
    #
    # Step 1: Tokens are dispatched to the rank hosting the corresponding expert.
    # --------------------------------------------------------------------------
    n_tokens_for_expert_from_rank = torch.empty(
        d, num_experts_per_rank, dtype=torch.int64, device="cuda"
    )
    # The following all_to_all_single communicates the number of tokens each
    # rank sends to each expert. This way, we can compute `output_split_sizes`
    # of the next all_to_all_single. In a more efficient implementation (e.g.
    # https://www.perplexity.ai/hub/blog/efficient-and-portable-mixture-of-experts-communication),
    # the two `all_to_all_single`s can be fused into one kernel.
    #
    # GPU 0: [[n_tokens_for_expert_0_from_rank_0, n_tokens_for_expert_1_from_rank_0, n_tokens_for_expert_2_from_rank_0],
    #         [n_tokens_for_expert_0_from_rank_1, n_tokens_for_expert_1_from_rank_1, n_tokens_for_expert_2_from_rank_1]]
    # GPU 1: [[n_tokens_for_expert_3_from_rank_0, n_tokens_for_expert_4_from_rank_0, n_tokens_for_expert_5_from_rank_0],
    #         [n_tokens_for_expert_3_from_rank_1, n_tokens_for_expert_4_from_rank_1, n_tokens_for_expert_5_from_rank_1]]
    dist.all_to_all_single(n_tokens_for_expert_from_rank, n_tokens_for_expert)

    # Number of tokens to receive from each rank
    # GPU 0: [n_tokens_for_expert_012_from_rank_0, n_tokens_for_expert_012_from_rank_1]
    # GPU 1: [n_tokens_for_expert_345_from_rank_0, n_tokens_for_expert_345_from_rank_1]
    n_tokens_from_rank = n_tokens_for_expert_from_rank.sum(-1)

    # Number of tokens to send to each rank
    # GPU 0: [n_tokens_for_expert_012_from_rank_0, n_tokens_for_expert_345_from_rank_0]
    # GPU 1: [n_tokens_for_expert_012_from_rank_1, n_tokens_for_expert_345_from_rank_1]
    n_tokens_to_rank = n_tokens_for_expert.view(d, -1).sum(-1)

    tokens_by_rank = torch.empty(
        n_tokens_from_rank.sum(), dtype=torch.complex64, device="cuda"
    )
    # GPU 0: tokens_for_expert_0_from_rank_0 || tokens_for_expert_1_from_rank_0 || tokens_for_expert_2_from_rank_0 || tokens_for_expert_0_from_rank_1 || tokens_for_expert_1_from_rank_1 || tokens_for_expert_2_from_rank_1
    # GPU 1: tokens_for_expert_3_from_rank_0 || tokens_for_expert_4_from_rank_0 || tokens_for_expert_5_from_rank_0 || tokens_for_expert_3_from_rank_1 || tokens_for_expert_4_from_rank_1 || tokens_for_expert_5_from_rank_1
    dist.all_to_all_single(
        tokens_by_rank, tokens, n_tokens_from_rank.tolist(), n_tokens_to_rank.tolist()
    )

    # --------------------------------------------------------------------------
    # Step 2: Each rank sorts the received tokens by expert ID.
    # --------------------------------------------------------------------------
    # GPU 0: tokens_for_expert_0_from_rank_0 || tokens_for_expert_0_from_rank_1 || tokens_for_expert_1_from_rank_0 || tokens_for_expert_1_from_rank_1 || tokens_for_expert_2_from_rank_0 || tokens_for_expert_2_from_rank_1
    # GPU 1: tokens_for_expert_3_from_rank_0 || tokens_for_expert_3_from_rank_1 || tokens_for_expert_4_from_rank_0 || tokens_for_expert_4_from_rank_1 || tokens_for_expert_5_from_rank_0 || tokens_for_expert_5_from_rank_1
    tokens_by_expert = rank_first_to_expert_first(
        tokens_by_rank, n_tokens_for_expert_from_rank
    )

    imag = torch.imag(tokens_by_expert)
    assert (imag >= num_experts_per_rank * rank).all()
    assert (imag < num_experts_per_rank * (rank + 1)).all()
    assert (imag[:-1] <= imag[1:]).all()

    # --------------------------------------------------------------------------
    # Step 3: Each rank processes the received tokens with their corresponding
    # experts. This is skipped by the test for simplicity.
    # --------------------------------------------------------------------------
    processed_tokens_by_expert = tokens_by_expert

    # --------------------------------------------------------------------------
    # Step 4: Each rank sorts the processed tokens by rank ID.
    # --------------------------------------------------------------------------
    # GPU 0: tokens_for_expert_0_from_rank_0 || tokens_for_expert_1_from_rank_0 || tokens_for_expert_2_from_rank_0 || tokens_for_expert_0_from_rank_1 || tokens_for_expert_1_from_rank_1 || tokens_for_expert_2_from_rank_1
    # GPU 1: tokens_for_expert_3_from_rank_0 || tokens_for_expert_4_from_rank_0 || tokens_for_expert_5_from_rank_0 || tokens_for_expert_3_from_rank_1 || tokens_for_expert_4_from_rank_1 || tokens_for_expert_5_from_rank_1
    processed_tokens_by_rank = expert_first_to_rank_first(
        processed_tokens_by_expert, n_tokens_for_expert_from_rank
    )

    # --------------------------------------------------------------------------
    # Step 5: Processed tokens are sent back to the original ranks.
    # --------------------------------------------------------------------------
    processed_tokens = torch.empty(n_tokens, dtype=torch.complex64, device="cuda")
    # GPU 0: tokens_for_expert_0_from_rank_0 || tokens_for_expert_1_from_rank_0 || tokens_for_expert_2_from_rank_0 || tokens_for_expert_3_from_rank_0 || tokens_for_expert_4_from_rank_0 || tokens_for_expert_5_from_rank_0
    # GPU 1: tokens_for_expert_0_from_rank_1 || tokens_for_expert_1_from_rank_1 || tokens_for_expert_2_from_rank_1 || tokens_for_expert_3_from_rank_1 || tokens_for_expert_4_from_rank_1 || tokens_for_expert_5_from_rank_1
    dist.all_to_all_single(
        processed_tokens,
        processed_tokens_by_rank,
        n_tokens_to_rank.tolist(),
        n_tokens_from_rank.tolist(),
    )

    # Because we skipped expert processing, the processed tokens should be the
    # same as the original tokens.
    torch.testing.assert_close(processed_tokens, tokens, rtol=0, atol=0)