"""Check NCCL communication and token-weighted distributed gradient scaling.""" import copy import json import os import time from datetime import timedelta from pathlib import Path import torch import torch.distributed as dist import torch.nn.functional as F from torch.nn.parallel import DistributedDataParallel rank = int(os.environ["LOCAL_RANK"]) torch.cuda.set_device(rank) dist.init_process_group("nccl", timeout=timedelta(seconds=40)) world = dist.get_world_size() torch.manual_seed(3407) model = torch.nn.Linear(7, 11, bias=False, dtype=torch.float64, device="cuda") reference = copy.deepcopy(model) model = DistributedDataParallel(model, device_ids=[rank]) x = torch.randn(32, 13, 7, dtype=torch.float64, device="cuda") labels = torch.randint(0, 11, (32, 13), device="cuda") labels[torch.arange(13, device="cuda")[None, :] > torch.arange(32, device="cuda")[:, None] % 13] = -100 reference_loss = F.cross_entropy(reference(x).flatten(0, 1), labels.flatten(), reduction="sum") / labels.ne(-100).sum() reference_loss.backward() local_x = x.chunk(world)[rank] local_labels = labels.chunk(world)[rank] count = local_labels.ne(-100).sum() dist.all_reduce(count) for i in range(2): # DDP averages gradients across ranks. Summed token losses therefore need # the global token denominator and a world-size factor. output = model(local_x[i * 8:(i + 1) * 8]) loss = F.cross_entropy(output.flatten(0, 1), local_labels[i * 8:(i + 1) * 8].flatten(), reduction="sum") * world / count loss.backward() error = (model.module.weight.grad - reference.weight.grad).abs().max().item() assert error < 1e-12, error buffer = torch.ones(4 * 1024 * 1024, device="cuda", dtype=torch.float32) dist.barrier() torch.cuda.synchronize() start = time.perf_counter() for _ in range(5): dist.all_reduce(buffer) torch.cuda.synchronize() latency = (time.perf_counter() - start) / 5 if rank == 0: result = {"world_size": world, "max_gradient_error": error, "all_reduce_16MiB_seconds": latency, "passed": True} Path("distributed_correctness.json").write_text(json.dumps(result, indent=2)) print(json.dumps(result, indent=2), flush=True) dist.destroy_process_group()