File size: 2,176 Bytes
f5d6a7c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
"""Check NCCL communication and token-weighted distributed gradient scaling."""

import copy
import json
import os
import time
from datetime import timedelta
from pathlib import Path

import torch
import torch.distributed as dist
import torch.nn.functional as F
from torch.nn.parallel import DistributedDataParallel

rank = int(os.environ["LOCAL_RANK"])
torch.cuda.set_device(rank)
dist.init_process_group("nccl", timeout=timedelta(seconds=40))
world = dist.get_world_size()
torch.manual_seed(3407)
model = torch.nn.Linear(7, 11, bias=False, dtype=torch.float64, device="cuda")
reference = copy.deepcopy(model)
model = DistributedDataParallel(model, device_ids=[rank])
x = torch.randn(32, 13, 7, dtype=torch.float64, device="cuda")
labels = torch.randint(0, 11, (32, 13), device="cuda")
labels[torch.arange(13, device="cuda")[None, :] > torch.arange(32, device="cuda")[:, None] % 13] = -100
reference_loss = F.cross_entropy(reference(x).flatten(0, 1), labels.flatten(), reduction="sum") / labels.ne(-100).sum()
reference_loss.backward()

local_x = x.chunk(world)[rank]
local_labels = labels.chunk(world)[rank]
count = local_labels.ne(-100).sum()
dist.all_reduce(count)
for i in range(2):
    # DDP averages gradients across ranks. Summed token losses therefore need
    # the global token denominator and a world-size factor.
    output = model(local_x[i * 8:(i + 1) * 8])
    loss = F.cross_entropy(output.flatten(0, 1), local_labels[i * 8:(i + 1) * 8].flatten(), reduction="sum") * world / count
    loss.backward()
error = (model.module.weight.grad - reference.weight.grad).abs().max().item()
assert error < 1e-12, error

buffer = torch.ones(4 * 1024 * 1024, device="cuda", dtype=torch.float32)
dist.barrier()
torch.cuda.synchronize()
start = time.perf_counter()
for _ in range(5):
    dist.all_reduce(buffer)
torch.cuda.synchronize()
latency = (time.perf_counter() - start) / 5
if rank == 0:
    result = {"world_size": world, "max_gradient_error": error, "all_reduce_16MiB_seconds": latency, "passed": True}
    Path("distributed_correctness.json").write_text(json.dumps(result, indent=2))
    print(json.dumps(result, indent=2), flush=True)
dist.destroy_process_group()