File size: 8,471 Bytes
c8ba9c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 | """CPU/Gloo checks for DDP bucket execution and BF16 accumulation semantics."""
from contextlib import nullcontext
from copy import deepcopy
from datetime import timedelta
import json
import os
from pathlib import Path
import subprocess
import sys
import pytest
import torch
import torch.distributed as dist
from torch import nn
from torch.nn.parallel import DistributedDataParallel
from ops.distributed_execution import ddp_kwargs, ddp_runtime_settings
@pytest.mark.parametrize("changes", [
{"bucket_cap_mb": 0}, {"bucket_cap_mb": -1}, {"bucket_cap_mb": True},
{"bucket_cap_mb": float("nan")}, {"bucket_cap_mb": float("inf")},
{"bucket_cap_mb": "128"}, {"bucket_cap_mb": 1e-12},
{"batched_grad_copy": 1}, {"batched_grad_copy": "false"},
{"bucket_cap_mb_list": []}, {"bucket_cap_mb_list": "64,128"},
{"bucket_cap_mb_list": [64, 0]}, {"bucket_cap_mb_list": [64, False]},
])
def test_invalid_runtime_settings_are_rejected(changes):
with pytest.raises(ValueError):
ddp_runtime_settings(**changes)
def test_legacy_runtime_does_not_receive_unsupported_optional_arguments(monkeypatch):
def legacy_ddp(module, broadcast_buffers=False, gradient_as_bucket_view=False,
bucket_cap_mb=None):
pass
monkeypatch.setattr(torch.nn.parallel, "DistributedDataParallel", legacy_ddp)
assert ddp_kwargs({}) == {
"broadcast_buffers": False, "gradient_as_bucket_view": True,
"bucket_cap_mb": 128,
}
for requested in [{"batched_grad_copy": True}, {"bucket_cap_mb_list": [64, 128]}]:
with pytest.raises(RuntimeError, match="Installed PyTorch DDP does not support"):
ddp_kwargs(ddp_runtime_settings(**requested))
def test_capacity_list_is_copied_and_records_effective_capacity():
capacities = [16, 64, 32]
settings = ddp_runtime_settings(bucket_cap_mb=128, batched_grad_copy=True,
bucket_cap_mb_list=capacities)
capacities[0] = 999
assert settings["ddp_bucket_cap_mb"] == 64
assert settings["ddp_bucket_cap_mb_list"] == [16, 64, 32]
assert json.loads(json.dumps(settings)) == settings
assert ddp_kwargs(settings)["bucket_cap_mb_list"] == [16, 64, 32]
@pytest.mark.parametrize("requested", [
{"batched_grad_copy": True}, {"bucket_cap_mb_list": [64, 128]},
])
def test_python_reducer_rejects_inapplicable_cpp_bucket_options(requested):
with torch._dynamo.config.patch(optimize_ddp="python_reducer"):
with pytest.raises(ValueError, match="require the C\\+\\+ DDP reducer"):
ddp_kwargs(ddp_runtime_settings(**requested))
assert ddp_kwargs(ddp_runtime_settings(bucket_cap_mb=64))["bucket_cap_mb"] == 64
def _model():
layers = []
for _ in range(6):
layers.extend([nn.Linear(32, 32), nn.SiLU()])
layers.append(nn.Linear(32, 8))
return nn.Sequential(*layers)
def _worker(rank, directory):
torch.set_num_threads(1)
directory = Path(directory)
dist.init_process_group("gloo", init_method=(directory / "rendezvous").as_uri(),
rank=rank, world_size=2, timeout=timedelta(seconds=30))
reports = []
try:
cases = [(capacity, batched, None)
for capacity in [64, 128, 256, 0.002]
for batched in [False, True]]
cases.append((128, True, [0.002, 0.004]))
for capacity, batched, capacities in cases:
torch.manual_seed(321 + rank)
settings = ddp_runtime_settings(bucket_cap_mb=capacity,
batched_grad_copy=batched,
bucket_cap_mb_list=capacities)
model = DistributedDataParallel(_model(), **ddp_kwargs(settings))
reference = deepcopy(model.module)
optimizer = torch.optim.AdamW(model.parameters(), lr=0.001, foreach=False)
reference_optimizer = torch.optim.AdamW(reference.parameters(), lr=0.001,
foreach=False)
maximum_gradient_error = 0.0
for update in range(3):
# Reallocation on every update exercises the production
# set_to_none=True path both before and after bucket rebuilding.
optimizer.zero_grad(set_to_none=True)
reference_optimizer.zero_grad(set_to_none=True)
assert all(parameter.grad is None for parameter in model.parameters())
for microstep in range(8):
generator = torch.Generator().manual_seed(
10000 + rank * 1000 + update * 8 + microstep)
inputs = torch.randn(3, 32, generator=generator)
target = torch.randn(3, 8, generator=generator)
context = model.no_sync() if microstep < 7 else nullcontext()
with context:
with torch.autocast("cpu", dtype=torch.bfloat16):
output = model(inputs)
reference_output = reference(inputs)
assert output.dtype == reference_output.dtype == torch.bfloat16
loss = (output.float() - target).square().mean() / 8
reference_loss = (reference_output.float() - target).square().mean() / 8
loss.backward()
reference_loss.backward()
for actual, expected in zip(model.parameters(), reference.parameters()):
assert actual.dtype == expected.dtype == torch.float32
assert actual.grad.dtype == expected.grad.dtype == torch.float32
# Independent reference: accumulate each rank's distinct
# microbatches locally, then explicitly average their FP32
# gradients. This detects missing/repeated microsteps and
# incorrect reducer scaling without assuming bucket layout.
dist.all_reduce(expected.grad)
expected.grad.div_(2)
maximum_gradient_error = max(maximum_gradient_error,
(actual.grad - expected.grad).abs().max().item())
torch.testing.assert_close(actual.grad, expected.grad,
rtol=1e-6, atol=1e-7)
assert torch.isfinite(actual.grad).all()
optimizer.step()
reference_optimizer.step()
for actual, expected in zip(model.parameters(), reference.parameters()):
torch.testing.assert_close(actual, expected, rtol=1e-6, atol=1e-7)
for state in optimizer.state.values():
assert state["exp_avg"].dtype == state["exp_avg_sq"].dtype == torch.float32
reports.append({"settings": settings,
"maximum_gradient_error": maximum_gradient_error,
"updates": 3, "microsteps_per_update": 8})
del model, reference, optimizer, reference_optimizer
(directory / f"rank-{rank}.json").write_text(json.dumps(reports))
finally:
dist.destroy_process_group()
def test_two_rank_bf16_accumulation_matches_explicit_fp32_gradient_average(tmp_path):
environment = os.environ.copy()
environment.update(CUDA_VISIBLE_DEVICES="", OMP_NUM_THREADS="1",
MKL_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1")
processes = []
try:
for rank in range(2):
processes.append(subprocess.Popen(
[sys.executable, str(Path(__file__).resolve()), str(rank), str(tmp_path)],
env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
))
for rank, process in enumerate(processes):
output, _ = process.communicate(timeout=90)
assert process.returncode == 0, f"Rank {rank} failed:\n{output}"
finally:
for process in processes:
if process.poll() is None:
process.kill()
for process in processes:
process.wait(timeout=10)
for rank in range(2):
reports = json.loads((tmp_path / f"rank-{rank}.json").read_text())
assert len(reports) == 9
assert all(report["maximum_gradient_error"] <= 1e-7 for report in reports)
if __name__ == "__main__":
_worker(int(sys.argv[1]), sys.argv[2])
|