Instructions to use PriyaMishra/temp with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use PriyaMishra/temp with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- Unsloth Desktop
File size: 2,176 Bytes
f5d6a7c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | """Check NCCL communication and token-weighted distributed gradient scaling."""
import copy
import json
import os
import time
from datetime import timedelta
from pathlib import Path
import torch
import torch.distributed as dist
import torch.nn.functional as F
from torch.nn.parallel import DistributedDataParallel
rank = int(os.environ["LOCAL_RANK"])
torch.cuda.set_device(rank)
dist.init_process_group("nccl", timeout=timedelta(seconds=40))
world = dist.get_world_size()
torch.manual_seed(3407)
model = torch.nn.Linear(7, 11, bias=False, dtype=torch.float64, device="cuda")
reference = copy.deepcopy(model)
model = DistributedDataParallel(model, device_ids=[rank])
x = torch.randn(32, 13, 7, dtype=torch.float64, device="cuda")
labels = torch.randint(0, 11, (32, 13), device="cuda")
labels[torch.arange(13, device="cuda")[None, :] > torch.arange(32, device="cuda")[:, None] % 13] = -100
reference_loss = F.cross_entropy(reference(x).flatten(0, 1), labels.flatten(), reduction="sum") / labels.ne(-100).sum()
reference_loss.backward()
local_x = x.chunk(world)[rank]
local_labels = labels.chunk(world)[rank]
count = local_labels.ne(-100).sum()
dist.all_reduce(count)
for i in range(2):
# DDP averages gradients across ranks. Summed token losses therefore need
# the global token denominator and a world-size factor.
output = model(local_x[i * 8:(i + 1) * 8])
loss = F.cross_entropy(output.flatten(0, 1), local_labels[i * 8:(i + 1) * 8].flatten(), reduction="sum") * world / count
loss.backward()
error = (model.module.weight.grad - reference.weight.grad).abs().max().item()
assert error < 1e-12, error
buffer = torch.ones(4 * 1024 * 1024, device="cuda", dtype=torch.float32)
dist.barrier()
torch.cuda.synchronize()
start = time.perf_counter()
for _ in range(5):
dist.all_reduce(buffer)
torch.cuda.synchronize()
latency = (time.perf_counter() - start) / 5
if rank == 0:
result = {"world_size": world, "max_gradient_error": error, "all_reduce_16MiB_seconds": latency, "passed": True}
Path("distributed_correctness.json").write_text(json.dumps(result, indent=2))
print(json.dumps(result, indent=2), flush=True)
dist.destroy_process_group()
|