Instructions to use PriyaMishra/temp with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use PriyaMishra/temp with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- Unsloth Desktop
Download project/verify_distributed.py from PriyaMishra/temp: direct link, hf CLI and curl.
- Browser
- Download file 2.18 kB
-
https://huggingface.co/PriyaMishra/temp/resolve/main/project/verify_distributed.py
- Command line
-
hf download hf://PriyaMishra/temp/project/verify_distributed.py
-
curl -L -o verify_distributed.py https://huggingface.co/PriyaMishra/temp/resolve/main/project/verify_distributed.py
2.18 kB
| """Check NCCL communication and token-weighted distributed gradient scaling.""" | |
| import copy | |
| import json | |
| import os | |
| import time | |
| from datetime import timedelta | |
| from pathlib import Path | |
| import torch | |
| import torch.distributed as dist | |
| import torch.nn.functional as F | |
| from torch.nn.parallel import DistributedDataParallel | |
| rank = int(os.environ["LOCAL_RANK"]) | |
| torch.cuda.set_device(rank) | |
| dist.init_process_group("nccl", timeout=timedelta(seconds=40)) | |
| world = dist.get_world_size() | |
| torch.manual_seed(3407) | |
| model = torch.nn.Linear(7, 11, bias=False, dtype=torch.float64, device="cuda") | |
| reference = copy.deepcopy(model) | |
| model = DistributedDataParallel(model, device_ids=[rank]) | |
| x = torch.randn(32, 13, 7, dtype=torch.float64, device="cuda") | |
| labels = torch.randint(0, 11, (32, 13), device="cuda") | |
| labels[torch.arange(13, device="cuda")[None, :] > torch.arange(32, device="cuda")[:, None] % 13] = -100 | |
| reference_loss = F.cross_entropy(reference(x).flatten(0, 1), labels.flatten(), reduction="sum") / labels.ne(-100).sum() | |
| reference_loss.backward() | |
| local_x = x.chunk(world)[rank] | |
| local_labels = labels.chunk(world)[rank] | |
| count = local_labels.ne(-100).sum() | |
| dist.all_reduce(count) | |
| for i in range(2): | |
| # DDP averages gradients across ranks. Summed token losses therefore need | |
| # the global token denominator and a world-size factor. | |
| output = model(local_x[i * 8:(i + 1) * 8]) | |
| loss = F.cross_entropy(output.flatten(0, 1), local_labels[i * 8:(i + 1) * 8].flatten(), reduction="sum") * world / count | |
| loss.backward() | |
| error = (model.module.weight.grad - reference.weight.grad).abs().max().item() | |
| assert error < 1e-12, error | |
| buffer = torch.ones(4 * 1024 * 1024, device="cuda", dtype=torch.float32) | |
| dist.barrier() | |
| torch.cuda.synchronize() | |
| start = time.perf_counter() | |
| for _ in range(5): | |
| dist.all_reduce(buffer) | |
| torch.cuda.synchronize() | |
| latency = (time.perf_counter() - start) / 5 | |
| if rank == 0: | |
| result = {"world_size": world, "max_gradient_error": error, "all_reduce_16MiB_seconds": latency, "passed": True} | |
| Path("distributed_correctness.json").write_text(json.dumps(result, indent=2)) | |
| print(json.dumps(result, indent=2), flush=True) | |
| dist.destroy_process_group() | |