scripts/check_nccl.py

#!/usr/bin/env python3
"""
Check NCCL availability for distributed training.
Run from repo root (no torchrun needed):
  uv run python scripts/check_nccl.py
  # or: python scripts/check_nccl.py
"""
import sys
import subprocess

def main():
    print("=== NCCL / CUDA environment check ===\n")

    # Python / PyTorch
    print(f"Python: {sys.executable}")
    import torch
    print(f"PyTorch: {torch.__version__}")
    print(f"CUDA compiled with: {torch.version.cuda or 'N/A'}")
    print(f"cuDNN: {torch.backends.cudnn.version() if torch.backends.cudnn.is_available() else 'N/A'}")

    cuda_available = torch.cuda.is_available()
    print(f"CUDA available: {cuda_available}")
    if cuda_available:
        print(f"CUDA runtime: {torch.version.cuda}")
        print(f"Device count: {torch.cuda.device_count()}")
        for i in range(torch.cuda.device_count()):
            print(f"  [{i}] {torch.cuda.get_device_name(i)}")

    # NCCL via PyTorch
    import torch.distributed as dist
    nccl_available = dist.is_nccl_available()
    print(f"\nNCCL available (PyTorch): {nccl_available}")

    # Optional: nvidia-smi
    try:
        out = subprocess.run(
            ["nvidia-smi", "--query-gpu=driver_version,name", "--format=csv,noheader"],
            capture_output=True,
            text=True,
            timeout=5,
        )
        if out.returncode == 0 and out.stdout.strip():
            print("\nnvidia-smi (driver / GPU):")
            for line in out.stdout.strip().split("\n"):
                print(f"  {line}")
    except FileNotFoundError:
        print("\nnvidia-smi: not found (optional)")
    except subprocess.TimeoutExpired:
        print("\nnvidia-smi: timeout (optional)")

    print("\n--- What to do next ---")
    if not cuda_available:
        print("  CUDA not available. Install PyTorch with CUDA: uv sync --extra gpu (and use GPU index)")
    elif not nccl_available:
        print("  NCCL not available in this PyTorch build. Use a CUDA-enabled wheel (e.g. cu128).")
        print("  Reinstall: uv sync --extra gpu  (with GPU index in pyproject)")
    else:
        print("  NCCL is available in PyTorch. To verify it actually runs (multi-process), use:")
        print("    torchrun --standalone --nproc_per_node=8 -m scripts.test_8gpu")
        print("  If that segfaults, use gloo: NANOCHAT_DDP_BACKEND=gloo torchrun ...")

    return 0 if (cuda_available and nccl_available) else 1

if __name__ == "__main__":
    sys.exit(main())