File size: 1,314 Bytes
6e802a4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | # test_validate_no_ddp.py: Validation never runs through the DDP wrapper.
import pathlib
import re
TRAIN = pathlib.Path(__file__).resolve().parent.parent / "train.py"
def _validate_body():
src = TRAIN.read_text(encoding="utf-8")
start = src.index("def validate(")
nxt = re.search(r"\n(?:def |# =====)", src[start + 10:])
return src[start:start + 10 + (nxt.start() if nxt else len(src))]
def test_validate_forwards_through_the_unwrapped_module():
body = _validate_body()
assert "outputs = unwrapped(" in body, \
"validation must forward through the unwrapped module"
assert "outputs = model(" not in body, (
"validate() forwards through the DDP wrapper — under a rank-0-only "
"call this broadcasts buffers to ranks that never arrive and deadlocks "
"(NCCL BROADCAST watchdog after 600 s)")
def test_validate_still_unwraps_for_ddp():
body = _validate_body()
assert "model.module if isinstance(model, DDP)" in body, \
"the DDP unwrap disappeared; `unwrapped` would then be the wrapper"
def test_call_site_is_still_rank_zero_only():
src = TRAIN.read_text(encoding="utf-8")
assert re.search(r"rank == 0 and step % args\.val_every", src), \
"validation call-site guard changed — re-check the DDP reasoning above"
|