| |
|
|
| import pathlib |
| import re |
|
|
| TRAIN = pathlib.Path(__file__).resolve().parent.parent / "train.py" |
|
|
|
|
| def _validate_body(): |
| src = TRAIN.read_text(encoding="utf-8") |
| start = src.index("def validate(") |
| nxt = re.search(r"\n(?:def |# =====)", src[start + 10:]) |
| return src[start:start + 10 + (nxt.start() if nxt else len(src))] |
|
|
|
|
| def test_validate_forwards_through_the_unwrapped_module(): |
| body = _validate_body() |
| assert "outputs = unwrapped(" in body, \ |
| "validation must forward through the unwrapped module" |
| assert "outputs = model(" not in body, ( |
| "validate() forwards through the DDP wrapper — under a rank-0-only " |
| "call this broadcasts buffers to ranks that never arrive and deadlocks " |
| "(NCCL BROADCAST watchdog after 600 s)") |
|
|
|
|
| def test_validate_still_unwraps_for_ddp(): |
| body = _validate_body() |
| assert "model.module if isinstance(model, DDP)" in body, \ |
| "the DDP unwrap disappeared; `unwrapped` would then be the wrapper" |
|
|
|
|
| def test_call_site_is_still_rank_zero_only(): |
| src = TRAIN.read_text(encoding="utf-8") |
| assert re.search(r"rank == 0 and step % args\.val_every", src), \ |
| "validation call-site guard changed — re-check the DDP reasoning above" |
|
|
