WarpStateAI / test_training_optimization.py
zerdovzad's picture
Upload 139 files
c8ba9c6 verified
Raw History Blame Contribute Delete
8.3 kB
"""Two-GPU full-state continuation checks for optional training optimizations."""
import json
import math
import pytest
import torch
from test_training_execution import (
TWO_GPUS,
assert_tensors_equal,
fixture_runs,
run_training,
)
CASES = [
pytest.param(
["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule"],
{"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
"overlap_producer_schedule": True, "overlap_swap_collectives": False,
"overlap_runtime_estimates": False, "overlap_memory_budget": 0.0,
"ddp_bucket_cap_mb": 128},
id="producer-scheduled-overlap",
),
pytest.param(
["--gradient-mode", "compiled-autograd-overlap", "--overlap-swap-collectives",
"--overlap-runtime-estimates"],
{"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
"overlap_swap_collectives": True, "overlap_runtime_estimates": True,
"overlap_memory_budget": 0.0, "ddp_bucket_cap_mb": 128},
id="runtime-estimated-overlap",
),
pytest.param(
["--gradient-mode", "compiled-autograd-overlap", "--overlap-swap-collectives",
"--overlap-memory-budget", "0.02", "--ddp-bucket-cap-mb", "64"],
{"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
"overlap_swap_collectives": True, "overlap_runtime_estimates": False,
"overlap_memory_budget": 0.02, "ddp_bucket_cap_mb": 64},
id="waits-past-collectives",
),
pytest.param(
["--gradient-mode", "compiled-autograd-overlap", "--matmul-plan", "__fixture_plan__"],
{"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
"cache_core_weight_casts": False, "ddp_batched_grad_copy": False, "ddp_bucket_cap_mb": 128},
id="matmul-epilogue-with-communication-overlap",
),
pytest.param(
["--gradient-mode", "eager", "--activation-mode", "all",
"--cache-core-weight-casts", "--ddp-batched-grad-copy",
"--ddp-bucket-cap-mb", "64"],
{"gradient_mode": "eager", "activation_mode": "all",
"cache_core_weight_casts": True, "ddp_batched_grad_copy": True,
"ddp_bucket_cap_mb": 64},
id="activations-cached-casts-batched-copy",
),
pytest.param(
["--gradient-mode", "compiled-autograd-overlap", "--activation-mode", "baseline",
"--no-cache-core-weight-casts", "--no-ddp-batched-grad-copy"],
{"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
"cache_core_weight_casts": False, "ddp_batched_grad_copy": False,
"ddp_bucket_cap_mb": 128},
id="compiled-autograd-overlap",
),
]
@TWO_GPUS
@pytest.mark.parametrize(("flags", "selected"), CASES)
def test_optimized_two_gpu_async_resume_preserves_full_state(tmp_path, flags, selected):
(continuous, resumed), common = fixture_runs(tmp_path, ["continuous", "resumed"])
common += [
"--loss-mode", "compiled-full", "--compile-core", "--compile-mode", "default",
"--bf16-residuals", "--no-activation-checkpointing", "--autocast-cache",
"--eval-every", "2", "--save-every", "2", *flags,
]
selected = dict(selected)
if "__fixture_plan__" in common:
from ops.matmul_execution import load_matmul_plan
tile = {"block_m": 16, "block_n": 32, "block_k": 16, "num_warps": 4, "num_stages": 2}
document = {"format": "warpstate-matmul-plan-v1", "entries": [
{"kind": "swiglu", "m": 16, "n": 64, "k": 16,
"a_stride": [16, 1], "b_stride": [1, 16], "dtype": "bfloat16",
"algorithm": "persistent", "tile": tile},
{"kind": "mm", "m": 64, "n": 16, "k": 16,
"a_stride": [1, 64], "b_stride": [16, 1], "dtype": "bfloat16",
"algorithm": "persistent_tma", "tile": tile},
]}
recipe = tmp_path / "matmul-plan.json"
recipe.write_text(json.dumps(document))
common[common.index("__fixture_plan__")] = str(recipe)
selected["matmul_execution"] = load_matmul_plan(recipe).metadata
initial = torch.load(tmp_path / "initial.pt", map_location="cpu", weights_only=False)
# The 73-token fixture consumes 32, 32, then 9 tokens. On its final update,
# rank zero consumes the remaining tokens and rank one has entirely masked
# targets. Resume must preserve that weighting and every rank's RNG state.
expected = run_training(continuous, common, 2, timeout=300,
expect_validation_display=True)
async_common = common + ["--async-checkpoint"]
partial = run_training(resumed, async_common, 2, ["--max-steps", "1"],
timeout=300, expect_async=True, expect_validation_display=True)
assert partial["tokens_seen"] == 32 and partial["step"] == 1
assert partial["val_step"] == 1 and partial["val_tokens"] == 32
actual = run_training(resumed, async_common, 2, timeout=300, expect_async=True,
expect_validation_display=True)
assert actual["format"] == expected["format"] == "warpstate-server-v1"
assert actual["step"] == expected["step"] == 3
assert actual["tokens_seen"] == expected["tokens_seen"] == 73
assert actual["origin"] == partial["origin"] == expected["origin"]
assert actual["execution_history"] == partial["execution_history"] == expected["execution_history"]
assert actual["signature"] == partial["signature"] == expected["signature"]
settings = actual["signature"]["settings"]
for name, value in selected.items():
assert settings[name] == value
assert settings["world_size"] == 2
assert settings["compile_core"] is True
assert settings["compile_mode"] == "default"
assert settings["loss_mode"] == "compiled-full"
assert settings["bf16_residuals"] is True
assert settings["activation_checkpointing"] is False
assert settings["seq_len"] * settings["micro_batch"] * settings["grad_accum"] * settings["world_size"] == 32
assert len(actual["rng_by_rank"]) == len(partial["rng_by_rank"]) == 2
for name in ("model", "optimizer", "rng_by_rank"):
assert_tensors_equal(actual[name], expected[name])
assert actual["model"].keys() == initial["model"].keys()
assert all(not name.startswith(("module.", "_orig_mod.")) for name in actual["model"])
assert all(value.dtype == torch.float32 and torch.isfinite(value).all()
for value in actual["model"].values())
assert actual["optimizer"]["state"]
for state in actual["optimizer"]["state"].values():
assert state["step"].item() == 3
for name in ("exp_avg", "exp_avg_sq"):
assert state[name].dtype == torch.float32 and torch.isfinite(state[name]).all()
assert actual["val_loss"] == expected["val_loss"]
assert math.isfinite(actual["val_loss"])
assert actual["val_step"] == expected["val_step"] == 3
assert actual["val_tokens"] == expected["val_tokens"] == 73
for directory, async_enabled in [(continuous, False), (resumed, True)]:
runtime = json.loads((directory / "run/settings.json").read_text())["runtime"]
for name, value in selected.items():
assert runtime[name] == value
assert runtime["async_checkpoint"] is async_enabled
assert runtime["bf16_residuals"] is True
assert runtime["eval_every"] == runtime["save_every"] == 2
assert not (directory / "run/latest.pt.tmp").exists()
records = [json.loads(line) for line in
(directory / "run/metrics.jsonl").read_text().splitlines()]
assert [record["step"] for record in records] == [1, 2, 3]
assert [record["tokens"] for record in records] == [32, 64, 73]
assert [record["lr"] for record in records] == [0.001, 0.002, 0.002]
assert all(math.isfinite(record["loss"]) and math.isfinite(record["grad_norm"])
for record in records)
validations = [json.loads(line) for line in
(resumed / "run/validation.jsonl").read_text().splitlines()]
assert [record["tokens"] for record in validations] == [32, 64, 73]
assert all(math.isfinite(record["loss"]) for record in validations)