Download test_training_optimization.py from zerdovzad/WarpStateAI: direct link, hf CLI and curl.
- Browser
- Download file 8.3 kB
-
https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/test_training_optimization.py
- Command line
-
hf download hf://zerdovzad/WarpStateAI/test_training_optimization.py
-
curl -L -o test_training_optimization.py https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/test_training_optimization.py
8.3 kB
| """Two-GPU full-state continuation checks for optional training optimizations.""" | |
| import json | |
| import math | |
| import pytest | |
| import torch | |
| from test_training_execution import ( | |
| TWO_GPUS, | |
| assert_tensors_equal, | |
| fixture_runs, | |
| run_training, | |
| ) | |
| CASES = [ | |
| pytest.param( | |
| ["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule"], | |
| {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline", | |
| "overlap_producer_schedule": True, "overlap_swap_collectives": False, | |
| "overlap_runtime_estimates": False, "overlap_memory_budget": 0.0, | |
| "ddp_bucket_cap_mb": 128}, | |
| id="producer-scheduled-overlap", | |
| ), | |
| pytest.param( | |
| ["--gradient-mode", "compiled-autograd-overlap", "--overlap-swap-collectives", | |
| "--overlap-runtime-estimates"], | |
| {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline", | |
| "overlap_swap_collectives": True, "overlap_runtime_estimates": True, | |
| "overlap_memory_budget": 0.0, "ddp_bucket_cap_mb": 128}, | |
| id="runtime-estimated-overlap", | |
| ), | |
| pytest.param( | |
| ["--gradient-mode", "compiled-autograd-overlap", "--overlap-swap-collectives", | |
| "--overlap-memory-budget", "0.02", "--ddp-bucket-cap-mb", "64"], | |
| {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline", | |
| "overlap_swap_collectives": True, "overlap_runtime_estimates": False, | |
| "overlap_memory_budget": 0.02, "ddp_bucket_cap_mb": 64}, | |
| id="waits-past-collectives", | |
| ), | |
| pytest.param( | |
| ["--gradient-mode", "compiled-autograd-overlap", "--matmul-plan", "__fixture_plan__"], | |
| {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline", | |
| "cache_core_weight_casts": False, "ddp_batched_grad_copy": False, "ddp_bucket_cap_mb": 128}, | |
| id="matmul-epilogue-with-communication-overlap", | |
| ), | |
| pytest.param( | |
| ["--gradient-mode", "eager", "--activation-mode", "all", | |
| "--cache-core-weight-casts", "--ddp-batched-grad-copy", | |
| "--ddp-bucket-cap-mb", "64"], | |
| {"gradient_mode": "eager", "activation_mode": "all", | |
| "cache_core_weight_casts": True, "ddp_batched_grad_copy": True, | |
| "ddp_bucket_cap_mb": 64}, | |
| id="activations-cached-casts-batched-copy", | |
| ), | |
| pytest.param( | |
| ["--gradient-mode", "compiled-autograd-overlap", "--activation-mode", "baseline", | |
| "--no-cache-core-weight-casts", "--no-ddp-batched-grad-copy"], | |
| {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline", | |
| "cache_core_weight_casts": False, "ddp_batched_grad_copy": False, | |
| "ddp_bucket_cap_mb": 128}, | |
| id="compiled-autograd-overlap", | |
| ), | |
| ] | |
| def test_optimized_two_gpu_async_resume_preserves_full_state(tmp_path, flags, selected): | |
| (continuous, resumed), common = fixture_runs(tmp_path, ["continuous", "resumed"]) | |
| common += [ | |
| "--loss-mode", "compiled-full", "--compile-core", "--compile-mode", "default", | |
| "--bf16-residuals", "--no-activation-checkpointing", "--autocast-cache", | |
| "--eval-every", "2", "--save-every", "2", *flags, | |
| ] | |
| selected = dict(selected) | |
| if "__fixture_plan__" in common: | |
| from ops.matmul_execution import load_matmul_plan | |
| tile = {"block_m": 16, "block_n": 32, "block_k": 16, "num_warps": 4, "num_stages": 2} | |
| document = {"format": "warpstate-matmul-plan-v1", "entries": [ | |
| {"kind": "swiglu", "m": 16, "n": 64, "k": 16, | |
| "a_stride": [16, 1], "b_stride": [1, 16], "dtype": "bfloat16", | |
| "algorithm": "persistent", "tile": tile}, | |
| {"kind": "mm", "m": 64, "n": 16, "k": 16, | |
| "a_stride": [1, 64], "b_stride": [16, 1], "dtype": "bfloat16", | |
| "algorithm": "persistent_tma", "tile": tile}, | |
| ]} | |
| recipe = tmp_path / "matmul-plan.json" | |
| recipe.write_text(json.dumps(document)) | |
| common[common.index("__fixture_plan__")] = str(recipe) | |
| selected["matmul_execution"] = load_matmul_plan(recipe).metadata | |
| initial = torch.load(tmp_path / "initial.pt", map_location="cpu", weights_only=False) | |
| # The 73-token fixture consumes 32, 32, then 9 tokens. On its final update, | |
| # rank zero consumes the remaining tokens and rank one has entirely masked | |
| # targets. Resume must preserve that weighting and every rank's RNG state. | |
| expected = run_training(continuous, common, 2, timeout=300, | |
| expect_validation_display=True) | |
| async_common = common + ["--async-checkpoint"] | |
| partial = run_training(resumed, async_common, 2, ["--max-steps", "1"], | |
| timeout=300, expect_async=True, expect_validation_display=True) | |
| assert partial["tokens_seen"] == 32 and partial["step"] == 1 | |
| assert partial["val_step"] == 1 and partial["val_tokens"] == 32 | |
| actual = run_training(resumed, async_common, 2, timeout=300, expect_async=True, | |
| expect_validation_display=True) | |
| assert actual["format"] == expected["format"] == "warpstate-server-v1" | |
| assert actual["step"] == expected["step"] == 3 | |
| assert actual["tokens_seen"] == expected["tokens_seen"] == 73 | |
| assert actual["origin"] == partial["origin"] == expected["origin"] | |
| assert actual["execution_history"] == partial["execution_history"] == expected["execution_history"] | |
| assert actual["signature"] == partial["signature"] == expected["signature"] | |
| settings = actual["signature"]["settings"] | |
| for name, value in selected.items(): | |
| assert settings[name] == value | |
| assert settings["world_size"] == 2 | |
| assert settings["compile_core"] is True | |
| assert settings["compile_mode"] == "default" | |
| assert settings["loss_mode"] == "compiled-full" | |
| assert settings["bf16_residuals"] is True | |
| assert settings["activation_checkpointing"] is False | |
| assert settings["seq_len"] * settings["micro_batch"] * settings["grad_accum"] * settings["world_size"] == 32 | |
| assert len(actual["rng_by_rank"]) == len(partial["rng_by_rank"]) == 2 | |
| for name in ("model", "optimizer", "rng_by_rank"): | |
| assert_tensors_equal(actual[name], expected[name]) | |
| assert actual["model"].keys() == initial["model"].keys() | |
| assert all(not name.startswith(("module.", "_orig_mod.")) for name in actual["model"]) | |
| assert all(value.dtype == torch.float32 and torch.isfinite(value).all() | |
| for value in actual["model"].values()) | |
| assert actual["optimizer"]["state"] | |
| for state in actual["optimizer"]["state"].values(): | |
| assert state["step"].item() == 3 | |
| for name in ("exp_avg", "exp_avg_sq"): | |
| assert state[name].dtype == torch.float32 and torch.isfinite(state[name]).all() | |
| assert actual["val_loss"] == expected["val_loss"] | |
| assert math.isfinite(actual["val_loss"]) | |
| assert actual["val_step"] == expected["val_step"] == 3 | |
| assert actual["val_tokens"] == expected["val_tokens"] == 73 | |
| for directory, async_enabled in [(continuous, False), (resumed, True)]: | |
| runtime = json.loads((directory / "run/settings.json").read_text())["runtime"] | |
| for name, value in selected.items(): | |
| assert runtime[name] == value | |
| assert runtime["async_checkpoint"] is async_enabled | |
| assert runtime["bf16_residuals"] is True | |
| assert runtime["eval_every"] == runtime["save_every"] == 2 | |
| assert not (directory / "run/latest.pt.tmp").exists() | |
| records = [json.loads(line) for line in | |
| (directory / "run/metrics.jsonl").read_text().splitlines()] | |
| assert [record["step"] for record in records] == [1, 2, 3] | |
| assert [record["tokens"] for record in records] == [32, 64, 73] | |
| assert [record["lr"] for record in records] == [0.001, 0.002, 0.002] | |
| assert all(math.isfinite(record["loss"]) and math.isfinite(record["grad_norm"]) | |
| for record in records) | |
| validations = [json.loads(line) for line in | |
| (resumed / "run/validation.jsonl").read_text().splitlines()] | |
| assert [record["tokens"] for record in validations] == [32, 64, 73] | |
| assert all(math.isfinite(record["loss"]) for record in validations) | |