File size: 3,648 Bytes
c83c617 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | """Explicit, checkpoint-bound V4 budget extension; ordinary resume stays strict."""
import copy
import math
from vimeml.training.data import file_sha
def configure(config, settings):
receipt = config["continuation"]
start, previous = receipt["source_step"], receipt["source_max_steps"]
rate = receipt["source_learning_rate"]
if (config.get("architecture") != "tiny_gpt_v4" or
type(start) is not int or type(previous) is not int or
not 0 < start <= previous or not math.isfinite(rate) or
not settings["min_learning_rate"] <= rate <= settings["learning_rate"] or
settings.get("learning_rate_schedule", "cosine") != "cosine" or
not settings.get("epochs") or settings["gradient_accumulation"] != 1):
raise ValueError("Invalid V4 continuation schedule.")
if settings["max_steps"] and settings["max_steps"] <= previous:
raise ValueError("Continuation must extend the previous training budget.")
settings["continuation_start_step"] = start
settings["continuation_start_learning_rate"] = rate
def validate_resume(saved, config, signatures, precision, checkpoint):
"""Allow only more epochs, an explicit plateau policy, and the trainer update."""
receipt = config["continuation"]
if (saved.get("format") != "vimeml_tiny_gpt_v4" or
saved["precision"] != precision or saved["step"] != receipt["source_step"] or
saved["scheduler"]["max_steps"] != receipt["source_max_steps"] or
file_sha(checkpoint) != receipt["source_checkpoint_sha256"]):
raise ValueError("V4 continuation does not match its source checkpoint.")
normalized = copy.deepcopy(config)
normalized.pop("continuation")
expected = copy.deepcopy(saved["config"])
expected.pop("continuation", None)
current = normalized["training"]
previous = expected["training"]
if current["epochs"] <= previous["epochs"]:
raise ValueError("V4 continuation must increase epochs.")
current["epochs"] = previous["epochs"]
if current.get("early_stopping_mode") != "plateau":
raise ValueError("The explicit continuation policy must be plateau.")
if "early_stopping_mode" not in previous:
current.pop("early_stopping_mode")
if normalized != expected:
raise ValueError("Only epochs and plateau stopping may change in a V4 extension.")
before, after = saved["signatures"], signatures
if {k: v for k, v in before.items() if k != "training_code"} != {
k: v for k, v in after.items() if k != "training_code"}:
raise ValueError("V4 continuation requires identical data signatures.")
old_code, new_code = before["training_code"], after["training_code"]
old_helper = old_code.get("continuation_v4.py")
if (old_code["train.py"] != receipt["source_train_sha256"] or
(old_helper is not None and old_helper != receipt.get("source_continuation_sha256")) or
set(new_code) != set(old_code) | {"continuation_v4.py"} or
any(new_code[k] != v for k, v in old_code.items()
if k not in {"train.py", "continuation_v4.py"})):
raise ValueError("Unexpected model/runtime/data code change during extension.")
if any(not math.isclose(group["lr"], receipt["source_learning_rate"],
rel_tol=0, abs_tol=1e-15)
for group in saved["optimizer"]["param_groups"]):
raise ValueError("Continuation learning rate must match the saved optimizer.")
if saved.get("early_stopped"):
raise ValueError("An early-stopped experiment requires a separate decision.")
|