"""Versioned research checkpoints, exact single-device RNG/optimizer resume.""" import json from dataclasses import asdict from pathlib import Path import torch from .network import Config, DuplexModel, factorization, from_legacy # The one- and two-stream format of 14 November, before text tracks. LEGACY_FACTORIZATION = "delay-0-1-1-1-1-1-1-1" def save(path, model, optimizer, step, provenance, sampler): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) state = { "schema_version": 3, "factorization": factorization(model.config), "config": asdict(model.config), "model": model.state_dict(), "optimizer": optimizer.state_dict(), "step": step, "provenance": provenance, "torch_rng": torch.get_rng_state(), "sampler_rng": sampler.get_state(), "cuda_rng": torch.cuda.get_rng_state_all() if torch.cuda.is_available() else [], } temporary = path.with_suffix(path.suffix + ".tmp") torch.save(state, temporary) temporary.replace(path) def model_from(config, weights, stored): """A model from a stored config and weights; 14 November checkpoints convert.""" if stored == LEGACY_FACTORIZATION: return from_legacy(config, weights) # Checkpoints before 17 November 2026 have no shift or head fields and load # with the defaults they were trained under. config = Config(**config) if stored != factorization(config): raise ValueError("Unsupported codebook factorization") model = DuplexModel(config) model.load_state_dict(weights, strict=True) return model def load(path, device="cpu"): # weights_only rejects arbitrary pickled classes; only tensors/basic containers are used. state = torch.load(path, map_location="cpu", weights_only=True) if state["schema_version"] not in (2, 3): raise ValueError("Unsupported checkpoint schema") model = model_from(state["config"], state["model"], state["factorization"]) return model.to(device), state def load_published(folder, device="cpu"): """A checkpoint folder written by `publish.py` (config.json and model.safetensors).""" from safetensors.torch import load_file folder = Path(folder) config = json.loads((folder / "config.json").read_text()) stored = config.pop("factorization") weights = load_file(folder / "model.safetensors") return model_from(config, weights, stored).to(device).eval() def resume(state, optimizer, sampler): if state["schema_version"] != 3: raise ValueError("Resume needs a checkpoint of this trainer") optimizer.load_state_dict(state["optimizer"]) torch.set_rng_state(state["torch_rng"]) sampler.set_state(state["sampler_rng"]) if state["cuda_rng"]: if not torch.cuda.is_available(): raise ValueError("Exact CUDA resume requires CUDA") torch.cuda.set_rng_state_all(state["cuda_rng"])