SLWM-X450M-Memory / load_checkpoint.py
diogoantunes's picture
Release verified Wave 6 checkpoint (seed 2001)
5f24c75 verified
Raw History Blame Contribute Delete
4.42 kB
"""Read the original SLWM NumPy exports without executing pickle data.
The format follows scripts/train/x1b_train.py in ValueAlignedAI/slwm.
See SOURCE_LICENSE for the original source-code license.
"""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
from typing import Iterator
import numpy as np
def verify_sha256(path: str | Path, expected: str) -> None:
digest = hashlib.sha256()
with Path(path).open("rb") as stream:
for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""):
digest.update(chunk)
if digest.hexdigest() != expected:
raise ValueError(f"SHA-256 mismatch: {Path(path).name}")
def iter_parameters(path: str | Path) -> Iterator[tuple[str, np.ndarray]]:
"""Yield each named parameter as float32, one array at a time."""
with np.load(path, allow_pickle=False) as archive:
if "__dtypes_json__" in archive.files:
manifest = json.loads(str(archive["__dtypes_json__"].item()))
expected_keys = set(manifest) | {"__dtypes_json__"}
if set(archive.files) != expected_keys:
raise ValueError("The BF16 archive and dtype manifest disagree")
for key, kind in manifest.items():
array = archive[key]
if kind == "bf16_u16":
if array.dtype.kind != "u" or array.dtype.itemsize != 2:
raise ValueError(f"Expected uint16 BF16 bits for {key}")
# BF16 uses the upper 16 bits of an IEEE-754 float32 value.
value = (array.astype(np.uint32) << 16).view(np.float32)
elif kind == "float32":
value = array.astype(np.float32)
else:
raise ValueError(f"Unsupported stored dtype: {kind}")
yield key, value
elif "__quant_manifest__" in archive.files:
manifest = json.loads(str(archive["__quant_manifest__"].item()))
expected_keys = set(manifest) | {"__quant_manifest__"}
expected_keys.update(key + "__scale" for key, kind in manifest.items() if kind == "int8_pc")
if set(archive.files) != expected_keys:
raise ValueError("The INT8 archive and quantization manifest disagree")
for key, kind in manifest.items():
array = archive[key]
if kind == "int8_pc":
if array.dtype != np.int8:
raise ValueError(f"Expected int8 values for {key}")
value = array.astype(np.float32) * archive[key + "__scale"].astype(np.float32)
elif kind == "float16":
value = array.astype(np.float32)
else:
raise ValueError(f"Unsupported quantization kind: {kind}")
yield key, value
else:
raise ValueError("This file is not a recognized SLWM checkpoint")
def load_checkpoint(path: str | Path, *, expected_sha256: str | None = None) -> dict[str, np.ndarray]:
"""Load a NumPy float32 parameter dictionary; optionally verify its SHA-256.
This expands both formats to float32. INT8 storage does not imply INT8
computation or a similarly small in-memory parameter dictionary.
"""
if expected_sha256 is not None:
verify_sha256(path, expected_sha256)
return dict(iter_parameters(path))
def inspect_checkpoint(path: str | Path) -> dict:
"""Check finite decoded parameters and reproduce the source's strict count."""
buckets = {"adapters": 0, "processor_core": 0, "jepa_and_uncertainty": 0, "policy_and_heads": 0}
tensor_count = 0
for key, value in iter_parameters(path):
if not np.isfinite(value).all():
raise ValueError(f"Non-finite weights in {key}")
tensor_count += 1
if key.startswith(("vis_", "aud_", "prop_", "text_", "act_", "reg_", "slot_", "mod_")):
bucket = "adapters"
elif key.startswith("L") or key == "final_norm":
bucket = "processor_core"
elif key.startswith(("jepa_", "unc_", "align_")):
bucket = "jepa_and_uncertainty"
else:
bucket = "policy_and_heads"
buckets[bucket] += int(value.size)
buckets["strict_total"] = sum(buckets.values())
return {"parameter_counts": buckets, "tensor_count": tensor_count, "all_finite": True}