File size: 4,419 Bytes
5f24c75
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
"""Read the original SLWM NumPy exports without executing pickle data.

The format follows scripts/train/x1b_train.py in ValueAlignedAI/slwm.
See SOURCE_LICENSE for the original source-code license.
"""

from __future__ import annotations

import hashlib
import json
from pathlib import Path
from typing import Iterator

import numpy as np


def verify_sha256(path: str | Path, expected: str) -> None:
    digest = hashlib.sha256()
    with Path(path).open("rb") as stream:
        for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""):
            digest.update(chunk)
    if digest.hexdigest() != expected:
        raise ValueError(f"SHA-256 mismatch: {Path(path).name}")


def iter_parameters(path: str | Path) -> Iterator[tuple[str, np.ndarray]]:
    """Yield each named parameter as float32, one array at a time."""
    with np.load(path, allow_pickle=False) as archive:
        if "__dtypes_json__" in archive.files:
            manifest = json.loads(str(archive["__dtypes_json__"].item()))
            expected_keys = set(manifest) | {"__dtypes_json__"}
            if set(archive.files) != expected_keys:
                raise ValueError("The BF16 archive and dtype manifest disagree")
            for key, kind in manifest.items():
                array = archive[key]
                if kind == "bf16_u16":
                    if array.dtype.kind != "u" or array.dtype.itemsize != 2:
                        raise ValueError(f"Expected uint16 BF16 bits for {key}")
                    # BF16 uses the upper 16 bits of an IEEE-754 float32 value.
                    value = (array.astype(np.uint32) << 16).view(np.float32)
                elif kind == "float32":
                    value = array.astype(np.float32)
                else:
                    raise ValueError(f"Unsupported stored dtype: {kind}")
                yield key, value
        elif "__quant_manifest__" in archive.files:
            manifest = json.loads(str(archive["__quant_manifest__"].item()))
            expected_keys = set(manifest) | {"__quant_manifest__"}
            expected_keys.update(key + "__scale" for key, kind in manifest.items() if kind == "int8_pc")
            if set(archive.files) != expected_keys:
                raise ValueError("The INT8 archive and quantization manifest disagree")
            for key, kind in manifest.items():
                array = archive[key]
                if kind == "int8_pc":
                    if array.dtype != np.int8:
                        raise ValueError(f"Expected int8 values for {key}")
                    value = array.astype(np.float32) * archive[key + "__scale"].astype(np.float32)
                elif kind == "float16":
                    value = array.astype(np.float32)
                else:
                    raise ValueError(f"Unsupported quantization kind: {kind}")
                yield key, value
        else:
            raise ValueError("This file is not a recognized SLWM checkpoint")


def load_checkpoint(path: str | Path, *, expected_sha256: str | None = None) -> dict[str, np.ndarray]:
    """Load a NumPy float32 parameter dictionary; optionally verify its SHA-256.

    This expands both formats to float32. INT8 storage does not imply INT8
    computation or a similarly small in-memory parameter dictionary.
    """
    if expected_sha256 is not None:
        verify_sha256(path, expected_sha256)
    return dict(iter_parameters(path))


def inspect_checkpoint(path: str | Path) -> dict:
    """Check finite decoded parameters and reproduce the source's strict count."""
    buckets = {"adapters": 0, "processor_core": 0, "jepa_and_uncertainty": 0, "policy_and_heads": 0}
    tensor_count = 0
    for key, value in iter_parameters(path):
        if not np.isfinite(value).all():
            raise ValueError(f"Non-finite weights in {key}")
        tensor_count += 1
        if key.startswith(("vis_", "aud_", "prop_", "text_", "act_", "reg_", "slot_", "mod_")):
            bucket = "adapters"
        elif key.startswith("L") or key == "final_norm":
            bucket = "processor_core"
        elif key.startswith(("jepa_", "unc_", "align_")):
            bucket = "jepa_and_uncertainty"
        else:
            bucket = "policy_and_heads"
        buckets[bucket] += int(value.size)
    buckets["strict_total"] = sum(buckets.values())
    return {"parameter_counts": buckets, "tensor_count": tensor_count, "all_finite": True}