| """Portable Release 188 safetensors inference authority. |
| |
| This module is an explicit external-I/O boundary. It admits the public release |
| from the standard repository-root model.safetensors.index.json and reduces that |
| index, its complete flat safetensors closure, and the protected tokenizer |
| surface to frozen typed authorities. No checkpoint, optimizer, generation |
| manifest, session path, or training authority participates in this contract. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import hashlib |
| import json |
| import os |
| import stat |
| from collections import Counter |
| from dataclasses import dataclass |
| from pathlib import Path, PurePosixPath |
| from typing import Final, Mapping, Sequence |
|
|
|
|
| RELEASE_188: Final[int] = 188 |
| RELEASE_188_REPOSITORY: Final[str] = "namenotfoundai/Nucleus-Resynthesis" |
| RELEASE_188_REVISION: Final[str] = "main" |
| RELEASE_188_TENSOR_COUNT: Final[int] = 984_720 |
| RELEASE_188_TOTAL_SIZE: Final[int] = 2_331_441_584_048 |
| RELEASE_188_WEIGHT_FILE_COUNT: Final[int] = 55 |
|
|
| _MODEL_INDEX_NAME: Final[str] = "model.safetensors.index.json" |
| _MODEL_INDEX_MAX_BYTES: Final[int] = 96 * 1024 * 1024 |
| _EXPECTED_MODEL_INDEX_BYTES: int = 80_606_542 |
| _EXPECTED_MODEL_INDEX_SHA256: str = ( |
| "c6160ea5454531a0a73c7d75d86c97019247ecd66c9ec9f7f3639867d009a9ac" |
| ) |
| _EXPECTED_TENSOR_COUNT: int = RELEASE_188_TENSOR_COUNT |
| _EXPECTED_TOTAL_SIZE: int = RELEASE_188_TOTAL_SIZE |
| _HASH_WAVE_BYTES: Final[int] = 64 * 1024 * 1024 |
| _SHA256_HEX_LENGTH: Final[int] = hashlib.sha256().digest_size * 2 |
| _WEIGHT_ROOT: Final[PurePosixPath] = PurePosixPath("weights", "safetensors") |
| _TOKENIZER_ROOT: Final[PurePosixPath] = PurePosixPath("tokenizer") |
| _PROTECTED_TOKENIZER_PATHS: Final[frozenset[str]] = frozenset( |
| { |
| "tokenizer/chat_template.jinja", |
| "tokenizer/config.json", |
| "tokenizer/generation_config.json", |
| "tokenizer/tokenizer.json", |
| "tokenizer/tokenizer_config.json", |
| } |
| ) |
| _FORBIDDEN_TENSOR_NAME_PARTS: Final[tuple[str, ...]] = ( |
| "optimizer", |
| ".exp_avg", |
| ".exp_avg_sq", |
| ) |
|
|
|
|
| @dataclass(frozen=True, slots=True) |
| class ReleaseArtifactAuthority: |
| """One regular repository-relative file bound to exact bytes.""" |
|
|
| relative_path: str |
| path: Path |
| byte_count: int |
| sha256: str |
|
|
|
|
| @dataclass(frozen=True, slots=True) |
| class ReleaseWeightArtifactAuthority: |
| """One safetensors artifact and its exact index-owned tensor count.""" |
|
|
| relative_path: str |
| path: Path |
| byte_count: int |
| sha256: str |
| tensor_count: int |
|
|
|
|
| @dataclass(frozen=True, slots=True) |
| class ReleaseWeightArtifacts: |
| """The complete flat Release 188 safetensors closure.""" |
|
|
| lexical_projection_substrate: ReleaseWeightArtifactAuthority |
| resynthesis: ReleaseWeightArtifactAuthority |
| resident_runtime: ReleaseWeightArtifactAuthority |
| direct_page_index: ReleaseWeightArtifactAuthority |
| page_shards: tuple[ReleaseWeightArtifactAuthority, ...] |
|
|
| @property |
| def all_artifacts(self) -> tuple[ReleaseWeightArtifactAuthority, ...]: |
| """Return all four core files followed by all 51 page shards.""" |
|
|
| return ( |
| self.lexical_projection_substrate, |
| self.resynthesis, |
| self.resident_runtime, |
| self.direct_page_index, |
| *self.page_shards, |
| ) |
|
|
|
|
| @dataclass(frozen=True, slots=True) |
| class ReleaseTokenizerArtifacts: |
| """Exact tokenizer files protected by the public Release 188 contract.""" |
|
|
| root: Path |
| chat_template: ReleaseArtifactAuthority |
| config: ReleaseArtifactAuthority |
| generation_config: ReleaseArtifactAuthority |
| model: ReleaseArtifactAuthority |
| tokenizer_config: ReleaseArtifactAuthority |
|
|
| @property |
| def all_artifacts(self) -> tuple[ReleaseArtifactAuthority, ...]: |
| """Return the complete protected tokenizer closure.""" |
|
|
| return ( |
| self.chat_template, |
| self.config, |
| self.generation_config, |
| self.model, |
| self.tokenizer_config, |
| ) |
|
|
|
|
| @dataclass(frozen=True, slots=True) |
| class ReleaseInferenceAuthority: |
| """Cold-load authority for the public safetensors-only Release 188.""" |
|
|
| repository_root: Path |
| model_index: ReleaseArtifactAuthority |
| repository: str |
| revision: str |
| release: int |
| tensor_count: int |
| total_size: int |
| weights: ReleaseWeightArtifacts |
| tokenizer: ReleaseTokenizerArtifacts |
|
|
|
|
| @dataclass(frozen=True, slots=True) |
| class _ArtifactSpec: |
| relative_path: str |
| byte_count: int |
| sha256: str |
| tensor_count: int |
|
|
|
|
| @dataclass(frozen=True, slots=True) |
| class _TokenizerSpec: |
| relative_path: str |
| byte_count: int |
| sha256: str |
|
|
|
|
| @dataclass(frozen=True, slots=True) |
| class _BoundArtifact: |
| authority: ReleaseArtifactAuthority |
| captured_bytes: bytes | None |
|
|
|
|
| _CORE_WEIGHT_SPECS: Final[tuple[_ArtifactSpec, ...]] = ( |
| _ArtifactSpec( |
| "weights/safetensors/model.safetensors", |
| 25_906_076_250, |
| "6f6e0f2029c347d92e5631d1980c514e242b68e1f2c7cf6aa03afa6ce068e5d6", |
| 2_509, |
| ), |
| _ArtifactSpec( |
| "weights/safetensors/resynthesis.safetensors", |
| 27_031_507_140, |
| "ff4a5243be81a2f12fe21a4359912d5d3965d6ef5d5efd76171678ca6afdcea3", |
| 2_301, |
| ), |
| _ArtifactSpec( |
| "weights/safetensors/resident-runtime.safetensors", |
| 241_818_298, |
| "8061e1454b3efde477fe0f835534293348afc2e20b00c7d59db4c0e48dd9f1a2", |
| 666, |
| ), |
| _ArtifactSpec( |
| "weights/safetensors/direct-page-index.safetensors", |
| 6_529_984, |
| "214543f654bdeb97e820e342ec7732307265aa59489c85e15226ac68fee76dd6", |
| 20, |
| ), |
| ) |
| _PAGE_SHARD_SHA256: Final[tuple[str, ...]] = tuple( |
| """ |
| a86b6c90aa4fa51ebf7ff0e5583948631d51576eb6ea72633d18999e8c949949 |
| 17c1fff79f3a0ffa04f9ac25e9012f3e2ad006a248e88620f6783c475bd140cf |
| 64b3fecaf580892d79b71ade7d62eecb2223821f870c3f408d984536750a89b9 |
| cf47ee1f0d07339a9288bd81ca7a85fe6d7ccea6cc40cf379838ac56c62b7c72 |
| 5d8e2d6c5fd32b9d848004d5825013a5a42aef1c75a5708358fc0d18639bb832 |
| 29b99521b2ebecd99eac708900483cb5e9d7e0b9b8362faf7622dedd68137c82 |
| e68719cbf9d9cb56b0caf9a2220a39f6c8eac8a0d708c2b3c904940580593926 |
| 7b77847c56dd0799bc4026c1c1bd5114fa75ffc0d7fc7c6514951a34833a5904 |
| d96b22e3d8ae0a903f41c93a50274936dffa56862b9dd2e98d04a412d3eaeace |
| fd13e502bcd85b35deb96ee280d592776e24ee663936d494c558810be2b8beb3 |
| b2f4ec283c1731c59c4f6b9cd9aaa6b520563c15162862d501142cc07040763f |
| 46a54104c5aac1e063801c67b89144af9c9e5dcd8107176b5d4bc17d8cfc7221 |
| 68340f70b16cb2effdd923172be631aa1062700b42f185210056365a51108226 |
| 3b5060641f62a547dbc7408291d983b3d01dbaf7c0b70203644b39f95f2a5a77 |
| 9868d0c1e3e3a4a6f23803cd87f4808c9100e39eb952d2da0bf46df8934ea3f3 |
| dac7f337494d8e7ea44dda5b20c769c25d434e81314fd5490b6134019947005e |
| 8d149a5025eabcae21964bb9a7c92ce8577fbdbd3d873579414b33161951ee36 |
| 4f72dcb3f1882430e2aaae9a87b2652bb7e621a6e9f13230a4d5e65be4c3d556 |
| 5728aeedede791cf0da5ca9f960cd2187a5aa54b1b1da14f541eda84f25a252a |
| d0597fbe2c0c8b63d20b3c43b07186be526cdf6cc438e3e41006d7f9b3556890 |
| ffc359071b05f61d7ee3abf1b5b76afe90014cbcd9288bf113c75c6edcb31779 |
| 0660753344de30f617fdb15ee9aa47e049d1ffe97b8dad36c764d13ed4c1ac1c |
| aff12abaa5639ebc9caa219cfad1e1e816ecf6c2e371500a78a68ac26d4d35a2 |
| 65f83caece195373cf2caab3602505a9da8e4b9e59571ba02443fc14448a53b3 |
| 1682618077fe82ef3ff6a8d5912dc4caeacb4c014ed50093bc500afaebecb3bd |
| f316d672289631a7b33062bc86b892a565b61ae325404ad9ecb5c9c85d2e04e3 |
| 5b669c7bf410dba7736ba5e886f4c79f828dfb82496d68a62ef9066f78a4e610 |
| 43c902a6f96fa630068b643016f548cc4d069a14706976313c5dfca090217800 |
| 3dc41c375777b914bdcf6344cb5eb1510623bd426735ad8de9c83cdc45c2e632 |
| 3441f307edbfbfe5e79f3f32e11b830cf82276799eef30c7dbc303a670768b5f |
| 388586def3b9fb6029eb86430260d556396f57bc67b5a1025952e6d3ee700478 |
| 68c65473521eb28c17537f7a24b11007afe388b95e71da59b123ab74ddb13b41 |
| c4a33cb69d09bdb7f5f82bcc8855b40e82782f7932f588d8a26c579ba10bf3dc |
| cd71ed8281b3a7dcc7c42d04382ea367c9a281cbb97134466bf7df9d0761a66f |
| 160be2a02c5640d308545888d8ef85cf69e193e5db0ab3911c5575092ab8646e |
| 49780379df7098c3fd4f0bc2a86814e7164a96ad3215a14036d4e566b93a527a |
| 20736796b5c974c89fe918f30a47bb4a26686d31f6fd6edf22c8cee3457d0aec |
| f9fdd53c2d62941792a608b0c486182717b7b773ccc2b59066de0fd78f898c79 |
| c1e0d0fd56e34c3649943b63357007224b94163cbaf76b5d6444f16a7281f66f |
| ce9b96beea4ed0688e8775bf6c17fc1e2681573163c83446e4ce42826b08b34b |
| df80e21f57ed1f7c443d62d7fb71acfc2a65263324314c6f95ebdf3975ebc9d9 |
| 3c9f05c8a97b11bf08de001ce287c91f795005c188bdde4a430f63b485f574fb |
| b807b6c28e88c76d67293870fe09589cc90cf52654a21b27009014cbeee9e442 |
| 54d09c5f2d3f5508652d0dfc7c629cfb10cd9899d65edb7ed79e688eff13ffa3 |
| 21f721fcbdf6b6e96c46f0feecd763f807c68985eef7c088b4a60b8f02317a51 |
| c04e26f645654392334ca2c955e096d3b0975c1edd86894d3c76b540f4d5e1f9 |
| 495a4a41f0b9b6c16ed4ef2b1ef59c0dfbf39c9b33bccfa3e501c8fc74f9b2c1 |
| 14e8e4956499ec6157748081adf85492cf8b98ffdc4f094f9778979bc7cbe70a |
| 4a6069463e3bb95ef8ae34042690a65630b0b6987cd2ebbf4e403e66041c8784 |
| cd45bac8c53d64510c92b123c9f49dcae57e0c951236aa888598a09481e7d003 |
| beb64481f93c781525d58fcb09bd0c9de727461f84b00d0a31004c719747a676 |
| """.split() |
| ) |
| _PAGE_WEIGHT_SPECS: Final[tuple[_ArtifactSpec, ...]] = tuple( |
| _ArtifactSpec( |
| ( |
| "weights/safetensors/" |
| f"pages-{index + 1:05d}-of-{len(_PAGE_SHARD_SHA256):05d}.safetensors" |
| ), |
| 44_979_669_860 if index < 50 else 29_372_195_888, |
| sha256, |
| 19_332 if index < 50 else 12_624, |
| ) |
| for index, sha256 in enumerate(_PAGE_SHARD_SHA256) |
| ) |
| _RELEASE_WEIGHT_SPECS: tuple[_ArtifactSpec, ...] = ( |
| *_CORE_WEIGHT_SPECS, |
| *_PAGE_WEIGHT_SPECS, |
| ) |
| _TOKENIZER_SPECS: tuple[_TokenizerSpec, ...] = ( |
| _TokenizerSpec( |
| "tokenizer/chat_template.jinja", |
| 7_695, |
| "23c95d888602bb8d610055a4eb855d25a035b0b7666635b3c3510ccde5488012", |
| ), |
| _TokenizerSpec( |
| "tokenizer/config.json", |
| 2_944, |
| "9b823a0199f4f4aedbcd0b967a83c7978b5b35f8d0916352cb93ded86f0c1446", |
| ), |
| _TokenizerSpec( |
| "tokenizer/generation_config.json", |
| 164, |
| "a78aebbc7804389b2f7863eaaac64c0bbe0b8a3fceb3d7ca71d539d3a3d96a83", |
| ), |
| _TokenizerSpec( |
| "tokenizer/tokenizer.json", |
| 19_989_343, |
| "87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4", |
| ), |
| _TokenizerSpec( |
| "tokenizer/tokenizer_config.json", |
| 1_232, |
| "ec78693b955edb09edfe01b9ebcf83f28242aa3a78d9b961964ee8036bc77a72", |
| ), |
| ) |
|
|
|
|
| def _object_without_duplicate_keys( |
| rows: Sequence[tuple[str, object]], |
| ) -> dict[str, object]: |
| payload: dict[str, object] = {} |
| for key, value in rows: |
| if key in payload: |
| raise ValueError(f"release index repeats field: {key}") |
| payload[key] = value |
| return payload |
|
|
|
|
| def _decode_json(payload: bytes) -> dict[str, object]: |
| try: |
| decoded = json.loads( |
| payload, |
| object_pairs_hook=_object_without_duplicate_keys, |
| ) |
| except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as error: |
| raise ValueError("release index is not strict JSON") from error |
| if not isinstance(decoded, dict): |
| raise ValueError("release index must be an object") |
| return decoded |
|
|
|
|
| def _mapping(value: object, *, name: str) -> Mapping[str, object]: |
| if not isinstance(value, dict): |
| raise ValueError(f"release index {name} must be an object") |
| return value |
|
|
|
|
| def _exact_keys( |
| payload: Mapping[str, object], |
| *, |
| required: frozenset[str], |
| name: str, |
| ) -> None: |
| missing = required.difference(payload) |
| unexpected = set(payload).difference(required) |
| if missing or unexpected: |
| raise ValueError( |
| f"release index {name} fields differ: " |
| f"missing={sorted(missing)} unexpected={sorted(unexpected)}" |
| ) |
|
|
|
|
| def _sha256(value: str, *, name: str) -> str: |
| if ( |
| len(value) != _SHA256_HEX_LENGTH |
| or value.lower() != value |
| or any(character not in "0123456789abcdef" for character in value) |
| ): |
| raise RuntimeError(f"Release 188 {name} is not lowercase SHA256") |
| return value |
|
|
|
|
| def _relative_path(value: object, *, name: str) -> PurePosixPath: |
| if not isinstance(value, str) or not value or "\x00" in value or "\\" in value: |
| raise ValueError(f"release index {name} is not a portable relative path") |
| relative_path = PurePosixPath(value) |
| if ( |
| relative_path.is_absolute() |
| or relative_path.as_posix() != value |
| or any(part in {"", ".", ".."} for part in relative_path.parts) |
| ): |
| raise ValueError(f"release index {name} escapes the repository") |
| return relative_path |
|
|
|
|
| def _open_repository_root(repository_root: Path) -> int: |
| try: |
| identity = repository_root.lstat() |
| except OSError as error: |
| raise RuntimeError("release repository root is unavailable") from error |
| if stat.S_ISLNK(identity.st_mode) or not stat.S_ISDIR(identity.st_mode): |
| raise RuntimeError("release repository root is not a real directory") |
| try: |
| return os.open( |
| repository_root, |
| os.O_RDONLY |
| | getattr(os, "O_CLOEXEC", 0) |
| | getattr(os, "O_DIRECTORY", 0) |
| | getattr(os, "O_NOFOLLOW", 0), |
| ) |
| except OSError as error: |
| raise RuntimeError("release repository root cannot be opened") from error |
|
|
|
|
| def _open_relative_regular_file( |
| root_descriptor: int, |
| relative_path: PurePosixPath, |
| ) -> int: |
| current_descriptor = os.dup(root_descriptor) |
| try: |
| for directory in relative_path.parts[:-1]: |
| next_descriptor = os.open( |
| directory, |
| os.O_RDONLY |
| | getattr(os, "O_CLOEXEC", 0) |
| | getattr(os, "O_DIRECTORY", 0) |
| | getattr(os, "O_NOFOLLOW", 0), |
| dir_fd=current_descriptor, |
| ) |
| os.close(current_descriptor) |
| current_descriptor = next_descriptor |
| descriptor = os.open( |
| relative_path.parts[-1], |
| os.O_RDONLY |
| | getattr(os, "O_CLOEXEC", 0) |
| | getattr(os, "O_NOFOLLOW", 0), |
| dir_fd=current_descriptor, |
| ) |
| except OSError as error: |
| raise RuntimeError( |
| f"release artifact is unavailable or symbolic: {relative_path}" |
| ) from error |
| finally: |
| os.close(current_descriptor) |
| identity = os.fstat(descriptor) |
| if not stat.S_ISREG(identity.st_mode): |
| os.close(descriptor) |
| raise RuntimeError(f"release artifact is not a regular file: {relative_path}") |
| return descriptor |
|
|
|
|
| def _bind_artifact( |
| *, |
| repository_root: Path, |
| root_descriptor: int, |
| relative_path: PurePosixPath, |
| expected_sha256: str, |
| expected_byte_count: int, |
| capture_limit: int | None = None, |
| ) -> _BoundArtifact: |
| descriptor = _open_relative_regular_file(root_descriptor, relative_path) |
| try: |
| before = os.fstat(descriptor) |
| if before.st_size != expected_byte_count: |
| raise RuntimeError(f"release artifact size differs: {relative_path}") |
| capture = ( |
| bytearray() |
| if capture_limit is not None and before.st_size <= capture_limit |
| else None |
| ) |
| digest = hashlib.sha256() |
| while True: |
| payload = os.read(descriptor, _HASH_WAVE_BYTES) |
| if not payload: |
| break |
| digest.update(payload) |
| if capture is not None: |
| capture.extend(payload) |
| after = os.fstat(descriptor) |
| if ( |
| before.st_dev != after.st_dev |
| or before.st_ino != after.st_ino |
| or before.st_size != after.st_size |
| or before.st_mtime_ns != after.st_mtime_ns |
| or before.st_ctime_ns != after.st_ctime_ns |
| ): |
| raise RuntimeError(f"release artifact changed during binding: {relative_path}") |
| actual_sha256 = digest.hexdigest() |
| if actual_sha256 != expected_sha256: |
| raise RuntimeError(f"release artifact SHA256 differs: {relative_path}") |
| return _BoundArtifact( |
| authority=ReleaseArtifactAuthority( |
| relative_path=relative_path.as_posix(), |
| path=repository_root.joinpath(*relative_path.parts), |
| byte_count=before.st_size, |
| sha256=actual_sha256, |
| ), |
| captured_bytes=bytes(capture) if capture is not None else None, |
| ) |
| finally: |
| os.close(descriptor) |
|
|
|
|
| def _read_exact(descriptor: int, byte_count: int) -> bytes: |
| payload = bytearray() |
| while len(payload) < byte_count: |
| chunk = os.read( |
| descriptor, |
| min(_HASH_WAVE_BYTES, byte_count - len(payload)), |
| ) |
| if not chunk: |
| break |
| payload.extend(chunk) |
| return bytes(payload) |
|
|
|
|
| def _bind_safetensors_artifact( |
| *, |
| repository_root: Path, |
| root_descriptor: int, |
| spec: _ArtifactSpec, |
| expected_tensor_names: frozenset[str], |
| ) -> ReleaseWeightArtifactAuthority: |
| """Bind safetensors structure without reading its multi-terabyte payload.""" |
|
|
| relative_path = _relative_path(spec.relative_path, name="artifact path") |
| descriptor = _open_relative_regular_file(root_descriptor, relative_path) |
| try: |
| before = os.fstat(descriptor) |
| if before.st_size != spec.byte_count: |
| raise RuntimeError(f"release artifact size differs: {relative_path}") |
| encoded_header_size = _read_exact(descriptor, 8) |
| if len(encoded_header_size) != 8: |
| raise RuntimeError( |
| f"release safetensors header is truncated: {relative_path}" |
| ) |
| header_size = int.from_bytes(encoded_header_size, byteorder="little") |
| if header_size < 2 or header_size > before.st_size - 8: |
| raise RuntimeError( |
| f"release safetensors header size differs: {relative_path}" |
| ) |
| encoded_header = _read_exact(descriptor, header_size) |
| if len(encoded_header) != header_size: |
| raise RuntimeError( |
| f"release safetensors header is truncated: {relative_path}" |
| ) |
| header = _decode_json(encoded_header) |
| header.pop("__metadata__", None) |
| if set(header) != expected_tensor_names: |
| raise RuntimeError( |
| f"release safetensors tensor key set differs: {relative_path}" |
| ) |
|
|
| spans: list[tuple[int, int, str]] = [] |
| for tensor_name, descriptor_payload in header.items(): |
| descriptor_mapping = _mapping( |
| descriptor_payload, |
| name=f"safetensors[{tensor_name!r}]", |
| ) |
| _exact_keys( |
| descriptor_mapping, |
| required=frozenset({"dtype", "shape", "data_offsets"}), |
| name=f"safetensors[{tensor_name!r}]", |
| ) |
| dtype = descriptor_mapping["dtype"] |
| shape = descriptor_mapping["shape"] |
| offsets = descriptor_mapping["data_offsets"] |
| if ( |
| not isinstance(dtype, str) |
| or not dtype |
| or not isinstance(shape, list) |
| or any( |
| isinstance(dimension, bool) |
| or not isinstance(dimension, int) |
| or dimension < 0 |
| for dimension in shape |
| ) |
| or not isinstance(offsets, list) |
| or len(offsets) != 2 |
| or any( |
| isinstance(offset, bool) or not isinstance(offset, int) |
| for offset in offsets |
| ) |
| ): |
| raise RuntimeError( |
| f"release safetensors descriptor differs: {relative_path}" |
| ) |
| start, end = offsets |
| if start < 0 or end < start: |
| raise RuntimeError( |
| f"release safetensors offsets differ: {relative_path}" |
| ) |
| spans.append((start, end, tensor_name)) |
|
|
| cursor = 0 |
| for start, end, tensor_name in sorted(spans): |
| if start != cursor: |
| raise RuntimeError( |
| "release safetensors payload is not contiguous: " |
| f"{relative_path}:{tensor_name}" |
| ) |
| cursor = end |
| if cursor != before.st_size - 8 - header_size: |
| raise RuntimeError( |
| f"release safetensors payload size differs: {relative_path}" |
| ) |
| after = os.fstat(descriptor) |
| if ( |
| before.st_dev != after.st_dev |
| or before.st_ino != after.st_ino |
| or before.st_size != after.st_size |
| or before.st_mtime_ns != after.st_mtime_ns |
| or before.st_ctime_ns != after.st_ctime_ns |
| ): |
| raise RuntimeError( |
| f"release artifact changed during binding: {relative_path}" |
| ) |
| return ReleaseWeightArtifactAuthority( |
| relative_path=relative_path.as_posix(), |
| path=repository_root.joinpath(*relative_path.parts), |
| byte_count=before.st_size, |
| sha256=spec.sha256, |
| tensor_count=spec.tensor_count, |
| ) |
| finally: |
| os.close(descriptor) |
|
|
|
|
| def _validate_specifications() -> None: |
| if ( |
| len(_RELEASE_WEIGHT_SPECS) != RELEASE_188_WEIGHT_FILE_COUNT |
| or len(_PAGE_SHARD_SHA256) != 51 |
| or len(_TOKENIZER_SPECS) != len(_PROTECTED_TOKENIZER_PATHS) |
| ): |
| raise RuntimeError("Release 188 artifact specification count differs") |
| weight_paths = [spec.relative_path for spec in _RELEASE_WEIGHT_SPECS] |
| if len(weight_paths) != len(set(weight_paths)): |
| raise RuntimeError("Release 188 artifact specification repeats a path") |
| for weight_spec in _RELEASE_WEIGHT_SPECS: |
| relative_path = _relative_path( |
| weight_spec.relative_path, |
| name="artifact path", |
| ) |
| if ( |
| relative_path.parent != _WEIGHT_ROOT |
| or relative_path.suffix != ".safetensors" |
| or weight_spec.byte_count < 1 |
| or weight_spec.tensor_count < 1 |
| ): |
| raise RuntimeError("Release 188 safetensors specification is invalid") |
| _sha256( |
| weight_spec.sha256, |
| name=f"artifact SHA256: {weight_spec.relative_path}", |
| ) |
| tokenizer_paths = {spec.relative_path for spec in _TOKENIZER_SPECS} |
| if tokenizer_paths != _PROTECTED_TOKENIZER_PATHS: |
| raise RuntimeError("Release 188 protected tokenizer path set differs") |
| for tokenizer_spec in _TOKENIZER_SPECS: |
| relative_path = _relative_path( |
| tokenizer_spec.relative_path, |
| name="tokenizer path", |
| ) |
| if ( |
| relative_path.parent != _TOKENIZER_ROOT |
| or tokenizer_spec.byte_count < 1 |
| ): |
| raise RuntimeError("Release 188 tokenizer specification is invalid") |
| _sha256( |
| tokenizer_spec.sha256, |
| name=f"tokenizer SHA256: {tokenizer_spec.relative_path}", |
| ) |
|
|
|
|
| def _forbidden_tensor_name(name: str) -> bool: |
| lowered = name.lower() |
| return ( |
| any(part in lowered for part in _FORBIDDEN_TENSOR_NAME_PARTS) |
| or lowered.rsplit(".", maxsplit=1)[-1] in {"step", "step_t"} |
| ) |
|
|
|
|
| def _validate_index(payload: bytes) -> dict[str, frozenset[str]]: |
| decoded = _decode_json(payload) |
| _exact_keys( |
| decoded, |
| required=frozenset({"metadata", "weight_map"}), |
| name="root", |
| ) |
| metadata = _mapping(decoded["metadata"], name="metadata") |
| _exact_keys( |
| metadata, |
| required=frozenset({"total_size"}), |
| name="metadata", |
| ) |
| total_size = metadata["total_size"] |
| if ( |
| isinstance(total_size, bool) |
| or not isinstance(total_size, int) |
| or total_size != _EXPECTED_TOTAL_SIZE |
| ): |
| raise RuntimeError("Release 188 index total_size differs") |
|
|
| weight_map = _mapping(decoded["weight_map"], name="weight_map") |
| if len(weight_map) != _EXPECTED_TENSOR_COUNT: |
| raise RuntimeError("Release 188 index tensor count differs") |
|
|
| specifications = { |
| spec.relative_path: spec for spec in _RELEASE_WEIGHT_SPECS |
| } |
| observed_counts: Counter[str] = Counter() |
| observed_names: dict[str, list[str]] = { |
| path: [] for path in specifications |
| } |
| for tensor_name, path_value in weight_map.items(): |
| if not tensor_name or _forbidden_tensor_name(tensor_name): |
| raise RuntimeError( |
| f"Release 188 index contains forbidden tensor: {tensor_name}" |
| ) |
| relative_path = _relative_path( |
| path_value, |
| name=f"weight_map[{tensor_name!r}]", |
| ) |
| if ( |
| relative_path.parent != _WEIGHT_ROOT |
| or relative_path.suffix != ".safetensors" |
| or relative_path.as_posix() not in specifications |
| ): |
| raise RuntimeError("Release 188 index weight file set differs") |
| path = relative_path.as_posix() |
| observed_counts[path] += 1 |
| observed_names[path].append(tensor_name) |
|
|
| expected_counts = Counter( |
| { |
| spec.relative_path: spec.tensor_count |
| for spec in _RELEASE_WEIGHT_SPECS |
| } |
| ) |
| if observed_counts != expected_counts: |
| raise RuntimeError("Release 188 index tensor distribution differs") |
| return { |
| path: frozenset(tensor_names) |
| for path, tensor_names in observed_names.items() |
| } |
|
|
|
|
| def load_release_inference_authority( |
| model_index_path: str | os.PathLike[str], |
| ) -> ReleaseInferenceAuthority: |
| """Bind the complete portable Release 188 repository. |
| |
| model_index_path must be the real repository-root |
| model.safetensors.index.json. The trusted index, all four core |
| safetensors and all 51 flat page shards are bound by exact size, published |
| SHA-256 authority, and safetensors header without reading tensor payloads. |
| The small index and five protected tokenizer files are hashed directly. |
| """ |
|
|
| index_path = Path(model_index_path).expanduser().absolute() |
| if index_path.name != _MODEL_INDEX_NAME: |
| raise ValueError("Release 188 model index name differs") |
| repository_root = index_path.parent |
| try: |
| index_identity = index_path.lstat() |
| except OSError as error: |
| raise RuntimeError("Release 188 model index is unavailable") from error |
| if stat.S_ISLNK(index_identity.st_mode) or not stat.S_ISREG( |
| index_identity.st_mode |
| ): |
| raise RuntimeError("Release 188 model index is symbolic or invalid") |
|
|
| _validate_specifications() |
| root_descriptor = _open_repository_root(repository_root) |
| try: |
| index_bound = _bind_artifact( |
| repository_root=repository_root, |
| root_descriptor=root_descriptor, |
| relative_path=PurePosixPath(_MODEL_INDEX_NAME), |
| expected_sha256=_sha256( |
| _EXPECTED_MODEL_INDEX_SHA256, |
| name="model index SHA256", |
| ), |
| expected_byte_count=_EXPECTED_MODEL_INDEX_BYTES, |
| capture_limit=_MODEL_INDEX_MAX_BYTES, |
| ) |
| if index_bound.captured_bytes is None: |
| raise RuntimeError("Release 188 model index exceeds its size boundary") |
| index_tensor_names = _validate_index(index_bound.captured_bytes) |
|
|
| bound_weights: dict[str, ReleaseWeightArtifactAuthority] = {} |
| for weight_spec in _RELEASE_WEIGHT_SPECS: |
| bound_weights[weight_spec.relative_path] = ( |
| _bind_safetensors_artifact( |
| repository_root=repository_root, |
| root_descriptor=root_descriptor, |
| spec=weight_spec, |
| expected_tensor_names=index_tensor_names[ |
| weight_spec.relative_path |
| ], |
| ) |
| ) |
|
|
| bound_tokenizer: dict[str, ReleaseArtifactAuthority] = {} |
| for tokenizer_spec in _TOKENIZER_SPECS: |
| relative_path = _relative_path( |
| tokenizer_spec.relative_path, |
| name="tokenizer path", |
| ) |
| bound = _bind_artifact( |
| repository_root=repository_root, |
| root_descriptor=root_descriptor, |
| relative_path=relative_path, |
| expected_sha256=tokenizer_spec.sha256, |
| expected_byte_count=tokenizer_spec.byte_count, |
| ) |
| bound_tokenizer[tokenizer_spec.relative_path] = bound.authority |
|
|
| page_paths = tuple( |
| spec.relative_path for spec in _RELEASE_WEIGHT_SPECS[4:] |
| ) |
| return ReleaseInferenceAuthority( |
| repository_root=repository_root, |
| model_index=index_bound.authority, |
| repository=RELEASE_188_REPOSITORY, |
| revision=RELEASE_188_REVISION, |
| release=RELEASE_188, |
| tensor_count=_EXPECTED_TENSOR_COUNT, |
| total_size=_EXPECTED_TOTAL_SIZE, |
| weights=ReleaseWeightArtifacts( |
| lexical_projection_substrate=bound_weights[ |
| "weights/safetensors/model.safetensors" |
| ], |
| resynthesis=bound_weights[ |
| "weights/safetensors/resynthesis.safetensors" |
| ], |
| resident_runtime=bound_weights[ |
| "weights/safetensors/resident-runtime.safetensors" |
| ], |
| direct_page_index=bound_weights[ |
| "weights/safetensors/direct-page-index.safetensors" |
| ], |
| page_shards=tuple(bound_weights[path] for path in page_paths), |
| ), |
| tokenizer=ReleaseTokenizerArtifacts( |
| root=repository_root / "tokenizer", |
| chat_template=bound_tokenizer[ |
| "tokenizer/chat_template.jinja" |
| ], |
| config=bound_tokenizer["tokenizer/config.json"], |
| generation_config=bound_tokenizer[ |
| "tokenizer/generation_config.json" |
| ], |
| model=bound_tokenizer["tokenizer/tokenizer.json"], |
| tokenizer_config=bound_tokenizer[ |
| "tokenizer/tokenizer_config.json" |
| ], |
| ), |
| ) |
| finally: |
| os.close(root_descriptor) |
|
|
|
|
| __all__ = [ |
| "RELEASE_188", |
| "RELEASE_188_REPOSITORY", |
| "RELEASE_188_REVISION", |
| "RELEASE_188_TENSOR_COUNT", |
| "RELEASE_188_TOTAL_SIZE", |
| "RELEASE_188_WEIGHT_FILE_COUNT", |
| "ReleaseArtifactAuthority", |
| "ReleaseInferenceAuthority", |
| "ReleaseTokenizerArtifacts", |
| "ReleaseWeightArtifactAuthority", |
| "ReleaseWeightArtifacts", |
| "load_release_inference_authority", |
| ] |
|
|
