"""Portable Release 188 safetensors inference authority. This module is an explicit external-I/O boundary. It admits the public release from the standard repository-root model.safetensors.index.json and reduces that index, its complete flat safetensors closure, and the protected tokenizer surface to frozen typed authorities. No checkpoint, optimizer, generation manifest, session path, or training authority participates in this contract. """ from __future__ import annotations import hashlib import json import os import stat from collections import Counter from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Final, Mapping, Sequence RELEASE_188: Final[int] = 188 RELEASE_188_REPOSITORY: Final[str] = "namenotfoundai/Nucleus-Resynthesis" RELEASE_188_REVISION: Final[str] = "main" RELEASE_188_TENSOR_COUNT: Final[int] = 984_720 RELEASE_188_TOTAL_SIZE: Final[int] = 2_331_441_584_048 RELEASE_188_WEIGHT_FILE_COUNT: Final[int] = 55 _MODEL_INDEX_NAME: Final[str] = "model.safetensors.index.json" _MODEL_INDEX_MAX_BYTES: Final[int] = 96 * 1024 * 1024 _EXPECTED_MODEL_INDEX_BYTES: int = 80_606_542 _EXPECTED_MODEL_INDEX_SHA256: str = ( "c6160ea5454531a0a73c7d75d86c97019247ecd66c9ec9f7f3639867d009a9ac" ) _EXPECTED_TENSOR_COUNT: int = RELEASE_188_TENSOR_COUNT _EXPECTED_TOTAL_SIZE: int = RELEASE_188_TOTAL_SIZE _HASH_WAVE_BYTES: Final[int] = 64 * 1024 * 1024 _SHA256_HEX_LENGTH: Final[int] = hashlib.sha256().digest_size * 2 _WEIGHT_ROOT: Final[PurePosixPath] = PurePosixPath("weights", "safetensors") _TOKENIZER_ROOT: Final[PurePosixPath] = PurePosixPath("tokenizer") _PROTECTED_TOKENIZER_PATHS: Final[frozenset[str]] = frozenset( { "tokenizer/chat_template.jinja", "tokenizer/config.json", "tokenizer/generation_config.json", "tokenizer/tokenizer.json", "tokenizer/tokenizer_config.json", } ) _FORBIDDEN_TENSOR_NAME_PARTS: Final[tuple[str, ...]] = ( "optimizer", ".exp_avg", ".exp_avg_sq", ) @dataclass(frozen=True, slots=True) class ReleaseArtifactAuthority: """One regular repository-relative file bound to exact bytes.""" relative_path: str path: Path byte_count: int sha256: str @dataclass(frozen=True, slots=True) class ReleaseWeightArtifactAuthority: """One safetensors artifact and its exact index-owned tensor count.""" relative_path: str path: Path byte_count: int sha256: str tensor_count: int @dataclass(frozen=True, slots=True) class ReleaseWeightArtifacts: """The complete flat Release 188 safetensors closure.""" lexical_projection_substrate: ReleaseWeightArtifactAuthority resynthesis: ReleaseWeightArtifactAuthority resident_runtime: ReleaseWeightArtifactAuthority direct_page_index: ReleaseWeightArtifactAuthority page_shards: tuple[ReleaseWeightArtifactAuthority, ...] @property def all_artifacts(self) -> tuple[ReleaseWeightArtifactAuthority, ...]: """Return all four core files followed by all 51 page shards.""" return ( self.lexical_projection_substrate, self.resynthesis, self.resident_runtime, self.direct_page_index, *self.page_shards, ) @dataclass(frozen=True, slots=True) class ReleaseTokenizerArtifacts: """Exact tokenizer files protected by the public Release 188 contract.""" root: Path chat_template: ReleaseArtifactAuthority config: ReleaseArtifactAuthority generation_config: ReleaseArtifactAuthority model: ReleaseArtifactAuthority tokenizer_config: ReleaseArtifactAuthority @property def all_artifacts(self) -> tuple[ReleaseArtifactAuthority, ...]: """Return the complete protected tokenizer closure.""" return ( self.chat_template, self.config, self.generation_config, self.model, self.tokenizer_config, ) @dataclass(frozen=True, slots=True) class ReleaseInferenceAuthority: """Cold-load authority for the public safetensors-only Release 188.""" repository_root: Path model_index: ReleaseArtifactAuthority repository: str revision: str release: int tensor_count: int total_size: int weights: ReleaseWeightArtifacts tokenizer: ReleaseTokenizerArtifacts @dataclass(frozen=True, slots=True) class _ArtifactSpec: relative_path: str byte_count: int sha256: str tensor_count: int @dataclass(frozen=True, slots=True) class _TokenizerSpec: relative_path: str byte_count: int sha256: str @dataclass(frozen=True, slots=True) class _BoundArtifact: authority: ReleaseArtifactAuthority captured_bytes: bytes | None _CORE_WEIGHT_SPECS: Final[tuple[_ArtifactSpec, ...]] = ( _ArtifactSpec( "weights/safetensors/model.safetensors", 25_906_076_250, "6f6e0f2029c347d92e5631d1980c514e242b68e1f2c7cf6aa03afa6ce068e5d6", 2_509, ), _ArtifactSpec( "weights/safetensors/resynthesis.safetensors", 27_031_507_140, "ff4a5243be81a2f12fe21a4359912d5d3965d6ef5d5efd76171678ca6afdcea3", 2_301, ), _ArtifactSpec( "weights/safetensors/resident-runtime.safetensors", 241_818_298, "8061e1454b3efde477fe0f835534293348afc2e20b00c7d59db4c0e48dd9f1a2", 666, ), _ArtifactSpec( "weights/safetensors/direct-page-index.safetensors", 6_529_984, "214543f654bdeb97e820e342ec7732307265aa59489c85e15226ac68fee76dd6", 20, ), ) _PAGE_SHARD_SHA256: Final[tuple[str, ...]] = tuple( """ a86b6c90aa4fa51ebf7ff0e5583948631d51576eb6ea72633d18999e8c949949 17c1fff79f3a0ffa04f9ac25e9012f3e2ad006a248e88620f6783c475bd140cf 64b3fecaf580892d79b71ade7d62eecb2223821f870c3f408d984536750a89b9 cf47ee1f0d07339a9288bd81ca7a85fe6d7ccea6cc40cf379838ac56c62b7c72 5d8e2d6c5fd32b9d848004d5825013a5a42aef1c75a5708358fc0d18639bb832 29b99521b2ebecd99eac708900483cb5e9d7e0b9b8362faf7622dedd68137c82 e68719cbf9d9cb56b0caf9a2220a39f6c8eac8a0d708c2b3c904940580593926 7b77847c56dd0799bc4026c1c1bd5114fa75ffc0d7fc7c6514951a34833a5904 d96b22e3d8ae0a903f41c93a50274936dffa56862b9dd2e98d04a412d3eaeace fd13e502bcd85b35deb96ee280d592776e24ee663936d494c558810be2b8beb3 b2f4ec283c1731c59c4f6b9cd9aaa6b520563c15162862d501142cc07040763f 46a54104c5aac1e063801c67b89144af9c9e5dcd8107176b5d4bc17d8cfc7221 68340f70b16cb2effdd923172be631aa1062700b42f185210056365a51108226 3b5060641f62a547dbc7408291d983b3d01dbaf7c0b70203644b39f95f2a5a77 9868d0c1e3e3a4a6f23803cd87f4808c9100e39eb952d2da0bf46df8934ea3f3 dac7f337494d8e7ea44dda5b20c769c25d434e81314fd5490b6134019947005e 8d149a5025eabcae21964bb9a7c92ce8577fbdbd3d873579414b33161951ee36 4f72dcb3f1882430e2aaae9a87b2652bb7e621a6e9f13230a4d5e65be4c3d556 5728aeedede791cf0da5ca9f960cd2187a5aa54b1b1da14f541eda84f25a252a d0597fbe2c0c8b63d20b3c43b07186be526cdf6cc438e3e41006d7f9b3556890 ffc359071b05f61d7ee3abf1b5b76afe90014cbcd9288bf113c75c6edcb31779 0660753344de30f617fdb15ee9aa47e049d1ffe97b8dad36c764d13ed4c1ac1c aff12abaa5639ebc9caa219cfad1e1e816ecf6c2e371500a78a68ac26d4d35a2 65f83caece195373cf2caab3602505a9da8e4b9e59571ba02443fc14448a53b3 1682618077fe82ef3ff6a8d5912dc4caeacb4c014ed50093bc500afaebecb3bd f316d672289631a7b33062bc86b892a565b61ae325404ad9ecb5c9c85d2e04e3 5b669c7bf410dba7736ba5e886f4c79f828dfb82496d68a62ef9066f78a4e610 43c902a6f96fa630068b643016f548cc4d069a14706976313c5dfca090217800 3dc41c375777b914bdcf6344cb5eb1510623bd426735ad8de9c83cdc45c2e632 3441f307edbfbfe5e79f3f32e11b830cf82276799eef30c7dbc303a670768b5f 388586def3b9fb6029eb86430260d556396f57bc67b5a1025952e6d3ee700478 68c65473521eb28c17537f7a24b11007afe388b95e71da59b123ab74ddb13b41 c4a33cb69d09bdb7f5f82bcc8855b40e82782f7932f588d8a26c579ba10bf3dc cd71ed8281b3a7dcc7c42d04382ea367c9a281cbb97134466bf7df9d0761a66f 160be2a02c5640d308545888d8ef85cf69e193e5db0ab3911c5575092ab8646e 49780379df7098c3fd4f0bc2a86814e7164a96ad3215a14036d4e566b93a527a 20736796b5c974c89fe918f30a47bb4a26686d31f6fd6edf22c8cee3457d0aec f9fdd53c2d62941792a608b0c486182717b7b773ccc2b59066de0fd78f898c79 c1e0d0fd56e34c3649943b63357007224b94163cbaf76b5d6444f16a7281f66f ce9b96beea4ed0688e8775bf6c17fc1e2681573163c83446e4ce42826b08b34b df80e21f57ed1f7c443d62d7fb71acfc2a65263324314c6f95ebdf3975ebc9d9 3c9f05c8a97b11bf08de001ce287c91f795005c188bdde4a430f63b485f574fb b807b6c28e88c76d67293870fe09589cc90cf52654a21b27009014cbeee9e442 54d09c5f2d3f5508652d0dfc7c629cfb10cd9899d65edb7ed79e688eff13ffa3 21f721fcbdf6b6e96c46f0feecd763f807c68985eef7c088b4a60b8f02317a51 c04e26f645654392334ca2c955e096d3b0975c1edd86894d3c76b540f4d5e1f9 495a4a41f0b9b6c16ed4ef2b1ef59c0dfbf39c9b33bccfa3e501c8fc74f9b2c1 14e8e4956499ec6157748081adf85492cf8b98ffdc4f094f9778979bc7cbe70a 4a6069463e3bb95ef8ae34042690a65630b0b6987cd2ebbf4e403e66041c8784 cd45bac8c53d64510c92b123c9f49dcae57e0c951236aa888598a09481e7d003 beb64481f93c781525d58fcb09bd0c9de727461f84b00d0a31004c719747a676 """.split() ) _PAGE_WEIGHT_SPECS: Final[tuple[_ArtifactSpec, ...]] = tuple( _ArtifactSpec( ( "weights/safetensors/" f"pages-{index + 1:05d}-of-{len(_PAGE_SHARD_SHA256):05d}.safetensors" ), 44_979_669_860 if index < 50 else 29_372_195_888, sha256, 19_332 if index < 50 else 12_624, ) for index, sha256 in enumerate(_PAGE_SHARD_SHA256) ) _RELEASE_WEIGHT_SPECS: tuple[_ArtifactSpec, ...] = ( *_CORE_WEIGHT_SPECS, *_PAGE_WEIGHT_SPECS, ) _TOKENIZER_SPECS: tuple[_TokenizerSpec, ...] = ( _TokenizerSpec( "tokenizer/chat_template.jinja", 7_695, "23c95d888602bb8d610055a4eb855d25a035b0b7666635b3c3510ccde5488012", ), _TokenizerSpec( "tokenizer/config.json", 2_944, "9b823a0199f4f4aedbcd0b967a83c7978b5b35f8d0916352cb93ded86f0c1446", ), _TokenizerSpec( "tokenizer/generation_config.json", 164, "a78aebbc7804389b2f7863eaaac64c0bbe0b8a3fceb3d7ca71d539d3a3d96a83", ), _TokenizerSpec( "tokenizer/tokenizer.json", 19_989_343, "87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4", ), _TokenizerSpec( "tokenizer/tokenizer_config.json", 1_232, "ec78693b955edb09edfe01b9ebcf83f28242aa3a78d9b961964ee8036bc77a72", ), ) def _object_without_duplicate_keys( rows: Sequence[tuple[str, object]], ) -> dict[str, object]: payload: dict[str, object] = {} for key, value in rows: if key in payload: raise ValueError(f"release index repeats field: {key}") payload[key] = value return payload def _decode_json(payload: bytes) -> dict[str, object]: try: decoded = json.loads( payload, object_pairs_hook=_object_without_duplicate_keys, ) except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as error: raise ValueError("release index is not strict JSON") from error if not isinstance(decoded, dict): raise ValueError("release index must be an object") return decoded def _mapping(value: object, *, name: str) -> Mapping[str, object]: if not isinstance(value, dict): raise ValueError(f"release index {name} must be an object") return value def _exact_keys( payload: Mapping[str, object], *, required: frozenset[str], name: str, ) -> None: missing = required.difference(payload) unexpected = set(payload).difference(required) if missing or unexpected: raise ValueError( f"release index {name} fields differ: " f"missing={sorted(missing)} unexpected={sorted(unexpected)}" ) def _sha256(value: str, *, name: str) -> str: if ( len(value) != _SHA256_HEX_LENGTH or value.lower() != value or any(character not in "0123456789abcdef" for character in value) ): raise RuntimeError(f"Release 188 {name} is not lowercase SHA256") return value def _relative_path(value: object, *, name: str) -> PurePosixPath: if not isinstance(value, str) or not value or "\x00" in value or "\\" in value: raise ValueError(f"release index {name} is not a portable relative path") relative_path = PurePosixPath(value) if ( relative_path.is_absolute() or relative_path.as_posix() != value or any(part in {"", ".", ".."} for part in relative_path.parts) ): raise ValueError(f"release index {name} escapes the repository") return relative_path def _open_repository_root(repository_root: Path) -> int: try: identity = repository_root.lstat() except OSError as error: raise RuntimeError("release repository root is unavailable") from error if stat.S_ISLNK(identity.st_mode) or not stat.S_ISDIR(identity.st_mode): raise RuntimeError("release repository root is not a real directory") try: return os.open( repository_root, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), ) except OSError as error: raise RuntimeError("release repository root cannot be opened") from error def _open_relative_regular_file( root_descriptor: int, relative_path: PurePosixPath, ) -> int: current_descriptor = os.dup(root_descriptor) try: for directory in relative_path.parts[:-1]: next_descriptor = os.open( directory, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), dir_fd=current_descriptor, ) os.close(current_descriptor) current_descriptor = next_descriptor descriptor = os.open( relative_path.parts[-1], os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), dir_fd=current_descriptor, ) except OSError as error: raise RuntimeError( f"release artifact is unavailable or symbolic: {relative_path}" ) from error finally: os.close(current_descriptor) identity = os.fstat(descriptor) if not stat.S_ISREG(identity.st_mode): os.close(descriptor) raise RuntimeError(f"release artifact is not a regular file: {relative_path}") return descriptor def _bind_artifact( *, repository_root: Path, root_descriptor: int, relative_path: PurePosixPath, expected_sha256: str, expected_byte_count: int, capture_limit: int | None = None, ) -> _BoundArtifact: descriptor = _open_relative_regular_file(root_descriptor, relative_path) try: before = os.fstat(descriptor) if before.st_size != expected_byte_count: raise RuntimeError(f"release artifact size differs: {relative_path}") capture = ( bytearray() if capture_limit is not None and before.st_size <= capture_limit else None ) digest = hashlib.sha256() while True: payload = os.read(descriptor, _HASH_WAVE_BYTES) if not payload: break digest.update(payload) if capture is not None: capture.extend(payload) after = os.fstat(descriptor) if ( before.st_dev != after.st_dev or before.st_ino != after.st_ino or before.st_size != after.st_size or before.st_mtime_ns != after.st_mtime_ns or before.st_ctime_ns != after.st_ctime_ns ): raise RuntimeError(f"release artifact changed during binding: {relative_path}") actual_sha256 = digest.hexdigest() if actual_sha256 != expected_sha256: raise RuntimeError(f"release artifact SHA256 differs: {relative_path}") return _BoundArtifact( authority=ReleaseArtifactAuthority( relative_path=relative_path.as_posix(), path=repository_root.joinpath(*relative_path.parts), byte_count=before.st_size, sha256=actual_sha256, ), captured_bytes=bytes(capture) if capture is not None else None, ) finally: os.close(descriptor) def _read_exact(descriptor: int, byte_count: int) -> bytes: payload = bytearray() while len(payload) < byte_count: chunk = os.read( descriptor, min(_HASH_WAVE_BYTES, byte_count - len(payload)), ) if not chunk: break payload.extend(chunk) return bytes(payload) def _bind_safetensors_artifact( *, repository_root: Path, root_descriptor: int, spec: _ArtifactSpec, expected_tensor_names: frozenset[str], ) -> ReleaseWeightArtifactAuthority: """Bind safetensors structure without reading its multi-terabyte payload.""" relative_path = _relative_path(spec.relative_path, name="artifact path") descriptor = _open_relative_regular_file(root_descriptor, relative_path) try: before = os.fstat(descriptor) if before.st_size != spec.byte_count: raise RuntimeError(f"release artifact size differs: {relative_path}") encoded_header_size = _read_exact(descriptor, 8) if len(encoded_header_size) != 8: raise RuntimeError( f"release safetensors header is truncated: {relative_path}" ) header_size = int.from_bytes(encoded_header_size, byteorder="little") if header_size < 2 or header_size > before.st_size - 8: raise RuntimeError( f"release safetensors header size differs: {relative_path}" ) encoded_header = _read_exact(descriptor, header_size) if len(encoded_header) != header_size: raise RuntimeError( f"release safetensors header is truncated: {relative_path}" ) header = _decode_json(encoded_header) header.pop("__metadata__", None) if set(header) != expected_tensor_names: raise RuntimeError( f"release safetensors tensor key set differs: {relative_path}" ) spans: list[tuple[int, int, str]] = [] for tensor_name, descriptor_payload in header.items(): descriptor_mapping = _mapping( descriptor_payload, name=f"safetensors[{tensor_name!r}]", ) _exact_keys( descriptor_mapping, required=frozenset({"dtype", "shape", "data_offsets"}), name=f"safetensors[{tensor_name!r}]", ) dtype = descriptor_mapping["dtype"] shape = descriptor_mapping["shape"] offsets = descriptor_mapping["data_offsets"] if ( not isinstance(dtype, str) or not dtype or not isinstance(shape, list) or any( isinstance(dimension, bool) or not isinstance(dimension, int) or dimension < 0 for dimension in shape ) or not isinstance(offsets, list) or len(offsets) != 2 or any( isinstance(offset, bool) or not isinstance(offset, int) for offset in offsets ) ): raise RuntimeError( f"release safetensors descriptor differs: {relative_path}" ) start, end = offsets if start < 0 or end < start: raise RuntimeError( f"release safetensors offsets differ: {relative_path}" ) spans.append((start, end, tensor_name)) cursor = 0 for start, end, tensor_name in sorted(spans): if start != cursor: raise RuntimeError( "release safetensors payload is not contiguous: " f"{relative_path}:{tensor_name}" ) cursor = end if cursor != before.st_size - 8 - header_size: raise RuntimeError( f"release safetensors payload size differs: {relative_path}" ) after = os.fstat(descriptor) if ( before.st_dev != after.st_dev or before.st_ino != after.st_ino or before.st_size != after.st_size or before.st_mtime_ns != after.st_mtime_ns or before.st_ctime_ns != after.st_ctime_ns ): raise RuntimeError( f"release artifact changed during binding: {relative_path}" ) return ReleaseWeightArtifactAuthority( relative_path=relative_path.as_posix(), path=repository_root.joinpath(*relative_path.parts), byte_count=before.st_size, sha256=spec.sha256, tensor_count=spec.tensor_count, ) finally: os.close(descriptor) def _validate_specifications() -> None: if ( len(_RELEASE_WEIGHT_SPECS) != RELEASE_188_WEIGHT_FILE_COUNT or len(_PAGE_SHARD_SHA256) != 51 or len(_TOKENIZER_SPECS) != len(_PROTECTED_TOKENIZER_PATHS) ): raise RuntimeError("Release 188 artifact specification count differs") weight_paths = [spec.relative_path for spec in _RELEASE_WEIGHT_SPECS] if len(weight_paths) != len(set(weight_paths)): raise RuntimeError("Release 188 artifact specification repeats a path") for weight_spec in _RELEASE_WEIGHT_SPECS: relative_path = _relative_path( weight_spec.relative_path, name="artifact path", ) if ( relative_path.parent != _WEIGHT_ROOT or relative_path.suffix != ".safetensors" or weight_spec.byte_count < 1 or weight_spec.tensor_count < 1 ): raise RuntimeError("Release 188 safetensors specification is invalid") _sha256( weight_spec.sha256, name=f"artifact SHA256: {weight_spec.relative_path}", ) tokenizer_paths = {spec.relative_path for spec in _TOKENIZER_SPECS} if tokenizer_paths != _PROTECTED_TOKENIZER_PATHS: raise RuntimeError("Release 188 protected tokenizer path set differs") for tokenizer_spec in _TOKENIZER_SPECS: relative_path = _relative_path( tokenizer_spec.relative_path, name="tokenizer path", ) if ( relative_path.parent != _TOKENIZER_ROOT or tokenizer_spec.byte_count < 1 ): raise RuntimeError("Release 188 tokenizer specification is invalid") _sha256( tokenizer_spec.sha256, name=f"tokenizer SHA256: {tokenizer_spec.relative_path}", ) def _forbidden_tensor_name(name: str) -> bool: lowered = name.lower() return ( any(part in lowered for part in _FORBIDDEN_TENSOR_NAME_PARTS) or lowered.rsplit(".", maxsplit=1)[-1] in {"step", "step_t"} ) def _validate_index(payload: bytes) -> dict[str, frozenset[str]]: decoded = _decode_json(payload) _exact_keys( decoded, required=frozenset({"metadata", "weight_map"}), name="root", ) metadata = _mapping(decoded["metadata"], name="metadata") _exact_keys( metadata, required=frozenset({"total_size"}), name="metadata", ) total_size = metadata["total_size"] if ( isinstance(total_size, bool) or not isinstance(total_size, int) or total_size != _EXPECTED_TOTAL_SIZE ): raise RuntimeError("Release 188 index total_size differs") weight_map = _mapping(decoded["weight_map"], name="weight_map") if len(weight_map) != _EXPECTED_TENSOR_COUNT: raise RuntimeError("Release 188 index tensor count differs") specifications = { spec.relative_path: spec for spec in _RELEASE_WEIGHT_SPECS } observed_counts: Counter[str] = Counter() observed_names: dict[str, list[str]] = { path: [] for path in specifications } for tensor_name, path_value in weight_map.items(): if not tensor_name or _forbidden_tensor_name(tensor_name): raise RuntimeError( f"Release 188 index contains forbidden tensor: {tensor_name}" ) relative_path = _relative_path( path_value, name=f"weight_map[{tensor_name!r}]", ) if ( relative_path.parent != _WEIGHT_ROOT or relative_path.suffix != ".safetensors" or relative_path.as_posix() not in specifications ): raise RuntimeError("Release 188 index weight file set differs") path = relative_path.as_posix() observed_counts[path] += 1 observed_names[path].append(tensor_name) expected_counts = Counter( { spec.relative_path: spec.tensor_count for spec in _RELEASE_WEIGHT_SPECS } ) if observed_counts != expected_counts: raise RuntimeError("Release 188 index tensor distribution differs") return { path: frozenset(tensor_names) for path, tensor_names in observed_names.items() } def load_release_inference_authority( model_index_path: str | os.PathLike[str], ) -> ReleaseInferenceAuthority: """Bind the complete portable Release 188 repository. model_index_path must be the real repository-root model.safetensors.index.json. The trusted index, all four core safetensors and all 51 flat page shards are bound by exact size, published SHA-256 authority, and safetensors header without reading tensor payloads. The small index and five protected tokenizer files are hashed directly. """ index_path = Path(model_index_path).expanduser().absolute() if index_path.name != _MODEL_INDEX_NAME: raise ValueError("Release 188 model index name differs") repository_root = index_path.parent try: index_identity = index_path.lstat() except OSError as error: raise RuntimeError("Release 188 model index is unavailable") from error if stat.S_ISLNK(index_identity.st_mode) or not stat.S_ISREG( index_identity.st_mode ): raise RuntimeError("Release 188 model index is symbolic or invalid") _validate_specifications() root_descriptor = _open_repository_root(repository_root) try: index_bound = _bind_artifact( repository_root=repository_root, root_descriptor=root_descriptor, relative_path=PurePosixPath(_MODEL_INDEX_NAME), expected_sha256=_sha256( _EXPECTED_MODEL_INDEX_SHA256, name="model index SHA256", ), expected_byte_count=_EXPECTED_MODEL_INDEX_BYTES, capture_limit=_MODEL_INDEX_MAX_BYTES, ) if index_bound.captured_bytes is None: raise RuntimeError("Release 188 model index exceeds its size boundary") index_tensor_names = _validate_index(index_bound.captured_bytes) bound_weights: dict[str, ReleaseWeightArtifactAuthority] = {} for weight_spec in _RELEASE_WEIGHT_SPECS: bound_weights[weight_spec.relative_path] = ( _bind_safetensors_artifact( repository_root=repository_root, root_descriptor=root_descriptor, spec=weight_spec, expected_tensor_names=index_tensor_names[ weight_spec.relative_path ], ) ) bound_tokenizer: dict[str, ReleaseArtifactAuthority] = {} for tokenizer_spec in _TOKENIZER_SPECS: relative_path = _relative_path( tokenizer_spec.relative_path, name="tokenizer path", ) bound = _bind_artifact( repository_root=repository_root, root_descriptor=root_descriptor, relative_path=relative_path, expected_sha256=tokenizer_spec.sha256, expected_byte_count=tokenizer_spec.byte_count, ) bound_tokenizer[tokenizer_spec.relative_path] = bound.authority page_paths = tuple( spec.relative_path for spec in _RELEASE_WEIGHT_SPECS[4:] ) return ReleaseInferenceAuthority( repository_root=repository_root, model_index=index_bound.authority, repository=RELEASE_188_REPOSITORY, revision=RELEASE_188_REVISION, release=RELEASE_188, tensor_count=_EXPECTED_TENSOR_COUNT, total_size=_EXPECTED_TOTAL_SIZE, weights=ReleaseWeightArtifacts( lexical_projection_substrate=bound_weights[ "weights/safetensors/model.safetensors" ], resynthesis=bound_weights[ "weights/safetensors/resynthesis.safetensors" ], resident_runtime=bound_weights[ "weights/safetensors/resident-runtime.safetensors" ], direct_page_index=bound_weights[ "weights/safetensors/direct-page-index.safetensors" ], page_shards=tuple(bound_weights[path] for path in page_paths), ), tokenizer=ReleaseTokenizerArtifacts( root=repository_root / "tokenizer", chat_template=bound_tokenizer[ "tokenizer/chat_template.jinja" ], config=bound_tokenizer["tokenizer/config.json"], generation_config=bound_tokenizer[ "tokenizer/generation_config.json" ], model=bound_tokenizer["tokenizer/tokenizer.json"], tokenizer_config=bound_tokenizer[ "tokenizer/tokenizer_config.json" ], ), ) finally: os.close(root_descriptor) __all__ = [ "RELEASE_188", "RELEASE_188_REPOSITORY", "RELEASE_188_REVISION", "RELEASE_188_TENSOR_COUNT", "RELEASE_188_TOTAL_SIZE", "RELEASE_188_WEIGHT_FILE_COUNT", "ReleaseArtifactAuthority", "ReleaseInferenceAuthority", "ReleaseTokenizerArtifacts", "ReleaseWeightArtifactAuthority", "ReleaseWeightArtifacts", "load_release_inference_authority", ]