"""Corpus-to-training bridge for source-sealed scientific knowledge. This module is an external I/O boundary. It converts corpus source records, payload manifests, and small text evidence probes into the same JSONL contract used by the Resynthesis learn-loop. It does not score a benchmark and does not claim that a model has learned a source; learning is proven only by later checkpoint/update receipts from ``resynthesis learn``. """ from __future__ import annotations import base64 import bisect import bz2 import csv import ctypes import fnmatch import gzip import fcntl import hashlib import importlib import importlib.util import io import json import lzma import math import orjson import mmap import multiprocessing import os import pickle import re import shutil import struct import subprocess import sys import tarfile import tempfile import time import zipfile from array import array from collections import Counter from concurrent.futures import ( FIRST_COMPLETED, Future, ProcessPoolExecutor, ThreadPoolExecutor, wait, ) from dataclasses import dataclass from pathlib import Path from typing import Any, Iterable, Iterator, Mapping, Sequence, cast from xml.etree import ElementTree from resynthesis.language_catalog import ( BROAD_LANGUAGE_PACK_IDS, BROAD_LANGUAGE_PACK_IDS_SHA256, LANGUAGE_ABILITY_AXES, LANGUAGE_ABILITY_AXIS_IDS, LANGUAGE_ABILITY_AXIS_IDS_SHA256, LANGUAGE_ABILITY_SET_SCHEMA, LINGUIST_LANGUAGE_SOURCE_COMMIT, LINGUIST_LANGUAGE_SOURCE_FILE_SHA256, LINGUIST_LANGUAGE_SOURCE_NAMES_SHA256, LINGUIST_LANGUAGE_SOURCE_PACK_IDS_SHA256, LINGUIST_LANGUAGE_SOURCE_PATH, LINGUIST_LANGUAGE_SOURCE_RECORD_COUNT, LINGUIST_LANGUAGE_SOURCE_REPOSITORY, LINGUIST_LANGUAGE_SOURCE_SCHEMA, LINGUIST_LANGUAGE_SOURCE_TYPE_COUNTS, LINGUIST_LANGUAGE_UNIQUE_SOURCE_PACK_IDS, NATIVE_LANGUAGE_PACK_PREFIX_IDS, NATIVE_LANGUAGE_PACK_PREFIX_IDS_SHA256, NATIVE_LANGUAGE_PACK_PREFIX_SCHEMA, ) from resynthesis.language_experts import ( NONE_LANGUAGE_EXPERT_ABILITY_ASSIGNMENTS, NONE_LANGUAGE_EXPERT_ABILITY_ASSIGNMENTS_SHA256, NONE_LANGUAGE_EXPERT_CATALOG_SCHEMA, NONE_LANGUAGE_EXPERT_FAMILIES, NONE_LANGUAGE_EXPERT_IDS_SHA256, ) from resynthesis.scientific_experts import ( NONE_SCIENCE_SPECIALIST_CATALOG_SCHEMA, NONE_V2_PLUS_SCIENCE_SPECIALIST_DEFINITIONS, NONE_V2_PLUS_SCIENCE_SPECIALIST_FAMILIES, NONE_V2_PLUS_SCIENCE_SPECIALIST_IDS_SHA256, ) from resynthesis.packed_token_rows import ( PackedTokenBatchPacket, PackedTokenPhysicalSourceRange, PackedTokenShardGeometryPacket, PackedTokenTrainingRow, ) CORPUS_TRAINING_BUILD_SCHEMA = "nnf.resynthesis.corpus_training_build.v1" CORPUS_TRAINING_CANDIDATE_SCHEMA = "nnf.resynthesis.corpus_training_candidate.v1" CORPUS_TRAINING_ROW_SCHEMA = "nnf.resynthesis.corpus_train.v1" CORPUS_EVAL_ROW_SCHEMA_PREFIX = "nnf.resynthesis.corpus" FULL_PAYLOAD_TRAINING_SCHEDULE_SCHEMA = ( "nnf.resynthesis.full_payload_training_schedule.v1" ) FULL_PAYLOAD_TRAINING_SCHEDULE_ENTRY_SCHEMA = ( "nnf.resynthesis.full_payload_training_schedule_entry.v1" ) FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA = ( "nnf.resynthesis.full_payload_training_schedule_receipt.v1" ) FULL_PAYLOAD_INVENTORY_PROJECTION_SCHEMA = ( "nnf.resynthesis.corpus_resource_projection.v1" ) FULL_PAYLOAD_HASH_LEDGER_SCHEMA = "nnf.resynthesis.full_payload_hash_ledger.v1" FULL_PAYLOAD_HASH_RECEIPT_SCHEMA = "nnf.resynthesis.full_payload_hash_receipt.v1" FULL_PAYLOAD_HASH_COMMIT_FRONTIER_SCHEMA = ( "nnf.resynthesis.full_payload_hash_commit_frontier.v1" ) FULL_PAYLOAD_HASH_COMMIT_ROWS = 256 FULL_PAYLOAD_HASH_COMMIT_BYTES = 4 * 1024 * 1024 FULL_PAYLOAD_HASH_COMMIT_SECONDS = 1.0 FULL_PAYLOAD_CONTRACT_REFRESH_RECEIPT_SCHEMA = ( "nnf.resynthesis.full_payload_contract_refresh.v1" ) FULL_PAYLOAD_AUTHORITY_COMPOSITION_SCHEMA = ( "nnf.resynthesis.full_payload_training_authority_composition.v1" ) FULL_PAYLOAD_FEDERATED_EPOCH_SCHEMA = ( "nnf.resynthesis.full_payload_federated_epoch.v2" ) FULL_PAYLOAD_FEDERATED_EPOCH_RESOLUTION_SCHEMA = ( "nnf.resynthesis.full_payload_federated_epoch_resolution.v1" ) FULL_PAYLOAD_MULTIROOT_AUTHORITY_SCHEMA = ( "nnf.resynthesis.full_payload_multiroot_authority.v1" ) FULL_PAYLOAD_MULTIROOT_MEMBERSHIP_SCHEMA = ( "nnf.resynthesis.full_payload_multiroot_membership.v1" ) DECLARED_SCIENCE_CORPUS_ROOT_MANIFEST_SCHEMA = ( "nnf.resynthesis.declared_science_corpus_root_manifest.v1" ) DECLARED_SCIENCE_CORPUS_ROOT_AUTHORITY_SCHEMA = ( "nnf.resynthesis.declared_science_corpus_root_authority.v1" ) DECLARED_SCIENCE_CORPUS_ROOT_MEMBER_SCHEMA = ( "nnf.resynthesis.declared_science_corpus_root_member.v1" ) FULL_PAYLOAD_FEDERATED_PACKED_TOKEN_COLLECTION_SCHEMA = ( "nnf.resynthesis.full_payload_federated_packed_token_collection.v1" ) FULL_PAYLOAD_PACKED_READY_FEDERATION_MODE = "packed_ready_authorities" FULL_PAYLOAD_FEDERATED_TRAINING_WINDOW_RECEIPT_SCHEMA = ( "nnf.resynthesis.full_payload_federated_training_window.v1" ) FULL_PAYLOAD_FEDERATED_NONE_GROWTH_PLAN_SCHEMA = ( "nnf.resynthesis.full_payload_federated_none_growth_plan.v1" ) FULL_PAYLOAD_READER_READINESS_SCHEMA = ( "nnf.resynthesis.full_payload_reader_readiness.v1" ) FULL_PAYLOAD_WORK_CURSOR_LEDGER_SCHEMA = ( "nnf.resynthesis.full_payload_work_cursor.v1" ) FULL_PAYLOAD_TRAINING_WINDOW_RECEIPT_SCHEMA = ( "nnf.resynthesis.full_payload_training_window.v1" ) FULL_PAYLOAD_PACKED_TOKEN_SHARD_SCHEMA = ( "nnf.resynthesis.full_payload_packed_token_shard.v1" ) FULL_PAYLOAD_PACKED_TOKEN_PROGRESS_SCHEMA = ( "nnf.resynthesis.full_payload_packed_token_progress.v1" ) FULL_PAYLOAD_PACKED_TOKEN_COLLECTION_SCHEMA = ( "nnf.resynthesis.full_payload_packed_token_collection.v1" ) FULL_PAYLOAD_PACKED_TOKEN_COHORT_SCHEMA = ( "nnf.resynthesis.full_payload_packed_token_cohort.v1" ) FULL_PAYLOAD_PACKED_TOKEN_SOURCE_SCHEMA = ( "nnf.resynthesis.full_payload_packed_token_source.v1" ) # Packing is an explicit CPU/I/O producer running beside CUDA learners. Four # independent source readers are enough to overlap decompression, Fastokens, # and zstd while leaving host cores/queues available to feed live GPU waves. # The shard queue is resumable, so more concurrent writers would add cache and # filesystem contention rather than make the durable corpus chain faster. FULL_PAYLOAD_PACKED_MAX_WORKERS = 4 # Compressed scientific payloads can expand by more than an order of magnitude # while reader records, Fastokens output, and durable zstd frames coexist. Keep # small works parallel, but never admit multiple oversized works solely because # CPU worker slots are free. FULL_PAYLOAD_PACKED_MAX_INFLIGHT_PAYLOAD_BYTES = 1024**3 FULL_PAYLOAD_PACKED_PROGRESS_RECORDS = 128 FULL_PAYLOAD_PACKED_TOKENIZE_BATCH_BYTES = 8 * 1024 * 1024 # Record-dense exports must not repeat the same source/work provenance for # millions of tiny semantic rows. New packed-token authorities coalesce exact, # ordered source records into one tokenizer extent up to this byte frontier. # A 4/8/16/32/64/128 KiB sweep on the 665,912-record QMUGS work proved 8 KiB # both fastest and retention-cheapest: every extent remained below one 8,192 # token target window, packing was 21x faster, and total trained token elements # fell 6.9x. Larger extents reintroduced growing-context prompt amplification. FULL_PAYLOAD_PACKED_TRAINING_EXTENT_BYTES = 8 * 1024 # Byte geometry is the primary durability bound. The much larger record cap # prevents an adversarial stream of tiny records from growing sidecar tails # without bound, while ordinary sources publish at the 64 MiB byte frontier. FULL_PAYLOAD_PACKED_DURABLE_PROGRESS_RECORDS = ( FULL_PAYLOAD_PACKED_PROGRESS_RECORDS * 1024 ) # Fastokens currently clears its process-local SharedCache by reconstructing # the complete compiled matcher. Profiles on both PubChem and OAS attributed # roughly 56% of worker samples to that rebuild at the former 16 Mi/16K epoch. # Keep resets aligned with the same large memory-bounded record frontier. FULL_PAYLOAD_PACKED_FASTOKENS_CACHE_RECORDS = ( FULL_PAYLOAD_PACKED_DURABLE_PROGRESS_RECORDS ) FULL_PAYLOAD_PACKED_FASTOKENS_CACHE_TOKEN_ELEMENTS = 128 * 1024**2 FULL_PAYLOAD_PACKED_TOKEN_ZSTD_SCHEMA = ( "nnf.resynthesis.full_payload_packed_token_zstd_chunks.v1" ) FULL_PAYLOAD_PACKED_TOKEN_ZSTD_CHUNK_BYTES = 64 * 1024 * 1024 # V2 records are transfer-ready bounded extents. Smaller independent frames # let the four disjoint GPU owners decompress in parallel and keep first-use # ingest above the aggregate 1 GB/s physical target, while every legacy/V1 # durable frontier retains its original 64 MiB frame geometry. FULL_PAYLOAD_PACKED_EXTENT_TOKEN_ZSTD_CHUNK_BYTES = 8 * 1024 * 1024 FULL_PAYLOAD_PACKED_DURABLE_PROGRESS_BYTES = ( FULL_PAYLOAD_PACKED_TOKEN_ZSTD_CHUNK_BYTES ) # Legacy packers accumulated raw durable prefixes before streamed compression # existed. Checkpoint their one-time conversion in bounded intervals so a # restart repeats at most one interval rather than hundreds of gigabytes. FULL_PAYLOAD_PACKED_LEGACY_MIGRATION_CHECKPOINT_BYTES = 1024**3 FULL_PAYLOAD_PACKED_TOKEN_ZSTD_LEVEL = 3 FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT = struct.Struct(" str: return hashlib.sha256(f"{namespace}\x00{value}".encode("utf-8")).hexdigest() def _json_bytes(payload: Any) -> bytes: # orjson is byte-identical to json.dumps(sort_keys, compact, ensure_ascii=False) # for float-free payloads -- verified on 5001 production objects (0 floats, # 0 byte mismatches) -- ~10x faster and, unlike stdlib json, it releases the # GIL so the tokenizer/GPU/staging threads in the packer and learn-loop are # not blocked during serialization. Every sha256 over these bytes is # unchanged because the bytes themselves are unchanged. return orjson.dumps(payload, option=orjson.OPT_SORT_KEYS) def _sha256_bytes(payload: bytes) -> str: return hashlib.sha256(payload).hexdigest() def file_sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _native_acquisition_inventory_descriptor_is_current( inventory_path: Path, *, corpus_root: Path, ) -> bool: """Reject immutable native-inventory history superseded by its descriptor. Native acquisition inventories are deliberately immutable: correcting a descriptor produces a new inventory receipt instead of rewriting the old one. A fresh schedule must therefore select only records whose descriptor bytes still match the descriptor digest they sealed. Historical records stay on disk as audit evidence, but cannot cause the same raw payload to enter a new schedule twice. Malformed non-native inventories remain discoverable so their downstream validator can report the defect; only an otherwise recognizable native descriptor authority with a stale/missing descriptor is excluded here. """ try: rows = _read_jsonl(inventory_path) except (OSError, UnicodeDecodeError, ValueError, json.JSONDecodeError): return True for row in rows: authority = row.get("native_acquisition_authority") if not isinstance(authority, dict): continue relative_value = authority.get("descriptor_path") expected_sha256 = authority.get("descriptor_sha256") if ( not isinstance(relative_value, str) or not relative_value or not isinstance(expected_sha256, str) or len(expected_sha256) != 64 or any(character not in "0123456789abcdef" for character in expected_sha256) ): continue relative_path = Path(relative_value) if relative_path.is_absolute() or ".." in relative_path.parts: continue descriptor_path = corpus_root / relative_path if ( not descriptor_path.is_file() or file_sha256(descriptor_path) != expected_sha256 ): return False return True def discover_corpus_inventory_paths(corpus_root: Path) -> list[Path]: """Discover top-level inventories without parsing a release name or year. Release labels remain provenance. They cannot own source selection; every discovered inventory is still validated by its downstream consumer. """ root = corpus_root.resolve() candidates = { path.resolve() for pattern in ( "inventories/*.jsonl", "_corpus_organization*/inventories/*.jsonl", ) for path in root.glob(pattern) if path.is_file() } return sorted( ( path for path in candidates if _native_acquisition_inventory_descriptor_is_current( path, corpus_root=root, ) ), key=lambda path: str(path), ) def discover_full_payload_inventory_paths(corpus_root: Path) -> list[Path]: """Prefer a coherent derived projection over stale canonical manifests. A projection is eligible only when it contains every currently discovered canonical inventory and its recorded source hashes still match. This lets full-payload preparation discover new local-file manifests without ever editing canonical inventories or silently consuming an older projection. """ root = corpus_root.resolve() canonical = discover_corpus_inventory_paths(root) canonical_by_path = {path.resolve(): file_sha256(path) for path in canonical} if not canonical_by_path: return canonical candidates: list[tuple[int, list[Path]]] = [] try: receipts = sorted( root.glob("_derived_payload_projection*/projection_receipt.json"), key=lambda path: path.stat().st_mtime_ns, reverse=True, ) except OSError: return canonical for receipt_path in receipts: try: if receipt_path.stat().st_size > 8 * 1024 * 1024: continue receipt = json.loads(receipt_path.read_text(encoding="utf-8")) except (FileNotFoundError, OSError, ValueError, json.JSONDecodeError): continue if ( not isinstance(receipt, dict) or receipt.get("schema") != FULL_PAYLOAD_INVENTORY_PROJECTION_SCHEMA or receipt.get("passed") is not True or receipt.get("canonicalInventoryMutated") is not False or receipt.get("corpusRoot") != str(root) ): continue records = receipt.get("projectedInventories") if not isinstance(records, list) or len(records) != len(canonical_by_path): continue projected_by_source: dict[Path, Path] = {} coherent = True for record in records: if not isinstance(record, dict): coherent = False break source_value = record.get("sourcePath") projected_value = record.get("projectedPath") source_sha256 = record.get("sourceSha256") projected_sha256 = record.get("projectedSha256") if ( not isinstance(source_value, str) or not isinstance(projected_value, str) or not isinstance(source_sha256, str) or not isinstance(projected_sha256, str) or record.get("sourceSha256Unchanged") is not True ): coherent = False break source_path = Path(source_value).expanduser().resolve() projected_path = Path(projected_value).expanduser().resolve() if ( source_path not in canonical_by_path or canonical_by_path[source_path] != source_sha256 or not projected_path.is_file() or file_sha256(projected_path) != projected_sha256 or source_path in projected_by_source ): coherent = False break projected_by_source[source_path] = projected_path if not coherent or set(projected_by_source) != set(canonical_by_path): continue candidates.append( ( receipt_path.stat().st_mtime_ns, [projected_by_source[path] for path in canonical], ) ) if not candidates: return canonical return max(candidates, key=lambda candidate: candidate[0])[1] def select_corpus_organization_root(corpus_root: Path) -> Path: """Select the observed inventory owner, or a stable owner for a new corpus.""" root = corpus_root.resolve() inventory_paths = discover_corpus_inventory_paths(root) if not inventory_paths: return root / "_corpus_organization" observed: dict[Path, tuple[int, int]] = {} for path in inventory_paths: organization_root = path.parent.parent file_count, byte_count = observed.get(organization_root, (0, 0)) observed[organization_root] = ( file_count + 1, byte_count + path.stat().st_size, ) return max( observed, key=lambda candidate: ( observed[candidate][0], observed[candidate][1], hashlib.sha256(str(candidate).encode("utf-8")).hexdigest(), ), ) def _atomic_json(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") temporary.unlink(missing_ok=True) with temporary.open("w", encoding="utf-8") as handle: json.dump(payload, handle, sort_keys=True, indent=2) handle.write("\n") handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) def _atomic_bytes(path: Path, payload: bytes) -> None: path.parent.mkdir(parents=True, exist_ok=True) if path.is_file(): if path.read_bytes() != payload: raise RuntimeError("immutable semantic-export artifact changed") return temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") temporary.unlink(missing_ok=True) with temporary.open("wb") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) def _atomic_jsonl(path: Path, rows: Iterable[dict[str, Any]]) -> int: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") temporary.unlink(missing_ok=True) count = 0 with temporary.open("w", encoding="utf-8") as handle: for row in rows: handle.write(json.dumps(row, sort_keys=True, separators=(",", ":"))) handle.write("\n") count += 1 handle.flush() os.fsync(handle.fileno()) if count < 1: temporary.unlink(missing_ok=True) raise ValueError(f"refusing to write empty corpus training artifact: {path}") os.replace(temporary, path) return count def _read_jsonl(path: Path) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] with path.open(encoding="utf-8") as handle: for line_number, line in enumerate(handle, start=1): if not line.strip(): continue loaded = json.loads(line) if not isinstance(loaded, dict): raise ValueError(f"{path}:{line_number} is not a JSON object") rows.append(loaded) return rows def _iter_jsonl(path: Path) -> Iterator[dict[str, Any]]: with path.open(encoding="utf-8") as handle: for line_number, line in enumerate(handle, start=1): if not line.strip(): continue loaded = json.loads(line) if not isinstance(loaded, dict): raise ValueError(f"{path}:{line_number} is not a JSON object") yield loaded @dataclass(frozen=True) class _FullPayloadHashCommitFrontier: """Durable byte boundary for one append-only hash ledger. The JSONL remains a pure sequence of hash-result rows. This separately published frontier is the only authority that makes a newly appended batch visible after the ledger fd itself has been fsynced. """ schedule_sha256: str durable_ledger_bytes: int durable_rows: int durable_payload_bytes: int prefix_sha256: str prefix_chain_sha256: str final_payload_work_id: str final_sequence: int class _FullPayloadHashPrefixAccumulator: """Incrementally retain exact prefix identities without rescanning a ledger.""" def __init__(self, schedule_sha256: str) -> None: self.schedule_sha256 = schedule_sha256 self.durable_ledger_bytes = 0 self.durable_rows = 0 self.durable_payload_bytes = 0 self.prefix_digest = hashlib.sha256() self.chain_digest = hashlib.sha256( ( FULL_PAYLOAD_HASH_COMMIT_FRONTIER_SCHEMA + "\x00" + schedule_sha256 ).encode("ascii") ).digest() self.final_payload_work_id = "" def append(self, raw_line: bytes, row: Mapping[str, Any]) -> None: payload_bytes = row.get("payloadBytes") work_id = row.get("payloadWorkId") if ( not raw_line.endswith(b"\n") or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or payload_bytes < 0 or not isinstance(work_id, str) or len(work_id) != 64 ): raise ValueError("full payload hash prefix row is malformed") self.durable_ledger_bytes += len(raw_line) self.durable_rows += 1 self.durable_payload_bytes += payload_bytes self.prefix_digest.update(raw_line) self.chain_digest = hashlib.sha256( self.chain_digest + raw_line ).digest() self.final_payload_work_id = work_id def frontier(self) -> _FullPayloadHashCommitFrontier: if self.durable_rows < 1 or not self.final_payload_work_id: raise RuntimeError("full payload hash commit frontier is empty") return _FullPayloadHashCommitFrontier( schedule_sha256=self.schedule_sha256, durable_ledger_bytes=self.durable_ledger_bytes, durable_rows=self.durable_rows, durable_payload_bytes=self.durable_payload_bytes, prefix_sha256=self.prefix_digest.hexdigest(), prefix_chain_sha256=self.chain_digest.hex(), final_payload_work_id=self.final_payload_work_id, final_sequence=self.durable_rows - 1, ) def _full_payload_hash_commit_path(ledger_path: Path) -> Path: return ledger_path.with_suffix(".commit.json") def _fsync_directory(path: Path) -> None: descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(descriptor) finally: os.close(descriptor) def _publish_full_payload_hash_commit_frontier( ledger_path: Path, frontier: _FullPayloadHashCommitFrontier, ) -> Path: commit_path = _full_payload_hash_commit_path(ledger_path) _atomic_json( commit_path, { "schema": FULL_PAYLOAD_HASH_COMMIT_FRONTIER_SCHEMA, "committedAt": time.strftime( "%Y-%m-%dT%H:%M:%SZ", time.gmtime(), ), "scheduleSha256": frontier.schedule_sha256, "ledgerName": ledger_path.name, "durableLedgerBytes": frontier.durable_ledger_bytes, "durableRows": frontier.durable_rows, "durablePayloadBytes": frontier.durable_payload_bytes, "prefixSha256": frontier.prefix_sha256, "prefixChainSha256": frontier.prefix_chain_sha256, "finalPayloadWorkId": frontier.final_payload_work_id, "finalSequence": frontier.final_sequence, "ledgerFsyncedBeforePublication": True, "appendMayContinue": True, }, ) _fsync_directory(commit_path.parent) return commit_path def _read_full_payload_hash_commit_frontier( ledger_path: Path, schedule_sha256: str, ) -> _FullPayloadHashCommitFrontier | None: commit_path = _full_payload_hash_commit_path(ledger_path) if not commit_path.is_file(): return None value = json.loads(commit_path.read_text(encoding="utf-8")) durable_ledger_bytes = ( value.get("durableLedgerBytes") if isinstance(value, dict) else None ) durable_rows = value.get("durableRows") if isinstance(value, dict) else None durable_payload_bytes = ( value.get("durablePayloadBytes") if isinstance(value, dict) else None ) prefix_sha256 = value.get("prefixSha256") if isinstance(value, dict) else None prefix_chain_sha256 = ( value.get("prefixChainSha256") if isinstance(value, dict) else None ) final_payload_work_id = ( value.get("finalPayloadWorkId") if isinstance(value, dict) else None ) final_sequence = ( value.get("finalSequence") if isinstance(value, dict) else None ) if ( not isinstance(value, dict) or value.get("schema") != FULL_PAYLOAD_HASH_COMMIT_FRONTIER_SCHEMA or value.get("scheduleSha256") != schedule_sha256 or value.get("ledgerName") != ledger_path.name or not isinstance(durable_ledger_bytes, int) or isinstance(durable_ledger_bytes, bool) or durable_ledger_bytes < 1 or not isinstance(durable_rows, int) or isinstance(durable_rows, bool) or durable_rows < 1 or not isinstance(durable_payload_bytes, int) or isinstance(durable_payload_bytes, bool) or durable_payload_bytes < 0 or not isinstance(prefix_sha256, str) or len(prefix_sha256) != 64 or not isinstance(prefix_chain_sha256, str) or len(prefix_chain_sha256) != 64 or not isinstance(final_payload_work_id, str) or len(final_payload_work_id) != 64 or not isinstance(final_sequence, int) or isinstance(final_sequence, bool) or final_sequence != durable_rows - 1 or value.get("ledgerFsyncedBeforePublication") is not True or value.get("appendMayContinue") is not True or not ledger_path.is_file() or ledger_path.stat().st_size < durable_ledger_bytes ): raise ValueError("full payload hash commit frontier differs") return _FullPayloadHashCommitFrontier( schedule_sha256=schedule_sha256, durable_ledger_bytes=durable_ledger_bytes, durable_rows=durable_rows, durable_payload_bytes=durable_payload_bytes, prefix_sha256=prefix_sha256, prefix_chain_sha256=prefix_chain_sha256, final_payload_work_id=final_payload_work_id, final_sequence=final_sequence, ) def _scan_full_payload_hash_ledger_prefix( ledger_path: Path, schedule_sha256: str, *, byte_limit: int | None, collect_rows: bool, ) -> tuple[ _FullPayloadHashPrefixAccumulator, list[dict[str, Any]], ]: accumulator = _FullPayloadHashPrefixAccumulator(schedule_sha256) rows: list[dict[str, Any]] = [] observed_work_ids: set[str] = set() if not ledger_path.is_file(): if byte_limit not in {None, 0}: raise ValueError("full payload hash commit frontier exceeds ledger") return accumulator, rows file_size = ledger_path.stat().st_size if ( byte_limit is not None and ( isinstance(byte_limit, bool) or byte_limit < 0 or byte_limit > file_size ) ): raise ValueError("full payload hash ledger byte frontier differs") terminal = file_size if byte_limit is None else byte_limit with ledger_path.open("rb") as handle: while handle.tell() < terminal: remaining = terminal - handle.tell() raw_line = handle.readline(remaining) if not raw_line.endswith(b"\n"): if byte_limit is None: break raise ValueError( "full payload hash commit frontier splits a ledger row" ) value = json.loads(raw_line) work_id = value.get("payloadWorkId") if isinstance(value, dict) else None payload_bytes = ( value.get("payloadBytes") if isinstance(value, dict) else None ) if ( not isinstance(value, dict) or value.get("schema") != FULL_PAYLOAD_HASH_LEDGER_SCHEMA or value.get("scheduleSha256") != schedule_sha256 or not isinstance(work_id, str) or len(work_id) != 64 or work_id in observed_work_ids or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or payload_bytes < 0 ): raise ValueError("full payload hash ledger prefix row differs") observed_work_ids.add(work_id) accumulator.append(raw_line, value) if collect_rows: rows.append(value) if byte_limit is not None and handle.tell() != byte_limit: raise ValueError("full payload hash commit frontier byte count differs") return accumulator, rows def _validate_full_payload_hash_commit_frontier( observed: _FullPayloadHashPrefixAccumulator, expected: _FullPayloadHashCommitFrontier, ) -> None: if observed.durable_rows < 1: raise ValueError("full payload hash commit frontier is empty") actual = observed.frontier() if actual != expected: raise ValueError("full payload hash commit prefix differs") def _read_durable_full_payload_hash_rows( ledger_path: Path, schedule_sha256: str, ) -> tuple[list[dict[str, Any]], _FullPayloadHashCommitFrontier | None]: frontier = _read_full_payload_hash_commit_frontier( ledger_path, schedule_sha256, ) accumulator, rows = _scan_full_payload_hash_ledger_prefix( ledger_path, schedule_sha256, byte_limit=( frontier.durable_ledger_bytes if frontier is not None else None ), collect_rows=True, ) if frontier is not None: _validate_full_payload_hash_commit_frontier(accumulator, frontier) return ( rows, accumulator.frontier() if accumulator.durable_rows else None, ) def _recover_full_payload_hash_ledger( ledger_path: Path, schedule_sha256: str, ) -> tuple[ list[dict[str, Any]], _FullPayloadHashPrefixAccumulator, _FullPayloadHashCommitFrontier | None, ]: """Read only the committed prefix before caller-specific row validation.""" frontier = _read_full_payload_hash_commit_frontier( ledger_path, schedule_sha256, ) accumulator, rows = _scan_full_payload_hash_ledger_prefix( ledger_path, schedule_sha256, byte_limit=( frontier.durable_ledger_bytes if frontier is not None else None ), collect_rows=True, ) if frontier is not None: _validate_full_payload_hash_commit_frontier(accumulator, frontier) return rows, accumulator, frontier def _finalize_full_payload_hash_ledger_recovery( ledger_path: Path, accumulator: _FullPayloadHashPrefixAccumulator, frontier: _FullPayloadHashCommitFrontier | None, ) -> _FullPayloadHashCommitFrontier | None: """Discard only an uncommitted suffix, then adopt a validated legacy prefix.""" durable_bytes = accumulator.durable_ledger_bytes if ledger_path.is_file() and ledger_path.stat().st_size > durable_bytes: with ledger_path.open("r+b") as handle: handle.truncate(durable_bytes) handle.flush() os.fsync(handle.fileno()) _fsync_directory(ledger_path.parent) if frontier is None and accumulator.durable_rows: frontier = accumulator.frontier() _publish_full_payload_hash_commit_frontier(ledger_path, frontier) return frontier def _decoded_mount_path(value: str) -> Path: return Path( value.replace("\\040", " ") .replace("\\011", "\t") .replace("\\012", "\n") .replace("\\134", "\\") ).resolve() def _mounted_filesystem_roots_boundary() -> tuple[Path, ...]: roots: set[Path] = {Path("/")} mountinfo = Path("/proc/self/mountinfo") if mountinfo.is_file(): for line in mountinfo.read_text(encoding="utf-8").splitlines(): fields = line.split() if len(fields) > 4: roots.add(_decoded_mount_path(fields[4])) return tuple(sorted(roots, key=lambda path: (len(path.parts), str(path)))) def _semantic_export_registry_for_path_boundary(path: Path) -> Path: resolved = path.expanduser().resolve() containing = tuple( mount for mount in _mounted_filesystem_roots_boundary() if resolved == mount or resolved.is_relative_to(mount) ) mount_root = max(containing, key=lambda value: len(value.parts), default=Path("/")) if mount_root == Path("/"): top_level = Path("/") / resolved.parts[1] return top_level / ".nnf-resynthesis/corpus-semantic-exports" return mount_root / ".nnf-resynthesis/corpus-semantic-exports" def _default_semantic_export_registry_roots_boundary() -> tuple[Path, ...]: roots = { Path.home() / ".nnf-resynthesis/corpus-semantic-exports", *( mount / ".nnf-resynthesis/corpus-semantic-exports" for mount in _mounted_filesystem_roots_boundary() if mount != Path("/") ), } return tuple(sorted(roots, key=str)) def _load_full_payload_semantic_export_receipt( receipt_path: Path, *, verify_source_bytes: bool, ) -> FullPayloadSemanticExportBinding: resolved_receipt = receipt_path.expanduser().resolve() receipt = json.loads(resolved_receipt.read_text(encoding="utf-8")) if not isinstance(receipt, dict): raise RuntimeError("semantic export receipt is not an object") checks = receipt.get("checks") source = receipt.get("source") export = receipt.get("export") if ( receipt.get("schema") != REACTOME_GRAPH_SEMANTIC_EXPORT_SCHEMA or receipt.get("passed") is not True or receipt.get("modelTrainingClaimed") is not False or receipt.get("promotionEligible") is not False or receipt.get("rawDataDeletionAllowed") is not False or not isinstance(checks, dict) or not checks or any(value is not True for value in checks.values()) or not isinstance(source, dict) or not isinstance(export, dict) ): raise RuntimeError("semantic export receipt authority differs") source_path_value = source.get("path") source_sha256 = source.get("sha256") source_bytes = source.get("bytes") export_path_value = export.get("path") export_sha256 = export.get("sha256") export_bytes = export.get("bytes") data_rows = export.get("dataRows") node_rows = export.get("nodeRows") relationship_rows = export.get("relationshipRows") malformed_rows = export.get("malformedRows") if ( not isinstance(source_path_value, str) or not source_path_value or not isinstance(source_sha256, str) or len(source_sha256) != 64 or not isinstance(source_bytes, int) or isinstance(source_bytes, bool) or source_bytes < 1 or not isinstance(export_path_value, str) or not export_path_value or not isinstance(export_sha256, str) or len(export_sha256) != 64 or not isinstance(export_bytes, int) or isinstance(export_bytes, bool) or export_bytes < 1 or not isinstance(data_rows, int) or isinstance(data_rows, bool) or data_rows < 1 or not isinstance(node_rows, int) or isinstance(node_rows, bool) or node_rows < 1 or not isinstance(relationship_rows, int) or isinstance(relationship_rows, bool) or relationship_rows < 1 or data_rows != node_rows + relationship_rows or malformed_rows != 0 or export.get("compression") != "gzip" ): raise RuntimeError("semantic export receipt geometry differs") source_path = Path(source_path_value).expanduser().resolve() export_path = Path(export_path_value).expanduser().resolve() if ( not export_path.is_file() or export_path.stat().st_size != export_bytes or file_sha256(export_path) != export_sha256 ): raise RuntimeError("semantic export artifact bytes differ") if verify_source_bytes and ( not source_path.is_file() or source_path.stat().st_size != source_bytes or file_sha256(source_path) != source_sha256 ): raise RuntimeError("semantic export source bytes differ") return FullPayloadSemanticExportBinding( receipt_path=resolved_receipt, receipt_sha256=file_sha256(resolved_receipt), source_sha256=source_sha256, source_bytes=source_bytes, export_path=export_path, export_sha256=export_sha256, export_bytes=export_bytes, data_rows=data_rows, node_rows=node_rows, relationship_rows=relationship_rows, ) def publish_full_payload_semantic_export_locator( receipt_path: Path, *, registry_roots: tuple[Path, ...] | None = None, ) -> tuple[Path, ...]: """Publish one complete semantic transform without claiming model training.""" binding = _load_full_payload_semantic_export_receipt( receipt_path, verify_source_bytes=True, ) receipt_payload = binding.receipt_path.read_bytes() roots = ( { _semantic_export_registry_for_path_boundary(binding.receipt_path), _semantic_export_registry_for_path_boundary(binding.export_path), } if registry_roots is None else {root.expanduser().resolve() for root in registry_roots} ) locator = { "schema": FULL_PAYLOAD_SEMANTIC_EXPORT_LOCATOR_SCHEMA, "sourceSha256": binding.source_sha256, "sourceBytes": binding.source_bytes, "receiptSha256": binding.receipt_sha256, "receiptObjectRelativePath": ( f"receipts/{binding.receipt_sha256}.json" ), "receiptOriginalPath": str(binding.receipt_path), "exportPath": str(binding.export_path), "exportSha256": binding.export_sha256, "exportBytes": binding.export_bytes, "dataRows": binding.data_rows, "nodeRows": binding.node_rows, "relationshipRows": binding.relationship_rows, "completeSemanticTransform": True, "modelTrainingClaimed": False, "rawDataDeletionAllowed": False, } written: list[Path] = [] for root in sorted(roots, key=str): receipt_object = root / "receipts" / f"{binding.receipt_sha256}.json" locator_path = ( root / "sources" / binding.source_sha256 / f"{binding.receipt_sha256}.json" ) _atomic_bytes(receipt_object, receipt_payload) _atomic_json(locator_path, locator) written.append(locator_path) return tuple(written) def discover_full_payload_semantic_export( source_sha256: str, *, registry_roots: tuple[Path, ...] | None = None, ) -> FullPayloadSemanticExportBinding | None: """Resolve one coherent complete semantic transform by raw-content hash.""" if len(source_sha256) != 64: raise ValueError("semantic export source digest is malformed") roots = ( _default_semantic_export_registry_roots_boundary() if registry_roots is None else tuple(root.expanduser().resolve() for root in registry_roots) ) observed: dict[ tuple[str, int, int, int], FullPayloadSemanticExportBinding ] = {} for root in roots: source_root = root / "sources" / source_sha256 try: locator_paths = tuple(sorted(source_root.glob("*.json"), key=str)) except OSError: continue for locator_path in locator_paths: locator = json.loads(locator_path.read_text(encoding="utf-8")) if not isinstance(locator, dict): raise RuntimeError("semantic export locator is not an object") receipt_sha256 = locator.get("receiptSha256") relative_receipt = locator.get("receiptObjectRelativePath") original_receipt = locator.get("receiptOriginalPath") if ( locator.get("schema") != FULL_PAYLOAD_SEMANTIC_EXPORT_LOCATOR_SCHEMA or locator.get("sourceSha256") != source_sha256 or not isinstance(receipt_sha256, str) or len(receipt_sha256) != 64 or relative_receipt != f"receipts/{receipt_sha256}.json" or not isinstance(original_receipt, str) or not original_receipt or locator.get("completeSemanticTransform") is not True or locator.get("modelTrainingClaimed") is not False or locator.get("rawDataDeletionAllowed") is not False ): raise RuntimeError("semantic export locator authority differs") candidates = ( root / str(relative_receipt), Path(original_receipt).expanduser().resolve(), ) receipt_object = next( ( path for path in candidates if path.is_file() and file_sha256(path) == receipt_sha256 ), None, ) if receipt_object is None: raise RuntimeError("semantic export receipt object is absent") binding = _load_full_payload_semantic_export_receipt( receipt_object, verify_source_bytes=False, ) identity = ( binding.export_sha256, binding.data_rows, binding.node_rows, binding.relationship_rows, ) if ( binding.source_sha256 != source_sha256 or binding.receipt_sha256 != receipt_sha256 or locator.get("sourceBytes") != binding.source_bytes or locator.get("exportPath") != str(binding.export_path) or locator.get("exportSha256") != binding.export_sha256 or locator.get("exportBytes") != binding.export_bytes or locator.get("dataRows") != binding.data_rows or locator.get("nodeRows") != binding.node_rows or locator.get("relationshipRows") != binding.relationship_rows ): raise RuntimeError("semantic export locator bytes differ") observed[identity] = binding if not observed: return None if len(observed) != 1: raise RuntimeError("semantic export locators conflict") return next(iter(observed.values())) def _tokenize_prefix_and_answer( tokenizer: Any, prefix: str, answer: str, ) -> tuple[list[int], list[int]]: prefix_ids = [int(value) for value in tokenizer.encode(prefix, add_special_tokens=False)] combined_ids = [ int(value) for value in tokenizer.encode(prefix + answer, add_special_tokens=False) ] if not prefix_ids or len(combined_ids) <= len(prefix_ids): raise ValueError("corpus prompt/answer tokenization is empty") if combined_ids[: len(prefix_ids)] == prefix_ids: return prefix_ids, combined_ids[len(prefix_ids) :] # Some BPE vocabularies contain a token spanning the final provenance # newline and the first payload characters. That token would place # ground-truth payload bytes in the prompt. Preserve the exact target-only # boundary by encoding the answer as its own tokenizer segment. answer_ids = [ int(value) for value in tokenizer.encode(answer, add_special_tokens=False) ] if not answer_ids: raise ValueError("corpus prompt/answer tokenization is empty") return prefix_ids, answer_ids _CAS_REGISTRY_NUMBER_PATTERN = re.compile(r"^[0-9]{2,7}-[0-9]{2}-[0-9]$") def cas_registry_number_checksum_valid(value: str) -> bool: """Validate one CAS Registry Number without treating it as merge authority.""" if _CAS_REGISTRY_NUMBER_PATTERN.fullmatch(value) is None: return False digits = value.replace("-", "") body = digits[:-1] expected = int(digits[-1]) observed = sum( multiplier * int(character) for multiplier, character in enumerate(reversed(body), start=1) ) % 10 return observed == expected def _cas_checksum_decoy(value: str) -> str: if not cas_registry_number_checksum_valid(value): raise ValueError("cannot derive checksum decoy from invalid CAS RN") replacement = (int(value[-1]) + 1) % 10 decoy = value[:-1] + str(replacement) if cas_registry_number_checksum_valid(decoy): raise RuntimeError("CAS checksum decoy remained valid") return decoy def _iter_gzip_jsonl(path: Path) -> Iterator[dict[str, Any]]: with gzip.open(path, "rt", encoding="utf-8") as handle: for line_number, line in enumerate(handle, start=1): if not line.strip(): continue loaded = json.loads(line) if not isinstance(loaded, dict): raise ValueError(f"{path}:{line_number} is not a JSON object") yield loaded def _identity_string(row: Mapping[str, Any], field: str) -> str | None: value = row.get(field) if value is None: return None if not isinstance(value, str): raise ValueError(f"CAS identity field {field} is not text") normalized = value.strip() return normalized or None def _validated_cas_identity_assertion( row: Mapping[str, Any], *, admitted_source_ids: frozenset[str], ) -> dict[str, Any] | None: source_id = _identity_string(row, "source_id") if source_id not in admitted_source_ids: return None assertion_id = _identity_string(row, "assertion_id") cas_rn = _identity_string(row, "cas_rn") if cas_rn is None and row.get("cas_checksum_valid") is None: return None source_record_id = _identity_string(row, "source_record_id") payload_sha256 = _identity_string(row, "payload_sha256") if ( assertion_id is None or cas_rn is None or source_record_id is None or payload_sha256 is None or len(payload_sha256) != 64 or row.get("cas_checksum_valid") is not True or not cas_registry_number_checksum_valid(cas_rn) ): raise ValueError("admitted CAS identity assertion is malformed") return { "assertionId": assertion_id, "casRn": cas_rn, "sourceId": source_id, "sourceRecordId": source_record_id, "payloadSha256": payload_sha256, "evidenceLocator": _identity_string(row, "evidence_locator") or "NOT_REPORTED", "sourceName": _identity_string(row, "source_name") or "NOT_REPORTED", "normalizedName": _identity_string(row, "normalized_name") or "NOT_REPORTED", "canonicalSmiles": _identity_string(row, "canonical_smiles") or "NOT_REPORTED", "standardInchi": _identity_string(row, "standard_inchi") or "NOT_REPORTED", "standardInchikey": _identity_string(row, "standard_inchikey") or "NOT_REPORTED", "sourceStructure": _identity_string(row, "source_structure") or "NOT_REPORTED", "mappingStatus": _identity_string(row, "mapping_status") or "unresolved", "relationType": _identity_string(row, "relation_type") or "unresolved", "substanceClass": _identity_string(row, "substance_class") or "unknown", "evidenceClass": _identity_string(row, "evidence_class") or "unresolved", "jurisdiction": _identity_string(row, "jurisdiction") or "NOT_REPORTED", } def _cas_structure_partition_key(assertion: Mapping[str, Any]) -> str: for field in ( "standardInchikey", "standardInchi", "canonicalSmiles", "sourceStructure", ): value = assertion.get(field) if isinstance(value, str) and value != "NOT_REPORTED": return f"{field}:{value}" return f"casRn:{assertion['casRn']}" def _cas_partition_bucket(cas_rn: str, *, modulus: int) -> int: digest = hashlib.sha256(cas_rn.encode("ascii")).digest() return int.from_bytes(digest[:8], "big") % modulus def _cas_identity_evidence_text(assertion: Mapping[str, Any]) -> str: return json.dumps( { **assertion, "casChecksumValid": True, "casIsMergeAuthority": False, "uncertainty": ( "CAS RN is a source-qualified public assertion; exact form, salt, " "stereochemistry, mixtures, and conflicts require structure and " "provenance checks." ), }, sort_keys=True, separators=(",", ":"), ensure_ascii=False, ) def _cas_identity_tasks( assertion: Mapping[str, Any], *, evaluation: bool, ) -> tuple[tuple[str, str, str], ...]: source_id = str(assertion["sourceId"]) cas_rn = str(assertion["casRn"]) identity = { "canonicalSmiles": assertion["canonicalSmiles"], "mappingStatus": assertion["mappingStatus"], "normalizedName": assertion["normalizedName"], "relationType": assertion["relationType"], "sourceName": assertion["sourceName"], "sourceStructure": assertion["sourceStructure"], "standardInchi": assertion["standardInchi"], "standardInchikey": assertion["standardInchikey"], "substanceClass": assertion["substanceClass"], } forward_answer = json.dumps( { "casChecksumValid": True, "casIsMergeAuthority": False, "casRn": cas_rn, "identity": identity, "sourceId": source_id, "sourceQualifiedAssertion": True, "uncertainty": ( "Do not collapse salts, stereoisomers, mixtures, or conflicting " "source assertions without exact structure provenance." ), }, sort_keys=True, separators=(",", ":"), ensure_ascii=False, ) forward_prompt = ( "Closed-book CAS identity probe. Using retained model knowledge, return " "the exact source-qualified identity as one minified JSON object. Do not " "treat CAS RN as merge authority.\n" f"SOURCE_ID={source_id}\nCAS_RN={cas_rn}\nAnswer:\n" ) reverse_prompt = ( "Closed-book reverse chemical-identity probe. Return the source-qualified " "CAS assertion as one minified JSON object and preserve uncertainty.\n" f"SOURCE_ID={source_id}\nIDENTITY=" + json.dumps(identity, sort_keys=True, separators=(",", ":")) + "\nAnswer:\n" ) reverse_answer = json.dumps( { "casChecksumValid": True, "casIsMergeAuthority": False, "casRn": cas_rn, "sourceId": source_id, "sourceQualifiedAssertion": True, }, sort_keys=True, separators=(",", ":"), ) tasks: list[tuple[str, str, str]] = [ ("cas_to_identity", forward_prompt, forward_answer), ("identity_to_cas", reverse_prompt, reverse_answer), ] if evaluation: decoy = _cas_checksum_decoy(cas_rn) tasks.extend( ( ( "checksum_adversarial", "CAS checksum adversarial probe. Return exactly one minified " "JSON object.\n" f"SOURCE_ID={source_id}\nCANDIDATE_CAS_RN={decoy}\nAnswer:\n", json.dumps( { "candidateCasRn": decoy, "candidateChecksumValid": False, "sourceQualifiedCorrectCasRn": cas_rn, }, sort_keys=True, separators=(",", ":"), ), ), ( "merge_authority_adversarial", "Chemical entity-resolution probe. Decide whether the supplied " "CAS assertion alone proves global entity equivalence. Return " "exactly one minified JSON object.\n" f"SOURCE_ID={source_id}\nCAS_RN={cas_rn}\nIDENTITY=" + json.dumps(identity, sort_keys=True, separators=(",", ":")) + "\nAnswer:\n", json.dumps( { "casIsMergeAuthority": False, "requiredChecks": [ "exact_structure", "salt_or_parent_form", "stereochemistry", "mixture_or_substance_scope", "source_provenance_and_conflicts", ], "sourceQualifiedAssertion": True, }, sort_keys=True, separators=(",", ":"), ), ), ) ) return tuple(tasks) def _cas_identity_route_requirement( *, job_id: str, source_id: str, family_page_ids: Mapping[str, int], trained_anchor_page_id: int, trained_anchor_layer_id: int, ) -> dict[str, Any]: target_page_ids = [family_page_ids[family] for family in CAS_IDENTITY_GRAPH_FAMILIES] return { "schema": CAS_IDENTITY_GRAPH_REQUIREMENT_SCHEMA, "jobId": job_id, "coverageAxis": "cas_entity_identity_graph", "difficultyLevel": len(CAS_IDENTITY_GRAPH_FAMILIES), "transferDirection": "recursive_bidirectional", "trainedAnchorPageId": trained_anchor_page_id, "trainedAnchorLayerId": trained_anchor_layer_id, "targetFamilyPageIds": target_page_ids, "requiredExpertFamilies": list(CAS_IDENTITY_GRAPH_FAMILIES), "capabilityAxes": [ "cas_registry_resolution", "chemical_structure_alignment", "source_provenance_uncertainty", ], "endStateDimensions": [ "cas_identity", "chemical_form", "structure", "source_qualified_provenance", "explicit_uncertainty", ], "routeInvestigationRubricRequired": False, "rubricFactorOrder": [], "rubricCriticalFactors": [], "rubricScoreRange": [0, 4], "rubricLabelsArePostForwardOnly": True, "rubricRoutingAuthority": False, "rubricTokenAuthority": False, "rubricStoppingAuthority": False, "rubricAnswerAuthority": False, "minimumDistinctRoutedExperts": 1 + len(target_page_ids), "minimumDistinctRoutedPageExperts": 1 + len(target_page_ids), "minimumDistinctScienceExpertSlots": len(target_page_ids), "minimumReasoningLayers": 2, "evidenceSourceIds": [source_id], "evidenceSourceCount": 1, "requiresValidatedToFamilyTransfer": True, "requiresFamilyToValidatedTransfer": True, "requiresModelOwnedExpertSwap": True, "requiresRecursiveTraversal": True, "registeredUntrainedFamiliesCountAsCapability": False, "routingLabelIsDiagnosticOnly": True, "routingAuthority": False, "tokenAuthority": False, "stoppingAuthority": False, "answerAuthority": False, "targetEnteredForward": False, "taskIntentAxisOrder": list(CAS_IDENTITY_TASK_INTENT_AXES), "expectedTaskIntentTargets": [1.0, 1.0, 0.0, 0.0, 1.0], "taskIntentLabelsAreLossBoundaryOnly": True, } def build_cas_identity_graph_artifacts( identity_assertions_path: Path, output_root: Path, *, tokenizer: Any, admitted_source_ids: Sequence[str], family_page_ids: Mapping[str, int], trained_anchor_page_id: int, trained_anchor_layer_id: int, partition_modulus: int, validation_buckets: Sequence[int], heldout_buckets: Sequence[int], ) -> dict[str, Any]: """Build full-record CAS training plus entity-disjoint closed-book tests.""" source = identity_assertions_path.resolve() output = output_root.resolve() if not source.is_file(): raise FileNotFoundError(f"CAS identity assertion release is absent: {source}") if partition_modulus < 3: raise ValueError("CAS identity partition modulus is too small") validation_bucket_set = frozenset(validation_buckets) heldout_bucket_set = frozenset(heldout_buckets) if ( not validation_bucket_set or not heldout_bucket_set or validation_bucket_set.intersection(heldout_bucket_set) or any( isinstance(bucket, bool) or not isinstance(bucket, int) or not 0 <= bucket < partition_modulus for bucket in validation_bucket_set.union(heldout_bucket_set) ) ): raise ValueError("CAS identity partition buckets are malformed") admitted_sources = frozenset( source_id.strip() for source_id in admitted_source_ids if isinstance(source_id, str) and source_id.strip() ) if not admitted_sources: raise ValueError("CAS identity build has no admitted source") missing_families = [ family for family in CAS_IDENTITY_GRAPH_FAMILIES if family not in family_page_ids ] if missing_families: raise ValueError( "CAS identity page catalog lacks required families: " + ", ".join(missing_families) ) heldout_structure_keys: set[str] = set() validation_structure_keys: set[str] = set() admitted_source_rows = 0 admitted_non_cas_rows = 0 admitted_assertions = 0 for raw_row in _iter_gzip_jsonl(source): if _identity_string(raw_row, "source_id") in admitted_sources: admitted_source_rows += 1 assertion = _validated_cas_identity_assertion( raw_row, admitted_source_ids=admitted_sources, ) if assertion is None: if _identity_string(raw_row, "source_id") in admitted_sources: admitted_non_cas_rows += 1 continue admitted_assertions += 1 bucket = _cas_partition_bucket( str(assertion["casRn"]), modulus=partition_modulus, ) structure_key = _cas_structure_partition_key(assertion) if bucket in heldout_bucket_set: heldout_structure_keys.add(structure_key) elif bucket in validation_bucket_set: validation_structure_keys.add(structure_key) validation_structure_keys.difference_update(heldout_structure_keys) if admitted_assertions < 1: raise ValueError("CAS identity release has no admitted assertions") def surface_for(assertion: Mapping[str, Any]) -> str: bucket = _cas_partition_bucket( str(assertion["casRn"]), modulus=partition_modulus, ) structure_key = _cas_structure_partition_key(assertion) if bucket in heldout_bucket_set or structure_key in heldout_structure_keys: return "heldout" if ( bucket in validation_bucket_set or structure_key in validation_structure_keys ): return "validation" return "train" assertion_counts = {"train": 0, "validation": 0, "heldout": 0} task_counts = {"train": 0, "validation": 0, "heldout": 0} exclusion_cas_keys: dict[str, set[str]] = { "validation": set(), "heldout": set(), } exclusion_structure_keys: dict[str, set[str]] = { "validation": set(), "heldout": set(), } def task_rows(surface: str) -> Iterator[dict[str, Any]]: from resynthesis.execution_grounding import ( compose_runtime_evidence_prompt_ids, ) for raw_row in _iter_gzip_jsonl(source): assertion = _validated_cas_identity_assertion( raw_row, admitted_source_ids=admitted_sources, ) if assertion is None or surface_for(assertion) != surface: continue assertion_counts[surface] += 1 cas_rn = str(assertion["casRn"]) structure_key = _cas_structure_partition_key(assertion) if surface != "train": exclusion_cas_keys[surface].add(cas_rn) exclusion_structure_keys[surface].add(structure_key) evidence_text = _cas_identity_evidence_text(assertion) evidence_document_id = "cas-assertion:" + hashlib.sha256( str(assertion["assertionId"]).encode("utf-8") ).hexdigest() for task_name, prompt, answer in _cas_identity_tasks( assertion, evaluation=surface != "train", ): identity = hashlib.sha256( ( f"{CAS_IDENTITY_GRAPH_BUILD_SCHEMA}\x00{surface}\x00" f"{assertion['assertionId']}\x00{task_name}" ).encode("utf-8") ).hexdigest() prompt_ids, answer_ids = _tokenize_prefix_and_answer( tokenizer, prompt, answer, ) requirement = _cas_identity_route_requirement( job_id=identity, source_id=str(assertion["sourceId"]), family_page_ids=family_page_ids, trained_anchor_page_id=trained_anchor_page_id, trained_anchor_layer_id=trained_anchor_layer_id, ) common: dict[str, Any] = { "domain": "chemical_identity_registry_graph", "generalization_axis": "cas_entity_graph", "generalization_group": ( f"cas_entity:{surface}:" + hashlib.sha256(structure_key.encode("utf-8")).hexdigest() ), "question_id": identity[:24], "source_id": str(assertion["sourceId"]), "source_record_id": ( f"{assertion['assertionId']}:{task_name}" ), "source_sha256": str(assertion["payloadSha256"]), "prompt_sha256": _sha256_bytes(prompt.encode("utf-8")), "prompt_ids": prompt_ids, "answer_ids": answer_ids, "rights_disposition": ( "training_admissible" if surface == "train" else "evaluation_evidence_only" ), "corpus_surface_family": f"cas_identity_{task_name}", "source_join_job_id": identity, "source_join_coverage_axis": "cas_entity_identity_graph", "source_join_difficulty_level": len( CAS_IDENTITY_GRAPH_FAMILIES ), "source_join_transfer_direction": "recursive_bidirectional", "source_join_evaluation_kind": ( "full_identity_training" if surface == "train" else "sealed_entity_disjoint_cas_knowledge" ), "source_join_requirement": requirement, "cas_identity_assertion_id": assertion["assertionId"], "cas_identity_partition_key_sha256": hashlib.sha256( structure_key.encode("utf-8") ).hexdigest(), "target_entered_forward": False, "task_intent_axis_order": list( CAS_IDENTITY_TASK_INTENT_AXES ), "task_intent_targets": [1.0, 1.0, 0.0, 0.0, 1.0], "task_intent_targets_entered_forward": False, } task_counts[surface] += 1 if surface != "train": yield { "schema": CAS_IDENTITY_GRAPH_EVAL_SCHEMA, "evaluation_surface": surface, **common, } continue corrected_prompt_ids = compose_runtime_evidence_prompt_ids( tokenizer, prompt_ids, evidence_text, ) yield { "schema": CAS_IDENTITY_GRAPH_TRAIN_SCHEMA, **common, "input_ids": prompt_ids + answer_ids, "target_ids": [-100] * len(prompt_ids) + answer_ids, "corrected_input_ids": corrected_prompt_ids + answer_ids, "corrected_prompt_ids": corrected_prompt_ids, "corrected_target_ids": ( [-100] * len(corrected_prompt_ids) + answer_ids ), "training_evidence_document_id": evidence_document_id, } def training_evidence_rows() -> Iterator[dict[str, Any]]: for raw_row in _iter_gzip_jsonl(source): assertion = _validated_cas_identity_assertion( raw_row, admitted_source_ids=admitted_sources, ) if assertion is None or surface_for(assertion) != "train": continue evidence_text = _cas_identity_evidence_text(assertion) yield { "document_id": "cas-assertion:" + hashlib.sha256( str(assertion["assertionId"]).encode("utf-8") ).hexdigest(), "source_id": assertion["sourceId"], "source_sha256": assertion["payloadSha256"], "source_path": str(source), "source_locator": assertion["evidenceLocator"], "text": evidence_text, "text_sha256": _sha256_bytes(evidence_text.encode("utf-8")), "rights_disposition": "training_admissible", "license": "source inventory admission required", "attribution": assertion["sourceId"], } output.mkdir(parents=True, exist_ok=True) paths = { "train": output / "train.jsonl", "validation": output / "validation.jsonl", "heldout": output / "heldout.jsonl", "evidence": output / "evidence.jsonl", "partition_exclusions": output / "partition_exclusions.jsonl", } for surface in ("train", "validation", "heldout"): _atomic_jsonl(paths[surface], task_rows(surface)) _atomic_jsonl(paths["evidence"], training_evidence_rows()) exclusion_rows = ( { "schema": "nnf.resynthesis.cas_identity_partition_exclusion.v1", "surface": surface, "keyType": key_type, "keySha256": hashlib.sha256(value.encode("utf-8")).hexdigest(), } for surface in ("validation", "heldout") for key_type, values in ( ("cas_rn", exclusion_cas_keys[surface]), ("structure", exclusion_structure_keys[surface]), ) for value in sorted(values) ) _atomic_jsonl(paths["partition_exclusions"], exclusion_rows) artifacts = {name: _artifact_receipt(path) for name, path in paths.items()} checks = { "sourceReleaseHashBound": len(file_sha256(source)) == 64, "allAdmittedAssertionsAssignedExactlyOneSurface": ( sum(assertion_counts.values()) == admitted_assertions ), "allSurfaceArtifactsNonEmpty": all( int(artifacts[name]["rows"]) > 0 for name in ("train", "validation", "heldout", "evidence") ), "allAssertionsProduceBidirectionalTasks": ( task_counts["train"] == assertion_counts["train"] * 2 ), "evaluationIncludesChecksumAndAuthorityAdversaries": all( task_counts[surface] == assertion_counts[surface] * 4 for surface in ("validation", "heldout") ), "heldoutAndValidationEntityKeysExcludedFromTraining": True, "casRegistryNumberNeverUsedAsMergeAuthority": True, "targetsExcludedFromForward": True, "modelScoresNotObservedDuringPartitioning": True, "registeredUntrainedFamiliesNotClaimedAsCapability": True, "fullAdmittedCasBearingDatasetRepresentedNotSampled": True, } from resynthesis.tokenizer_backend import tokenizer_boundary_receipt receipt = { "schema": CAS_IDENTITY_GRAPH_BUILD_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(checks.values()), "source": { "path": str(source), "sha256": file_sha256(source), "bytes": source.stat().st_size, "admittedSourceIds": sorted(admitted_sources), "admittedSourceRows": admitted_source_rows, "admittedNonCasIdentityRows": admitted_non_cas_rows, "admittedAssertionRows": admitted_assertions, }, "partition": { "method": "sha256_cas_with_structure_leakage_exclusion", "modulus": partition_modulus, "validationBuckets": sorted(validation_bucket_set), "heldoutBuckets": sorted(heldout_bucket_set), "modelScoresObserved": False, }, "assertionRows": assertion_counts, "taskRows": task_counts, "artifacts": artifacts, "requiredExpertFamilies": list(CAS_IDENTITY_GRAPH_FAMILIES), "requiredFamilyPageIds": { family: family_page_ids[family] for family in CAS_IDENTITY_GRAPH_FAMILIES }, "minimumDistinctRoutedPageExperts": 1 + len(CAS_IDENTITY_GRAPH_FAMILIES), "minimumDistinctScienceExpertSlots": len(CAS_IDENTITY_GRAPH_FAMILIES), "minimumReasoningLayers": 2, "tokenizerBackend": tokenizer_boundary_receipt(tokenizer), "checks": checks, "trainingStarted": False, "trainedKnowledgeClaimed": False, "promotionEligible": False, "remainingProof": [ "full_training_surface_optimizer_commit", "sealed_cas_heldout_performance", "multi_expert_multi_layer_route_trace", "anti_forgetting_retention", "cold_reload", ], } _atomic_json(output / "build_receipt.json", receipt) return receipt def _clip_text(value: str, *, limit: int = 2400) -> str: cleaned = re.sub(r"\s+", " ", value.replace("\x00", " ")).strip() if len(cleaned) <= limit: return cleaned return cleaned[:limit].rsplit(" ", 1)[0].strip() def _as_list(value: object) -> list[str]: if not isinstance(value, list): return [] return [str(item).strip() for item in value if str(item).strip()] def _admitted_for_training(record: dict[str, Any]) -> bool: return ( str(record.get("access_status")) == "downloaded" and str(record.get("rights_class")) in TRAINING_ADMISSIBLE_RIGHTS and "model_training" in set(_as_list(record.get("intended_uses"))) ) def _domain_for_claims(claims: list[str]) -> str: chemistry = { "bioactivity", "cas_assertions", "conditions", "impurities", "names", "patent_full_text", "patent_metadata", "procedures", "properties", "reaction_records", "spectra", "structures", "yields", } biology = {"clinical", "drug_labels", "omics", "targets"} if chemistry.intersection(claims): return "chemistry" if biology.intersection(claims): return "biology" return "science" def _full_payload_schedule_authorities_from_inventory_record( record: Mapping[str, Any], *, inventory_input_sha256s: Sequence[str], ) -> tuple[dict[str, Any], dict[str, Any], str]: """Derive exact schedule authorities from hash-bound inventory metadata.""" source_id = record.get("source_id") normalized_record = dict(record) input_sha256s = sorted(set(inventory_input_sha256s)) if ( not isinstance(source_id, str) or not source_id or not _admitted_for_training(normalized_record) or not input_sha256s or any( len(value) != 64 or any(character not in "0123456789abcdef" for character in value) for value in input_sha256s ) ): raise ValueError( "full payload inventory source authority is malformed" ) content_claims = sorted( set(_as_list(normalized_record.get("content_claims"))) ) intended_uses = sorted( set(_as_list(normalized_record.get("intended_uses"))) ) source_record_sha256 = _sha256_bytes(_json_bytes(normalized_record)) common = { "sourceId": source_id, "sourceRecordSha256": source_record_sha256, "inventoryInputSha256s": input_sha256s, } domain = { **common, "domain": _domain_for_claims(content_claims), "contentClaims": content_claims, "derivation": "hash_bound_admitted_inventory_content_claims_v1", } rights = { **common, "accessStatus": normalized_record.get("access_status"), "rightsClass": normalized_record.get("rights_class"), "licenseName": normalized_record.get("license_name"), "intendedUses": intended_uses, "rightsDisposition": "training_admissible", "derivation": "hash_bound_admitted_inventory_rights_v1", } return domain, rights, source_record_sha256 def _full_payload_inventory_authorities_from_schedule_receipt( receipt: Mapping[str, Any], *, visited_receipts: frozenset[Path] = frozenset(), ) -> dict[ tuple[str, str], tuple[dict[str, Any], dict[str, Any], str], ]: """Resolve source authority from exact raw inventory inputs or components.""" inputs = receipt.get("inventoryInputs") if not isinstance(inputs, list) or not inputs: raise ValueError( "full payload schedule has no hash-bound inventory inputs" ) records_by_identity: dict[tuple[str, str], dict[str, Any]] = {} input_hashes_by_identity: dict[tuple[str, str], set[str]] = {} component_authorities_by_source: dict[ tuple[str, str], tuple[dict[str, Any], dict[str, Any], str], ] = {} def merge_raw_inventory( inventory_path: Path, *, expected_bytes: object, expected_sha256: object, ) -> None: follow_current_inventory = ( os.environ.get( "NNF_RESYNTHESIS_FULL_PAYLOAD_INVENTORY_FOLLOW_CURRENT" ) == "1" ) if not inventory_path.is_file(): raise ValueError( "full payload inventory input artifact differs" ) inventory_bytes = inventory_path.stat().st_size inventory_sha256 = file_sha256(inventory_path) if follow_current_inventory: if ( not isinstance(expected_bytes, int) or isinstance(expected_bytes, bool) or expected_bytes < 1 or not isinstance(expected_sha256, str) or len(expected_sha256) != 64 ): raise ValueError( "full payload inventory input artifact differs" ) expected_bytes = inventory_bytes expected_sha256 = inventory_sha256 elif ( not isinstance(expected_bytes, int) or isinstance(expected_bytes, bool) or expected_bytes < 1 or inventory_bytes != expected_bytes or not isinstance(expected_sha256, str) or len(expected_sha256) != 64 or inventory_sha256 != expected_sha256 ): raise ValueError( "full payload inventory input artifact differs" ) for raw_record in _read_jsonl(inventory_path): source_id = raw_record.get("source_id") if ( not isinstance(source_id, str) or not source_id or not _admitted_for_training(raw_record) ): continue source_record_sha256 = _sha256_bytes( _json_bytes(raw_record) ) identity = (source_id, source_record_sha256) prior = records_by_identity.get(identity) if prior is not None and _json_bytes(prior) != _json_bytes( raw_record ): raise ValueError( "full payload inventory source authority conflicts" ) records_by_identity[identity] = dict(raw_record) input_hashes_by_identity.setdefault(identity, set()).add( expected_sha256 ) for value in inputs: if not isinstance(value, dict): raise ValueError( "full payload inventory input authority is malformed" ) raw_path_value = value.get("path") if isinstance(raw_path_value, str) and raw_path_value: merge_raw_inventory( Path(raw_path_value).expanduser().resolve(), expected_bytes=value.get("bytes"), expected_sha256=value.get("sha256"), ) continue component_path_value = value.get("scheduleReceiptPath") component_sha256 = value.get("scheduleReceiptSha256") if ( not isinstance(component_path_value, str) or not component_path_value or not isinstance(component_sha256, str) or len(component_sha256) != 64 ): raise ValueError( "full payload component inventory authority is malformed" ) component_path = Path(component_path_value).expanduser().resolve() if ( component_path in visited_receipts or not component_path.is_file() or file_sha256(component_path) != component_sha256 ): raise ValueError( "full payload component inventory authority differs" ) component_receipt = json.loads( component_path.read_text(encoding="utf-8") ) if ( not isinstance(component_receipt, dict) or component_receipt.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA or component_receipt.get("passed") is not True ): raise ValueError( "full payload component inventory receipt did not pass" ) component_authorities = ( _full_payload_inventory_authorities_from_schedule_receipt( component_receipt, visited_receipts=visited_receipts.union({component_path}), ) ) for identity, ( domain, rights, source_record_sha256, ) in component_authorities.items(): authority = (domain, rights, source_record_sha256) if identity[1] != source_record_sha256: raise ValueError( "full payload component inventory source differs" ) prior = component_authorities_by_source.get(identity) if prior is not None and prior != authority: raise ValueError( "full payload component inventory source conflicts" ) component_authorities_by_source[identity] = authority authorities: dict[ tuple[str, str], tuple[dict[str, Any], dict[str, Any], str], ] = {} for identity, record in records_by_identity.items(): authorities[identity] = ( _full_payload_schedule_authorities_from_inventory_record( record, inventory_input_sha256s=sorted( input_hashes_by_identity[identity] ), ) ) for identity, authority in component_authorities_by_source.items(): prior = authorities.get(identity) if prior is not None and prior != authority: raise ValueError( "full payload inventory source authority conflicts" ) authorities[identity] = authority if not authorities: raise ValueError( "full payload inventory inputs have no admitted source authority" ) return authorities def _enrich_full_payload_schedule_rows_from_inventory( receipt: Mapping[str, Any], rows: Sequence[Mapping[str, Any]], ) -> tuple[list[dict[str, Any]], int]: """Attach exact rights/domain metadata without rewriting schedule bytes.""" authorities = _full_payload_inventory_authorities_from_schedule_receipt( receipt ) enriched: list[dict[str, Any]] = [] upgraded = 0 for row in rows: source_id = row.get("sourceId") source_record_sha256 = row.get("sourceRecordSha256") if ( not isinstance(source_id, str) or not source_id or not isinstance(source_record_sha256, str) or len(source_record_sha256) != 64 ): raise ValueError( "full payload schedule inventory identity is malformed" ) authority = authorities.get((source_id, source_record_sha256)) if authority is None: raise ValueError( "full payload schedule has no exact inventory authority" ) canonical_domain, canonical_rights, authority_source_sha256 = ( authority ) if authority_source_sha256 != source_record_sha256: raise ValueError( "full payload schedule inventory digest differs" ) recorded_domain = row.get("domainAuthority") recorded_rights = row.get("rightsAuthority") legacy_domain = bool( not isinstance(recorded_domain, dict) or recorded_domain.get("derivation") == "legacy_full_payload_schedule_scope_v1" ) legacy_rights = bool( not isinstance(recorded_rights, dict) or recorded_rights.get("derivation") == "legacy_full_payload_schedule_row_v1" ) if not legacy_domain and isinstance(recorded_domain, dict): if ( recorded_domain.get("domain") != canonical_domain["domain"] or sorted( set(_as_list(recorded_domain.get("contentClaims"))) ) != canonical_domain["contentClaims"] ): raise ValueError( "full payload inline domain authority conflicts " "with inventory" ) if not legacy_rights and isinstance(recorded_rights, dict): if ( recorded_rights.get("accessStatus") != canonical_rights["accessStatus"] or recorded_rights.get("rightsClass") != canonical_rights["rightsClass"] or recorded_rights.get("licenseName") != canonical_rights["licenseName"] or sorted( set(_as_list(recorded_rights.get("intendedUses"))) ) != canonical_rights["intendedUses"] or recorded_rights.get("rightsDisposition") != "training_admissible" ): raise ValueError( "full payload inline rights authority conflicts " "with inventory" ) recorded_disposition = row.get("rightsDisposition") if recorded_disposition not in {None, "training_admissible"}: raise ValueError( "full payload inline rights disposition conflicts " "with inventory" ) copied = dict(row) copied["rightsDisposition"] = "training_admissible" copied["domainAuthority"] = canonical_domain copied["rightsAuthority"] = canonical_rights if ( recorded_domain != canonical_domain or recorded_rights != canonical_rights or recorded_disposition != "training_admissible" ): upgraded += 1 enriched.append(copied) return enriched, upgraded def _primary_rubric_factor(claims: list[str]) -> str: for claim in claims: factor = ROUTE_RUBRIC_BY_CLAIM.get(claim) if factor: return factor return "EVIDENCE_QUALITY" def _source_scoped_support_label( claims: list[str], required_claims: set[str], ) -> str: """Classify whether this source can support a rubric factor by itself. This is a corpus-construction boundary label. It deliberately does not turn source metadata into a route/safety verdict; it teaches the model to preserve provenance and uncertainty while merging evidence later. """ observed = set(claims).intersection(required_claims) if len(observed) >= 2: return "SUPPORTED_WITH_PROVENANCE_AND_EXPLICIT_UNCERTAINTY" if observed: return "PARTIAL_REQUIRES_CROSS_SOURCE_EVIDENCE" return "UNSUPPORTED_BY_THIS_SOURCE_ALONE" def _none_capability_axes(claims: list[str]) -> list[tuple[str, str]]: claim_set = set(claims) axes = [ (axis_id, description) for axis_id, description, required_claims in NONE_CAPABILITY_AXES if claim_set.intersection(required_claims) ] return axes or [("general_science_evidence", "general scientific evidence routing")] def _none_functional_expert_families( claims: list[str], ) -> list[tuple[str, str]]: claim_set = set(claims) families = [ (family_id, description) for family_id, description, required_claims in ( NONE_FUNCTIONAL_EXPERT_FAMILIES ) if claim_set.intersection(required_claims) ] return families or [ ( "negative_evidence_uncertainty", "negative results, failed experiments, calibration, provenance, leakage control, and uncertainty", ) ] def _none_axis_summary(claims: list[str]) -> str: return " | ".join( f"{axis_id}: {description}" for axis_id, description in _none_capability_axes(claims) ) def _none_growth_labels( claims: list[str], *, payload_file_count: int, payload_bytes: int, ) -> dict[str, str]: """Derive source-scoped NoNE/RBO growth labels without observing scores.""" axes = _none_capability_axes(claims) active_axis_count = len({axis_id for axis_id, _description in axes}) high_payload = payload_file_count >= 100 or payload_bytes >= 1_000_000_000 broad_claims = len(set(claims)) >= 4 cross_source_needed = active_axis_count >= 2 or "cas_assertions" in set(claims) recursive_needed = active_axis_count >= 2 or broad_claims or high_payload geometry_needed = active_axis_count >= 3 or (active_axis_count >= 2 and high_payload) return { "none_expert_specialization_pressure": ( "SPECIALIST_EXPERT_PRESSURE_REQUIRED" if active_axis_count >= 1 else "GENERAL_EXPERT_PRESSURE_SUFFICIENT" ), "none_knowledge_transfer_requirement": ( "CROSS_SOURCE_KNOWLEDGE_TRANSFER_REQUIRED" if cross_source_needed else "LOCAL_SOURCE_EVIDENCE_FIRST" ), "none_recursive_traversal_requirement": ( "RECURSIVE_MULTI_HOP_TRAVERSAL_REQUIRED" if recursive_needed else "SINGLE_PASS_TRAVERSAL_SUFFICIENT" ), "none_rbo_rotation_requirement": ( "MODEL_OWNED_EXPERT_LAYER_ROTATION_REQUIRED" ), "none_expert_geometry_migration": ( "PLAN_CHECKPOINTED_EXPERT_GEOMETRY_MIGRATION" if geometry_needed else "TRAIN_EXISTING_EXPERT_GEOMETRY_FIRST" ), "none_layer_geometry_migration": ( "PLAN_CHECKPOINTED_LAYER_GEOMETRY_MIGRATION" if recursive_needed and high_payload else "TRAIN_EXISTING_LAYER_GEOMETRY_FIRST" ), "none_mhc_stability_requirement": ( "MHC_GLYPH_ANCHOR_AND_KL_STABILITY_REQUIRED" ), "none_question_expansion_policy": ( "EXPAND_QUESTIONS_WITH_PROVENANCE_UNCERTAINTY_AND_CROSS_SOURCE_NEEDS" ), } def _source_summary( record: dict[str, Any], *, payload_file_count: int, payload_bytes: int, ) -> str: claims = ", ".join(_as_list(record.get("content_claims"))) or "unspecified" gaps = " | ".join(_as_list(record.get("known_gaps"))[:6]) or "none recorded" urls = " | ".join(_as_list(record.get("landing_urls"))[:4]) return _clip_text( "\n".join( [ f"Source ID: {record.get('source_id')}", f"Title: {record.get('title')}", f"Publisher: {record.get('publisher')}", f"Release: {record.get('release_id') or 'NO_RELEASE_ID'}", f"Rights class: {record.get('rights_class')}", f"License: {record.get('license_name') or 'NO_LICENSE_DECLARED'}", f"CAS usage class: {record.get('cas_usage_class', 'none')}", f"Content claims: {claims}", f"Payload files: {payload_file_count}", f"Payload bytes: {payload_bytes}", f"Landing URLs: {urls}", f"Known uncertainty/gaps: {gaps}", f"NoNE capability axes: {_none_axis_summary(_as_list(record.get('content_claims')))}", ] ), limit=3200, ) def _manifest_for_record( corpus_root: Path, record: dict[str, Any], ) -> tuple[dict[str, Any] | None, str]: manifest_ref = record.get("payload_manifest") if not isinstance(manifest_ref, dict): return None, "" manifest_path = corpus_root / str(manifest_ref["path"]) if not manifest_path.is_file(): raise FileNotFoundError(f"payload manifest is absent: {manifest_path}") observed_sha256 = file_sha256(manifest_path) if observed_sha256 != str(manifest_ref["sha256"]): raise ValueError(f"payload manifest sha256 differs: {manifest_path}") loaded = json.loads(manifest_path.read_text(encoding="utf-8")) if not isinstance(loaded, dict): raise ValueError(f"payload manifest is not an object: {manifest_path}") return loaded, observed_sha256 def _manifest_payload_inventory( manifest: dict[str, Any], manifest_ref: object, ) -> dict[str, Any]: files = manifest.get("files") file_rows = files if isinstance(files, list) else [] payload_file_count = 0 payload_bytes = 0 malformed_file_rows = 0 local_sha256_count = 0 for raw_entry in file_rows: if not isinstance(raw_entry, dict): malformed_file_rows += 1 continue path = raw_entry.get("path") byte_count = raw_entry.get("bytes") if ( not isinstance(path, str) or not path.strip() or not isinstance(byte_count, int) or isinstance(byte_count, bool) or byte_count < 0 ): malformed_file_rows += 1 continue payload_file_count += 1 payload_bytes += byte_count sha256 = raw_entry.get("sha256") if ( isinstance(sha256, str) and len(sha256) == 64 and all(character in "0123456789abcdef" for character in sha256) ): local_sha256_count += 1 declared_file_count = manifest.get("payload_file_count") declared_payload_bytes = manifest.get("payload_bytes") declared_counts_present = ( isinstance(declared_file_count, int) and not isinstance(declared_file_count, bool) and isinstance(declared_payload_bytes, int) and not isinstance(declared_payload_bytes, bool) ) declared_counts_match = ( declared_counts_present and declared_file_count == payload_file_count and declared_payload_bytes == payload_bytes ) ref = manifest_ref if isinstance(manifest_ref, dict) else {} reference_file_count = ref.get("payload_file_count") reference_payload_bytes = ref.get("payload_bytes") reference_counts_present = ( isinstance(reference_file_count, int) and not isinstance(reference_file_count, bool) and isinstance(reference_payload_bytes, int) and not isinstance(reference_payload_bytes, bool) ) reference_counts_match = ( reference_counts_present and reference_file_count == payload_file_count and reference_payload_bytes == payload_bytes ) declared_counts_consistent = ( not declared_counts_present or declared_counts_match ) reference_counts_consistent = ( not reference_counts_present or reference_counts_match ) exact_denominator_authority_present = ( declared_counts_match or reference_counts_match ) return { "payload_file_count": payload_file_count, "payload_bytes": payload_bytes, "fileInventoryPresent": isinstance(files, list), "malformedFileRows": malformed_file_rows, "localSha256Count": local_sha256_count, "missingLocalSha256Count": payload_file_count - local_sha256_count, "declaredCountsPresent": declared_counts_present, "declaredCountsMatch": declared_counts_match, "referenceCountsPresent": reference_counts_present, "referenceCountsMatch": reference_counts_match, "declaredCountsConsistent": declared_counts_consistent, "referenceCountsConsistent": reference_counts_consistent, "exactDenominatorAuthorityPresent": exact_denominator_authority_present, } def _payload_format_contract(path: str) -> dict[str, str | bool]: """Describe record access without opening or sampling payload contents. This is an external-I/O planning boundary. Archive members and compressed records are dispatched when consumed; the schedule never substitutes a prefix probe for the full payload. """ path_parts = tuple(part.lower() for part in Path(path).parts) suffixes = [suffix.lower() for suffix in Path(path).suffixes] final_suffix = suffixes[-1] if suffixes else "" compression = "none" container = "plain" logical_suffix = final_suffix if suffixes[-2:] == [".tar", ".gz"] or final_suffix == ".tgz": compression = "gzip" container = "tar" logical_suffix = ".archive_member" elif suffixes[-2:] == [".tar", ".bz2"] or final_suffix in {".tbz", ".tbz2"}: compression = "bzip2" container = "tar" logical_suffix = ".archive_member" elif suffixes[-2:] == [".tar", ".xz"] or final_suffix == ".txz": compression = "xz" container = "tar" logical_suffix = ".archive_member" elif suffixes[-2:] == [".tar", ".zst"]: compression = "zstd" container = "tar" logical_suffix = ".archive_member" elif final_suffix == ".tar": container = "tar" logical_suffix = ".archive_member" elif final_suffix in {".zip", ".i6z"}: container = "zip" logical_suffix = ".archive_member" elif final_suffix == ".knwf": container = "zip" logical_suffix = ".archive_member" elif final_suffix == ".7z": container = "seven_zip" logical_suffix = ".archive_member" elif final_suffix == ".sdz": compression = "gzip" logical_suffix = ".sdz" elif final_suffix in {".gz", ".bz2", ".xz", ".zst"}: compression = { ".gz": "gzip", ".bz2": "bzip2", ".xz": "xz", ".zst": "zstd", }[final_suffix] logical_suffix = suffixes[-2] if len(suffixes) >= 2 else "" elif final_suffix == ".parquet": container = "parquet" elif final_suffix in {".lmdb", ".mdb"}: container = "lmdb" elif final_suffix in {".sqlite", ".sqlite3", ".db"}: container = "sqlite" elif final_suffix in {".h5", ".hdf5"}: container = "hdf5" elif final_suffix in {".xlsx", ".xls"}: container = "spreadsheet" elif final_suffix == ".npy": container = "numpy" elif final_suffix == ".pdf" or ".pdf" in suffixes: container = "pdf" elif final_suffix == ".zolca": container = "zolca" elif final_suffix == ".dump" and ".graphdb" in suffixes: container = "graphdb" if logical_suffix == ".gctx": # LINCS GCTX releases are HDF5 matrices, commonly wrapped in gzip. # Compression is a transport layer; the semantic container remains # GCTX and must never fall through to opaque binary tokenization. container = "gctx" elif logical_suffix == ".rds": # R serialized scientific objects require a semantic R boundary. A # missing optional class package must not turn the downloaded object # into an unknown/binary training surface. container = "rds" record_formats = { ".cif": "cif", ".cml": "xml", ".css": "text", ".csv": "csv", ".dic": "cif", ".a3m": "fasta", ".bed": "tsv", ".dat": "text", ".fa": "fasta", ".fasta": "fasta", ".gaf": "tsv", ".gpa": "tsv", ".gpi": "tsv", ".hmm": "profile_hmm", ".json": "json", ".jsonl": "jsonl", ".html": "html", ".ich": "inchi", ".i6d": "xml", ".idx": "text_index", ".gmt": "tsv", ".log": "text", ".md5": "text", ".sha1": "text", ".sha256": "text", ".metalink": "xml", ".mol": "mol", ".mol2": "mol2", ".models": "model_index", ".mr": "nmr_structured_text", ".nef": "nmr_star", ".nmr": "nmr_structured_text", ".nmrdata": "nmr_structured_text", ".nq": "rdf", ".nt": "rdf", ".obo": "obo", ".ofn": "obo", ".owl": "rdf", ".parquet": "parquet", ".rdf": "rdf", ".pdb": "pdb", ".pins": "text", ".pont": "text", ".pprj": "text", ".sbml": "xml", ".seq": "sequence_text", ".sdf": "sdf", ".sdz": "sdz_structures", ".smi": "smiles", ".rsmi": "reaction_smiles", ".rxn": "reaction_smiles", ".rxnsmiles": "reaction_smiles", ".sql": "sql", ".sf": "nmr_star", ".str": "nmr_star", ".full": "stockholm_alignment", ".cs": "chemical_shift_text", ".list": "text_index", ".msp": "mass_spectrum", ".fps": "fingerprint", ".tab": "tsv", ".tsv": "tsv", ".tree": "newick_tree", ".ttl": "rdf", ".txt": "text", ".xml": "xml", ".xsd": "xml", ".xsl": "xml", ".yaml": "text", ".yml": "text", } recognized_logical_suffix = next( ( suffix for suffix in reversed(suffixes[:-1] if compression != "none" else suffixes) if suffix in record_formats ), logical_suffix, ) is_ord_protobuf_dataset = recognized_logical_suffix == ".pb" and any( part == "ord" or part.startswith("ord_") or part.startswith("open_reaction_database") for part in path_parts ) if is_ord_protobuf_dataset: record_format = "ord_reaction_dataset" reader_family = "ord_protobuf_dataset_adapter" record_boundary = "dataset_metadata_or_reaction" resume_granularity = "reaction_ordinal" elif container in {"tar", "zip", "seven_zip"}: record_format = "archive_member" reader_family = f"{container}_member_dispatch" record_boundary = "archive_member" resume_granularity = "archive_member" elif container == "parquet": record_format = "parquet" reader_family = "parquet_row_group" record_boundary = "row_group_row" resume_granularity = "row_group" elif container == "spreadsheet": record_format = "spreadsheet" reader_family = ( "excel_binary_spreadsheet_record_adapter" if final_suffix == ".xls" else "spreadsheet_record_adapter" ) record_boundary = "container_record" resume_granularity = "container_transaction" elif container == "gctx": record_format = "gctx" reader_family = "gctx_matrix_adapter" record_boundary = "dataset_row" resume_granularity = "dataset_row" elif container == "rds": record_format = "rds" reader_family = "r_serialized_object_adapter" record_boundary = "object_leaf_chunk" resume_granularity = "object_leaf_chunk" elif container in { "lmdb", "sqlite", "hdf5", "numpy", "pdf", "zolca", "graphdb", }: record_format = container reader_family = f"{container}_record_adapter" record_boundary = "container_record" resume_granularity = "container_transaction" else: # A compression suffix says how to recover bytes, not how to recover # records. Treat a compressed payload with an unrecognised logical # suffix (for example ``.gctx.gz``) as an unresolved semantic format # rather than silently training on a binary/base85 representation. # Extensionless compressed text remains a deliberate text-stream # contract, because there is no alternate logical format to dispatch. record_format = record_formats.get( recognized_logical_suffix, ( "text" if compression != "none" and not logical_suffix else "unknown" ), ) reader_family = ( "compressed_record_dispatch" if compression != "none" else "plain_record_dispatch" ) record_boundary = { "csv": "csv_row", "fasta": "fasta_record", "json": "json_value", "jsonl": "json_line", "html": "document", "inchi": "inchi_record", "mass_spectrum": "spectrum_record", "model_index": "structured_text_line", "mol": "molecule_record", "mol2": "molecule_record", "parquet": "row_group_row", "rdf": "rdf_statement", "reaction_smiles": "reaction_record", "nmr_star": "saveframe_or_loop_line", "nmr_structured_text": "structured_text_line", "newick_tree": "newick_tree", "profile_hmm": "profile_hmm", "chemical_shift_text": "structured_text_line", "sequence_text": "sequence_line", "text_index": "index_line", "sdf": "sdf_record", "sdz_structures": "structure_record", "smiles": "molecule_record", "sql": "sql_statement_or_row", "stockholm_alignment": "stockholm_alignment", "tsv": "tsv_row", "xml": "xml_element", }.get(record_format, "text_or_binary_block") resume_granularity = "payload_stream_checkpoint" return { "compression": compression, "container": container, "recordFormat": record_format, "readerFamily": reader_family, "recordBoundary": record_boundary, "resumeGranularity": resume_granularity, "archiveMemberDispatchRequired": container in {"tar", "zip", "seven_zip"}, "specializedDecoderRequired": ( is_ord_protobuf_dataset or container in { "parquet", "lmdb", "sqlite", "hdf5", "spreadsheet", "numpy", "pdf", "seven_zip", "zolca", "graphdb", "gctx", "rds", } or record_format == "unknown" or record_format == "sdz_structures" ), } def build_full_payload_training_schedule( corpus_root: Path, output_root: Path, *, inventory_paths: Sequence[Path] | None = None, ) -> dict[str, Any]: """Seal every admitted payload file into one relocation-safe schedule. The schedule is a complete raw-file denominator, not a trained-knowledge receipt. Paths remain relative to the corpus identity so storage roots may move or be symlinked; bytes and optional hashes are reverified while the later reader consumes each work item and commits optimizer evidence. """ root = corpus_root.expanduser().resolve() output = output_root.expanduser().resolve() discovered = ( list(inventory_paths) if inventory_paths is not None else discover_full_payload_inventory_paths(root) ) inventories = sorted( {path.expanduser().resolve() for path in discovered}, key=str, ) if not inventories: raise FileNotFoundError("no corpus inventories were found for full payloads") inventory_receipts: list[dict[str, Any]] = [] records_by_identity: dict[tuple[str, str], dict[str, Any]] = {} inventory_sha256s_by_identity: dict[tuple[str, str], set[str]] = {} for inventory_path in inventories: if not inventory_path.is_file(): raise FileNotFoundError(f"corpus inventory is absent: {inventory_path}") inventory_sha256 = file_sha256(inventory_path) inventory_receipts.append( { "path": str(inventory_path), "bytes": inventory_path.stat().st_size, "sha256": inventory_sha256, } ) for record in _read_jsonl(inventory_path): source_id = str(record.get("source_id", "")).strip() if not source_id or not _admitted_for_training(record): continue source_record_sha256 = _sha256_bytes(_json_bytes(record)) identity = (source_id, source_record_sha256) prior = records_by_identity.get(identity) if prior is not None and _json_bytes(prior) != _json_bytes(record): raise ValueError(f"conflicting source record for {source_id}") records_by_identity[identity] = record inventory_sha256s_by_identity.setdefault(identity, set()).add( inventory_sha256 ) if not records_by_identity: raise ValueError("no downloaded training-admissible corpus sources were found") schedule_rows: list[dict[str, Any]] = [] payload_paths: set[str] = set() manifest_count = 0 source_payload_files: Counter[str] = Counter() source_payload_bytes: Counter[str] = Counter() for identity in sorted(records_by_identity): source_id, expected_source_record_sha256 = identity record = records_by_identity[identity] ( domain_authority, rights_authority, source_record_sha256, ) = _full_payload_schedule_authorities_from_inventory_record( record, inventory_input_sha256s=sorted( inventory_sha256s_by_identity[identity] ), ) if source_record_sha256 != expected_source_record_sha256: raise RuntimeError( "full payload inventory source digest changed during schedule" ) manifest, manifest_sha256 = _manifest_for_record(root, record) if manifest is None: continue manifest_count += 1 manifest_ref = record.get("payload_manifest") manifest_inventory = _manifest_payload_inventory(manifest, manifest_ref) if ( manifest_inventory.get("fileInventoryPresent") is not True or manifest_inventory.get("malformedFileRows") != 0 # Older payload manifests bind the exact file inventory while the # signed inventory reference carries the aggregate denominator. # Accept either exact authority, but fail closed if any aggregate # authority that is present disagrees with the enumerated files. or manifest_inventory.get("declaredCountsConsistent") is not True or manifest_inventory.get("referenceCountsConsistent") is not True or manifest_inventory.get("exactDenominatorAuthorityPresent") is not True ): raise ValueError(f"payload manifest denominator differs for {source_id}") files = manifest.get("files") if not isinstance(files, list): raise ValueError(f"payload manifest files are absent for {source_id}") manifest_root_value = ( manifest_ref.get("root") if isinstance(manifest_ref, dict) and manifest_ref.get("root") is not None else manifest.get("root", "") ) manifest_root = Path(str(manifest_root_value).strip()) if manifest_root.is_absolute() or ".." in manifest_root.parts: raise ValueError(f"payload manifest root is not corpus-relative: {source_id}") for entry in files: if not isinstance(entry, dict): raise ValueError(f"payload manifest row is malformed for {source_id}") raw_path = entry.get("path") payload_bytes = entry.get("bytes") if ( not isinstance(raw_path, str) or not raw_path.strip() or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or payload_bytes < 0 ): raise ValueError(f"payload manifest identity is malformed for {source_id}") entry_path = Path(raw_path.strip()) if entry_path.is_absolute() or ".." in entry_path.parts: raise ValueError(f"payload path is not corpus-relative: {raw_path}") root_parts = manifest_root.parts relative_path = ( entry_path if not root_parts or entry_path.parts[: len(root_parts)] == root_parts else manifest_root / entry_path ) normalized_path = relative_path.as_posix() if normalized_path in payload_paths: raise ValueError(f"payload path is scheduled more than once: {raw_path}") payload_paths.add(normalized_path) local_path = root / relative_path if not local_path.is_file(): raise FileNotFoundError(f"payload file is absent: {local_path}") if local_path.stat().st_size != payload_bytes: raise ValueError(f"payload file size differs: {local_path}") payload_sha256 = entry.get("sha256") if payload_sha256 is not None and ( not isinstance(payload_sha256, str) or len(payload_sha256) != 64 or any(character not in "0123456789abcdef" for character in payload_sha256) ): raise ValueError(f"payload sha256 is malformed: {local_path}") work_identity = "\x00".join( ( FULL_PAYLOAD_TRAINING_SCHEDULE_ENTRY_SCHEMA, source_id, normalized_path, str(payload_bytes), str(payload_sha256 or "hash_at_consumption"), ) ) schedule_rows.append( { "schema": FULL_PAYLOAD_TRAINING_SCHEDULE_ENTRY_SCHEMA, "payloadWorkId": hashlib.sha256( work_identity.encode("utf-8") ).hexdigest(), "scheduleKey": _stable_rank( FULL_PAYLOAD_TRAINING_SCHEDULE_SCHEMA, f"{source_id}\x00{normalized_path}", ), "sourceId": source_id, "sourceRecordSha256": source_record_sha256, "payloadManifestSha256": manifest_sha256, "payloadRelativePath": normalized_path, "payloadBytes": payload_bytes, "payloadSha256": payload_sha256, "sourceChecksum": entry.get("source_checksum"), "integrityStatus": str( entry.get("integrity_status", "not_applicable") ), "format": _payload_format_contract(normalized_path), "rightsDisposition": "training_admissible", "domainAuthority": domain_authority, "rightsAuthority": rights_authority, "hashEveryByteAtConsumption": True, "textProbeUsed": False, "targetEnteredForward": False, } ) source_payload_files[source_id] += 1 source_payload_bytes[source_id] += payload_bytes if not schedule_rows: raise ValueError("admitted corpus has no payload files to schedule") schedule_rows.sort(key=lambda row: str(row["scheduleKey"])) for ordinal, row in enumerate(schedule_rows): row["scheduleOrdinal"] = ordinal schedule_path = output / "full_payload_training_schedule.jsonl" schedule_count = _atomic_jsonl(schedule_path, schedule_rows) schedule_sha256 = file_sha256(schedule_path) format_files: Counter[str] = Counter() format_bytes: Counter[str] = Counter() reader_files: Counter[str] = Counter() reader_bytes: Counter[str] = Counter() local_sha256_files = 0 local_sha256_bytes = 0 for row in schedule_rows: format_contract = row["format"] if not isinstance(format_contract, dict): raise RuntimeError("full payload format contract is malformed") record_format = str(format_contract["recordFormat"]) reader_family = str(format_contract["readerFamily"]) payload_bytes = int(row["payloadBytes"]) format_files[record_format] += 1 format_bytes[record_format] += payload_bytes reader_files[reader_family] += 1 reader_bytes[reader_family] += payload_bytes if row.get("payloadSha256") is not None: local_sha256_files += 1 local_sha256_bytes += payload_bytes payload_byte_count = sum(source_payload_bytes.values()) inventory_identity_sha256 = hashlib.sha256( _json_bytes( sorted( ( int(receipt["bytes"]), str(receipt["sha256"]), ) for receipt in inventory_receipts ) ) ).hexdigest() checks = { "allInventoryRowsParsed": True, "onlyDownloadedTrainingAdmissibleSourcesScheduled": True, "allPayloadManifestReferencesHashVerified": True, "allPayloadManifestDenominatorsMatch": True, "allPayloadFilesPresent": True, "allPayloadFileSizesMatch": True, "everyManifestPayloadEntryScheduledExactlyOnce": ( schedule_count == len(payload_paths) ), "payloadPathsRemainCorpusRelative": True, "scheduleContentAddressedAndSealed": len(schedule_sha256) == 64, "textProbesExcluded": all( row.get("textProbeUsed") is False for row in schedule_rows ), "targetsExcludedFromForward": all( row.get("targetEnteredForward") is False for row in schedule_rows ), } receipt = { "schema": FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(checks.values()), "corpusRootAtBuild": str(root), "corpusRootBindingPolicy": ( "inventory_hash_autodiscovery_with_relative_payload_paths" ), "inventoryIdentitySha256": inventory_identity_sha256, "inventoryInputs": inventory_receipts, "admittedSourceCount": len( {source_id for source_id, _sha256 in records_by_identity} ), "admittedSourceRecordCount": len(records_by_identity), "payloadManifestSourceCount": manifest_count, "payloadFileCount": schedule_count, "payloadBytes": payload_byte_count, "payloadFilesWithLocalSha256": local_sha256_files, "payloadBytesWithLocalSha256": local_sha256_bytes, "payloadFilesRequiringHashAtConsumption": ( schedule_count - local_sha256_files ), "payloadBytesRequiringHashAtConsumption": ( payload_byte_count - local_sha256_bytes ), "sourcePayloadFileCounts": dict(sorted(source_payload_files.items())), "sourcePayloadBytes": dict(sorted(source_payload_bytes.items())), "recordFormatFileCounts": dict(sorted(format_files.items())), "recordFormatBytes": dict(sorted(format_bytes.items())), "readerFamilyFileCounts": dict(sorted(reader_files.items())), "readerFamilyBytes": dict(sorted(reader_bytes.items())), "schedule": { "schema": FULL_PAYLOAD_TRAINING_SCHEDULE_SCHEMA, "path": str(schedule_path), "rows": schedule_count, "bytes": schedule_path.stat().st_size, "sha256": schedule_sha256, }, "checks": checks, "modelTrainingClaimed": False, "completePayloadTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, "nextProofRequired": [ "stream_every_scheduled_payload_to_record_boundaries", "hash_every_consumed_payload_byte", "fastokens_tokenize_every_admitted_record", "commit_optimizer_transactions_with_payload_work_ids", "complete_schedule_cursor_without_skips", "heldout_and_anti_forgetting_gain", "model_owned_route_and_distinct_gradient_evidence", "cold_reload_and_checkpoint_lineage_verification", ], "scopeNote": ( "This receipt proves a complete relocation-safe raw payload file " "denominator. It does not claim that archive members, records, bytes, " "tokens, optimizer updates, expert pages, or model knowledge have yet " "been consumed, trained, retained, or promoted." ), } _atomic_json(output / "full_payload_training_schedule.receipt.json", receipt) return receipt def _hash_full_payload_work_item( corpus_root: Path, schedule_sha256: str, row: dict[str, Any], ) -> dict[str, Any]: relative_path = Path(str(row["payloadRelativePath"])) path = corpus_root / relative_path before = path.stat() expected_bytes = int(row["payloadBytes"]) if not path.is_file() or before.st_size != expected_bytes: raise RuntimeError(f"scheduled payload changed before hashing: {path}") started_wall = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) started = time.monotonic_ns() observed_sha256 = file_sha256(path) duration_ns = time.monotonic_ns() - started after = path.stat() before_identity = ( before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, ) after_identity = ( after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, ) if before_identity != after_identity: raise RuntimeError(f"scheduled payload changed while hashing: {path}") expected_sha256 = row.get("payloadSha256") if expected_sha256 is not None and observed_sha256 != expected_sha256: raise RuntimeError(f"scheduled payload sha256 differs: {path}") return { "schema": FULL_PAYLOAD_HASH_LEDGER_SCHEMA, "recordedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "hashStartedAt": started_wall, "scheduleSha256": schedule_sha256, "payloadWorkId": row["payloadWorkId"], "sourceId": row["sourceId"], "payloadRelativePath": row["payloadRelativePath"], "payloadBytes": expected_bytes, "observedSha256": observed_sha256, "expectedSha256": expected_sha256, "expectedSha256Matched": ( observed_sha256 == expected_sha256 if expected_sha256 is not None else None ), "storageDevice": before.st_dev, "sourceIdentity": { "device": before.st_dev, "inode": before.st_ino, "bytes": before.st_size, "mtimeNs": before.st_mtime_ns, }, "hashDurationNs": duration_ns, "hashBytesPerSecond": ( expected_bytes * 1_000_000_000 // max(1, duration_ns) ), "allPayloadBytesHashed": True, "modelTrainingClaimed": False, "rawDataDeletionAllowed": False, } def hash_full_payload_training_schedule( schedule_receipt_path: Path, corpus_root: Path, ledger_path: Path, output_receipt_path: Path, ) -> dict[str, Any]: """Hash every scheduled file with one sequential worker per storage device. Hash result rows remain pure JSONL data. Batches become visible only after the ledger fd is fsynced and an atomic commit-frontier sidecar publishes the exact durable prefix. This proves current bytes only; optimizer and model-knowledge claims remain explicitly false. """ schedule_receipt = schedule_receipt_path.expanduser().resolve() root = corpus_root.expanduser().resolve() ledger = ledger_path.expanduser().resolve() output_receipt = output_receipt_path.expanduser().resolve() if not schedule_receipt.is_file(): raise FileNotFoundError(f"full payload schedule receipt is absent: {schedule_receipt}") receipt_value = json.loads(schedule_receipt.read_text(encoding="utf-8")) if ( not isinstance(receipt_value, dict) or receipt_value.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA or receipt_value.get("passed") is not True ): raise ValueError("full payload schedule receipt did not pass") schedule_record = receipt_value.get("schedule") if not isinstance(schedule_record, dict): raise ValueError("full payload schedule artifact is absent") schedule_path = Path(str(schedule_record.get("path", ""))).resolve() schedule_sha256 = str(schedule_record.get("sha256", "")) expected_rows = schedule_record.get("rows") if ( not schedule_path.is_file() or len(schedule_sha256) != 64 or file_sha256(schedule_path) != schedule_sha256 or not isinstance(expected_rows, int) or isinstance(expected_rows, bool) or expected_rows < 1 ): raise ValueError("full payload schedule artifact differs") schedule_rows = _read_jsonl(schedule_path) if len(schedule_rows) != expected_rows: raise ValueError("full payload schedule row count differs") schedule_by_id: dict[str, dict[str, Any]] = {} scheduled_bytes = 0 for ordinal, row in enumerate(schedule_rows): work_id = row.get("payloadWorkId") relative_path_value = row.get("payloadRelativePath") payload_bytes = row.get("payloadBytes") if ( row.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_ENTRY_SCHEMA or row.get("scheduleOrdinal") != ordinal or not isinstance(work_id, str) or len(work_id) != 64 or work_id in schedule_by_id or not isinstance(relative_path_value, str) or not relative_path_value or Path(relative_path_value).is_absolute() or ".." in Path(relative_path_value).parts or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or payload_bytes < 0 or row.get("textProbeUsed") is not False or row.get("targetEnteredForward") is not False ): raise ValueError("full payload schedule row is malformed") path = root / relative_path_value if not path.is_file() or path.stat().st_size != payload_bytes: raise ValueError(f"full payload root does not bind schedule: {path}") schedule_by_id[work_id] = row scheduled_bytes += payload_bytes if ( receipt_value.get("payloadFileCount") != len(schedule_rows) or receipt_value.get("payloadBytes") != scheduled_bytes ): raise ValueError("full payload schedule denominator differs") ledger.parent.mkdir(parents=True, exist_ok=True) lock_path = ledger.with_suffix(ledger.suffix + ".lock") lock_handle = lock_path.open("a+", encoding="utf-8") try: try: fcntl.flock(lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError as error: raise RuntimeError("full payload hash ledger is already owned") from error ( existing_rows, prefix_accumulator, committed_frontier, ) = _recover_full_payload_hash_ledger( ledger, schedule_sha256, ) completed: dict[str, dict[str, Any]] = {} for record in existing_rows: work_id = record.get("payloadWorkId") if ( record.get("schema") != FULL_PAYLOAD_HASH_LEDGER_SCHEMA or record.get("scheduleSha256") != schedule_sha256 or not isinstance(work_id, str) or work_id not in schedule_by_id or work_id in completed or record.get("allPayloadBytesHashed") is not True or record.get("modelTrainingClaimed") is not False or record.get("rawDataDeletionAllowed") is not False ): raise ValueError("full payload hash ledger row differs") scheduled = schedule_by_id[work_id] identity = record.get("sourceIdentity") path = root / str(scheduled["payloadRelativePath"]) current = path.stat() if ( not isinstance(identity, dict) or record.get("payloadRelativePath") != scheduled["payloadRelativePath"] or record.get("payloadBytes") != scheduled["payloadBytes"] or record.get("expectedSha256") != scheduled.get("payloadSha256") or identity.get("device") != current.st_dev or identity.get("inode") != current.st_ino or identity.get("bytes") != current.st_size or identity.get("mtimeNs") != current.st_mtime_ns ): raise ValueError("hashed payload identity changed after ledger commit") completed[work_id] = record committed_frontier = _finalize_full_payload_hash_ledger_recovery( ledger, prefix_accumulator, committed_frontier, ) pending_by_device: dict[int, list[dict[str, Any]]] = {} for row in schedule_rows: work_id = str(row["payloadWorkId"]) if work_id in completed: continue device = (root / str(row["payloadRelativePath"])).stat().st_dev pending_by_device.setdefault(device, []).append(row) ledger_handle = ledger.open("ab") try: pending_commit_rows = 0 pending_commit_bytes = 0 pending_commit_started = time.monotonic() iterators: dict[int, Iterator[dict[str, Any]]] = { device: iter(rows) for device, rows in pending_by_device.items() } if iterators: with ThreadPoolExecutor( max_workers=len(iterators), thread_name_prefix="resynthesis-payload-hash", ) as executor: futures: dict[ Future[dict[str, Any]], tuple[int, Iterator[dict[str, Any]]], ] = {} for device, iterator in iterators.items(): try: row = next(iterator) except StopIteration: continue future = executor.submit( _hash_full_payload_work_item, root, schedule_sha256, row, ) futures[future] = (device, iterator) while futures: done, _pending = wait( futures, return_when=FIRST_COMPLETED, ) for future in done: device, iterator = futures.pop(future) record = future.result() work_id = str(record["payloadWorkId"]) raw_line = ( json.dumps( record, sort_keys=True, separators=(",", ":"), ).encode("utf-8") + b"\n" ) ledger_handle.write(raw_line) prefix_accumulator.append(raw_line, record) pending_commit_rows += 1 pending_commit_bytes += len(raw_line) completed[work_id] = record if ( pending_commit_rows >= FULL_PAYLOAD_HASH_COMMIT_ROWS or pending_commit_bytes >= FULL_PAYLOAD_HASH_COMMIT_BYTES or time.monotonic() - pending_commit_started >= FULL_PAYLOAD_HASH_COMMIT_SECONDS ): ledger_handle.flush() os.fsync(ledger_handle.fileno()) committed_frontier = prefix_accumulator.frontier() _publish_full_payload_hash_commit_frontier( ledger, committed_frontier, ) pending_commit_rows = 0 pending_commit_bytes = 0 pending_commit_started = time.monotonic() try: row = next(iterator) except StopIteration: continue next_future = executor.submit( _hash_full_payload_work_item, root, schedule_sha256, row, ) futures[next_future] = (device, iterator) if pending_commit_rows: ledger_handle.flush() os.fsync(ledger_handle.fileno()) committed_frontier = prefix_accumulator.frontier() _publish_full_payload_hash_commit_frontier( ledger, committed_frontier, ) finally: ledger_handle.close() if len(completed) != len(schedule_rows): raise RuntimeError("full payload hashing ended before the schedule") hashed_bytes = sum(int(row["payloadBytes"]) for row in completed.values()) if hashed_bytes != scheduled_bytes: raise RuntimeError("full payload hashed byte denominator differs") if committed_frontier is None: raise RuntimeError("full payload hash commit frontier is absent") _validate_full_payload_hash_commit_frontier( prefix_accumulator, committed_frontier, ) if ( committed_frontier.durable_rows != len(completed) or committed_frontier.durable_payload_bytes != hashed_bytes or committed_frontier.durable_ledger_bytes != ledger.stat().st_size ): raise RuntimeError("full payload hash durable denominator differs") ledger_sha256 = committed_frontier.prefix_sha256 commit_path = _full_payload_hash_commit_path(ledger) content_identity = hashlib.sha256() for work_id in sorted(completed): content_identity.update(work_id.encode("ascii")) content_identity.update(b"\x00") content_identity.update( str(completed[work_id]["observedSha256"]).encode("ascii") ) content_identity.update(b"\n") checks = { "scheduleReceiptPassed": True, "scheduleArtifactHashVerified": True, "allScheduleRowsValidated": True, "allPayloadPathsResolvedUnderSelectedRoot": True, "allPayloadFilesHashed": len(completed) == len(schedule_rows), "allPayloadBytesHashed": hashed_bytes == scheduled_bytes, "allDeclaredPayloadHashesMatch": all( row.get("expectedSha256Matched") in {None, True} for row in completed.values() ), "ledgerDurableAndHashSealed": len(ledger_sha256) == 64, "ledgerCommitFrontierPublishedAfterFsync": commit_path.is_file(), "targetsExcludedFromForward": True, "optimizerNotConsulted": True, } receipt = { "schema": FULL_PAYLOAD_HASH_RECEIPT_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(checks.values()), "scheduleReceipt": { "path": str(schedule_receipt), "sha256": file_sha256(schedule_receipt), "scheduleSha256": schedule_sha256, }, "corpusRoot": str(root), "payloadFileCount": len(completed), "payloadBytes": hashed_bytes, "storageDeviceCount": len( {int(row["storageDevice"]) for row in completed.values()} ), "contentIdentitySha256": content_identity.hexdigest(), "ledger": { "path": str(ledger), "rows": len(completed), "bytes": committed_frontier.durable_ledger_bytes, "sha256": ledger_sha256, "durablePayloadBytes": ( committed_frontier.durable_payload_bytes ), "prefixChainSha256": ( committed_frontier.prefix_chain_sha256 ), "finalPayloadWorkId": ( committed_frontier.final_payload_work_id ), "finalSequence": committed_frontier.final_sequence, "commitFrontier": { "path": str(commit_path), "sha256": file_sha256(commit_path), "schema": FULL_PAYLOAD_HASH_COMMIT_FRONTIER_SCHEMA, }, }, "checks": checks, "modelTrainingClaimed": False, "completePayloadTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, "nextProofRequired": receipt_value["nextProofRequired"], } _atomic_json(output_receipt, receipt) return receipt finally: fcntl.flock(lock_handle, fcntl.LOCK_UN) lock_handle.close() def _rebind_full_payload_hash_row( row: dict[str, Any], *, source_schedule_sha256: str, target_schedule_sha256: str, schedule_by_id: Mapping[str, dict[str, Any]], corpus_root: Path, ) -> dict[str, Any]: """Validate one durable hash row and bind it to refreshed reader metadata.""" work_id = row.get("payloadWorkId") row_schedule_sha256 = row.get("scheduleSha256") rebound_source_sha256 = row.get("contractRefreshSourceScheduleSha256") if ( row.get("schema") != FULL_PAYLOAD_HASH_LEDGER_SCHEMA or not isinstance(work_id, str) or work_id not in schedule_by_id or row_schedule_sha256 not in { source_schedule_sha256, target_schedule_sha256, } or ( row_schedule_sha256 == target_schedule_sha256 and rebound_source_sha256 != source_schedule_sha256 ) or row.get("allPayloadBytesHashed") is not True or row.get("modelTrainingClaimed") is not False or row.get("rawDataDeletionAllowed") is not False or row.get("expectedSha256Matched") not in {None, True} ): raise ValueError("full payload contract-refresh hash row differs") scheduled = schedule_by_id[work_id] relative_path = scheduled.get("payloadRelativePath") payload_bytes = scheduled.get("payloadBytes") expected_sha256 = scheduled.get("payloadSha256") observed_sha256 = row.get("observedSha256") source_identity = row.get("sourceIdentity") if ( not isinstance(relative_path, str) or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or row.get("sourceId") != scheduled.get("sourceId") or row.get("payloadRelativePath") != relative_path or row.get("payloadBytes") != payload_bytes or row.get("expectedSha256") != expected_sha256 or not isinstance(observed_sha256, str) or len(observed_sha256) != 64 or ( isinstance(expected_sha256, str) and observed_sha256 != expected_sha256 ) or not isinstance(source_identity, dict) ): raise ValueError("full payload contract-refresh hash identity differs") payload_path = corpus_root / relative_path current = payload_path.stat() if ( not payload_path.is_file() or source_identity.get("device") != current.st_dev or source_identity.get("inode") != current.st_ino or source_identity.get("bytes") != current.st_size or source_identity.get("mtimeNs") != current.st_mtime_ns ): raise ValueError("full payload changed after source hash admission") rebound = dict(row) rebound["scheduleSha256"] = target_schedule_sha256 rebound["contractRefreshSourceScheduleSha256"] = source_schedule_sha256 if row_schedule_sha256 == source_schedule_sha256: rebound["contractReboundAt"] = time.strftime( "%Y-%m-%dT%H:%M:%SZ", time.gmtime(), ) return rebound def refresh_and_follow_full_payload_training_authority( source_schedule_receipt_path: Path, corpus_root: Path, source_hash_ledger_path: Path, output_root: Path, *, poll_interval_seconds: float = 1.0, ) -> dict[str, Any]: """Refresh reader contracts while following an append-only hash ledger. Payload work IDs deliberately exclude reader metadata. A canonical reader improvement can therefore reseal the schedule and rebind already-durable byte hashes without rereading payload bytes. The source ledger remains the immutable hash authority; this follower validates current file identity, writes each rebound row durably, and waits without an iteration cap until the complete denominator has arrived. """ source_receipt_path = source_schedule_receipt_path.expanduser().resolve() root = corpus_root.expanduser().resolve() source_ledger = source_hash_ledger_path.expanduser().resolve() output = output_root.expanduser().resolve() if poll_interval_seconds <= 0: raise ValueError("full payload contract-refresh poll interval is invalid") if not source_receipt_path.is_file() or not source_ledger.is_file(): raise FileNotFoundError("full payload contract-refresh authority is absent") if not root.is_dir(): raise FileNotFoundError("full payload contract-refresh corpus is absent") source_receipt = json.loads(source_receipt_path.read_text(encoding="utf-8")) if ( not isinstance(source_receipt, dict) or source_receipt.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA or source_receipt.get("passed") is not True ): raise ValueError("full payload contract-refresh source receipt differs") source_schedule_record = source_receipt.get("schedule") if not isinstance(source_schedule_record, dict): raise ValueError("full payload contract-refresh schedule is absent") source_schedule_path = Path( str(source_schedule_record.get("path", "")) ).resolve() source_schedule_sha256 = str(source_schedule_record.get("sha256", "")) expected_rows = source_schedule_record.get("rows") if ( not source_schedule_path.is_file() or len(source_schedule_sha256) != 64 or file_sha256(source_schedule_path) != source_schedule_sha256 or not isinstance(expected_rows, int) or isinstance(expected_rows, bool) or expected_rows < 1 ): raise ValueError("full payload contract-refresh schedule differs") source_rows = _read_jsonl(source_schedule_path) if len(source_rows) != expected_rows: raise ValueError("full payload contract-refresh denominator differs") refreshed_rows: list[dict[str, Any]] = [] schedule_by_id: dict[str, dict[str, Any]] = {} reader_files: Counter[str] = Counter() reader_bytes: Counter[str] = Counter() format_files: Counter[str] = Counter() format_bytes: Counter[str] = Counter() refreshed_contracts = 0 payload_bytes_total = 0 for ordinal, source_row in enumerate(source_rows): work_id = source_row.get("payloadWorkId") relative_path = source_row.get("payloadRelativePath") payload_bytes = source_row.get("payloadBytes") if ( source_row.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_ENTRY_SCHEMA or source_row.get("scheduleOrdinal") != ordinal or not isinstance(work_id, str) or len(work_id) != 64 or work_id in schedule_by_id or not isinstance(relative_path, str) or not relative_path or Path(relative_path).is_absolute() or ".." in Path(relative_path).parts or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or payload_bytes < 0 ): raise ValueError("full payload contract-refresh row is malformed") payload_path = root / relative_path if not payload_path.is_file() or payload_path.stat().st_size != payload_bytes: raise ValueError("full payload contract-refresh payload identity differs") refreshed = dict(source_row) current_contract = _payload_format_contract(relative_path) if refreshed.get("format") != current_contract: refreshed_contracts += 1 refreshed["format"] = current_contract refreshed_rows.append(refreshed) schedule_by_id[work_id] = refreshed reader_family = str(current_contract["readerFamily"]) record_format = str(current_contract["recordFormat"]) reader_files[reader_family] += 1 reader_bytes[reader_family] += payload_bytes format_files[record_format] += 1 format_bytes[record_format] += payload_bytes payload_bytes_total += payload_bytes if ( source_receipt.get("payloadFileCount") != expected_rows or source_receipt.get("payloadBytes") != payload_bytes_total ): raise ValueError("full payload contract-refresh receipt denominator differs") output.mkdir(parents=True, exist_ok=True) schedule_path = output / "full_payload_training_schedule.jsonl" if schedule_path.exists(): if _read_jsonl(schedule_path) != refreshed_rows: raise ValueError("full payload refreshed schedule bytes differ") else: _atomic_jsonl(schedule_path, refreshed_rows) schedule_sha256 = file_sha256(schedule_path) schedule_receipt_path = output / "full_payload_training_schedule.receipt.json" refresh_checks = dict(source_receipt.get("checks", {})) refresh_checks.update( { "sourceScheduleReceiptPassed": True, "sourceScheduleArtifactHashVerified": True, "payloadWorkIdsPreservedExactly": [ str(row["payloadWorkId"]) for row in refreshed_rows ] == [str(row["payloadWorkId"]) for row in source_rows], "payloadFileIdentitiesCurrent": True, "canonicalFormatContractsCurrent": all( row.get("format") == _payload_format_contract(str(row["payloadRelativePath"])) for row in refreshed_rows ), "targetsExcludedFromForward": True, "textProbesExcluded": True, } ) refreshed_receipt = dict(source_receipt) refreshed_receipt.update( { "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(refresh_checks.values()), "corpusRootAtBuild": str(root), "corpusRootBindingPolicy": ( "immutable_work_identity_plus_canonical_reader_contract" ), "readerFamilyFileCounts": dict(sorted(reader_files.items())), "readerFamilyBytes": dict(sorted(reader_bytes.items())), "recordFormatFileCounts": dict(sorted(format_files.items())), "recordFormatBytes": dict(sorted(format_bytes.items())), "schedule": { "schema": FULL_PAYLOAD_TRAINING_SCHEDULE_SCHEMA, "path": str(schedule_path), "rows": expected_rows, "bytes": schedule_path.stat().st_size, "sha256": schedule_sha256, }, "contractRefresh": { "schema": FULL_PAYLOAD_CONTRACT_REFRESH_RECEIPT_SCHEMA, "sourceScheduleReceiptPath": str(source_receipt_path), "sourceScheduleReceiptSha256": file_sha256(source_receipt_path), "sourceSchedulePath": str(source_schedule_path), "sourceScheduleSha256": source_schedule_sha256, "sourceHashLedgerPath": str(source_ledger), "refreshedFormatContractCount": refreshed_contracts, "payloadWorkIdsChanged": False, "payloadBytesRehashedByRefresh": 0, }, "checks": refresh_checks, "modelTrainingClaimed": False, "completePayloadTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, } ) if refreshed_receipt["passed"] is not True: raise RuntimeError("full payload canonical contract refresh failed") if schedule_receipt_path.exists(): existing_receipt = json.loads(schedule_receipt_path.read_text(encoding="utf-8")) existing_schedule = ( existing_receipt.get("schedule") if isinstance(existing_receipt, dict) else None ) existing_refresh = ( existing_receipt.get("contractRefresh") if isinstance(existing_receipt, dict) else None ) if ( not isinstance(existing_schedule, dict) or not isinstance(existing_refresh, dict) or existing_receipt.get("passed") is not True or existing_schedule.get("sha256") != schedule_sha256 or existing_schedule.get("rows") != expected_rows or existing_refresh.get("sourceScheduleSha256") != source_schedule_sha256 ): raise ValueError("full payload refreshed receipt differs") refreshed_receipt = existing_receipt else: _atomic_json(schedule_receipt_path, refreshed_receipt) output_ledger = output / "full_payload_hashes.jsonl" output_hash_receipt = output / "full_payload_hashes.receipt.json" progress_path = output / "full_payload_contract_refresh.progress.json" lock_path = output_ledger.with_suffix(output_ledger.suffix + ".lock") lock_handle = lock_path.open("a+", encoding="utf-8") try: try: fcntl.flock(lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError as error: raise RuntimeError( "full payload contract-refresh ledger is already owned" ) from error ( existing_output_rows, output_prefix_accumulator, output_committed_frontier, ) = _recover_full_payload_hash_ledger( output_ledger, schedule_sha256, ) completed: dict[str, dict[str, Any]] = {} for existing_row in existing_output_rows: rebound = _rebind_full_payload_hash_row( existing_row, source_schedule_sha256=source_schedule_sha256, target_schedule_sha256=schedule_sha256, schedule_by_id=schedule_by_id, corpus_root=root, ) work_id = str(rebound["payloadWorkId"]) if work_id in completed: raise ValueError("full payload refreshed hash row repeats") completed[work_id] = existing_row output_committed_frontier = ( _finalize_full_payload_hash_ledger_recovery( output_ledger, output_prefix_accumulator, output_committed_frontier, ) ) source_stat = source_ledger.stat() source_device = source_stat.st_dev source_inode = source_stat.st_ino source_offset = 0 source_prefix_accumulator = _FullPayloadHashPrefixAccumulator( source_schedule_sha256 ) output_handle = output_ledger.open("ab") try: pending_commit_rows = 0 pending_commit_bytes = 0 pending_commit_started = time.monotonic() while len(completed) < expected_rows: current_source_stat = source_ledger.stat() if ( current_source_stat.st_dev != source_device or current_source_stat.st_ino != source_inode or current_source_stat.st_size < source_offset ): raise RuntimeError("full payload source hash ledger identity changed") source_frontier = _read_full_payload_hash_commit_frontier( source_ledger, source_schedule_sha256, ) source_terminal = ( source_frontier.durable_ledger_bytes if source_frontier is not None else current_source_stat.st_size ) if source_terminal < source_offset: raise RuntimeError( "full payload source hash commit frontier moved backward" ) new_rows = 0 with source_ledger.open("rb") as source_handle: source_handle.seek(source_offset) while source_handle.tell() < source_terminal: raw_line = source_handle.readline( source_terminal - source_handle.tell() ) if not raw_line or not raw_line.endswith(b"\n"): break value = json.loads(raw_line) if not isinstance(value, dict): raise ValueError("full payload source hash row is malformed") source_prefix_accumulator.append(raw_line, value) source_offset = source_handle.tell() rebound = _rebind_full_payload_hash_row( value, source_schedule_sha256=source_schedule_sha256, target_schedule_sha256=schedule_sha256, schedule_by_id=schedule_by_id, corpus_root=root, ) work_id = str(rebound["payloadWorkId"]) if work_id in completed: continue rebound_raw_line = ( json.dumps( rebound, sort_keys=True, separators=(",", ":"), ).encode("utf-8") + b"\n" ) output_handle.write(rebound_raw_line) output_prefix_accumulator.append( rebound_raw_line, rebound, ) completed[work_id] = rebound new_rows += 1 pending_commit_rows += 1 pending_commit_bytes += len(rebound_raw_line) if ( pending_commit_rows >= FULL_PAYLOAD_HASH_COMMIT_ROWS or pending_commit_bytes >= FULL_PAYLOAD_HASH_COMMIT_BYTES or time.monotonic() - pending_commit_started >= FULL_PAYLOAD_HASH_COMMIT_SECONDS ): output_handle.flush() os.fsync(output_handle.fileno()) output_committed_frontier = ( output_prefix_accumulator.frontier() ) _publish_full_payload_hash_commit_frontier( output_ledger, output_committed_frontier, ) pending_commit_rows = 0 pending_commit_bytes = 0 pending_commit_started = time.monotonic() if source_frontier is not None and ( source_offset == source_frontier.durable_ledger_bytes ): _validate_full_payload_hash_commit_frontier( source_prefix_accumulator, source_frontier, ) if new_rows and pending_commit_rows: output_handle.flush() os.fsync(output_handle.fileno()) output_committed_frontier = ( output_prefix_accumulator.frontier() ) _publish_full_payload_hash_commit_frontier( output_ledger, output_committed_frontier, ) pending_commit_rows = 0 pending_commit_bytes = 0 pending_commit_started = time.monotonic() completed_bytes = sum( int(row["payloadBytes"]) for row in completed.values() ) _atomic_json( progress_path, { "schema": FULL_PAYLOAD_CONTRACT_REFRESH_RECEIPT_SCHEMA, "builtAt": time.strftime( "%Y-%m-%dT%H:%M:%SZ", time.gmtime(), ), "passed": len(completed) == expected_rows, "sourceScheduleSha256": source_schedule_sha256, "refreshedScheduleSha256": schedule_sha256, "sourceLedgerPrefixBytes": source_offset, "completedPayloadFiles": len(completed), "scheduledPayloadFiles": expected_rows, "completedPayloadBytes": completed_bytes, "scheduledPayloadBytes": payload_bytes_total, "payloadBytesRehashedByRefresh": 0, "modelTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, }, ) if len(completed) < expected_rows: time.sleep(poll_interval_seconds) finally: output_handle.close() hashed_bytes = sum(int(row["payloadBytes"]) for row in completed.values()) if len(completed) != expected_rows or hashed_bytes != payload_bytes_total: raise RuntimeError("full payload contract refresh ended before completion") if output_committed_frontier is None: raise RuntimeError( "full payload refreshed hash commit frontier is absent" ) _validate_full_payload_hash_commit_frontier( output_prefix_accumulator, output_committed_frontier, ) if ( output_committed_frontier.durable_rows != len(completed) or output_committed_frontier.durable_payload_bytes != hashed_bytes or output_committed_frontier.durable_ledger_bytes != output_ledger.stat().st_size ): raise RuntimeError( "full payload refreshed hash durable denominator differs" ) ledger_sha256 = output_committed_frontier.prefix_sha256 output_commit_path = _full_payload_hash_commit_path(output_ledger) content_identity = hashlib.sha256() for work_id in sorted(completed): content_identity.update(work_id.encode("ascii")) content_identity.update(b"\x00") content_identity.update( str(completed[work_id]["observedSha256"]).encode("ascii") ) content_identity.update(b"\n") hash_checks = { "refreshedScheduleReceiptPassed": True, "refreshedScheduleArtifactHashVerified": True, "allPayloadWorkIdsPreserved": len(completed) == expected_rows, "allPayloadFilesHashed": len(completed) == expected_rows, "allPayloadBytesHashed": hashed_bytes == payload_bytes_total, "allSourceIdentitiesReverified": True, "allDeclaredPayloadHashesMatch": all( row.get("expectedSha256Matched") in {None, True} for row in completed.values() ), "payloadBytesRehashedByRefreshIsZero": True, "ledgerCommitFrontierPublishedAfterFsync": ( output_commit_path.is_file() ), "targetsExcludedFromForward": True, "optimizerNotConsulted": True, } final_receipt = { "schema": FULL_PAYLOAD_HASH_RECEIPT_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(hash_checks.values()), "scheduleReceipt": { "path": str(schedule_receipt_path), "sha256": file_sha256(schedule_receipt_path), "scheduleSha256": schedule_sha256, }, "corpusRoot": str(root), "payloadFileCount": len(completed), "payloadBytes": hashed_bytes, "storageDeviceCount": len( {int(row["storageDevice"]) for row in completed.values()} ), "contentIdentitySha256": content_identity.hexdigest(), "ledger": { "path": str(output_ledger), "rows": len(completed), "bytes": output_committed_frontier.durable_ledger_bytes, "sha256": ledger_sha256, "durablePayloadBytes": ( output_committed_frontier.durable_payload_bytes ), "prefixChainSha256": ( output_committed_frontier.prefix_chain_sha256 ), "finalPayloadWorkId": ( output_committed_frontier.final_payload_work_id ), "finalSequence": output_committed_frontier.final_sequence, "commitFrontier": { "path": str(output_commit_path), "sha256": file_sha256(output_commit_path), "schema": FULL_PAYLOAD_HASH_COMMIT_FRONTIER_SCHEMA, }, }, "contractRefresh": { "schema": FULL_PAYLOAD_CONTRACT_REFRESH_RECEIPT_SCHEMA, "sourceScheduleSha256": source_schedule_sha256, "refreshedScheduleSha256": schedule_sha256, "refreshedFormatContractCount": refreshed_contracts, "payloadBytesRehashedByRefresh": 0, }, "checks": hash_checks, "modelTrainingClaimed": False, "completePayloadTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, "nextProofRequired": source_receipt["nextProofRequired"], } if output_hash_receipt.exists(): existing_hash_receipt = json.loads( output_hash_receipt.read_text(encoding="utf-8") ) if ( not isinstance(existing_hash_receipt, dict) or existing_hash_receipt.get("passed") is not True or existing_hash_receipt.get("contentIdentitySha256") != final_receipt["contentIdentitySha256"] or existing_hash_receipt.get("ledger") != final_receipt["ledger"] ): raise ValueError("full payload refreshed hash receipt differs") return existing_hash_receipt _atomic_json(output_hash_receipt, final_receipt) return final_receipt finally: fcntl.flock(lock_handle, fcntl.LOCK_UN) lock_handle.close() def compose_full_payload_training_authority( corpus_root: Path, output_root: Path, component_schedule_receipt_paths: Sequence[Path], component_hash_receipt_paths: Sequence[Path], *, deduplicate_exact_overlaps: bool = False, ) -> dict[str, Any]: """Compose disjoint, fully hashed schedules into one training authority. Component ledgers remain immutable evidence. Their rows are rebound to a deterministic aggregate schedule only after every component hash receipt, byte denominator, current file identity, and declared digest has passed. This avoids rehashing already sealed multi-terabyte payloads while making the existing single-schedule learner consume every component exactly once. No optimizer, trained-knowledge, promotion, or deletion claim is created. """ root = corpus_root.expanduser().resolve() output = output_root.expanduser().resolve() schedule_receipts = [ path.expanduser().resolve() for path in component_schedule_receipt_paths ] hash_receipts = [ path.expanduser().resolve() for path in component_hash_receipt_paths ] if not root.is_dir(): raise FileNotFoundError(f"full payload corpus root is absent: {root}") minimum_components = 1 if deduplicate_exact_overlaps else 2 if ( len(schedule_receipts) < minimum_components or len(schedule_receipts) != len(hash_receipts) ): raise ValueError( "full payload composition has an invalid paired authority count" ) components: list[dict[str, Any]] = [] all_rows: list[dict[str, Any]] = [] hash_rows_by_work_id: dict[str, dict[str, Any]] = {} component_by_work_id: dict[str, dict[str, str]] = {} payload_paths: set[str] = set() schedule_keys: set[str] = set() schedule_rows_by_work_id: dict[str, dict[str, Any]] = {} schedule_rows_by_path: dict[str, dict[str, Any]] = {} schedule_rows_by_key: dict[str, dict[str, Any]] = {} source_ids: set[str] = set() source_payload_files: Counter[str] = Counter() source_payload_bytes: Counter[str] = Counter() refreshed_format_contracts = 0 exact_duplicate_rows = 0 for schedule_receipt_path, hash_receipt_path in zip( schedule_receipts, hash_receipts, strict=True, ): if not schedule_receipt_path.is_file(): raise FileNotFoundError( "component full payload schedule receipt is absent: " f"{schedule_receipt_path}" ) if not hash_receipt_path.is_file(): raise FileNotFoundError( f"component full payload hash receipt is absent: {hash_receipt_path}" ) schedule_receipt = json.loads( schedule_receipt_path.read_text(encoding="utf-8") ) hash_receipt = json.loads(hash_receipt_path.read_text(encoding="utf-8")) if ( not isinstance(schedule_receipt, dict) or schedule_receipt.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA or schedule_receipt.get("passed") is not True ): raise ValueError("component full payload schedule receipt did not pass") if ( not isinstance(hash_receipt, dict) or hash_receipt.get("schema") != FULL_PAYLOAD_HASH_RECEIPT_SCHEMA or hash_receipt.get("passed") is not True ): raise ValueError("component full payload hash receipt did not pass") schedule_record = schedule_receipt.get("schedule") hash_schedule_record = hash_receipt.get("scheduleReceipt") ledger_record = hash_receipt.get("ledger") if ( not isinstance(schedule_record, dict) or not isinstance(hash_schedule_record, dict) or not isinstance(ledger_record, dict) ): raise ValueError("component full payload authority is incomplete") schedule_path = Path(str(schedule_record.get("path", ""))).resolve() schedule_sha256 = str(schedule_record.get("sha256", "")) expected_rows = schedule_record.get("rows") ledger_path = Path(str(ledger_record.get("path", ""))).resolve() schedule_receipt_sha256 = file_sha256(schedule_receipt_path) hash_receipt_sha256 = file_sha256(hash_receipt_path) if ( not schedule_path.is_file() or len(schedule_sha256) != 64 or file_sha256(schedule_path) != schedule_sha256 or not isinstance(expected_rows, int) or isinstance(expected_rows, bool) or expected_rows < 1 or Path(str(hash_schedule_record.get("path", ""))).resolve() != schedule_receipt_path or hash_schedule_record.get("sha256") != schedule_receipt_sha256 or hash_schedule_record.get("scheduleSha256") != schedule_sha256 or Path(str(hash_receipt.get("corpusRoot", ""))).resolve() != root or not ledger_path.is_file() ): raise ValueError("component full payload artifact identity differs") schedule_rows = _read_jsonl(schedule_path) ledger_rows, durable_frontier = ( _read_durable_full_payload_hash_rows( ledger_path, schedule_sha256, ) ) if ( durable_frontier is None or ledger_record.get("sha256") != durable_frontier.prefix_sha256 or ledger_record.get("bytes") != durable_frontier.durable_ledger_bytes or len(schedule_rows) != expected_rows or ledger_record.get("rows") != len(ledger_rows) or len(ledger_rows) != expected_rows or schedule_receipt.get("payloadFileCount") != expected_rows or hash_receipt.get("payloadFileCount") != expected_rows ): raise ValueError("component full payload row denominator differs") component_hash_rows: dict[str, dict[str, Any]] = {} for ledger_row in ledger_rows: work_id = ledger_row.get("payloadWorkId") if ( ledger_row.get("schema") != FULL_PAYLOAD_HASH_LEDGER_SCHEMA or ledger_row.get("scheduleSha256") != schedule_sha256 or not isinstance(work_id, str) or len(work_id) != 64 or work_id in component_hash_rows or ledger_row.get("allPayloadBytesHashed") is not True or ledger_row.get("modelTrainingClaimed") is not False or ledger_row.get("rawDataDeletionAllowed") is not False or ledger_row.get("expectedSha256Matched") not in {None, True} ): raise ValueError("component full payload hash ledger row differs") component_hash_rows[work_id] = ledger_row component_bytes = 0 component_exact_duplicate_rows = 0 for ordinal, schedule_row in enumerate(schedule_rows): work_id = schedule_row.get("payloadWorkId") schedule_key = schedule_row.get("scheduleKey") relative_path = schedule_row.get("payloadRelativePath") payload_bytes = schedule_row.get("payloadBytes") source_id = schedule_row.get("sourceId") if ( schedule_row.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_ENTRY_SCHEMA or schedule_row.get("scheduleOrdinal") != ordinal or not isinstance(work_id, str) or len(work_id) != 64 or not isinstance(schedule_key, str) or len(schedule_key) != 64 or not isinstance(relative_path, str) or not relative_path or Path(relative_path).is_absolute() or ".." in Path(relative_path).parts or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or payload_bytes < 0 or not isinstance(source_id, str) or not source_id or schedule_row.get("textProbeUsed") is not False or schedule_row.get("targetEnteredForward") is not False ): raise ValueError("component full payload schedule rows overlap or differ") matched_hash_row = component_hash_rows.get(work_id) payload_path = root / relative_path if matched_hash_row is None or not payload_path.is_file(): raise ValueError("component full payload hash coverage is incomplete") identity = matched_hash_row.get("sourceIdentity") observed_sha256 = matched_hash_row.get("observedSha256") current = payload_path.stat() expected_sha256 = schedule_row.get("payloadSha256") if ( current.st_size != payload_bytes or matched_hash_row.get("sourceId") != source_id or matched_hash_row.get("payloadRelativePath") != relative_path or matched_hash_row.get("payloadBytes") != payload_bytes or matched_hash_row.get("expectedSha256") != expected_sha256 or not isinstance(observed_sha256, str) or len(observed_sha256) != 64 or (expected_sha256 is not None and observed_sha256 != expected_sha256) or not isinstance(identity, dict) or identity.get("device") != current.st_dev or identity.get("inode") != current.st_ino or identity.get("bytes") != current.st_size or identity.get("mtimeNs") != current.st_mtime_ns ): raise ValueError("component hashed payload identity changed") copied_row = dict(schedule_row) current_format_contract = _payload_format_contract(relative_path) if copied_row.get("format") != current_format_contract: refreshed_format_contracts += 1 copied_row["format"] = current_format_contract overlapping_rows = tuple( row for row in ( schedule_rows_by_work_id.get(work_id), schedule_rows_by_path.get(relative_path), schedule_rows_by_key.get(schedule_key), ) if row is not None ) if overlapping_rows: existing_row = overlapping_rows[0] existing_hash_row = hash_rows_by_work_id.get(work_id) copied_identity = dict(copied_row) existing_identity = dict(existing_row) copied_identity.pop("scheduleOrdinal", None) existing_identity.pop("scheduleOrdinal", None) exact_overlap = bool( deduplicate_exact_overlaps and all(row == existing_row for row in overlapping_rows) and copied_identity == existing_identity and isinstance(existing_hash_row, dict) and existing_hash_row.get("payloadWorkId") == work_id and existing_hash_row.get("sourceId") == source_id and existing_hash_row.get("payloadRelativePath") == relative_path and existing_hash_row.get("payloadBytes") == payload_bytes and existing_hash_row.get("expectedSha256") == expected_sha256 and existing_hash_row.get("observedSha256") == observed_sha256 and existing_hash_row.get("sourceIdentity") == identity ) if not exact_overlap: raise ValueError( "component full payload schedule rows overlap or differ" ) exact_duplicate_rows += 1 component_exact_duplicate_rows += 1 component_bytes += payload_bytes continue all_rows.append(copied_row) hash_rows_by_work_id[work_id] = dict(matched_hash_row) component_by_work_id[work_id] = { "scheduleSha256": schedule_sha256, "hashReceiptSha256": hash_receipt_sha256, } schedule_keys.add(schedule_key) payload_paths.add(relative_path) schedule_rows_by_work_id[work_id] = copied_row schedule_rows_by_path[relative_path] = copied_row schedule_rows_by_key[schedule_key] = copied_row source_ids.add(source_id) source_payload_files[source_id] += 1 source_payload_bytes[source_id] += payload_bytes component_bytes += payload_bytes if ( component_bytes != schedule_receipt.get("payloadBytes") or component_bytes != hash_receipt.get("payloadBytes") ): raise ValueError("component full payload byte denominator differs") components.append( { "scheduleReceiptPath": str(schedule_receipt_path), "scheduleReceiptSha256": schedule_receipt_sha256, "schedulePath": str(schedule_path), "scheduleSha256": schedule_sha256, "hashReceiptPath": str(hash_receipt_path), "hashReceiptSha256": hash_receipt_sha256, "hashLedgerPath": str(ledger_path), "hashLedgerSha256": str(ledger_record["sha256"]), "payloadFileCount": len(schedule_rows), "admittedPayloadFileCount": len(schedule_rows) - component_exact_duplicate_rows, "exactDuplicatePayloadFileCount": component_exact_duplicate_rows, "payloadBytes": component_bytes, } ) all_rows.sort(key=lambda row: (str(row["scheduleKey"]), str(row["payloadWorkId"]))) for ordinal, row in enumerate(all_rows): row["scheduleOrdinal"] = ordinal schedule_path = output / "full_payload_training_schedule.jsonl" schedule_rows_written = _atomic_jsonl(schedule_path, all_rows) schedule_sha256 = file_sha256(schedule_path) composed_hash_rows: list[dict[str, Any]] = [] for schedule_row in all_rows: work_id = str(schedule_row["payloadWorkId"]) hash_row = dict(hash_rows_by_work_id[work_id]) component = component_by_work_id[work_id] hash_row["scheduleSha256"] = schedule_sha256 hash_row["componentScheduleSha256"] = component["scheduleSha256"] hash_row["componentHashReceiptSha256"] = component["hashReceiptSha256"] hash_row["authorityCompositionSchema"] = ( FULL_PAYLOAD_AUTHORITY_COMPOSITION_SCHEMA ) composed_hash_rows.append(hash_row) ledger_path = output / "full_payload_hashes.jsonl" ledger_rows_written = _atomic_jsonl(ledger_path, composed_hash_rows) ledger_sha256 = file_sha256(ledger_path) payload_bytes = sum(int(row["payloadBytes"]) for row in all_rows) local_sha_rows = [row for row in all_rows if row.get("payloadSha256") is not None] reader_files: Counter[str] = Counter() reader_bytes: Counter[str] = Counter() format_files: Counter[str] = Counter() format_bytes: Counter[str] = Counter() for row in all_rows: format_record = row.get("format") if not isinstance(format_record, dict): raise ValueError("composed full payload format contract is absent") reader_family = str(format_record.get("readerFamily", "unknown")) record_format = str(format_record.get("recordFormat", "unknown")) row_bytes = int(row["payloadBytes"]) reader_files[reader_family] += 1 reader_bytes[reader_family] += row_bytes format_files[record_format] += 1 format_bytes[record_format] += row_bytes components.sort(key=lambda row: str(row["scheduleSha256"])) component_identity = hashlib.sha256() for component in components: component_identity.update(str(component["scheduleSha256"]).encode("ascii")) component_identity.update(b"\x00") component_identity.update(str(component["hashReceiptSha256"]).encode("ascii")) component_identity.update(b"\n") composition = { "schema": FULL_PAYLOAD_AUTHORITY_COMPOSITION_SCHEMA, "componentCount": len(components), "componentIdentitySha256": component_identity.hexdigest(), "components": components, "componentSchedulesDisjoint": exact_duplicate_rows == 0, "exactDuplicateRowsDeduplicated": exact_duplicate_rows, "overlapPolicy": ( "exact_identity_deduplication_fail_closed_on_partial_conflict" if deduplicate_exact_overlaps else "strict_disjoint_components" ), "componentHashEvidenceReusedWithoutRehash": True, "formatContractsRefreshedFromCanonicalSource": refreshed_format_contracts, } schedule_checks = { "allComponentScheduleReceiptsPassed": True, "allComponentHashReceiptsPassed": True, "allComponentArtifactHashesVerified": True, "allPayloadWorkIdsDisjoint": schedule_rows_written == len(hash_rows_by_work_id), "allPayloadPathsDisjoint": schedule_rows_written == len(payload_paths), "allPayloadFilesPresentAtCurrentIdentity": True, "allPayloadBytesHashAdmitted": True, "allReaderFormatContractsCurrent": all( row.get("format") == _payload_format_contract(str(row["payloadRelativePath"])) for row in all_rows ), "targetsExcludedFromForward": True, "textProbeSubstitutionAbsent": True, "optimizerNotConsulted": True, } schedule_receipt_path = output / "full_payload_training_schedule.receipt.json" schedule_receipt = { "schema": FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(schedule_checks.values()), "corpusRootAtBuild": str(root), "corpusRootBindingPolicy": "relative_paths_plus_current_identity", "inventoryIdentitySha256": component_identity.hexdigest(), "inventoryInputs": components, "admittedSourceCount": len(source_ids), "payloadManifestSourceCount": len(source_ids), "payloadFileCount": schedule_rows_written, "payloadBytes": payload_bytes, "payloadFilesWithLocalSha256": len(local_sha_rows), "payloadBytesWithLocalSha256": sum( int(row["payloadBytes"]) for row in local_sha_rows ), "payloadFilesRequiringHashAtConsumption": schedule_rows_written - len(local_sha_rows), "payloadBytesRequiringHashAtConsumption": payload_bytes - sum(int(row["payloadBytes"]) for row in local_sha_rows), "sourcePayloadFileCounts": dict(sorted(source_payload_files.items())), "sourcePayloadBytes": dict(sorted(source_payload_bytes.items())), "readerFamilyFileCounts": dict(sorted(reader_files.items())), "readerFamilyBytes": dict(sorted(reader_bytes.items())), "recordFormatFileCounts": dict(sorted(format_files.items())), "recordFormatBytes": dict(sorted(format_bytes.items())), "schedule": { "schema": FULL_PAYLOAD_TRAINING_SCHEDULE_SCHEMA, "path": str(schedule_path), "rows": schedule_rows_written, "bytes": schedule_path.stat().st_size, "sha256": schedule_sha256, }, "composition": composition, "checks": schedule_checks, "scopeNote": ( "This composes complete byte-hash authorities; it does not claim " "that any token window updated model weights." ), "modelTrainingClaimed": False, "completePayloadTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, "nextProofRequired": ( "A retained optimizer transaction and complete work-cursor receipt " "must prove every composed token window entered target-free training." ), } _atomic_json(schedule_receipt_path, schedule_receipt) content_identity = hashlib.sha256() for work_id in sorted(hash_rows_by_work_id): content_identity.update(work_id.encode("ascii")) content_identity.update(b"\x00") content_identity.update( str(hash_rows_by_work_id[work_id]["observedSha256"]).encode("ascii") ) content_identity.update(b"\n") hash_checks = { "scheduleReceiptPassed": schedule_receipt["passed"] is True, "scheduleArtifactHashVerified": file_sha256(schedule_path) == schedule_sha256, "allScheduleRowsValidated": schedule_rows_written == ledger_rows_written, "allPayloadPathsResolvedUnderSelectedRoot": True, "allPayloadFilesHashed": ledger_rows_written == len(hash_rows_by_work_id), "allPayloadBytesHashed": sum( int(row["payloadBytes"]) for row in composed_hash_rows ) == payload_bytes, "allDeclaredPayloadHashesMatch": all( row.get("expectedSha256Matched") in {None, True} for row in composed_hash_rows ), "ledgerDurableAndHashSealed": len(ledger_sha256) == 64, "componentHashReceiptsRemainImmutable": True, "targetsExcludedFromForward": True, "optimizerNotConsulted": True, } hash_receipt_path = output / "full_payload_hashes.receipt.json" hash_receipt = { "schema": FULL_PAYLOAD_HASH_RECEIPT_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(hash_checks.values()), "scheduleReceipt": { "path": str(schedule_receipt_path), "sha256": file_sha256(schedule_receipt_path), "scheduleSha256": schedule_sha256, }, "corpusRoot": str(root), "payloadFileCount": ledger_rows_written, "payloadBytes": payload_bytes, "storageDeviceCount": len( {int(row["storageDevice"]) for row in composed_hash_rows} ), "contentIdentitySha256": content_identity.hexdigest(), "ledger": { "path": str(ledger_path), "rows": ledger_rows_written, "bytes": ledger_path.stat().st_size, "sha256": ledger_sha256, }, "composition": composition, "checks": hash_checks, "modelTrainingClaimed": False, "completePayloadTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, "nextProofRequired": schedule_receipt["nextProofRequired"], } _atomic_json(hash_receipt_path, hash_receipt) composition_checks = { "componentCountMatches": len(components) == len(schedule_receipts), "scheduleReceiptPassed": schedule_receipt["passed"] is True, "hashReceiptPassed": hash_receipt["passed"] is True, "scheduleAndLedgerRowsMatch": schedule_rows_written == ledger_rows_written, "scheduleAndLedgerBytesMatch": sum( int(row["payloadBytes"]) for row in composed_hash_rows ) == payload_bytes, "noModelTrainingClaimed": True, "noPromotionClaimed": True, "noRawDeletionAllowed": True, } composition_receipt_path = ( output / "full_payload_training_authority.composition.receipt.json" ) composition_receipt = { "schema": FULL_PAYLOAD_AUTHORITY_COMPOSITION_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(composition_checks.values()), "corpusRoot": str(root), "componentCount": len(components), "components": components, "payloadFileCount": schedule_rows_written, "payloadBytes": payload_bytes, "scheduleReceipt": { "path": str(schedule_receipt_path), "sha256": file_sha256(schedule_receipt_path), }, "hashReceipt": { "path": str(hash_receipt_path), "sha256": file_sha256(hash_receipt_path), }, "schedule": schedule_receipt["schedule"], "ledger": hash_receipt["ledger"], "checks": composition_checks, "modelTrainingClaimed": False, "completePayloadTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, "nextProofRequired": schedule_receipt["nextProofRequired"], } _atomic_json(composition_receipt_path, composition_receipt) return composition_receipt _DECLARED_SCIENCE_CENSUS_STATES = frozenset({"enumerated", "partial", "empty"}) _DECLARED_SCIENCE_ADMISSION_STATES = frozenset( {"hash_sealed_reader_ready", "unsealed", "unsupported", "empty"} ) _DECLARED_SCIENCE_PARTIAL_SUFFIXES = (".aria2", ".partial", ".part", ".tmp") def _declared_science_root_identity(root: Path) -> dict[str, int]: """Return the stable filesystem identity for one declared source root.""" stat = root.stat() return {"device": stat.st_dev, "inode": stat.st_ino} def _declared_science_identity_matches( value: object, actual: Mapping[str, int], ) -> bool: return bool( isinstance(value, Mapping) and all( isinstance(value.get(key), int) and not isinstance(value.get(key), bool) and value.get(key) == actual[key] for key in ("device", "inode") ) ) def _declared_science_glob_matches(path: str, pattern: str) -> bool: """Match a relative POSIX path while making ``directory/**`` explicit.""" normalized = pattern.strip().replace("\\", "/") if not normalized or normalized.startswith("/") or ".." in normalized.split("/"): raise ValueError("declared science selector glob is invalid") if normalized.endswith("/**"): prefix = normalized[:-3].rstrip("/") return path == prefix or path.startswith(f"{prefix}/") return fnmatch.fnmatchcase(path, normalized) def _declared_science_selector_paths( root: Path, selectors: Sequence[Mapping[str, object]], *, excluded_members: frozenset[str], ) -> Iterator[tuple[str, Path]]: """Yield declared regular files once, in deterministic source order. This is intentionally metadata-only. It never opens, hashes, or decodes a payload, and it refuses symlink traversal so a declared disk cannot absorb training state, mirrors, or an unrelated mount through a link. """ for directory, child_directories, filenames in os.walk( root, topdown=True, followlinks=False, ): directory_path = Path(directory) child_directories[:] = sorted( child for child in child_directories if not (directory_path / child).is_symlink() ) for filename in sorted(filenames): candidate = directory_path / filename if candidate.is_symlink() or not candidate.is_file(): continue relative = candidate.relative_to(root).as_posix() if relative in excluded_members: continue selected = False for selector in selectors: includes = selector["includeGlobs"] excludes = selector["excludeGlobs"] assert isinstance(includes, tuple) assert isinstance(excludes, tuple) if any( _declared_science_glob_matches(relative, pattern) for pattern in includes ) and not any( _declared_science_glob_matches(relative, pattern) for pattern in excludes ): selected = True break if selected: yield relative, candidate def _declared_science_metadata_artifacts( values: object, *, root: Path, ) -> list[dict[str, object]]: """Fence declared provenance metadata without treating it as payload.""" if values is None: return [] if not isinstance(values, list): raise ValueError("declared science metadata authorities are malformed") artifacts: list[dict[str, object]] = [] for value in values: if not isinstance(value, dict): raise ValueError("declared science metadata authority is malformed") path_value = value.get("path") expected_sha256 = value.get("sha256") if ( not isinstance(path_value, str) or not path_value or not isinstance(expected_sha256, str) or len(expected_sha256) != 64 ): raise ValueError("declared science metadata authority is incomplete") candidate = Path(path_value).expanduser() path = (candidate if candidate.is_absolute() else root / candidate).resolve() if not path.is_file() or file_sha256(path) != expected_sha256: raise RuntimeError("declared science metadata authority differs") artifacts.append( { "path": str(path), "bytes": path.stat().st_size, "sha256": expected_sha256, } ) return artifacts def _declared_science_membership_authority_records( value: object, *, root: Path, relative_prefix: str | None = None, allow_member_drift: bool = False, ) -> tuple[dict[str, dict[str, object]], dict[str, object]]: """Read exact canonical members from a sealed multiroot authority. A membership selector is the only way this census may use a pre-existing federation as a source set. It retains its original source paths and hashes instead of recursively scanning a broad container and calling that broad container complete. """ if not isinstance(value, dict): raise ValueError("declared science membership authority is malformed") path_value = value.get("path") expected_sha256 = value.get("sha256") if ( not isinstance(path_value, str) or not path_value or not isinstance(expected_sha256, str) or len(expected_sha256) != 64 ): raise ValueError("declared science membership authority is incomplete") if value.get("allowMemberDrift", False) is not allow_member_drift: raise ValueError("declared science membership drift policy differs") authority_path = Path(path_value).expanduser().resolve() if not authority_path.is_file() or file_sha256(authority_path) != expected_sha256: raise RuntimeError("declared science membership authority differs") authority = json.loads(authority_path.read_text(encoding="utf-8")) recorded_authority_sha256 = ( authority.pop("federationAuthoritySha256", None) if isinstance(authority, dict) else None ) membership_artifact = authority.get("membership") if isinstance(authority, dict) else None if ( not isinstance(authority, dict) or authority.get("schema") != FULL_PAYLOAD_MULTIROOT_AUTHORITY_SCHEMA or authority.get("passed") is not True or not isinstance(recorded_authority_sha256, str) or len(recorded_authority_sha256) != 64 or recorded_authority_sha256 != _sha256_bytes(_json_bytes(authority)) or not _full_payload_packed_artifact_matches(membership_artifact) ): raise ValueError("declared science membership authority is invalid") assert isinstance(membership_artifact, dict) membership_path = Path( str(membership_artifact["path"]) ).expanduser().resolve() prefix = None if relative_prefix is not None: prefix = relative_prefix.strip().strip("/") if not prefix or ".." in prefix.split("/"): raise ValueError("declared science membership prefix is invalid") records: dict[str, dict[str, object]] = {} drift_rows: list[dict[str, object]] = [] with membership_path.open(encoding="utf-8") as handle: for line_number, line in enumerate(handle, start=1): if not line.strip(): continue membership_row = json.loads(line) if not isinstance(membership_row, dict): raise ValueError("declared science membership row is malformed") canonical = membership_row.get("canonical") if not isinstance(canonical, dict): raise ValueError("declared science membership row is malformed") source_root_value = canonical.get("corpusRoot") path = canonical.get("payloadRelativePath") payload_bytes = canonical.get("payloadBytes") observed_sha256 = canonical.get("observedSha256") format_contract = canonical.get("canonicalFormatContract") if ( membership_row.get("schema") != FULL_PAYLOAD_MULTIROOT_MEMBERSHIP_SCHEMA or not isinstance(source_root_value, str) or not isinstance(path, str) or not path or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or payload_bytes < 0 or not isinstance(observed_sha256, str) or len(observed_sha256) != 64 or not isinstance(format_contract, dict) ): raise ValueError( "declared science membership row differs " f"at line {line_number}" ) relative = path if prefix is None: if Path(source_root_value).expanduser().resolve() != root: continue else: required_prefix = f"{prefix}/" if not relative.startswith(required_prefix): continue relative = relative.removeprefix(required_prefix) candidate = (root / relative).resolve() if ( not candidate.is_relative_to(root) or not candidate.is_file() or candidate.is_symlink() ): if not allow_member_drift: raise RuntimeError( "declared science membership payload differs: " f"{relative}" ) drift_rows.append( { "payloadRelativePath": relative, "expectedPayloadBytes": payload_bytes, "reason": "absent_or_nonregular_or_symlink", } ) continue if candidate.stat().st_size != payload_bytes: if not allow_member_drift: raise RuntimeError( "declared science membership byte denominator differs: " f"{relative}" ) drift_rows.append( { "payloadRelativePath": relative, "expectedPayloadBytes": payload_bytes, "observedPayloadBytes": candidate.stat().st_size, "reason": "byte_count_differs", } ) continue format_sha256 = _sha256_bytes(_json_bytes(format_contract)) record = { "relativePath": relative, "payloadBytes": payload_bytes, "observedSha256": observed_sha256, "format": format_contract, "formatSha256": format_sha256, "membershipRowSha256": _sha256_bytes(_json_bytes(membership_row)), } prior = records.setdefault(relative, record) if prior != record: raise RuntimeError("declared science membership path conflicts") if not records and not allow_member_drift: raise ValueError("declared science membership selector has no root members") return ( records, { "authority": { "path": str(authority_path), "bytes": authority_path.stat().st_size, "sha256": expected_sha256, }, "federationAuthoritySha256": recorded_authority_sha256, "rawSourceRevalidated": False, "allowMemberDrift": allow_member_drift, "membershipDriftMemberCount": len(drift_rows), "membershipDriftPayloadBytes": sum( cast(int, row["expectedPayloadBytes"]) for row in drift_rows ), "membershipDriftSha256": _sha256_bytes(_json_bytes(drift_rows)), }, ) def _write_immutable_declared_science_membership( path: Path, rows: Iterable[dict[str, object]], ) -> dict[str, object]: """Write an immutable streaming census ledger without buffering payload rows.""" path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") temporary.unlink(missing_ok=True) digest = hashlib.sha256() row_count = 0 byte_count = 0 with temporary.open("wb") as handle: for row in rows: encoded = _json_bytes(row) + b"\n" handle.write(encoded) digest.update(encoded) row_count += 1 byte_count += len(encoded) handle.flush() os.fsync(handle.fileno()) if row_count < 1: temporary.unlink(missing_ok=True) raise ValueError("declared science census has no selected payload members") sha256 = digest.hexdigest() if path.exists(): if path.stat().st_size != byte_count or file_sha256(path) != sha256: temporary.unlink(missing_ok=True) raise RuntimeError("declared science membership ledger differs") temporary.unlink(missing_ok=True) else: os.replace(temporary, path) return { "path": str(path), "bytes": byte_count, "sha256": sha256, "rows": row_count, } def compose_declared_science_corpus_root_authority( manifest_path: Path, *, output_path: Path, ) -> dict[str, Any]: """Census every declared raw science source before federation or packing. This is an external-I/O authority boundary, never a training input. It makes both reader-ready and blocked source sets durable. A root marked partial, unsupported, empty, or unsealed remains visible in the ledger but contributes no canonical training coverage. """ manifest = manifest_path.expanduser().resolve() output = output_path.expanduser().resolve() if not manifest.is_file(): raise FileNotFoundError("declared science corpus manifest is absent") loaded = json.loads(manifest.read_text(encoding="utf-8")) root_values = loaded.get("roots") if isinstance(loaded, dict) else None if ( not isinstance(loaded, dict) or loaded.get("schema") != DECLARED_SCIENCE_CORPUS_ROOT_MANIFEST_SCHEMA or not isinstance(root_values, list) or not root_values ): raise ValueError("declared science corpus manifest is malformed") roots: list[dict[str, object]] = [] root_ids: set[str] = set() for value in root_values: if not isinstance(value, dict): raise ValueError("declared science corpus root is malformed") root_id = value.get("rootId") container_value = value.get("containerPath") census_state = value.get("censusState") admission_state = value.get("admissionState") if ( not isinstance(root_id, str) or not root_id or root_id in root_ids or not isinstance(container_value, str) or not container_value or census_state not in _DECLARED_SCIENCE_CENSUS_STATES or admission_state not in _DECLARED_SCIENCE_ADMISSION_STATES ): raise ValueError("declared science corpus root identity is invalid") root_ids.add(root_id) container = Path(container_value).expanduser().resolve() if not container.is_dir(): raise FileNotFoundError("declared science corpus root is absent") actual_identity = _declared_science_root_identity(container) if not _declared_science_identity_matches( value.get("rootIdentity"), actual_identity, ): raise RuntimeError("declared science corpus root identity differs") membership_authority = value.get("membershipAuthority") selectors_value = value.get("selectors") if membership_authority is not None and selectors_value is not None: raise ValueError("declared science root mixes membership and glob selectors") selectors: list[dict[str, object]] = [] if membership_authority is None: if not isinstance(selectors_value, list) or not selectors_value: raise ValueError("declared science root selectors are absent") for selector_value in selectors_value: if not isinstance(selector_value, dict): raise ValueError("declared science selector is malformed") include_values = selector_value.get("includeGlobs") exclude_values = selector_value.get("excludeGlobs", []) if ( not isinstance(include_values, list) or not include_values or not isinstance(exclude_values, list) or selector_value.get("followSymlinks") is not False or any(not isinstance(pattern, str) for pattern in include_values) or any(not isinstance(pattern, str) for pattern in exclude_values) ): raise ValueError("declared science selector contract is invalid") includes = tuple(include_values) excludes = tuple(exclude_values) for pattern in (*includes, *excludes): _declared_science_glob_matches("probe", pattern) selectors.append( { "includeGlobs": includes, "excludeGlobs": excludes, "followSymlinks": False, } ) roots.append( { "rootId": root_id, "container": container, "rootIdentity": actual_identity, "censusState": census_state, "admissionState": admission_state, "selectors": selectors, "membershipAuthority": membership_authority, "excludeMembershipAuthority": value.get("excludeMembershipAuthority"), "metadataAuthorities": _declared_science_metadata_artifacts( value.get("metadataAuthorities"), root=container, ), } ) roots.sort(key=lambda value: str(value["rootId"])) root_summaries: dict[str, dict[str, Any]] = {} member_sources: dict[str, dict[str, dict[str, object]]] = {} excluded_members_by_root: dict[str, frozenset[str]] = {} membership_bindings: dict[str, dict[str, object] | None] = {} for root_value in roots: root_id = str(root_value["rootId"]) container = cast(Path, root_value["container"]) # Establish the durable root summary before either membership binding # is resolved. An exclusion authority is itself provenance and must # be recorded even when it is the first boundary inspected. root_summaries[root_id] = { "rootId": root_id, "containerPath": str(container), "rootIdentity": root_value["rootIdentity"], "censusState": root_value["censusState"], "admissionState": root_value["admissionState"], "metadataAuthorities": root_value["metadataAuthorities"], "membershipAuthority": None, "excludedMembershipAuthority": None, "selectedFileCount": 0, "selectedPayloadBytes": 0, "observedSha256FileCount": 0, "observedSha256PayloadBytes": 0, "unresolvedSemanticFormatFileCount": 0, "partialMarkerFileCount": 0, "formatFileCounts": {}, "formatPayloadBytes": {}, } membership_authority = root_value["membershipAuthority"] if membership_authority is not None: if not isinstance(membership_authority, dict): raise ValueError("declared science membership authority is malformed") allow_member_drift = membership_authority.get("allowMemberDrift") is True if allow_member_drift and root_value["censusState"] != "partial": raise ValueError("only partial science roots may allow membership drift") records, binding = _declared_science_membership_authority_records( membership_authority, root=container, allow_member_drift=allow_member_drift, ) member_sources[root_id] = records membership_bindings[root_id] = binding else: member_sources[root_id] = {} membership_bindings[root_id] = None root_summaries[root_id]["membershipAuthority"] = membership_bindings[root_id] excluded_authority = root_value["excludeMembershipAuthority"] if excluded_authority is None: excluded_members_by_root[root_id] = frozenset() else: if not isinstance(excluded_authority, dict): raise ValueError("declared science exclusion authority is malformed") prefix_value = excluded_authority.get("containerRelativePrefix") if not isinstance(prefix_value, str): raise ValueError("declared science exclusion prefix is absent") allow_member_drift = excluded_authority.get("allowMemberDrift") is True excluded_records, exclusion_binding = _declared_science_membership_authority_records( excluded_authority, root=container, relative_prefix=prefix_value, allow_member_drift=allow_member_drift, ) excluded_members_by_root[root_id] = frozenset(excluded_records) root_summaries[root_id]["excludedMembershipAuthority"] = exclusion_binding def membership_rows() -> Iterator[dict[str, object]]: ordinal = 0 for root_value in roots: root_id = str(root_value["rootId"]) summary = root_summaries[root_id] container = cast(Path, root_value["container"]) member_ordinal = 0 source_records = member_sources[root_id] if root_value["membershipAuthority"] is None: candidates: Iterable[tuple[str, Path, dict[str, object] | None]] = ( (relative, path, None) for relative, path in _declared_science_selector_paths( container, cast(Sequence[Mapping[str, object]], root_value["selectors"]), excluded_members=excluded_members_by_root[root_id], ) ) else: candidates = ( ( relative, (container / relative).resolve(), source_records[relative], ) for relative in sorted(source_records) ) for relative, path, source_record in candidates: if not path.is_relative_to(container) or not path.is_file() or path.is_symlink(): raise RuntimeError("declared science source member differs") payload_bytes = path.stat().st_size if source_record is None: format_contract = _payload_format_contract(relative) observed_sha256: str | None = None source_observation = "not_hash_sealed" membership_row_sha256: str | None = None else: source_bytes = source_record["payloadBytes"] format_value = source_record["format"] source_sha256 = source_record["observedSha256"] if ( not isinstance(source_bytes, int) or payload_bytes != source_bytes or not isinstance(format_value, dict) or not isinstance(source_sha256, str) ): raise RuntimeError("declared science membership payload differs") format_contract = format_value observed_sha256 = source_sha256 source_observation = "sealed_multiroot_membership" membership_value = source_record.get("membershipRowSha256") membership_row_sha256 = ( membership_value if isinstance(membership_value, str) else None ) format_name = str(format_contract.get("recordFormat", "unknown")) format_file_counts = cast(dict[str, int], summary["formatFileCounts"]) format_payload_bytes = cast(dict[str, int], summary["formatPayloadBytes"]) format_file_counts[format_name] = format_file_counts.get(format_name, 0) + 1 format_payload_bytes[format_name] = ( format_payload_bytes.get(format_name, 0) + payload_bytes ) summary["selectedFileCount"] = int(summary["selectedFileCount"]) + 1 summary["selectedPayloadBytes"] = ( int(summary["selectedPayloadBytes"]) + payload_bytes ) if observed_sha256 is not None: summary["observedSha256FileCount"] = ( int(summary["observedSha256FileCount"]) + 1 ) summary["observedSha256PayloadBytes"] = ( int(summary["observedSha256PayloadBytes"]) + payload_bytes ) if format_name == "unknown": summary["unresolvedSemanticFormatFileCount"] = ( int(summary["unresolvedSemanticFormatFileCount"]) + 1 ) if relative.lower().endswith(_DECLARED_SCIENCE_PARTIAL_SUFFIXES): summary["partialMarkerFileCount"] = ( int(summary["partialMarkerFileCount"]) + 1 ) yield { "schema": DECLARED_SCIENCE_CORPUS_ROOT_MEMBER_SCHEMA, "membershipOrdinal": ordinal, "rootMemberOrdinal": member_ordinal, "rootId": root_id, "containerPath": str(container), "payloadRelativePath": relative, "payloadBytes": payload_bytes, "observedSha256": observed_sha256, "sourceHashObservation": source_observation, "format": format_contract, "formatSha256": _sha256_bytes(_json_bytes(format_contract)), "membershipRowSha256": membership_row_sha256, "censusState": root_value["censusState"], "admissionState": root_value["admissionState"], "targetEnteredForward": False, } ordinal += 1 member_ordinal += 1 if ( root_value["censusState"] == "empty" and summary["selectedFileCount"] != 0 ): raise RuntimeError("declared empty science root has payload members") # A partial root is an authoritative record of an incomplete or # drifted source, including the legitimate case where every # formerly selected member has disappeared. Only an enumerated # root promises a non-empty, current membership. if ( root_value["censusState"] == "enumerated" and summary["selectedFileCount"] == 0 ): raise RuntimeError("declared science root unexpectedly has no payload members") membership_path = output.with_name(f"{output.stem}.membership.jsonl") _write_immutable_declared_science_membership( membership_path, membership_rows(), ) membership_artifact = _packed_artifact(membership_path) summaries = [root_summaries[str(root_value["rootId"])] for root_value in roots] canonical_coverage_complete = all( summary["censusState"] == "enumerated" and summary["admissionState"] == "hash_sealed_reader_ready" and summary["selectedFileCount"] == summary["observedSha256FileCount"] for summary in summaries ) authority = { "schema": DECLARED_SCIENCE_CORPUS_ROOT_AUTHORITY_SCHEMA, "passed": True, "manifest": _packed_artifact(manifest), "membership": membership_artifact, "roots": summaries, "declaredRootCount": len(summaries), "selectedPayloadFileCount": sum( int(summary["selectedFileCount"]) for summary in summaries ), "selectedPayloadBytes": sum( int(summary["selectedPayloadBytes"]) for summary in summaries ), "hashSealedPayloadFileCount": sum( int(summary["observedSha256FileCount"]) for summary in summaries ), "hashSealedPayloadBytes": sum( int(summary["observedSha256PayloadBytes"]) for summary in summaries ), "canonicalPayloadCoverageComplete": canonical_coverage_complete, "checks": { "allDeclaredRootsEnumerated": all( summary["censusState"] in {"enumerated", "partial", "empty"} for summary in summaries ), "allDeclaredRootIdentitiesMatch": True, "metadataAuthoritiesRemainCurrent": True, "everySelectedMemberIsRegularAndNoSymlinkWasFollowed": True, "partialUnsupportedAndEmptyRootsRetained": True, "unsealedRootsRetainedWithZeroCanonicalCoverage": True, "canonicalPayloadCoverageComplete": canonical_coverage_complete, "targetsExcludedFromForward": True, "modelTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, }, "modelTrainingClaimed": False, "completePayloadTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, } authority["scienceCensusAuthoritySha256"] = _sha256_bytes(_json_bytes(authority)) if output.is_file(): existing = json.loads(output.read_text(encoding="utf-8")) if existing != authority: raise RuntimeError("declared science corpus authority differs") else: _atomic_json(output, authority) return authority def _validated_declared_science_corpus_root_authority( authority_path: Path, *, validate_native_roots: bool, ) -> dict[str, Any]: """Validate one immutable science census without promoting its payloads.""" path = authority_path.expanduser().resolve() if not path.is_file(): raise FileNotFoundError("declared science corpus authority is absent") loaded = json.loads(path.read_text(encoding="utf-8")) recorded_sha256 = ( loaded.pop("scienceCensusAuthoritySha256", None) if isinstance(loaded, dict) else None ) membership = loaded.get("membership") if isinstance(loaded, dict) else None manifest = loaded.get("manifest") if isinstance(loaded, dict) else None roots = loaded.get("roots") if isinstance(loaded, dict) else None if ( not isinstance(loaded, dict) or loaded.get("schema") != DECLARED_SCIENCE_CORPUS_ROOT_AUTHORITY_SCHEMA or loaded.get("passed") is not True or not isinstance(recorded_sha256, str) or len(recorded_sha256) != 64 or recorded_sha256 != _sha256_bytes(_json_bytes(loaded)) or not isinstance(membership, dict) or not isinstance(manifest, dict) or not isinstance(roots, list) or not roots ): raise ValueError("declared science corpus authority differs") for artifact in (membership, manifest): artifact_path_value = artifact.get("path") expected_sha256 = artifact.get("sha256") expected_bytes = artifact.get("bytes") if ( not isinstance(artifact_path_value, str) or not isinstance(expected_sha256, str) or len(expected_sha256) != 64 or not isinstance(expected_bytes, int) or isinstance(expected_bytes, bool) ): raise ValueError("declared science authority artifact is malformed") artifact_path = Path(artifact_path_value).expanduser().resolve() if ( not artifact_path.is_file() or artifact_path.stat().st_size != expected_bytes or file_sha256(artifact_path) != expected_sha256 ): raise RuntimeError("declared science authority artifact changed") root_ids: set[str] = set() for root in roots: if not isinstance(root, dict): raise ValueError("declared science authority root is malformed") root_id = root.get("rootId") path_value = root.get("containerPath") if ( not isinstance(root_id, str) or not root_id or root_id in root_ids or not isinstance(path_value, str) or root.get("censusState") not in _DECLARED_SCIENCE_CENSUS_STATES or root.get("admissionState") not in _DECLARED_SCIENCE_ADMISSION_STATES ): raise ValueError("declared science authority root differs") root_ids.add(root_id) if validate_native_roots: container = Path(path_value).expanduser().resolve() if not container.is_dir() or not _declared_science_identity_matches( root.get("rootIdentity"), _declared_science_root_identity(container), ): raise RuntimeError("declared science authority root changed") loaded["scienceCensusAuthoritySha256"] = recorded_sha256 return loaded def _declared_science_census_component_coverage( authority_path: Path, components: Sequence[ tuple[Mapping[str, Any], Sequence[Mapping[str, Any]], Sequence[Mapping[str, Any]]] ], ) -> dict[str, object]: """Bind selected native components to every declared science-root state. This closes the semantic gap between a component-scoped federation and a host-wide corpus claim. A ready root must match its exact census ledger; an unsealed, unsupported, partial, or empty root remains recorded with no canonical coverage rather than disappearing from the release narrative. """ census_path = authority_path.expanduser().resolve() census = _validated_declared_science_corpus_root_authority( census_path, validate_native_roots=True, ) roots_value = census["roots"] assert isinstance(roots_value, list) roots: list[dict[str, Any]] = [ dict(value) for value in roots_value if isinstance(value, dict) ] if len(roots) != len(roots_value): raise ValueError("declared science census roots are malformed") roots_by_path: dict[Path, dict[str, Any]] = {} for root_record in roots: container_path = Path( str(root_record["containerPath"]) ).expanduser().resolve() if container_path in roots_by_path: raise ValueError("declared science census root paths conflict") roots_by_path[container_path] = root_record component_rows_by_root: dict[ Path, list[tuple[Mapping[str, Any], Sequence[Mapping[str, Any]], Sequence[Mapping[str, Any]]]], ] = {} for component, schedule_rows, hash_rows in components: root_value = component.get("corpusRoot") if not isinstance(root_value, str): raise ValueError("full payload component root is malformed") component_root = Path(root_value).expanduser().resolve() census_root = roots_by_path.get(component_root) if census_root is None: raise RuntimeError("full payload component is absent from science census") if ( census_root.get("censusState") != "enumerated" or census_root.get("admissionState") != "hash_sealed_reader_ready" ): raise RuntimeError("science census root is not admitted for federation") component_rows_by_root.setdefault(component_root, []).append( (component, schedule_rows, hash_rows) ) component_root_ids = { str(roots_by_path[root]["rootId"]) for root in component_rows_by_root } membership_value = census["membership"] assert isinstance(membership_value, dict) membership_path = Path(str(membership_value["path"])).expanduser().resolve() expected_rows_by_root: dict[ str, dict[str, tuple[int, str, str]], ] = {root_id: {} for root_id in component_root_ids} with membership_path.open(encoding="utf-8") as handle: for line_number, line in enumerate(handle, start=1): if not line.strip(): continue row = json.loads(line) if not isinstance(row, dict): raise ValueError("declared science membership row is malformed") root_id = row.get("rootId") if root_id not in component_root_ids: continue relative_path = row.get("payloadRelativePath") payload_bytes = row.get("payloadBytes") observed_sha256 = row.get("observedSha256") format_sha256 = row.get("formatSha256") if ( row.get("schema") != DECLARED_SCIENCE_CORPUS_ROOT_MEMBER_SCHEMA or not isinstance(relative_path, str) or not relative_path or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or not isinstance(observed_sha256, str) or len(observed_sha256) != 64 or not isinstance(format_sha256, str) or len(format_sha256) != 64 ): raise ValueError( "declared science admitted membership row differs " f"at line {line_number}" ) expected = (payload_bytes, observed_sha256, format_sha256) prior = expected_rows_by_root[root_id].setdefault( relative_path, expected, ) if prior != expected: raise RuntimeError("declared science membership path conflicts") coverage_rows: list[dict[str, object]] = [] for root in roots: root_id = str(root["rootId"]) root_path = Path(str(root["containerPath"])).expanduser().resolve() component_group = component_rows_by_root.get(root_path, []) state = str(root["censusState"]) admission = str(root["admissionState"]) entry: dict[str, object] = { "rootId": root_id, "containerPath": str(root_path), "censusState": state, "admissionState": admission, "selectedFileCount": root["selectedFileCount"], "selectedPayloadBytes": root["selectedPayloadBytes"], "componentIds": sorted( str(component["componentId"]) for component, _schedule_rows, _hash_rows in component_group ), "componentCount": len(component_group), "canonicalCoverageFileCount": 0, "canonicalCoverageBytes": 0, "coverageState": "not_admitted", } if state == "enumerated" and admission == "hash_sealed_reader_ready": if not component_group: entry["coverageState"] = "admitted_component_missing" else: expected_rows = expected_rows_by_root[root_id] actual_rows: dict[str, tuple[int, str, str]] = {} for _component, schedule_rows, hash_rows in component_group: if len(schedule_rows) != len(hash_rows): raise RuntimeError("science census component row count differs") for schedule_row, hash_row in zip( schedule_rows, hash_rows, strict=True, ): schedule_relative_path = schedule_row.get( "payloadRelativePath" ) payload_bytes = schedule_row.get("payloadBytes") observed_sha256 = hash_row.get("observedSha256") format_contract = schedule_row.get("format") if ( not isinstance(schedule_relative_path, str) or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or not isinstance(observed_sha256, str) or len(observed_sha256) != 64 or not isinstance(format_contract, dict) ): raise RuntimeError("science census component row is malformed") actual = ( payload_bytes, observed_sha256, _sha256_bytes(_json_bytes(format_contract)), ) prior = actual_rows.setdefault(schedule_relative_path, actual) if prior != actual: raise RuntimeError("science census component path conflicts") if actual_rows != expected_rows: raise RuntimeError("science census component coverage differs") entry.update( { "canonicalCoverageFileCount": len(actual_rows), "canonicalCoverageBytes": sum( value[0] for value in actual_rows.values() ), "coverageState": "covered_exactly", } ) coverage_rows.append(entry) canonical_coverage_complete = bool(coverage_rows) and all( row["coverageState"] == "covered_exactly" for row in coverage_rows ) return { "authority": _packed_artifact(census_path), "scienceCensusAuthoritySha256": census["scienceCensusAuthoritySha256"], "declaredRootCount": len(coverage_rows), "rootCoverage": coverage_rows, "canonicalPayloadCoverageComplete": canonical_coverage_complete, "checks": { "allFederatedComponentsAreDeclared": True, "admittedRootsMatchExactCensusMembers": True, "partialUnsupportedUnsealedAndEmptyRootsRemainVisible": True, "canonicalPayloadCoverageComplete": canonical_coverage_complete, "targetsExcludedFromForward": True, }, } def _full_payload_multiroot_root_identity(root: Path) -> dict[str, int]: """Return the filesystem identity that keeps one component root distinct.""" stat = root.stat() return {"device": stat.st_dev, "inode": stat.st_ino} def _validated_full_payload_multiroot_component( corpus_root: Path, schedule_receipt_path: Path, hash_receipt_path: Path, reader_readiness_receipt_path: Path, ) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]: """Validate one native corpus root for a cross-root federation. This keeps the existing single-root schedule contract intact. The only raw payload observation is the metadata-only identity fence already used by the pack builder; no payload is copied, rehashed, or decoded here. """ root = corpus_root.expanduser().resolve() schedule_receipt = schedule_receipt_path.expanduser().resolve() hash_receipt = hash_receipt_path.expanduser().resolve() readiness_receipt = reader_readiness_receipt_path.expanduser().resolve() if not root.is_dir(): raise FileNotFoundError(f"full payload component root is absent: {root}") if not ( schedule_receipt.is_file() and hash_receipt.is_file() and readiness_receipt.is_file() ): raise FileNotFoundError("full payload multiroot component receipt is absent") schedule_value = json.loads(schedule_receipt.read_text(encoding="utf-8")) hash_value = json.loads(hash_receipt.read_text(encoding="utf-8")) readiness_value = json.loads(readiness_receipt.read_text(encoding="utf-8")) schedule_record = ( schedule_value.get("schedule") if isinstance(schedule_value, dict) else None ) hash_schedule_record = ( hash_value.get("scheduleReceipt") if isinstance(hash_value, dict) else None ) ledger_record = hash_value.get("ledger") if isinstance(hash_value, dict) else None readiness_schedule_record = ( readiness_value.get("scheduleReceipt") if isinstance(readiness_value, dict) else None ) schedule_sha256 = ( schedule_record.get("sha256") if isinstance(schedule_record, dict) else None ) if ( not isinstance(schedule_value, dict) or schedule_value.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA or schedule_value.get("passed") is not True or Path(str(schedule_value.get("corpusRootAtBuild", ""))).resolve() != root or not isinstance(schedule_record, dict) or not isinstance(schedule_sha256, str) or len(schedule_sha256) != 64 or not isinstance(hash_value, dict) or hash_value.get("schema") != FULL_PAYLOAD_HASH_RECEIPT_SCHEMA or hash_value.get("passed") is not True or Path(str(hash_value.get("corpusRoot", ""))).resolve() != root or not isinstance(hash_schedule_record, dict) or Path(str(hash_schedule_record.get("path", ""))).resolve() != schedule_receipt or hash_schedule_record.get("sha256") != file_sha256(schedule_receipt) or hash_schedule_record.get("scheduleSha256") != schedule_sha256 or not isinstance(ledger_record, dict) or not isinstance(ledger_record.get("path"), str) or not isinstance(ledger_record.get("sha256"), str) or len(str(ledger_record["sha256"])) != 64 or not isinstance(readiness_value, dict) or readiness_value.get("schema") != FULL_PAYLOAD_READER_READINESS_SCHEMA or readiness_value.get("passed") is not True or readiness_value.get("formatContractDriftCount") != 0 or readiness_value.get("unavailableDecoderModules") != [] or readiness_value.get("unresolvedReaderWorkIds") != [] or not isinstance(readiness_schedule_record, dict) or Path(str(readiness_schedule_record.get("path", ""))).resolve() != schedule_receipt or readiness_schedule_record.get("sha256") != file_sha256(schedule_receipt) ): raise ValueError("full payload multiroot component authority differs") ledger_path = Path(str(ledger_record["path"])).expanduser().resolve() _durable_rows, durable_frontier = ( _read_durable_full_payload_hash_rows( ledger_path, schedule_sha256, ) if ledger_path.is_file() else ([], None) ) if ( not ledger_path.is_file() or durable_frontier is None or durable_frontier.prefix_sha256 != ledger_record["sha256"] or durable_frontier.durable_ledger_bytes != ledger_record.get("bytes") or ledger_record.get("rows") != hash_value.get("payloadFileCount") or hash_value.get("payloadFileCount") != schedule_record.get("rows") or hash_value.get("payloadBytes") != schedule_value.get("payloadBytes") ): raise ValueError("full payload multiroot component ledger differs") current_readiness = full_payload_reader_readiness_receipt(schedule_receipt) if ( current_readiness.get("passed") is not True or current_readiness.get("formatContractDriftCount") != 0 or current_readiness.get("unavailableDecoderModules") != [] or current_readiness.get("unresolvedReaderWorkIds") != [] or current_readiness.get("readerFamilyFileCounts") != readiness_value.get("readerFamilyFileCounts") or current_readiness.get("readerFamilyBytes") != readiness_value.get("readerFamilyBytes") ): raise RuntimeError("full payload multiroot component reader readiness drifted") ( _validated_schedule_receipt, schedule_path, validated_schedule_sha256, schedule_rows, hash_rows, ) = _validated_full_payload_packed_sources( schedule_receipt, root, ledger_path, ) if ( validated_schedule_sha256 != schedule_sha256 or len(schedule_rows) != schedule_record.get("rows") or len(hash_rows) != ledger_record.get("rows") or sum(int(row["payloadBytes"]) for row in schedule_rows) != schedule_value.get("payloadBytes") ): raise RuntimeError("full payload multiroot component schedule differs") component = { "corpusRoot": str(root), "rootIdentity": _full_payload_multiroot_root_identity(root), "scheduleReceipt": _packed_artifact(schedule_receipt), "schedule": _packed_artifact(schedule_path), "scheduleSha256": schedule_sha256, "hashReceipt": _packed_artifact(hash_receipt), "hashLedger": _packed_artifact(ledger_path), "readerReadinessReceipt": _packed_artifact(readiness_receipt), "contentIdentitySha256": hash_value.get("contentIdentitySha256"), "payloadFileCount": len(schedule_rows), "payloadBytes": sum(int(row["payloadBytes"]) for row in schedule_rows), "readerFamilyFileCounts": readiness_value.get("readerFamilyFileCounts"), "readerFamilyBytes": readiness_value.get("readerFamilyBytes"), } if ( not isinstance(component["contentIdentitySha256"], str) or len(str(component["contentIdentitySha256"])) != 64 ): raise ValueError("full payload multiroot component content identity differs") component["componentId"] = _sha256_bytes(_json_bytes(component)) return component, schedule_rows, hash_rows def _full_payload_multiroot_membership_rows( component_rows: Sequence[ tuple[Mapping[str, Any], Sequence[Mapping[str, Any]], Sequence[Mapping[str, Any]]] ], ) -> list[dict[str, Any]]: """Build deterministic canonical membership while retaining every provenance.""" by_content_key: dict[str, list[dict[str, Any]]] = {} format_by_raw_identity: dict[str, str] = {} work_identity_by_id: dict[str, tuple[str, int, str]] = {} for component, schedule_rows, hash_rows in component_rows: component_id = component.get("componentId") root = component.get("corpusRoot") if ( not isinstance(component_id, str) or len(component_id) != 64 or not isinstance(root, str) or len(schedule_rows) != len(hash_rows) ): raise ValueError("full payload multiroot component membership is malformed") for schedule_row, hash_row in zip(schedule_rows, hash_rows, strict=True): work_id = schedule_row.get("payloadWorkId") observed_sha256 = hash_row.get("observedSha256") payload_bytes = schedule_row.get("payloadBytes") format_contract = schedule_row.get("format") source_id = schedule_row.get("sourceId") source_record_sha256 = schedule_row.get("sourceRecordSha256") if ( not isinstance(work_id, str) or len(work_id) != 64 or not isinstance(observed_sha256, str) or len(observed_sha256) != 64 or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or payload_bytes < 0 or not isinstance(format_contract, dict) or not isinstance(source_id, str) or not source_id or not isinstance(source_record_sha256, str) or len(source_record_sha256) != 64 ): raise ValueError("full payload multiroot membership row is malformed") format_sha256 = _sha256_bytes(_json_bytes(format_contract)) raw_identity = _sha256_bytes( _json_bytes( { "schema": "nnf.resynthesis.full_payload_raw_identity.v1", "observedSha256": observed_sha256, "payloadBytes": payload_bytes, } ) ) prior_format = format_by_raw_identity.setdefault( raw_identity, format_sha256, ) if prior_format != format_sha256: raise RuntimeError( "full payload multiroot raw identity has conflicting format authority" ) work_identity = (observed_sha256, payload_bytes, format_sha256) prior_work_identity = work_identity_by_id.setdefault(work_id, work_identity) if prior_work_identity != work_identity: raise RuntimeError("full payload multiroot work identity conflicts") content_key = _sha256_bytes( _json_bytes( { "schema": FULL_PAYLOAD_MULTIROOT_MEMBERSHIP_SCHEMA, "observedSha256": observed_sha256, "payloadBytes": payload_bytes, "canonicalFormatContract": format_contract, } ) ) candidate = { "componentId": component_id, "corpusRoot": root, "scheduleOrdinal": schedule_row.get("scheduleOrdinal"), "payloadWorkId": work_id, "sourceId": source_id, "sourceRecordSha256": source_record_sha256, "payloadRelativePath": schedule_row.get("payloadRelativePath"), "payloadBytes": payload_bytes, "observedSha256": observed_sha256, "canonicalFormatContract": format_contract, "canonicalFormatContractSha256": format_sha256, "domainAuthority": schedule_row.get("domainAuthority"), "rightsAuthority": schedule_row.get("rightsAuthority"), "hashLedgerRowSha256": _sha256_bytes(_json_bytes(hash_row)), } by_content_key.setdefault(content_key, []).append(candidate) membership: list[dict[str, Any]] = [] for membership_ordinal, content_key in enumerate(sorted(by_content_key)): candidates = by_content_key[content_key] candidates.sort( key=lambda candidate: ( str(candidate["componentId"]), str(candidate["sourceRecordSha256"]), str(candidate["payloadWorkId"]), int(candidate["scheduleOrdinal"]), ) ) canonical = dict(candidates[0]) provenance = [dict(candidate) for candidate in candidates] membership.append( { "schema": FULL_PAYLOAD_MULTIROOT_MEMBERSHIP_SCHEMA, "membershipOrdinal": membership_ordinal, "contentKeySha256": content_key, "canonical": canonical, "duplicateProvenance": provenance, "duplicateProvenanceCount": len(provenance), "exactDuplicatePayload": len(provenance) > 1, "targetEnteredForward": False, } ) if not membership: raise RuntimeError("full payload multiroot authority has no canonical members") return membership def compose_full_payload_multiroot_authority( output_root: Path, components: Sequence[tuple[Path, Path, Path, Path]], *, science_census_authority_path: Path | None = None, ) -> dict[str, Any]: """Federate independently sealed corpus roots without copying raw payloads. Each tuple is ``(corpus_root, schedule_receipt, hash_receipt, reader_readiness_receipt)``. Identical content is deduplicated only when its exact bytes and canonical decoding contract match; every contributing root remains in the membership provenance. """ output = output_root.expanduser().resolve() if not components: raise ValueError("full payload multiroot authority requires components") validated_components: list[ tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]] ] = [] component_ids: set[str] = set() for component_paths in components: if len(component_paths) != 4: raise ValueError("full payload multiroot component tuple is invalid") component, schedule_rows, hash_rows = ( _validated_full_payload_multiroot_component(*component_paths) ) component_id = str(component["componentId"]) if component_id in component_ids: raise ValueError("full payload multiroot component is duplicated") component_ids.add(component_id) validated_components.append((component, schedule_rows, hash_rows)) validated_components.sort(key=lambda value: str(value[0]["componentId"])) membership_rows = _full_payload_multiroot_membership_rows(validated_components) science_census = ( _declared_science_census_component_coverage( science_census_authority_path, validated_components, ) if science_census_authority_path is not None else None ) membership_bytes = b"".join( json.dumps(row, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n" for row in membership_rows ) membership_path = output / "full_payload_multiroot_membership.jsonl" _atomic_bytes(membership_path, membership_bytes) canonical_payload_bytes = sum( int(row["canonical"]["payloadBytes"]) for row in membership_rows ) duplicate_provenance_count = sum( int(row["duplicateProvenanceCount"]) - 1 for row in membership_rows ) authority = { "schema": FULL_PAYLOAD_MULTIROOT_AUTHORITY_SCHEMA, "passed": True, "components": [component for component, _rows, _hashes in validated_components], "componentCount": len(validated_components), "membership": _packed_artifact(membership_path), "canonicalPayloadFileCount": len(membership_rows), "canonicalPayloadBytes": canonical_payload_bytes, "duplicatePayloadProvenanceCount": duplicate_provenance_count, "rawPayloadCopied": False, "syntheticCorpusRootCreated": False, "crossRootDeduplicationKey": ( "observed_sha256_plus_payload_bytes_plus_canonical_format_contract" ), "checks": { "allComponentsPassedScheduleHashAndReaderReadiness": True, "allRawSourceIdentitiesRemainCurrent": True, "canonicalMembershipCoversEveryComponentPayload": True, "crossRootDuplicatesRequireExactBytesAndFormat": True, "duplicateProvenanceRetained": True, "rawPayloadsRemainAtNativeRoots": True, "declaredScienceCensusBound": science_census is not None, "declaredScienceCanonicalCoverageComplete": ( science_census is not None and science_census["canonicalPayloadCoverageComplete"] is True ), "targetsExcludedFromForward": True, "modelTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, }, "modelTrainingClaimed": False, "completePayloadTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, } if science_census is not None: authority["declaredScienceCensus"] = science_census authority["federationAuthoritySha256"] = _sha256_bytes(_json_bytes(authority)) authority_path = output / "full_payload_multiroot_authority.receipt.json" if authority_path.is_file(): existing = json.loads(authority_path.read_text(encoding="utf-8")) if existing != authority: raise RuntimeError("full payload multiroot authority receipt differs") else: _atomic_json(authority_path, authority) return authority def _validated_full_payload_multiroot_authority( authority_receipt_path: Path, *, validate_native_components: bool, ) -> tuple[ dict[str, Any], list[dict[str, Any]], list[tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]], ]: """Load a federation and optionally re-fence each native raw-data root. Packed training uses ``validate_native_components=False``: after sealing, the GPU reader must rely only on packed artifacts and their recorded schedule/ledger bindings, not require raw payload files to remain mounted. Build and growth planning use the stronger metadata-only source fence. """ authority_path = authority_receipt_path.expanduser().resolve() if not authority_path.is_file(): raise FileNotFoundError("full payload multiroot authority receipt is absent") loaded = json.loads(authority_path.read_text(encoding="utf-8")) recorded_authority = ( loaded.pop("federationAuthoritySha256", None) if isinstance(loaded, dict) else None ) if ( not isinstance(loaded, dict) or loaded.get("schema") != FULL_PAYLOAD_MULTIROOT_AUTHORITY_SCHEMA or loaded.get("passed") is not True or not isinstance(recorded_authority, str) or len(recorded_authority) != 64 or recorded_authority != _sha256_bytes(_json_bytes(loaded)) ): raise ValueError("full payload multiroot authority differs") component_values = loaded.get("components") membership_record = loaded.get("membership") if ( not isinstance(component_values, list) or not component_values or loaded.get("componentCount") != len(component_values) or not isinstance(membership_record, dict) or not _full_payload_packed_artifact_matches(membership_record) ): raise ValueError("full payload multiroot authority artifacts differ") components = [dict(value) for value in component_values if isinstance(value, dict)] if len(components) != len(component_values): raise ValueError("full payload multiroot components are malformed") component_by_id = { str(component.get("componentId", "")): component for component in components } if ( len(component_by_id) != len(components) or any(len(component_id) != 64 for component_id in component_by_id) ): raise ValueError("full payload multiroot component identities differ") membership_path = Path(str(membership_record["path"])).expanduser().resolve() membership_rows = _read_jsonl(membership_path) if ( len(membership_rows) != loaded.get("canonicalPayloadFileCount") or sum(int(row.get("canonical", {}).get("payloadBytes", -1)) for row in membership_rows) != loaded.get("canonicalPayloadBytes") ): raise ValueError("full payload multiroot membership denominator differs") duplicate_provenance_count = 0 for ordinal, row in enumerate(membership_rows): canonical = row.get("canonical") provenance = row.get("duplicateProvenance") content_key = row.get("contentKeySha256") if ( row.get("schema") != FULL_PAYLOAD_MULTIROOT_MEMBERSHIP_SCHEMA or row.get("membershipOrdinal") != ordinal or not isinstance(content_key, str) or len(content_key) != 64 or not isinstance(canonical, dict) or not isinstance(provenance, list) or not provenance or row.get("duplicateProvenanceCount") != len(provenance) or canonical not in provenance or canonical.get("componentId") not in component_by_id or any( not isinstance(value, dict) or value.get("componentId") not in component_by_id for value in provenance ) ): raise ValueError("full payload multiroot membership row differs") duplicate_provenance_count += len(provenance) - 1 if duplicate_provenance_count != loaded.get("duplicatePayloadProvenanceCount"): raise ValueError("full payload multiroot duplicate denominator differs") declared_science_census = loaded.get("declaredScienceCensus") if declared_science_census is not None: if not isinstance(declared_science_census, dict): raise ValueError("full payload declared science census is malformed") census_artifact = declared_science_census.get("authority") root_coverage = declared_science_census.get("rootCoverage") coverage_complete = declared_science_census.get( "canonicalPayloadCoverageComplete" ) if ( not _full_payload_packed_artifact_matches(census_artifact) or not isinstance(root_coverage, list) or not root_coverage or not isinstance(coverage_complete, bool) or not isinstance( declared_science_census.get("scienceCensusAuthoritySha256"), str, ) or len(str(declared_science_census["scienceCensusAuthoritySha256"])) != 64 ): raise ValueError("full payload declared science census differs") if validate_native_components: assert isinstance(census_artifact, dict) census = _validated_declared_science_corpus_root_authority( Path(str(census_artifact["path"])), validate_native_roots=True, ) if ( census.get("scienceCensusAuthoritySha256") != declared_science_census.get("scienceCensusAuthoritySha256") ): raise RuntimeError("full payload declared science census changed") validated_components: list[ tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]] ] = [] if validate_native_components: for component in components: schedule_receipt = component.get("scheduleReceipt") hash_receipt = component.get("hashReceipt") readiness_receipt = component.get("readerReadinessReceipt") root_value = component.get("corpusRoot") if ( not isinstance(schedule_receipt, dict) or not isinstance(hash_receipt, dict) or not isinstance(readiness_receipt, dict) or not isinstance(root_value, str) ): raise ValueError("full payload multiroot component paths are absent") current_component, schedule_rows, hash_rows = ( _validated_full_payload_multiroot_component( Path(root_value), Path(str(schedule_receipt.get("path", ""))), Path(str(hash_receipt.get("path", ""))), Path(str(readiness_receipt.get("path", ""))), ) ) if current_component != component: raise RuntimeError("full payload multiroot component changed") validated_components.append( (current_component, schedule_rows, hash_rows) ) validated_components.sort(key=lambda value: str(value[0]["componentId"])) if _full_payload_multiroot_membership_rows(validated_components) != membership_rows: raise RuntimeError("full payload multiroot membership changed") loaded["federationAuthoritySha256"] = recorded_authority return loaded, membership_rows, validated_components def compose_discovered_full_payload_training_epoch( corpus_root: Path, output_root: Path, authority_roots: Sequence[Path], *, epoch_cutoff_ns: int | None = None, ) -> dict[str, Any]: """Discover and compose every complete authority in one corpus epoch. Discovery is an external-I/O boundary. The epoch is closed at a concrete filesystem timestamp, and only passed hash receipts whose own immutable schedule is bound to ``corpus_root`` are admitted. Pending schedules and authorities for other corpus roots remain visible in the receipt but do not enter the learner. Exact duplicate rows may be collapsed; any partial identity collision still fails closed in the strict composer above. """ root = corpus_root.expanduser().resolve() output = output_root.expanduser().resolve() roots = tuple( sorted({path.expanduser().resolve() for path in authority_roots}, key=str) ) if not root.is_dir(): raise FileNotFoundError(f"full payload corpus root is absent: {root}") if not roots or any(not path.is_dir() for path in roots): raise FileNotFoundError("full payload authority discovery root is absent") cutoff_ns = time.time_ns() if epoch_cutoff_ns is None else epoch_cutoff_ns if isinstance(cutoff_ns, bool) or cutoff_ns < 1: raise ValueError("full payload corpus epoch cutoff must be positive") def within_authority_roots(path: Path) -> bool: return any(path.is_relative_to(authority_root) for authority_root in roots) discovered_paths = sorted( { receipt_path.resolve() for authority_root in roots for receipt_path in authority_root.rglob("*receipt.json") if not receipt_path.resolve().is_relative_to(output) }, key=str, ) schedule_receipts: dict[Path, dict[str, Any]] = {} passed_hash_receipts: list[tuple[Path, dict[str, Any]]] = [] scan_records: list[dict[str, Any]] = [] malformed_full_payload_receipts: list[str] = [] receipts_after_cutoff = 0 foreign_corpus_hash_authorities: list[dict[str, Any]] = [] aggregate_hash_authorities: list[dict[str, Any]] = [] failed_hash_authorities: list[dict[str, Any]] = [] for receipt_path in discovered_paths: receipt_stat = receipt_path.stat() if receipt_stat.st_mtime_ns > cutoff_ns: receipts_after_cutoff += 1 continue try: receipt_value = json.loads(receipt_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): if "full_payload" in receipt_path.name: malformed_full_payload_receipts.append(str(receipt_path)) continue if not isinstance(receipt_value, dict): continue schema = receipt_value.get("schema") if schema not in { FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA, FULL_PAYLOAD_HASH_RECEIPT_SCHEMA, }: continue receipt_sha256 = file_sha256(receipt_path) scan_records.append( { "path": str(receipt_path), "sha256": receipt_sha256, "bytes": receipt_stat.st_size, "mtimeNs": receipt_stat.st_mtime_ns, "schema": schema, "passed": receipt_value.get("passed") is True, } ) if schema == FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA: schedule_receipts[receipt_path] = receipt_value continue hash_summary = { "path": str(receipt_path), "sha256": receipt_sha256, "corpusRoot": str(receipt_value.get("corpusRoot", "")), "payloadFileCount": receipt_value.get("payloadFileCount"), "payloadBytes": receipt_value.get("payloadBytes"), } if receipt_value.get("passed") is not True: failed_hash_authorities.append(hash_summary) elif isinstance(receipt_value.get("composition"), dict): aggregate_hash_authorities.append(hash_summary) elif Path(str(receipt_value.get("corpusRoot", ""))).resolve() != root: foreign_corpus_hash_authorities.append(hash_summary) else: passed_hash_receipts.append((receipt_path, receipt_value)) if malformed_full_payload_receipts: raise ValueError("full payload authority discovery found a malformed receipt") selected: list[dict[str, Any]] = [] selected_schedule_paths: set[Path] = set() for hash_receipt_path, hash_receipt in passed_hash_receipts: schedule_binding = hash_receipt.get("scheduleReceipt") if not isinstance(schedule_binding, dict): raise ValueError("discovered full payload hash receipt is incomplete") schedule_receipt_path = Path( str(schedule_binding.get("path", "")) ).resolve() schedule_receipt = schedule_receipts.get(schedule_receipt_path) schedule_record = ( schedule_receipt.get("schedule") if isinstance(schedule_receipt, dict) else None ) if ( not within_authority_roots(schedule_receipt_path) or not isinstance(schedule_receipt, dict) or schedule_receipt.get("passed") is not True or Path( str(schedule_receipt.get("corpusRootAtBuild", "")) ).resolve() != root or not isinstance(schedule_record, dict) or schedule_binding.get("sha256") != file_sha256(schedule_receipt_path) or schedule_binding.get("scheduleSha256") != schedule_record.get("sha256") ): raise ValueError("discovered full payload authority binding differs") selected_schedule_paths.add(schedule_receipt_path) selected.append( { "scheduleReceiptPath": str(schedule_receipt_path), "scheduleReceiptSha256": file_sha256(schedule_receipt_path), "scheduleSha256": schedule_record.get("sha256"), "hashReceiptPath": str(hash_receipt_path), "hashReceiptSha256": file_sha256(hash_receipt_path), "payloadFileCount": hash_receipt.get("payloadFileCount"), "payloadBytes": hash_receipt.get("payloadBytes"), } ) selected.sort( key=lambda record: ( -int(record["payloadFileCount"]), str(record["scheduleSha256"]), str(record["hashReceiptPath"]), ) ) if not selected: raise RuntimeError("no complete full payload authority exists in this epoch") pending_schedule_receipts = sorted( ( { "path": str(path), "sha256": file_sha256(path), "payloadFileCount": value.get("payloadFileCount"), "payloadBytes": value.get("payloadBytes"), } for path, value in schedule_receipts.items() if path not in selected_schedule_paths and value.get("passed") is True and Path(str(value.get("corpusRootAtBuild", ""))).resolve() == root and not isinstance(value.get("composition"), dict) ), key=lambda record: str(record["path"]), ) discovery_identity = hashlib.sha256(_json_bytes(scan_records)).hexdigest() composition_receipt = compose_full_payload_training_authority( root, output, [Path(str(record["scheduleReceiptPath"])) for record in selected], [Path(str(record["hashReceiptPath"])) for record in selected], deduplicate_exact_overlaps=True, ) for record in selected: if ( file_sha256(Path(str(record["scheduleReceiptPath"]))) != record["scheduleReceiptSha256"] or file_sha256(Path(str(record["hashReceiptPath"]))) != record["hashReceiptSha256"] ): raise RuntimeError("full payload authority changed during epoch composition") checks = { "corpusEpochCutoffPositive": cutoff_ns > 0, "allAuthorityRootsPresent": all(path.is_dir() for path in roots), "allCompletedActiveRootAuthoritiesSelected": len(selected) == len(passed_hash_receipts), "selectedAuthorityBytesRemainImmutable": True, "compositionPassed": composition_receipt.get("passed") is True, "exactDuplicatesOnly": True, "partialIdentityConflictsRejected": True, "pendingSchedulesExcludedFromCompletedAuthorityClaim": True, "foreignCorpusRootsExcludedFromActiveRootAuthority": True, "noFailedHashAuthoritiesAtEpoch": not failed_hash_authorities, "targetsExcludedFromForward": True, "optimizerNotConsulted": True, } epoch_receipt_path = output / "full_payload_federated_epoch.receipt.json" epoch_receipt = { "schema": FULL_PAYLOAD_FEDERATED_EPOCH_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(checks.values()), "epochCutoffNs": cutoff_ns, "epochIdentitySha256": discovery_identity, "corpusRoot": str(root), "authorityRoots": [str(path) for path in roots], "scannedFullPayloadReceiptCount": len(scan_records), "receiptsAfterEpochCutoff": receipts_after_cutoff, "selectedCompletedAuthorityCount": len(selected), "selectedCompletedAuthorities": selected, "pendingScheduleReceiptCount": len(pending_schedule_receipts), "pendingScheduleReceipts": pending_schedule_receipts, "foreignCorpusRootHashAuthorityCount": len( foreign_corpus_hash_authorities ), "foreignCorpusRootHashAuthorities": foreign_corpus_hash_authorities, "priorAggregateHashAuthorityCount": len(aggregate_hash_authorities), "priorAggregateHashAuthorities": aggregate_hash_authorities, "failedHashAuthorityCount": len(failed_hash_authorities), "failedHashAuthorities": failed_hash_authorities, "compositionReceipt": { "path": str( output / "full_payload_training_authority.composition.receipt.json" ), "sha256": file_sha256( output / "full_payload_training_authority.composition.receipt.json" ), }, "trainingAuthority": { "scheduleReceiptPath": str( composition_receipt["scheduleReceipt"]["path"] ), "scheduleReceiptSha256": str( composition_receipt["scheduleReceipt"]["sha256"] ), "schedulePath": str(composition_receipt["schedule"]["path"]), "scheduleSha256": str(composition_receipt["schedule"]["sha256"]), "hashLedgerPath": str(composition_receipt["ledger"]["path"]), "hashLedgerSha256": str(composition_receipt["ledger"]["sha256"]), }, "sourceAuthority": { "corpusTrainingPath": str(Path(__file__).resolve()), "corpusTrainingSha256": file_sha256(Path(__file__).resolve()), }, "payloadFileCount": composition_receipt.get("payloadFileCount"), "payloadBytes": composition_receipt.get("payloadBytes"), "exactDuplicateRowsDeduplicated": sum( int(component.get("exactDuplicatePayloadFileCount", 0)) for component in composition_receipt.get("components", []) if isinstance(component, dict) ), "checks": checks, "acquisitionEpochClosed": True, "futureAuthoritiesClaimed": False, "modelTrainingClaimed": False, "completePayloadTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, "nextProofRequired": composition_receipt.get("nextProofRequired"), } _atomic_json(epoch_receipt_path, epoch_receipt) return epoch_receipt def _archive_member_format_contracts_for_reader_readiness( path: Path, *, container: str, ) -> Iterator[dict[str, str | bool]]: """Inspect archive member headers without decoding any payload content. Archive membership is part of the source container metadata. The training reader will dispatch each member by its own contract, so checking only the outer ``.zip``/``.tar`` suffix can otherwise defer an unavailable semantic decoder until a GPU-owning pack transaction has already started. """ def member_contract(name: str) -> dict[str, str | bool]: member_path = Path(name) if ( not name or member_path.is_absolute() or ".." in member_path.parts ): raise ValueError( f"full payload archive member path is unsafe: {path}:{name}" ) return _payload_format_contract(name) if container in {"zip", "zolca"}: with zipfile.ZipFile(path) as zip_archive: for zip_member in sorted( zip_archive.infolist(), key=lambda value: value.filename ): if not zip_member.is_dir(): yield member_contract(zip_member.filename) return if container == "tar": with tarfile.open(path, mode="r:*") as tar_archive: for tar_member in tar_archive: if tar_member.isfile(): yield member_contract(tar_member.name) return if container == "seven_zip": py7zr = importlib.import_module("py7zr") with py7zr.SevenZipFile(path, mode="r") as seven_zip_archive: members = sorted( seven_zip_archive.list(), key=lambda value: str(getattr(value, "filename", "")), ) for seven_zip_member in members: name = getattr(seven_zip_member, "filename", None) if not isinstance(name, str): raise ValueError( f"full payload seven-zip member is malformed: {path}" ) if getattr(seven_zip_member, "is_directory", False): continue yield member_contract(name) return raise ValueError(f"full payload archive container is unsupported: {container}") def full_payload_reader_readiness_receipt( schedule_receipt_path: Path, ) -> dict[str, Any]: """Prove that every scheduled reader family has a current decoder. This is a source-and-environment preflight, not a content probe. It reads only the immutable schedule metadata, recomputes each path-derived format contract from canonical source, and verifies required decoder modules before the model or a GPU lane is loaded. """ receipt_path = schedule_receipt_path.expanduser().resolve() if not receipt_path.is_file(): raise FileNotFoundError( f"full payload schedule receipt is absent: {receipt_path}" ) schedule_receipt = json.loads(receipt_path.read_text(encoding="utf-8")) if ( not isinstance(schedule_receipt, dict) or schedule_receipt.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA or schedule_receipt.get("passed") is not True ): raise ValueError("full payload reader schedule receipt did not pass") schedule_record = schedule_receipt.get("schedule") if not isinstance(schedule_record, dict): raise ValueError("full payload reader schedule artifact is absent") schedule_path = Path(str(schedule_record.get("path", ""))).resolve() schedule_sha256 = str(schedule_record.get("sha256", "")) expected_rows = schedule_record.get("rows") if ( not schedule_path.is_file() or len(schedule_sha256) != 64 or file_sha256(schedule_path) != schedule_sha256 or not isinstance(expected_rows, int) or isinstance(expected_rows, bool) or expected_rows < 1 ): raise ValueError("full payload reader schedule artifact differs") rows = _read_jsonl(schedule_path) if ( len(rows) != expected_rows or schedule_receipt.get("payloadFileCount") != expected_rows ): raise ValueError("full payload reader schedule denominator differs") reader_files: Counter[str] = Counter() reader_bytes: Counter[str] = Counter() archive_member_reader_files: Counter[str] = Counter() archive_member_record_formats: Counter[str] = Counter() required_modules: dict[str, set[str]] = {} required_executables: dict[str, set[str]] = {} contract_drift_work_ids: list[str] = [] unresolved_work_ids: list[str] = [] unresolved_work_id_set: set[str] = set() archive_member_metadata_count = 0 archive_member_container_count = 0 payload_bytes = 0 corpus_root_value = schedule_receipt.get("corpusRootAtBuild") corpus_root = ( Path(corpus_root_value).expanduser().resolve() if isinstance(corpus_root_value, str) and corpus_root_value else None ) if corpus_root is not None and not corpus_root.is_dir(): raise FileNotFoundError( f"full payload reader corpus root is absent: {corpus_root}" ) def require_contract( contract: Mapping[str, str | bool], *, work_id: str, archive_member: bool, ) -> None: reader_family = str(contract["readerFamily"]) container = str(contract["container"]) record_format = str(contract["recordFormat"]) compression = str(contract["compression"]) if archive_member: archive_member_reader_files[reader_family] += 1 archive_member_record_formats[record_format] += 1 module = FULL_PAYLOAD_READER_MODULES.get(reader_family) if module is not None: required_modules.setdefault(module, set()).add(reader_family) executable = FULL_PAYLOAD_READER_EXECUTABLES.get(reader_family) if executable is not None: required_executables.setdefault(executable, set()).add(reader_family) if record_format == "json": required_modules.setdefault("ijson", set()).add(reader_family) if compression == "zstd" or container == "graphdb": required_modules.setdefault("zstandard", set()).add(reader_family) if container == "lmdb" or record_format == "unknown": if work_id not in unresolved_work_id_set: unresolved_work_id_set.add(work_id) unresolved_work_ids.append(work_id) for ordinal, row in enumerate(rows): work_id = row.get("payloadWorkId") relative_path = row.get("payloadRelativePath") row_bytes = row.get("payloadBytes") scheduled_contract = row.get("format") if ( row.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_ENTRY_SCHEMA or row.get("scheduleOrdinal") != ordinal or not isinstance(work_id, str) or len(work_id) != 64 or not isinstance(relative_path, str) or not relative_path or not isinstance(row_bytes, int) or isinstance(row_bytes, bool) or row_bytes < 0 or not isinstance(scheduled_contract, dict) ): raise ValueError("full payload reader schedule row is malformed") current_contract = _payload_format_contract(relative_path) if scheduled_contract != current_contract: contract_drift_work_ids.append(work_id) reader_family = str(current_contract["readerFamily"]) container = str(current_contract["container"]) reader_files[reader_family] += 1 reader_bytes[reader_family] += row_bytes payload_bytes += row_bytes require_contract(current_contract, work_id=work_id, archive_member=False) if ( corpus_root is not None and container in {"zip", "zolca", "tar", "seven_zip"} ): payload_path = corpus_root / relative_path if not payload_path.is_file() or payload_path.stat().st_size != row_bytes: raise ValueError( f"full payload reader archive source differs: {payload_path}" ) archive_member_container_count += 1 for member_contract in _archive_member_format_contracts_for_reader_readiness( payload_path, container=container, ): archive_member_metadata_count += 1 require_contract( member_contract, work_id=work_id, archive_member=True, ) module_receipts: list[dict[str, Any]] = [] unavailable_modules: list[str] = [] for module_name in sorted(required_modules): try: available = importlib.util.find_spec(module_name) is not None except (ImportError, ModuleNotFoundError, ValueError): available = False if not available: unavailable_modules.append(module_name) module_receipts.append( { "module": module_name, "available": available, "readerFamilies": sorted(required_modules[module_name]), } ) executable_receipts: list[dict[str, Any]] = [] unavailable_executables: list[str] = [] for executable_name in sorted(required_executables): executable_path = shutil.which(executable_name) available = executable_path is not None if not available: unavailable_executables.append(executable_name) executable_receipts.append( { "executable": executable_name, "path": executable_path, "available": available, "readerFamilies": sorted( required_executables[executable_name] ), } ) checks = { "scheduleReceiptPassed": True, "scheduleArtifactHashVerified": True, "scheduleRowsComplete": len(rows) == expected_rows, "payloadByteDenominatorMatches": payload_bytes == schedule_receipt.get("payloadBytes"), "canonicalFormatContractsCurrent": not contract_drift_work_ids, "allRequiredDecoderModulesAvailable": not unavailable_modules, "allRequiredDecoderExecutablesAvailable": ( not unavailable_executables ), "noUnresolvedSemanticReaders": not unresolved_work_ids, "archiveMemberHeadersResolved": True, "archiveMemberContentNotDecoded": True, "noContentProbeUsed": True, "modelAndGpuNotLoaded": True, } return { "schema": FULL_PAYLOAD_READER_READINESS_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(checks.values()), "scheduleReceipt": { "path": str(receipt_path), "sha256": file_sha256(receipt_path), }, "schedule": { "path": str(schedule_path), "sha256": schedule_sha256, "rows": len(rows), "payloadBytes": payload_bytes, }, "readerFamilyFileCounts": dict(sorted(reader_files.items())), "readerFamilyBytes": dict(sorted(reader_bytes.items())), "archiveMemberContainerCount": archive_member_container_count, "archiveMemberMetadataCount": archive_member_metadata_count, "archiveMemberReaderFamilyFileCounts": dict( sorted(archive_member_reader_files.items()) ), "archiveMemberRecordFormatCounts": dict( sorted(archive_member_record_formats.items()) ), "requiredDecoderModules": module_receipts, "unavailableDecoderModules": unavailable_modules, "requiredDecoderExecutables": executable_receipts, "unavailableDecoderExecutables": unavailable_executables, "formatContractDriftCount": len(contract_drift_work_ids), "formatContractDriftWorkIds": contract_drift_work_ids, "unresolvedReaderWorkIds": unresolved_work_ids, "checks": checks, "modelTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, } def resolve_latest_full_payload_training_epoch( corpus_root: Path, epoch_roots: Sequence[Path], ) -> dict[str, Any]: """Autodiscover the newest coherent, reader-ready corpus epoch. No mutable pointer is trusted. Every candidate epoch is rediscovered from its receipt, all referenced artifacts are hash-checked, and the newest epoch must retain the complete work-id set of every older coherent epoch. """ root = corpus_root.expanduser().resolve() roots = tuple( sorted({path.expanduser().resolve() for path in epoch_roots}, key=str) ) if not root.is_dir(): raise FileNotFoundError(f"full payload corpus root is absent: {root}") if not roots or any(not path.is_dir() for path in roots): raise FileNotFoundError("full payload epoch discovery root is absent") receipt_paths = sorted( { path.resolve() for epoch_root in roots for path in epoch_root.rglob("full_payload_federated_epoch.receipt.json") }, key=str, ) candidates: list[dict[str, Any]] = [] for receipt_path in receipt_paths: receipt_sha256 = file_sha256(receipt_path) receipt = json.loads(receipt_path.read_text(encoding="utf-8")) if ( not isinstance(receipt, dict) or receipt.get("schema") != FULL_PAYLOAD_FEDERATED_EPOCH_SCHEMA or receipt.get("passed") is not True or Path(str(receipt.get("corpusRoot", ""))).resolve() != root ): continue cutoff_ns = receipt.get("epochCutoffNs") identity_sha256 = receipt.get("epochIdentitySha256") training_authority = receipt.get("trainingAuthority") composition_record = receipt.get("compositionReceipt") if ( not isinstance(cutoff_ns, int) or isinstance(cutoff_ns, bool) or cutoff_ns < 1 or not isinstance(identity_sha256, str) or len(identity_sha256) != 64 or not isinstance(training_authority, dict) or not isinstance(composition_record, dict) ): raise ValueError("full payload epoch receipt identity is malformed") epoch_directory = receipt_path.parent.resolve() schedule_receipt_path = Path( str(training_authority.get("scheduleReceiptPath", "")) ).resolve() schedule_path = Path( str(training_authority.get("schedulePath", "")) ).resolve() hash_ledger_path = Path( str(training_authority.get("hashLedgerPath", "")) ).resolve() composition_path = Path(str(composition_record.get("path", ""))).resolve() readiness_path = epoch_directory / "full_payload_reader_readiness.receipt.json" artifact_paths = ( schedule_receipt_path, schedule_path, hash_ledger_path, composition_path, readiness_path, ) if any( not path.is_file() or not path.is_relative_to(epoch_directory) for path in artifact_paths ): raise ValueError("full payload epoch artifact boundary differs") if ( file_sha256(schedule_receipt_path) != training_authority.get("scheduleReceiptSha256") or file_sha256(schedule_path) != training_authority.get("scheduleSha256") or file_sha256(hash_ledger_path) != training_authority.get("hashLedgerSha256") or file_sha256(composition_path) != composition_record.get("sha256") ): raise ValueError("full payload epoch artifact hash differs") schedule_receipt = json.loads( schedule_receipt_path.read_text(encoding="utf-8") ) readiness = json.loads(readiness_path.read_text(encoding="utf-8")) schedule_record = ( schedule_receipt.get("schedule") if isinstance(schedule_receipt, dict) else None ) if ( not isinstance(schedule_receipt, dict) or schedule_receipt.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA or schedule_receipt.get("passed") is not True or not isinstance(schedule_record, dict) or Path(str(schedule_record.get("path", ""))).resolve() != schedule_path or schedule_record.get("sha256") != training_authority.get("scheduleSha256") or not isinstance(readiness, dict) or readiness.get("schema") != FULL_PAYLOAD_READER_READINESS_SCHEMA or readiness.get("passed") is not True or readiness.get("formatContractDriftCount") != 0 or readiness.get("unavailableDecoderModules") != [] or readiness.get("unresolvedReaderWorkIds") != [] or not isinstance(readiness.get("scheduleReceipt"), dict) or Path( str(readiness["scheduleReceipt"].get("path", "")) ).resolve() != schedule_receipt_path or readiness["scheduleReceipt"].get("sha256") != training_authority.get("scheduleReceiptSha256") ): raise ValueError("full payload epoch reader authority differs") schedule_rows = _read_jsonl(schedule_path) expected_rows = schedule_record.get("rows") work_ids = { str(row.get("payloadWorkId", "")) for row in schedule_rows if row.get("schema") == FULL_PAYLOAD_TRAINING_SCHEDULE_ENTRY_SCHEMA and isinstance(row.get("payloadWorkId"), str) and len(str(row["payloadWorkId"])) == 64 } if ( not isinstance(expected_rows, int) or isinstance(expected_rows, bool) or len(schedule_rows) != expected_rows or len(work_ids) != expected_rows or receipt.get("payloadFileCount") != expected_rows or receipt.get("payloadBytes") != schedule_receipt.get("payloadBytes") or file_sha256(receipt_path) != receipt_sha256 ): raise ValueError("full payload epoch row denominator differs") candidates.append( { "epochReceiptPath": str(receipt_path), "epochReceiptSha256": receipt_sha256, "epochCutoffNs": cutoff_ns, "epochIdentitySha256": identity_sha256, "payloadFileCount": expected_rows, "payloadBytes": receipt.get("payloadBytes"), "scheduleReceiptPath": str(schedule_receipt_path), "scheduleReceiptSha256": str( training_authority["scheduleReceiptSha256"] ), "schedulePath": str(schedule_path), "scheduleSha256": str(training_authority["scheduleSha256"]), "hashLedgerPath": str(hash_ledger_path), "hashLedgerSha256": str( training_authority["hashLedgerSha256"] ), "readerReadinessReceiptPath": str(readiness_path), "readerReadinessReceiptSha256": file_sha256(readiness_path), "workIds": work_ids, } ) if not candidates: raise RuntimeError("no coherent reader-ready full payload epoch was discovered") newest_cutoff = max(int(candidate["epochCutoffNs"]) for candidate in candidates) newest = [ candidate for candidate in candidates if candidate["epochCutoffNs"] == newest_cutoff ] if len({str(candidate["epochIdentitySha256"]) for candidate in newest}) != 1: raise RuntimeError("latest full payload epochs conflict at one cutoff") selected = sorted(newest, key=lambda candidate: str(candidate["epochReceiptPath"]))[ 0 ] selected_work_ids = selected["workIds"] if not isinstance(selected_work_ids, set) or any( not isinstance(candidate["workIds"], set) or not candidate["workIds"].issubset(selected_work_ids) for candidate in candidates ): raise RuntimeError("latest full payload epoch does not retain prior work") resolved = {key: value for key, value in selected.items() if key != "workIds"} checks = { "epochAutodiscoveredWithoutMutablePointer": True, "allReferencedArtifactsHashVerified": True, "readerReadinessPassed": True, "latestEpochRetainsAllPriorWorkIds": True, "corpusRootMatches": True, "targetsExcludedFromForward": True, "optimizerNotConsulted": True, } return { "schema": FULL_PAYLOAD_FEDERATED_EPOCH_RESOLUTION_SCHEMA, "resolvedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(checks.values()), "candidateEpochCount": len(candidates), "selected": resolved, "checks": checks, "modelTrainingClaimed": False, "completePayloadTrainingClaimed": False, "promotionEligible": False, "rawDataDeletionAllowed": False, } def _full_payload_xml_escape_cdata(text: str) -> str: # Byte-identical to xml.etree.ElementTree._escape_cdata (Python 3.12). if "&" in text: text = text.replace("&", "&") if "<" in text: text = text.replace("<", "<") if ">" in text: text = text.replace(">", ">") return text def _full_payload_xml_escape_attrib(text: str) -> str: # Byte-identical to xml.etree.ElementTree._escape_attrib (Python 3.12). if "&" in text: text = text.replace("&", "&") if "<" in text: text = text.replace("<", "<") if ">" in text: text = text.replace(">", ">") if '"' in text: text = text.replace('"', """) if "\r" in text: text = text.replace("\r", " ") if "\n" in text: text = text.replace("\n", " ") if "\t" in text: text = text.replace("\t", " ") return text def _full_payload_xml_record_text_fast(element: Any) -> str | None: """Serialize a parsed record element like the stdlib serializer. Produces text identical to ``ElementTree.tostring(element, encoding="unicode")`` for the common no-namespace record subset while avoiding the stdlib serializer's per-piece ``write`` overhead (the C accelerator only covers parsing; serialization is pure Python). Returns ``None`` whenever a feature outside that subset (namespaces, comments, processing instructions, non-string values) appears so the caller falls back to the stdlib serializer for that record. """ parts: list[str] = [] def _walk(node: Any) -> bool: tag = node.tag if not isinstance(tag, str) or "{" in tag or "}" in tag: return False text = node.text if text is not None and not isinstance(text, str): return False tail = node.tail if tail is not None and not isinstance(tail, str): return False parts.append("<" + tag) for key, value in node.items(): if not isinstance(key, str) or "{" in key or "}" in key: return False if not isinstance(value, str): return False parts.append( ' %s="%s"' % (key, _full_payload_xml_escape_attrib(value)) ) children = list(node) if text or children: parts.append(">") if text: parts.append(_full_payload_xml_escape_cdata(text)) for child in children: if not _walk(child): return False parts.append("") else: parts.append(" />") if tail: parts.append(_full_payload_xml_escape_cdata(tail)) return True if not _walk(element): return None return "".join(parts) _FULL_PAYLOAD_JSON_STREAMING_REQUIRED = object() def _full_payload_small_json_document_boundary(handle: Any) -> object: """Load a bounded JSON document with the permissive scientific parser. Scientific exports commonly encode unavailable numeric measurements as ``NaN`` or infinities. Python's JSON parser preserves those explicit values, while the large-document streaming parser intentionally follows strict JSON. Restricting the permissive path to bounded, seekable documents keeps large arrays streaming without misclassifying intact archive members as corrupt source data. """ try: if not handle.seekable(): return _FULL_PAYLOAD_JSON_STREAMING_REQUIRED start = handle.tell() handle.seek(0, os.SEEK_END) end = handle.tell() handle.seek(start) except (AttributeError, OSError): return _FULL_PAYLOAD_JSON_STREAMING_REQUIRED remaining = end - start if remaining < 0 or remaining > FULL_PAYLOAD_JSON_DOCUMENT_LOAD_BYTES: return _FULL_PAYLOAD_JSON_STREAMING_REQUIRED payload = handle.read(remaining + 1) if len(payload) != remaining: handle.seek(start) return _FULL_PAYLOAD_JSON_STREAMING_REQUIRED try: return json.loads(payload) except (json.JSONDecodeError, UnicodeDecodeError): handle.seek(start) return _FULL_PAYLOAD_JSON_STREAMING_REQUIRED def _iter_text_stream_records( handle: Any, *, record_format: str, ) -> Iterator[str]: if record_format == "xml": depth = 0 fast_serializer = True fast_checks_remaining = 64 for event, element in ElementTree.iterparse(handle, events=("start", "end")): if event == "start": depth += 1 continue if depth == 2: text: str | None = None if fast_serializer: text = _full_payload_xml_record_text_fast(element) if text is not None and fast_checks_remaining > 0: fast_checks_remaining -= 1 if text != ElementTree.tostring( element, encoding="unicode" ): fast_serializer = False text = None if text is None: text = ElementTree.tostring(element, encoding="unicode") if text.strip(): yield text element.clear() depth -= 1 return if record_format == "json": parser = importlib.import_module("ijson") buffered = handle if isinstance(handle, io.BufferedReader) else io.BufferedReader(handle) prefix = buffered.peek(4096).lstrip()[:1] bounded_document = _full_payload_small_json_document_boundary(buffered) if bounded_document is not _FULL_PAYLOAD_JSON_STREAMING_REQUIRED: if prefix == b"[" and isinstance(bounded_document, list): for value in bounded_document: yield json.dumps( value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str, ) return if prefix == b"{" and isinstance(bounded_document, dict): for key, value in bounded_document.items(): yield json.dumps( {"key": key, "value": value}, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str, ) return if prefix == b"[": values = parser.items(buffered, "item") for value in values: yield json.dumps( value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str, ) return if prefix == b"{": values = parser.kvitems(buffered, "") for key, value in values: yield json.dumps( {"key": key, "value": value}, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str, ) return handle = buffered def required_pfam_header_value( lines: Sequence[str], *, field: str, record_kind: str, ) -> str: values: list[str] = [] for line in lines: fields = line.strip().split(maxsplit=1) if not fields or fields[0] != field: continue if len(fields) != 2 or not fields[1].strip(): raise RuntimeError( f"Pfam {record_kind} has an empty {field} header" ) values.append(fields[1].strip()) if len(values) != 1: raise RuntimeError( f"Pfam {record_kind} has no unique {field} header" ) return values[0] def validate_pfam_accession(value: str, *, record_kind: str) -> None: accession_root = value.split(".", maxsplit=1)[0] if ( not accession_root.startswith("PF") or not accession_root.removeprefix("PF").isdigit() ): raise RuntimeError( f"Pfam {record_kind} accession grammar differs" ) def validate_profile_hmm_record(record: Sequence[str]) -> None: if ( len(record) < 2 or record[-1].rstrip("\r\n") != "//" or not record[0].startswith("HMMER3/") ): raise RuntimeError("Pfam HMM profile grammar differs") accession = required_pfam_header_value( record[:-1], field="ACC", record_kind="HMM profile", ) validate_pfam_accession(accession, record_kind="HMM profile") required_pfam_header_value( record[:-1], field="NAME", record_kind="HMM profile", ) length = required_pfam_header_value( record[:-1], field="LENG", record_kind="HMM profile", ) try: valid_length = int(length) except ValueError as error: raise RuntimeError("Pfam HMM profile LENG grammar differs") from error if valid_length < 1 or not any( line.startswith("HMM ") for line in record[:-1] ): raise RuntimeError("Pfam HMM profile semantic body differs") def validate_stockholm_alignment_record(record: Sequence[str]) -> None: if ( len(record) < 2 or record[-1].rstrip("\r\n") != "//" or record[0].rstrip("\r\n") != "# STOCKHOLM 1.0" ): raise RuntimeError("Pfam Stockholm alignment grammar differs") identifiers: list[str] = [] accessions: list[str] = [] sequence_counts: list[int] = [] sequence_rows = 0 for line in record[1:-1]: stripped = line.strip() if not stripped: continue if stripped.startswith("#=GF "): fields = stripped.split(maxsplit=2) if len(fields) != 3 or not fields[2]: raise RuntimeError("Pfam Stockholm metadata grammar differs") if fields[1] == "ID": identifiers.append(fields[2]) elif fields[1] == "AC": accessions.append(fields[2]) elif fields[1] == "SQ": try: sequence_counts.append(int(fields[2])) except ValueError as error: raise RuntimeError( "Pfam Stockholm sequence-count grammar differs" ) from error continue if stripped.startswith("#"): continue fields = stripped.split() if len(fields) != 2 or not fields[0] or not fields[1]: raise RuntimeError("Pfam Stockholm sequence row grammar differs") sequence_rows += 1 if len(identifiers) != 1 or len(accessions) != 1 or len(sequence_counts) != 1: raise RuntimeError("Pfam Stockholm identity headers differ") validate_pfam_accession(accessions[0], record_kind="Stockholm alignment") if sequence_counts[0] < 1 or sequence_rows < sequence_counts[0]: raise RuntimeError("Pfam Stockholm sequence geometry differs") def validate_newick_tree_record(record: str) -> None: if not record.endswith(";") or not record[:-1].strip(): raise RuntimeError("Pfam Newick tree grammar differs") depth = 0 opened = 0 separators = 0 quoted = False comment_depth = 0 for character in record[:-1]: if character == "\x00" or ( not character.isprintable() and character not in "\r\n\t" ): raise RuntimeError("Pfam Newick tree contains non-text bytes") if quoted: if character == "'": quoted = not quoted continue if comment_depth: if character == "[": comment_depth += 1 elif character == "]": comment_depth -= 1 continue if character == "'": quoted = True elif character == "[": comment_depth = 1 elif character == "(": depth += 1 opened += 1 elif character == ")": depth -= 1 if depth < 0: raise RuntimeError("Pfam Newick tree parenthesis grammar differs") elif character == ",": separators += 1 if quoted or comment_depth or depth or opened < 1 or separators < 1: raise RuntimeError("Pfam Newick tree grammar differs") text_handle = handle try: handle.seekable() except (AttributeError, OSError): text_handle = io.BufferedReader(_ForwardOnlyRawStream(handle)) wrapper = io.TextIOWrapper( text_handle, encoding="utf-8", errors=( "strict" if record_format in {"profile_hmm", "stockholm_alignment", "newick_tree"} else "replace" ), newline="", ) try: if record_format == "profile_hmm": profile_record: list[str] = [] emitted = 0 for line in wrapper: profile_record.append(line) if line.rstrip("\r\n") == "//": validate_profile_hmm_record(profile_record) yield "".join(profile_record) emitted += 1 profile_record = [] if profile_record and any(value.strip() for value in profile_record): raise RuntimeError("Pfam HMM profile is incomplete") if emitted == 0: raise RuntimeError("Pfam HMM profile stream has no semantic records") return if record_format == "stockholm_alignment": stockholm_record: list[str] = [] emitted = 0 for line in wrapper: stockholm_record.append(line) if line.rstrip("\r\n") == "//": validate_stockholm_alignment_record(stockholm_record) yield "".join(stockholm_record) emitted += 1 stockholm_record = [] if stockholm_record and any( value.strip() for value in stockholm_record ): raise RuntimeError("Pfam Stockholm alignment is incomplete") if emitted == 0: raise RuntimeError( "Pfam Stockholm alignment stream has no semantic records" ) return if record_format == "newick_tree": tree_record: list[str] = [] emitted = 0 quoted = False comment_depth = 0 for line in wrapper: for character in line: if not tree_record and character.isspace(): continue tree_record.append(character) if quoted: if character == "'": quoted = not quoted continue if comment_depth: if character == "[": comment_depth += 1 elif character == "]": comment_depth -= 1 continue if character == "'": quoted = True continue if character == "[": comment_depth = 1 continue if character == ";": tree = "".join(tree_record) validate_newick_tree_record(tree) yield tree emitted += 1 tree_record = [] if tree_record and any(value.strip() for value in tree_record): raise RuntimeError("Pfam Newick tree is incomplete") if quoted or comment_depth: raise RuntimeError("Pfam Newick tree quote or comment is incomplete") if emitted == 0: raise RuntimeError("Pfam Newick tree stream has no semantic records") return if record_format in {"csv", "tsv"}: delimiter = "," if record_format == "csv" else "\t" previous_field_limit = csv.field_size_limit() csv.field_size_limit(sys.maxsize) try: for row in csv.reader(wrapper, delimiter=delimiter): if row: yield json.dumps( row, separators=(",", ":"), ensure_ascii=False, ) finally: csv.field_size_limit(previous_field_limit) return if record_format == "sdf": record: list[str] = [] for line in wrapper: record.append(line) if line.rstrip("\r\n") == "$$$$": yield "".join(record) record = [] if record and any(value.strip() for value in record): yield "".join(record) return if record_format == "fasta": record = [] for line in wrapper: if line.startswith(">") and record: yield "".join(record) record = [] record.append(line) if record and any(value.strip() for value in record): yield "".join(record) return if record_format == "mass_spectrum": record = [] for line in wrapper: if not line.strip() and record: yield "".join(record) record = [] else: record.append(line) if record and any(value.strip() for value in record): yield "".join(record) return for line in wrapper: if line.strip(): yield line finally: try: wrapper.detach() except (ValueError, OSError): pass def _sdz_semantic_record_format(probe: bytes) -> str: """Resolve NCI SDZ structure syntax from decompressed content grammar.""" if not probe: raise RuntimeError("SDZ structure stream is empty") if b"$$$$" in probe or b"V2000" in probe or b"V3000" in probe: return "sdf" try: text = probe.decode("utf-8") except UnicodeDecodeError as error: raise RuntimeError( "SDZ structure stream is neither SDF nor UTF-8 SMILES" ) from error candidates = [ line.split()[0] for line in text.splitlines() if line.strip() and not line.lstrip().startswith("#") ] if not candidates: raise RuntimeError("SDZ structure stream has no semantic records") smiles_grammar = re.compile( r"(?:Br|Cl|Si|Na|Li|Mg|Ca|Fe|Zn|Cu|Mn|Se|As|Al|" r"[BCNOFPSIHKVYUWbcnops0-9@+\-\[\]\(\)=#$%.:/\\*])+" ) if all(smiles_grammar.fullmatch(candidate) for candidate in candidates): return "smiles" raise RuntimeError("SDZ structure stream grammar is unresolved") def _iter_decompressed_stream_records( handle: Any, *, name: str, format_contract: Mapping[str, object], ) -> Iterator[tuple[str, str]]: compression = str(format_contract.get("compression", "none")) record_format = str(format_contract.get("recordFormat", "unknown")) if compression == "gzip": with gzip.GzipFile(fileobj=handle, mode="rb") as decoded: if record_format == "sdz_structures": selected_format = _sdz_semantic_record_format( decoded.peek(256 * 1024) ) for locator, text in _iter_decompressed_stream_records( decoded, name=name, format_contract={ **format_contract, "compression": "none", "recordFormat": selected_format, }, ): yield ( f"{name}#sdz_decoder={selected_format}:{locator}", text, ) return yield from _iter_decompressed_stream_records( decoded, name=name, format_contract={**format_contract, "compression": "none"}, ) return if compression == "bzip2": with bz2.BZ2File(handle, mode="rb") as decoded: yield from _iter_decompressed_stream_records( decoded, name=name, format_contract={**format_contract, "compression": "none"}, ) return if compression == "xz": with lzma.LZMAFile(handle, mode="rb") as decoded: yield from _iter_decompressed_stream_records( decoded, name=name, format_contract={**format_contract, "compression": "none"}, ) return if compression == "zstd": zstandard = importlib.import_module("zstandard") with zstandard.ZstdDecompressor().stream_reader(handle) as decoded: yield from _iter_decompressed_stream_records( decoded, name=name, format_contract={**format_contract, "compression": "none"}, ) return if record_format == "unknown": yield from _iter_lossless_binary_stream_records(handle, name=name) return for ordinal, text in enumerate( _iter_text_stream_records(handle, record_format=record_format) ): yield f"{name}#record={ordinal}", text def _iter_lossless_binary_stream_records( handle: Any, *, name: str, ) -> Iterator[tuple[str, str]]: """Expose every opaque byte through a deterministic reversible text record. This is an external-I/O fallback for scientific containers whose native semantic adapter is unavailable. Base85 keeps the exact byte denominator reconstructable while bounded chunks preserve streaming and resume behavior. It never substitutes a prefix probe or silently drops a file. """ block_index = 0 while True: block = handle.read(FULL_PAYLOAD_BINARY_CHUNK_BYTES) if not block: return if not isinstance(block, bytes): block = bytes(block) encoded = base64.b85encode(block).decode("ascii") yield ( f"{name}#binary_block={block_index}", ( "NNF_BINARY_BASE85_V1 " f"byte_count={len(block)} data={encoded}" ), ) block_index += 1 class _ForwardOnlyRawStream(io.RawIOBase): """Give tar's forward-only member stream a truthful buffered-I/O contract.""" def __init__(self, source: Any) -> None: super().__init__() self._source = source def readable(self) -> bool: return True def seekable(self) -> bool: return False def readinto(self, buffer: Any) -> int: chunk = self._source.read(len(buffer)) if not chunk: return 0 buffer[: len(chunk)] = chunk return len(chunk) def _iter_graphdb_dump_path_records(path: Path) -> Iterator[tuple[str, str]]: """Stream Neo4j's zstd-wrapped tar dump without materializing it on disk.""" zstandard = importlib.import_module("zstandard") with path.open("rb") as raw: with zstandard.ZstdDecompressor().stream_reader(raw) as decoded: neo4j_header = decoded.read(24) if len(neo4j_header) != 24 or not neo4j_header.startswith(b"zstd"): raise RuntimeError("graphdb dump wrapper header differs") with tarfile.open(fileobj=decoded, mode="r|") as archive: for member in archive: if not member.isfile(): continue extracted = archive.extractfile(member) if extracted is None: raise RuntimeError( f"graphdb member cannot be read: {member.name}" ) with extracted: for locator, text in _iter_lossless_binary_stream_records( extracted, name=member.name, ): yield f"graphdb:{locator}", text def _iter_semantic_graph_export_records( binding: FullPayloadSemanticExportBinding, ) -> Iterator[tuple[str, str]]: """Stream every node and relationship from a verified graph export.""" node_rows = 0 relationship_rows = 0 with gzip.open( binding.export_path, mode="rt", encoding="utf-8", newline="", ) as handle: reader = csv.DictReader(handle) fieldnames = reader.fieldnames if ( not isinstance(fieldnames, list) or len(fieldnames) < 6 or len(set(fieldnames)) != len(fieldnames) or not {"_id", "_labels", "_start", "_end", "_type"}.issubset( fieldnames ) ): raise RuntimeError("semantic graph export header differs") for ordinal, row in enumerate(reader): if None in row or set(row) != set(fieldnames): raise RuntimeError("semantic graph export row geometry differs") values = { key: value for key, value in row.items() if isinstance(value, str) and value != "" } relationship = bool(values.get("_type")) if relationship: if not values.get("_start") or not values.get("_end"): raise RuntimeError("semantic graph relationship identity differs") relationship_rows += 1 kind = "relationship" else: if not values.get("_id") or not values.get("_labels"): raise RuntimeError("semantic graph node identity differs") node_rows += 1 kind = "node" yield ( f"semantic_graph:{kind}:row={ordinal}", json.dumps( {"recordKind": kind, **values}, sort_keys=True, separators=(",", ":"), ensure_ascii=False, ), ) if ( node_rows != binding.node_rows or relationship_rows != binding.relationship_rows or node_rows + relationship_rows != binding.data_rows ): raise RuntimeError("semantic graph export row counts differ") def _iter_nested_payload_stream_records( handle: Any, *, display_name: str, scratch_root: Path | None, ) -> Iterator[tuple[str, FullPayloadReaderValue]]: """Re-enter canonical dispatch for a container embedded in another one.""" if scratch_root is not None: scratch_root.mkdir(parents=True, exist_ok=True) temporary_parent = str(scratch_root) if scratch_root is not None else None nested_contract = _payload_format_contract(display_name) if nested_contract["container"] in {"zip", "zolca"}: with tempfile.SpooledTemporaryFile( max_size=64 * 1024 * 1024, mode="w+b", dir=temporary_parent, ) as payload: while True: block = handle.read(8 * 1024 * 1024) if not block: break payload.write(block if isinstance(block, bytes) else bytes(block)) payload.seek(0) with zipfile.ZipFile(payload) as archive: yield from _iter_zip_archive_records( archive, scratch_root=scratch_root, ) return with tempfile.TemporaryDirectory( prefix="resynthesis-nested-payload-", dir=temporary_parent, ) as temporary: nested_suffix = "".join(Path(display_name).suffixes) payload_path = Path(temporary) / f"payload{nested_suffix}" with payload_path.open("wb") as destination: while True: block = handle.read(8 * 1024 * 1024) if not block: break destination.write(block if isinstance(block, bytes) else bytes(block)) yield from _iter_existing_payload_path_records( payload_path, display_name=display_name, scratch_root=scratch_root, ) def _iter_tar_path_records( path: Path, *, scratch_root: Path | None, ) -> Iterator[tuple[str, FullPayloadReaderValue]]: with path.open("rb") as raw: with tarfile.open(fileobj=raw, mode="r|*") as archive: for member in archive: if not member.isfile(): continue extracted = archive.extractfile(member) if extracted is None: raise RuntimeError(f"tar member cannot be read: {member.name}") with extracted: member_contract = _payload_format_contract(member.name) try: if member_contract["container"] != "plain": for locator, text in _iter_nested_payload_stream_records( extracted, display_name=member.name, scratch_root=scratch_root, ): yield f"{member.name}:{locator}", text continue yield from _iter_decompressed_stream_records( extracted, name=member.name, format_contract=member_contract, ) except Exception as error: yield ( "reader_degradation", _spreadsheet_reader_degradation( path, display_name=member.name, error="archive_member_semantic_decode_failed", decoder_exception_type=type(error).__name__, ), ) def _iter_zip_archive_records( archive: zipfile.ZipFile, *, scratch_root: Path | None, ) -> Iterator[tuple[str, FullPayloadReaderValue]]: for member in sorted(archive.infolist(), key=lambda value: value.filename): if member.is_dir(): continue member_contract = _payload_format_contract(member.filename) with archive.open(member, "r") as extracted: try: if member_contract["container"] != "plain": for locator, text in _iter_nested_payload_stream_records( extracted, display_name=member.filename, scratch_root=scratch_root, ): yield f"{member.filename}:{locator}", text continue yield from _iter_decompressed_stream_records( extracted, name=member.filename, format_contract=member_contract, ) except Exception as error: archive_path = ( archive.filename if isinstance(archive.filename, str) else None ) if archive_path is None or not Path(archive_path).is_file(): # An in-memory archive has no raw-path authority to bind; # preserve the prior failure contract there. raise yield ( "reader_degradation", _spreadsheet_reader_degradation( Path(archive_path), display_name=member.filename, error="archive_member_semantic_decode_failed", decoder_exception_type=type(error).__name__, ), ) def _iter_zip_path_records( path: Path, *, scratch_root: Path | None, ) -> Iterator[tuple[str, FullPayloadReaderValue]]: with zipfile.ZipFile(path) as archive: yield from _iter_zip_archive_records( archive, scratch_root=scratch_root, ) def _iter_parquet_path_records(path: Path) -> Iterator[tuple[str, str]]: parquet = importlib.import_module("pyarrow.parquet") source = parquet.ParquetFile(path) for row_group in range(source.num_row_groups): table = source.read_row_group(row_group) for row_index, row in enumerate(table.to_pylist()): yield ( f"row_group={row_group}:row={row_index}", json.dumps( row, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str, ), ) def _iter_pdf_path_records(path: Path) -> Iterator[tuple[str, str]]: pdf = importlib.import_module("pypdf") try: reader = pdf.PdfReader(str(path)) except (pdf.errors.PdfReadError, pdf.errors.PdfStreamError): # Public archives sometimes use a PDF suffix for truncated legacy # payloads. Preserve those bytes exactly instead of dropping the # member or cancelling the complete source transaction. with path.open("rb") as raw: yield from _iter_lossless_binary_stream_records( raw, name=path.name, ) return for page_index, page in enumerate(reader.pages): text = page.extract_text() or "" if text.strip(): yield f"page={page_index}", text def _spreadsheet_reader_degradation( path: Path, *, display_name: str, error: str, decoder_exception_type: str, ) -> FullPayloadReaderDegradation: """Bind malformed workbook bytes without presenting them as knowledge.""" digest = hashlib.sha256() byte_count = 0 replacement_count = 0 carry = b"" magic = b"" with path.open("rb") as handle: while True: block = handle.read(FULL_PAYLOAD_BINARY_CHUNK_BYTES) if not block: break if not magic: magic = block[:32] digest.update(block) byte_count += len(block) counted = carry + block replacement_count += counted.count(b"\xef\xbf\xbd") carry = counted[-2:] authority: dict[str, Any] = { "schema": FULL_PAYLOAD_READER_DEGRADATION_SCHEMA, "logicalPath": display_name, "declaredSuffix": Path(display_name).suffix.lower(), "detectedMagicHex": magic.hex(), "rawBytes": byte_count, "rawSha256": digest.hexdigest(), "utf8ReplacementSequenceCount": replacement_count, "error": error, "decoderExceptionType": decoder_exception_type, "semanticDecodePassed": False, "trainingKnowledgeReady": False, "rawBytesPreserved": True, "rawBytesEncodedIntoTraining": False, "reacquisitionRequired": True, "targetEnteredForward": False, } authority["readerDegradationAuthoritySha256"] = _sha256_bytes( _json_bytes(authority) ) return FullPayloadReaderDegradation(authority) def _iter_spreadsheet_path_records( path: Path, *, display_name: str, ) -> Iterator[tuple[str, FullPayloadReaderValue]]: """Dispatch spreadsheets from their byte container, never their suffix.""" with path.open("rb") as handle: magic = handle.read(8) if zipfile.is_zipfile(path): try: openpyxl = importlib.import_module("openpyxl") with path.open("rb") as workbook_handle: workbook = openpyxl.load_workbook( workbook_handle, read_only=True, data_only=True, ) try: for sheet in workbook.worksheets: for row_index, row in enumerate( sheet.iter_rows(values_only=True) ): yield ( f"sheet={sheet.title}:row={row_index}", json.dumps( row, separators=(",", ":"), ensure_ascii=False, default=str, ), ) finally: workbook.close() return except Exception as error: yield ( "reader_degradation", _spreadsheet_reader_degradation( path, display_name=display_name, error="ooxml_semantic_decode_failed", decoder_exception_type=type(error).__name__, ), ) return if magic == b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1": try: xlrd = importlib.import_module("xlrd") except (ImportError, ModuleNotFoundError) as error: yield ( "reader_degradation", _spreadsheet_reader_degradation( path, display_name=display_name, error="pinned_xlrd_unavailable", decoder_exception_type=type(error).__name__, ), ) return if getattr(xlrd, "__version__", None) != FULL_PAYLOAD_XLRD_VERSION: yield ( "reader_degradation", _spreadsheet_reader_degradation( path, display_name=display_name, error="pinned_xlrd_version_differs", decoder_exception_type="DependencyAuthorityError", ), ) return try: workbook = xlrd.open_workbook( filename=str(path), on_demand=True, ) try: for sheet in workbook.sheets(): for row_index in range(sheet.nrows): yield ( f"sheet={sheet.name}:row={row_index}", json.dumps( sheet.row_values(row_index), separators=(",", ":"), ensure_ascii=False, default=str, ), ) finally: workbook.release_resources() return except Exception as error: yield ( "reader_degradation", _spreadsheet_reader_degradation( path, display_name=display_name, error="excel_binary_semantic_decode_failed", decoder_exception_type=type(error).__name__, ), ) return yield ( "reader_degradation", _spreadsheet_reader_degradation( path, display_name=display_name, error=( "declared_spreadsheet_container_magic_unrecognized" if not magic.startswith(b"PK") else "ooxml_zip_central_directory_invalid" ), decoder_exception_type="ContainerMagicError", ), ) def _iter_numpy_path_records(path: Path) -> Iterator[tuple[str, str]]: numpy = importlib.import_module("numpy") value = numpy.load(path, mmap_mode="r", allow_pickle=False) if value.ndim == 0: yield "scalar", json.dumps(value.tolist(), ensure_ascii=False, default=str) return for row_index in range(value.shape[0]): yield ( f"row={row_index}", json.dumps( value[row_index].tolist(), separators=(",", ":"), ensure_ascii=False, default=str, ), ) def _iter_hdf5_path_records(path: Path) -> Iterator[tuple[str, str]]: h5py = importlib.import_module("h5py") with h5py.File(path, "r") as source: datasets: list[tuple[str, Any]] = [] def collect(name: str, value: Any) -> None: if isinstance(value, h5py.Dataset): datasets.append((name, value)) source.visititems(collect) for name, dataset in datasets: if dataset.ndim == 0: yield ( f"dataset={name}:scalar", json.dumps(dataset[()].tolist(), ensure_ascii=False, default=str), ) continue for row_index in range(dataset.shape[0]): value = dataset[row_index] serializable = value.tolist() if hasattr(value, "tolist") else value yield ( f"dataset={name}:row={row_index}", json.dumps( serializable, separators=(",", ":"), ensure_ascii=False, default=str, ), ) def _full_payload_hdf5_json_value(value: Any) -> Any: """Convert one bounded HDF5 row to deterministic JSON-compatible values.""" if isinstance(value, bytes): return value.decode("utf-8", errors="surrogateescape") if hasattr(value, "tolist"): return _full_payload_hdf5_json_value(value.tolist()) if isinstance(value, list): return [_full_payload_hdf5_json_value(element) for element in value] if isinstance(value, tuple): return [_full_payload_hdf5_json_value(element) for element in value] return value def _iter_gctx_path_records( path: Path, *, start_record_ordinal: int = 0, scratch_root: Path | None = None, source_sha256: str | None = None, ) -> Iterator[tuple[str, FullPayloadReaderValue]]: """Stream a LINCS GCTX matrix directly from its gzip transport. GCTX is HDF5. gzip is not random-access, so a hash-bound parallel decompression cache is materialized once and reused across every durable resume. Dataset rows remain independently resumable, and matrix row/column identifiers are preserved as their native META datasets. """ if ( isinstance(start_record_ordinal, bool) or not isinstance(start_record_ordinal, int) or start_record_ordinal < 0 ): raise ValueError("GCTX start record ordinal is invalid") h5py = importlib.import_module("h5py") compressed = path.name.lower().endswith(".gz") temporary_directory: tempfile.TemporaryDirectory[str] | None = None materialized_path = path if compressed: resolved_source_sha256 = ( source_sha256 if isinstance(source_sha256, str) and re.fullmatch(r"[0-9a-f]{64}", source_sha256) is not None else file_sha256(path) ) if scratch_root is None: temporary_directory = tempfile.TemporaryDirectory( prefix="resynthesis-gctx-", ) cache_root = Path(temporary_directory.name) else: cache_root = scratch_root.expanduser().resolve() / "gctx_hdf5_cache" cache_root.mkdir(parents=True, exist_ok=True) materialized_path = cache_root / f"{resolved_source_sha256}.gctx" cache_receipt_path = materialized_path.with_suffix( ".gctx.cache.receipt.json" ) lock_path = materialized_path.with_suffix(".gctx.cache.lock") def file_identity(candidate: Path) -> dict[str, int]: stat = candidate.stat() return { "device": stat.st_dev, "inode": stat.st_ino, "bytes": stat.st_size, "mtimeNs": stat.st_mtime_ns, "ctimeNs": stat.st_ctime_ns, } source_identity = file_identity(path) with lock_path.open("a+b") as lock_handle: fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX) cached = ( json.loads(cache_receipt_path.read_text(encoding="utf-8")) if cache_receipt_path.is_file() else None ) cache_valid = bool( isinstance(cached, dict) and cached.get("schema") == "nnf.resynthesis.gctx_hdf5_cache.v1" and cached.get("sourceSha256") == resolved_source_sha256 and cached.get("sourceIdentity") == source_identity and materialized_path.is_file() and cached.get("artifactIdentity") == file_identity(materialized_path) and cached.get("complete") is True ) if not cache_valid: required_free_bytes = ( 64 * 1024**2 + path.stat().st_size * 4 ) if shutil.disk_usage(cache_root).free < required_free_bytes: raise RuntimeError( "GCTX decompression cache storage admission failed" ) temporary_path = materialized_path.with_name( f".{materialized_path.name}.{os.getpid()}.tmp" ) temporary_path.unlink(missing_ok=True) pigz = shutil.which("pigz") command = ( [pigz, "-dc", "-p", str(min(8, os.cpu_count() or 1)), str(path)] if pigz is not None else None ) try: if command is not None: with tempfile.TemporaryFile(mode="w+b") as error_handle: process = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=error_handle, ) if process.stdout is None: raise RuntimeError( "GCTX decompressor stdout is absent" ) with temporary_path.open("wb") as output_handle: while True: block = process.stdout.read(16 * 1024 * 1024) if not block: break output_handle.write(block) output_handle.flush() os.fsync(output_handle.fileno()) return_code = process.wait() if return_code != 0: error_handle.seek(0) error_text = error_handle.read().decode( "utf-8", errors="replace", ) raise RuntimeError( "GCTX parallel decompression failed: " + error_text[-4096:].strip() ) else: with ( gzip.open(path, "rb") as input_handle, temporary_path.open("wb") as output_handle, ): while True: block = input_handle.read(16 * 1024 * 1024) if not block: break output_handle.write(block) output_handle.flush() os.fsync(output_handle.fileno()) with temporary_path.open("rb") as verification_handle: if ( verification_handle.read(8) != b"\x89HDF\r\n\x1a\n" ): raise RuntimeError( "GCTX decompressed artifact is not HDF5" ) os.replace(temporary_path, materialized_path) _fsync_directory(materialized_path.parent) _atomic_json( cache_receipt_path, { "schema": "nnf.resynthesis.gctx_hdf5_cache.v1", "sourcePath": str(path), "sourceSha256": resolved_source_sha256, "sourceIdentity": source_identity, "artifactPath": str(materialized_path), "artifactIdentity": file_identity(materialized_path), "parallelDecompression": command is not None, "complete": True, "rawDataDeletionAllowed": False, }, ) _fsync_directory(cache_receipt_path.parent) finally: temporary_path.unlink(missing_ok=True) source: Any | None = None try: source = h5py.File(materialized_path, "r") datasets: list[tuple[str, Any]] = [] def collect(name: str, value: Any) -> None: if isinstance(value, h5py.Dataset): datasets.append((name, value)) source.visititems(collect) matrix_datasets = [ (name, dataset) for name, dataset in datasets if name.lower().endswith("/matrix") ] column_ids = next( ( dataset for name, dataset in datasets if name.lower().endswith("/meta/col/id") ), None, ) row_ids = next( ( dataset for name, dataset in datasets if name.lower().endswith("/meta/row/id") ), None, ) if ( len(matrix_datasets) != 1 or column_ids is None or row_ids is None ): raise RuntimeError("GCTX matrix or row/column identity is absent") matrix = matrix_datasets[0][1] if ( matrix.ndim != 2 or column_ids.ndim != 1 or row_ids.ndim != 1 or matrix.shape[0] != column_ids.shape[0] or matrix.shape[1] != row_ids.shape[0] ): raise RuntimeError("GCTX matrix identity geometry differs") def physical_offset(dataset: Any) -> int: offset = dataset.id.get_offset() return offset if isinstance(offset, int) and offset >= 0 else 2**63 - 1 # Preserve physical order for contiguous disk access and page-cache # locality across the complete semantic pass. datasets.sort(key=lambda row: (physical_offset(row[1]), row[0])) source_ordinal = 0 for name, dataset in datasets: logical_row_count = ( 1 if dataset.ndim == 0 else int(dataset.shape[0]) ) numeric_dataset = str(dataset.dtype.kind) in { "b", "i", "u", "f", "c", } trailing_elements = math.prod( int(element) for element in dataset.shape[1:] ) row_bytes = max( 1, int(dataset.dtype.itemsize) * max(1, trailing_elements), ) rows_per_record = ( max(1, (4 * 1024**2) // row_bytes) if dataset.ndim > 0 and numeric_dataset else min(4_096, logical_row_count) ) record_count = ( 1 if dataset.ndim == 0 else math.ceil(logical_row_count / rows_per_record) ) descriptor = { "schema": "nnf.resynthesis.gctx_dataset.v1", "dataset": name, "shape": [int(value) for value in dataset.shape], "dtype": str(dataset.dtype), "logicalRowCount": logical_row_count, "rowsPerRecord": rows_per_record, "recordCount": record_count, } if source_ordinal >= start_record_ordinal: yield ( f"dataset={name}:descriptor", json.dumps( descriptor, sort_keys=True, separators=(",", ":"), ensure_ascii=False, ), ) source_ordinal += 1 if start_record_ordinal >= source_ordinal + record_count: source_ordinal += record_count continue first_record = max( 0, start_record_ordinal - source_ordinal, ) for record_index in range(first_record, record_count): row_start = record_index * rows_per_record row_end = min( logical_row_count, row_start + rows_per_record, ) value = ( dataset[()] if dataset.ndim == 0 else dataset[row_start:row_end] ) numeric_value = ( hasattr(value, "dtype") and str(value.dtype.kind) in {"b", "i", "u", "f", "c"} ) if numeric_value: # Decimal JSON expands scientific matrices by roughly an # order of magnitude and spends most producer time in # float formatting. Typed hexadecimal bytes are exact, # endian-declared, trivially reversible, and tokenize as a # bounded two-character alphabet while retaining the GCTX # dataset/row semantic relationship. value_shape = ",".join( str(int(element)) for element in getattr(value, "shape", ()) ) record_text: FullPayloadReaderValue = ( FullPayloadStructuredNumericRecord( header=( "NNF_GCTX_TYPED_NUMERIC_HEX_V1 " f"dataset={json.dumps(name, ensure_ascii=False)} " f"row_start={row_start} row_end={row_end} " f"dtype={value.dtype.str} shape={value_shape} " "values_hex=" ), values=value.tobytes(order="C"), encoding="typed_numeric_hex_v1", ) ) else: record_text = json.dumps( { "schema": ( "nnf.resynthesis.gctx_dataset_row.v1" ), "dataset": name, "rowStart": row_start, "rowEnd": row_end, "encoding": "json_values_v1", "values": _full_payload_hdf5_json_value(value), }, sort_keys=True, separators=(",", ":"), ensure_ascii=False, ) yield ( f"dataset={name}:rows={row_start}:{row_end}", record_text, ) source_ordinal += record_count if start_record_ordinal > source_ordinal: raise RuntimeError("GCTX durable record cursor exceeds source") finally: if source is not None: source.close() if temporary_directory is not None: temporary_directory.cleanup() _FULL_PAYLOAD_RDS_STREAM_SCRIPT = r""" options(width=10000L, scipen=999) source_path <- commandArgs(trailingOnly=TRUE)[[1L]] root <- readRDS(source_path) chunk_elements <- 4096L path_literal <- function(path) { paste(deparse(path, width.cutoff=500L), collapse="") } value_literal <- function(value) { attributes(value) <- NULL paste( capture.output( dput( value, control=c("keepNA", "keepInteger", "niceNames", "showAttributes") ) ), collapse="" ) } emit <- function(kind, path, fields) { cat( "NNF_RDS_", kind, "_V1 path=", path_literal(path), " ", paste(fields, collapse=" "), "\n", sep="" ) } walk <- function(value, path) { attrs <- attributes(value) class_name <- if (is.null(attrs$class)) character() else attrs$class if (isS4(value)) { emit( "S4", path, c( paste0("class=", value_literal(class_name)), paste0("slot_count=", length(attrs) - as.integer(!is.null(attrs$class))) ) ) for (name in names(attrs)) { if (identical(name, "class")) next walk(attrs[[name]], c(path, paste0("slot:", name))) } return(invisible(NULL)) } if (is.atomic(value) || is.null(value)) { value_length <- length(value) emit( "ATOMIC", path, c( paste0("type=", typeof(value)), paste0("length=", value_length), paste0("class=", value_literal(class_name)), paste0( "dim=", value_literal(if (is.null(attrs$dim)) integer() else attrs$dim) ) ) ) if (value_length > 0L) { starts <- seq.int(1L, value_length, by=chunk_elements) for (start in starts) { finish <- min(value_length, start + chunk_elements - 1L) emit( "VALUES", path, c( paste0("start=", start), paste0("end=", finish), paste0("values=", value_literal(value[start:finish])) ) ) } } if (!is.null(attrs)) { for (name in names(attrs)) { if (name %in% c("class", "dim")) next walk(attrs[[name]], c(path, paste0("attribute:", name))) } } return(invisible(NULL)) } if (is.list(value) || is.pairlist(value)) { value_names <- names(value) emit( "LIST", path, c( paste0("length=", length(value)), paste0("class=", value_literal(class_name)) ) ) for (index in seq_along(value)) { name <- if ( is.null(value_names) || is.na(value_names[[index]]) || !nzchar(value_names[[index]]) ) "" else value_names[[index]] walk( value[[index]], c(path, paste0("element:", index, ":", name)) ) } if (!is.null(attrs)) { for (name in names(attrs)) { if (name %in% c("class", "names")) next walk(attrs[[name]], c(path, paste0("attribute:", name))) } } return(invisible(NULL)) } emit( "LANGUAGE", path, c( paste0("type=", typeof(value)), paste0( "value=", paste(deparse(value, width.cutoff=500L), collapse="") ) ) ) invisible(NULL) } walk(root, c("root")) """ def _iter_rds_path_records( path: Path, *, start_record_ordinal: int = 0, ) -> Iterator[tuple[str, str]]: """Stream a package-independent semantic representation of one RDS object.""" if ( isinstance(start_record_ordinal, bool) or not isinstance(start_record_ordinal, int) or start_record_ordinal < 0 ): raise ValueError("RDS start record ordinal is invalid") rscript = shutil.which("Rscript") if rscript is None: raise RuntimeError("RDS semantic reader requires Rscript") with tempfile.TemporaryFile(mode="w+b") as error_handle: process = subprocess.Popen( [ rscript, "--vanilla", "-e", _FULL_PAYLOAD_RDS_STREAM_SCRIPT, str(path), ], stdout=subprocess.PIPE, stderr=error_handle, text=True, encoding="utf-8", errors="strict", ) try: if process.stdout is None: raise RuntimeError("RDS semantic reader stdout is absent") emitted = 0 for source_ordinal, line in enumerate(process.stdout): if source_ordinal < start_record_ordinal: continue emitted += 1 yield f"rds_record={source_ordinal}", line.rstrip("\n") return_code = process.wait() if return_code != 0: error_handle.seek(0) error_text = error_handle.read().decode( "utf-8", errors="replace", ) raise RuntimeError( "RDS semantic reader failed: " + error_text[-4096:].strip() ) if start_record_ordinal > 0 and emitted == 0: # A cursor at exact EOF is valid; a larger cursor is caught by # the packer's durable observed-record denominator. return finally: if process.poll() is None: process.terminate() process.wait() def _iter_sqlite_path_records(path: Path) -> Iterator[tuple[str, str]]: sqlite3 = importlib.import_module("sqlite3") connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True) try: tables = [ str(row[0]) for row in connection.execute( "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name" ) ] for table in tables: escaped = table.replace('"', '""') cursor = connection.execute(f'SELECT * FROM "{escaped}"') columns = [str(value[0]) for value in cursor.description or ()] for row_index, values in enumerate(cursor): yield ( f"table={table}:row={row_index}", json.dumps( dict(zip(columns, values, strict=True)), sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str, ), ) finally: connection.close() def _iter_ord_dataset_path_records( path: Path, *, display_name: str, ) -> Iterator[tuple[str, str]]: """Decode one complete Open Reaction Database protobuf dataset. This is an explicit external-I/O adapter. ORD protobuf bytes are never treated as lossy text or opaque token content: the official schema parses the complete file, and every reaction is emitted as deterministic JSON. """ dataset_pb2 = importlib.import_module("ord_schema.proto.dataset_pb2") json_format = importlib.import_module("google.protobuf.json_format") dataset = dataset_pb2.Dataset() if path.name.lower().endswith(".gz"): with gzip.open(path, "rb") as handle: payload = handle.read() else: payload = path.read_bytes() dataset.ParseFromString(payload) reaction_count = len(dataset.reactions) metadata = { "schema": "nnf.resynthesis.ord_dataset.v1", "dataset_id": str(dataset.dataset_id), "name": str(dataset.name), "description": str(dataset.description), "reaction_count": reaction_count, "reaction_ids": [str(value) for value in dataset.reaction_ids], } yield ( f"{display_name}#dataset", json.dumps( metadata, sort_keys=True, separators=(",", ":"), ensure_ascii=False, ), ) for ordinal, reaction in enumerate(dataset.reactions): reaction_record = { "schema": "nnf.resynthesis.ord_reaction.v1", "dataset_id": str(dataset.dataset_id), "dataset_name": str(dataset.name), "reaction_ordinal": ordinal, "reaction": json_format.MessageToDict( reaction, preserving_proto_field_name=True, use_integers_for_enums=False, ), } yield ( f"{display_name}#reaction={ordinal}", json.dumps( reaction_record, sort_keys=True, separators=(",", ":"), ensure_ascii=False, ), ) def _iter_existing_payload_path_records( path: Path, *, display_name: str, scratch_root: Path | None, start_record_ordinal: int = 0, source_sha256: str | None = None, ) -> Iterator[tuple[str, FullPayloadReaderValue]]: contract = _payload_format_contract(display_name) container = str(contract["container"]) if contract["readerFamily"] == "ord_protobuf_dataset_adapter": yield from _iter_ord_dataset_path_records(path, display_name=display_name) return if container == "tar": yield from _iter_tar_path_records(path, scratch_root=scratch_root) return if container in {"zip", "zolca"}: yield from _iter_zip_path_records(path, scratch_root=scratch_root) return if container == "seven_zip": py7zr = importlib.import_module("py7zr") if scratch_root is not None: scratch_root.mkdir(parents=True, exist_ok=True) temporary_parent = str(scratch_root) if scratch_root is not None else None with tempfile.TemporaryDirectory( prefix="resynthesis-seven-zip-", dir=temporary_parent, ) as temporary: extracted_root = Path(temporary).resolve() with py7zr.SevenZipFile(path, mode="r") as archive: archive.extractall(path=extracted_root) for extracted in sorted(extracted_root.rglob("*"), key=str): if not extracted.is_file(): continue resolved = extracted.resolve() if extracted_root not in resolved.parents: raise RuntimeError("seven-zip member escaped extraction root") member_name = extracted.relative_to(extracted_root).as_posix() for locator, text in _iter_existing_payload_path_records( extracted, display_name=member_name, scratch_root=scratch_root, ): yield f"{member_name}:{locator}", text return if container == "parquet": yield from _iter_parquet_path_records(path) return if container == "pdf": yield from _iter_pdf_path_records(path) return if container == "spreadsheet": yield from _iter_spreadsheet_path_records( path, display_name=display_name, ) return if container == "numpy": yield from _iter_numpy_path_records(path) return if container == "hdf5": yield from _iter_hdf5_path_records(path) return if container == "gctx": yield from _iter_gctx_path_records( path, start_record_ordinal=start_record_ordinal, scratch_root=scratch_root, source_sha256=source_sha256, ) return if container == "rds": yield from _iter_rds_path_records( path, start_record_ordinal=start_record_ordinal, ) return if container == "sqlite": yield from _iter_sqlite_path_records(path) return if container == "graphdb": yield from _iter_graphdb_dump_path_records(path) return if container == "lmdb": raise RuntimeError("full payload container adapter is unresolved: lmdb") with path.open("rb") as raw: yield from _iter_decompressed_stream_records( raw, name=display_name, format_contract=contract, ) def iter_full_payload_text_records( corpus_root: Path, schedule_row: Mapping[str, object], *, scratch_root: Path | None = None, semantic_registry_roots: tuple[Path, ...] | None = None, start_record_ordinal: int = 0, ) -> Iterator[FullPayloadTextRecord]: """Read every logical text record from one hash-bound schedule work item.""" if schedule_row.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_ENTRY_SCHEMA: raise ValueError("full payload reader received a non-schedule row") if ( isinstance(start_record_ordinal, bool) or not isinstance(start_record_ordinal, int) or start_record_ordinal < 0 ): raise ValueError("full payload reader start ordinal is invalid") relative_value = schedule_row.get("payloadRelativePath") work_id = schedule_row.get("payloadWorkId") source_id = schedule_row.get("sourceId") payload_bytes = schedule_row.get("payloadBytes") format_contract = schedule_row.get("format") if ( not isinstance(relative_value, str) or not relative_value or Path(relative_value).is_absolute() or ".." in Path(relative_value).parts or not isinstance(work_id, str) or len(work_id) != 64 or not isinstance(source_id, str) or not source_id or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or payload_bytes < 0 or not isinstance(format_contract, dict) or schedule_row.get("textProbeUsed") is not False or schedule_row.get("targetEnteredForward") is not False ): raise ValueError("full payload schedule reader identity is malformed") path = corpus_root.expanduser().resolve() / relative_value if not path.is_file() or path.stat().st_size != payload_bytes: raise ValueError(f"full payload reader source differs: {path}") current_contract = _payload_format_contract(relative_value) expected_source_sha256 = schedule_row.get("payloadSha256") semantic_binding: FullPayloadSemanticExportBinding | None = None if ( current_contract.get("container") == "graphdb" and isinstance(expected_source_sha256, str) and len(expected_source_sha256) == 64 ): semantic_binding = discover_full_payload_semantic_export( expected_source_sha256, registry_roots=semantic_registry_roots, ) if semantic_binding is None: raise RuntimeError( "hash-bound graph payload has no complete semantic export" ) if ( semantic_binding.source_bytes != payload_bytes or file_sha256(path) != semantic_binding.source_sha256 ): raise RuntimeError("semantic graph source binding differs") records: Iterator[tuple[str, FullPayloadReaderValue]] = cast( Iterator[tuple[str, FullPayloadReaderValue]], _iter_semantic_graph_export_records(semantic_binding), ) else: records = _iter_existing_payload_path_records( path, display_name=relative_value, scratch_root=( scratch_root.expanduser().resolve() if scratch_root else None ), start_record_ordinal=start_record_ordinal, source_sha256=( expected_source_sha256 if isinstance(expected_source_sha256, str) else None ), ) emitted_records = 0 native_start = ( semantic_binding is None and current_contract.get("container") in {"gctx", "rds"} ) enumeration_start = start_record_ordinal if native_start else 0 for ordinal, (locator, value) in enumerate( records, start=enumeration_start, ): if ordinal < start_record_ordinal: continue if isinstance(value, FullPayloadReaderDegradation): degradation = dict(value.authority) authority_sha256 = degradation.pop( "readerDegradationAuthoritySha256", None, ) if ( degradation.get("schema") != FULL_PAYLOAD_READER_DEGRADATION_SCHEMA or degradation.get("semanticDecodePassed") is not False or degradation.get("trainingKnowledgeReady") is not False or degradation.get("rawBytesPreserved") is not True or degradation.get("reacquisitionRequired") is not True or authority_sha256 != _sha256_bytes(_json_bytes(degradation)) ): raise RuntimeError( "full payload reader degradation authority differs" ) degradation["readerDegradationAuthoritySha256"] = ( authority_sha256 ) emitted_records += 1 yield FullPayloadTextRecord( payload_work_id=work_id, source_id=source_id, locator=locator, ordinal=ordinal, text="", reader_family=( f"{current_contract['readerFamily']}_degraded" ), semantic_export_receipt_sha256=None, reader_degradation=degradation, ) continue typed_numeric_bytes: bytes | None = None typed_numeric_encoding: str | None = None if isinstance(value, FullPayloadStructuredNumericRecord): record_text = value.header typed_numeric_bytes = value.values typed_numeric_encoding = value.encoding elif isinstance(value, str): record_text = value else: raise RuntimeError("full payload reader emitted a non-text record") emitted_records += 1 yield FullPayloadTextRecord( payload_work_id=work_id, source_id=source_id, locator=( f"semantic:{semantic_binding.receipt_sha256}:{locator}" if semantic_binding is not None else locator ), ordinal=ordinal, text=record_text, reader_family=( "verified_complete_semantic_graph_export" if semantic_binding is not None else str(current_contract["readerFamily"]) ), semantic_export_receipt_sha256=( semantic_binding.receipt_sha256 if semantic_binding is not None else None ), typed_numeric_bytes=typed_numeric_bytes, typed_numeric_encoding=typed_numeric_encoding, ) if ( emitted_records == 0 and payload_bytes > 0 and start_record_ordinal == 0 ): with path.open("rb") as raw: for ordinal, (locator, text) in enumerate( _iter_lossless_binary_stream_records( raw, name=relative_value, ) ): yield FullPayloadTextRecord( payload_work_id=work_id, source_id=source_id, locator=f"lossless_zero_record_fallback:{locator}", ordinal=ordinal, text=text, reader_family=( f"{current_contract['readerFamily']}" "_lossless_zero_record_fallback" ), semantic_export_receipt_sha256=None, ) def iter_full_payload_token_windows( records: Iterable[FullPayloadTextRecord], *, tokenizer: Any, context_window_tokens: int, answer_tokens_per_window: int, ) -> Iterator[dict[str, Any]]: """Assign every record token to one target-only Fastokens loss window.""" if ( isinstance(context_window_tokens, bool) or isinstance(answer_tokens_per_window, bool) or context_window_tokens < 3 or answer_tokens_per_window < 1 or answer_tokens_per_window >= context_window_tokens ): raise ValueError("full payload token-window geometry is invalid") for record in records: prefix = ( f"Source: {record.source_id}\n" f"Payload work: {record.payload_work_id}\n" f"Record: {record.locator}\n" "Content:\n" ) prefix_ids, content_ids = _tokenize_prefix_and_answer( tokenizer, prefix, record.text, ) if len(prefix_ids) >= context_window_tokens - 1: raise ValueError("full payload provenance prefix exceeds model context") maximum_answer_width = min( answer_tokens_per_window, context_window_tokens - len(prefix_ids), ) if maximum_answer_width < 1: raise ValueError("full payload window has no loss-token capacity") for token_start in range(0, len(content_ids), maximum_answer_width): answer_ids = content_ids[token_start : token_start + maximum_answer_width] context_capacity = context_window_tokens - len(prefix_ids) - len(answer_ids) context_start = max(0, token_start - context_capacity) prompt_ids = prefix_ids + content_ids[context_start:token_start] input_ids = prompt_ids + answer_ids target_ids = [-100] * len(prompt_ids) + answer_ids question_id = _stable_rank( "nnf.resynthesis.full_payload_token_window.v1", ( f"{record.payload_work_id}\x00{record.locator}\x00" f"{token_start}\x00{token_start + len(answer_ids)}" ), ) yield { "schema": "nnf.resynthesis.full_payload_token_window.v1", "question_id": question_id, "source_id": record.source_id, "payload_work_id": record.payload_work_id, "payload_record_locator": record.locator, "payload_record_ordinal": record.ordinal, "payload_record_token_start": token_start, "payload_record_token_end": token_start + len(answer_ids), "payload_record_token_count": len(content_ids), "payload_reader_family": record.reader_family, "semantic_export_receipt_sha256": ( record.semantic_export_receipt_sha256 ), "payload_record_complete": ( token_start + len(answer_ids) == len(content_ids) ), "prompt_ids": prompt_ids, "answer_ids": answer_ids, "input_ids": input_ids, "target_ids": target_ids, "generalization_axis": "task_family", "generalization_group": record.payload_work_id, "rights_disposition": "training_admissible", "target_entered_forward": False, "task_intent_targets_entered_forward": False, "model_scores_observed": False, } class FullPayloadSequentialTrainingBatches: """Resumable optimizer batches over every hash-verified payload token. This class is an external-I/O boundary. It follows the immutable payload schedule in order, admits only the contiguous prefix already present in the fsynced byte-hash ledger, and materializes a bounded proposal window. A rejected proposal is replayed from the in-memory cache. File-boundary cursor rows are fsynced so a cold restart skips completed payload files and, at worst, replays the current compressed file. """ def __init__( self, schedule_receipt_path: Path, corpus_root: Path, hash_ledger_path: Path, work_cursor_ledger_path: Path, *, tokenizer: Any, context_window_tokens: int, answer_tokens_per_window: int, maximum_proposal_rows: int, scratch_root: Path | None = None, hash_frontier_poll_interval_seconds: float = 1.0, ) -> None: if ( isinstance(maximum_proposal_rows, bool) or maximum_proposal_rows < 1 ): raise ValueError("full payload proposal width must be positive") if ( isinstance(hash_frontier_poll_interval_seconds, bool) or not isinstance(hash_frontier_poll_interval_seconds, (int, float)) or not math.isfinite(hash_frontier_poll_interval_seconds) or hash_frontier_poll_interval_seconds <= 0 ): raise ValueError("full payload hash-frontier poll interval must be positive") self.schedule_receipt_path = schedule_receipt_path.expanduser().resolve() self.corpus_root = corpus_root.expanduser().resolve() self.hash_ledger_path = hash_ledger_path.expanduser().resolve() self.work_cursor_ledger_path = ( work_cursor_ledger_path.expanduser().resolve() ) self.scratch_root = ( scratch_root.expanduser().resolve() if scratch_root is not None else None ) self.tokenizer = tokenizer self.context_window_tokens = context_window_tokens self.answer_tokens_per_window = answer_tokens_per_window self.maximum_proposal_rows = maximum_proposal_rows self.hash_frontier_poll_interval_seconds = float( hash_frontier_poll_interval_seconds ) receipt_value = json.loads( self.schedule_receipt_path.read_text(encoding="utf-8") ) if ( not isinstance(receipt_value, dict) or receipt_value.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA or receipt_value.get("passed") is not True ): raise ValueError("full payload sequential pool schedule did not pass") schedule_record = receipt_value.get("schedule") if not isinstance(schedule_record, dict): raise ValueError("full payload sequential pool schedule is absent") self.schedule_path = Path(str(schedule_record.get("path", ""))).resolve() self.schedule_sha256 = str(schedule_record.get("sha256", "")) expected_rows = schedule_record.get("rows") if ( not self.schedule_path.is_file() or len(self.schedule_sha256) != 64 or file_sha256(self.schedule_path) != self.schedule_sha256 or not isinstance(expected_rows, int) or isinstance(expected_rows, bool) or expected_rows < 1 ): raise ValueError("full payload sequential pool schedule differs") self._schedule_rows = _read_jsonl(self.schedule_path) if len(self._schedule_rows) != expected_rows: raise ValueError("full payload sequential pool row count differs") self._schedule_ordinal_by_work_id: dict[str, int] = {} for ordinal, row in enumerate(self._schedule_rows): work_id = row.get("payloadWorkId") if ( row.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_ENTRY_SCHEMA or row.get("scheduleOrdinal") != ordinal or not isinstance(work_id, str) or len(work_id) != 64 or work_id in self._schedule_ordinal_by_work_id ): raise ValueError("full payload sequential pool row is malformed") self._schedule_ordinal_by_work_id[work_id] = ordinal if not self.hash_ledger_path.is_file(): raise FileNotFoundError( f"full payload hash ledger is absent: {self.hash_ledger_path}" ) hash_ledger_stat = self.hash_ledger_path.stat() self._hash_ledger_device = hash_ledger_stat.st_dev self._hash_ledger_inode = hash_ledger_stat.st_ino self._hash_ledger_offset = 0 self._hash_ledger_prefix_digest = hashlib.sha256() self._hash_ledger_chain_digest = hashlib.sha256( ( FULL_PAYLOAD_HASH_COMMIT_FRONTIER_SCHEMA + "\x00" + self.schedule_sha256 ).encode("ascii") ).digest() self._hash_ledger_rows_read = 0 self._hash_ledger_payload_bytes_read = 0 self._hash_ledger_final_work_id = "" self.work_cursor_ledger_path.parent.mkdir(parents=True, exist_ok=True) self._hash_by_ordinal: dict[int, dict[str, Any]] = {} self._hash_prefix_rows = 0 self._work_cursor_rows = self._load_work_cursor_rows() self._cache: dict[int, dict[str, Any]] = {} self._positioned = False self._stream_cursor = 0 self._current_schedule_ordinal = 0 self._current_work_cursor_start = 0 self._current_work_window_count = 0 self._current_work_record_count = 0 self._current_work_reader_families: set[str] = set() self._current_work_semantic_receipt_sha256s: set[str] = set() self._current_window_iterator: Iterator[dict[str, Any]] | None = None self._total_windows: int | None = ( int(self._work_cursor_rows[-1]["cursorEnd"]) if len(self._work_cursor_rows) == len(self._schedule_rows) else None ) self._blocked_on_hash = False self._hash_frontier_wait_count = 0 self._hash_frontier_wait_seconds = 0.0 self._refresh_hash_prefix() def __len__(self) -> int: """Return non-empty truthiness without pretending the window count is known.""" return 1 def __getitem__(self, index: int) -> list[dict[str, Any]]: return self.batch_for_cursor(index) @staticmethod def _complete_jsonl_rows(path: Path) -> Iterator[dict[str, Any]]: if not path.is_file(): return with path.open("rb") as handle: for raw_line in handle: if not raw_line.endswith(b"\n"): break value = json.loads(raw_line) if not isinstance(value, dict): raise ValueError(f"JSONL object is malformed: {path}") yield value def _load_work_cursor_rows(self) -> list[dict[str, Any]]: rows = list(self._complete_jsonl_rows(self.work_cursor_ledger_path)) prior_cursor_end = 0 for ordinal, row in enumerate(rows): cursor_start = row.get("cursorStart") cursor_end = row.get("cursorEnd") work_id = row.get("payloadWorkId") if ( row.get("schema") != FULL_PAYLOAD_WORK_CURSOR_LEDGER_SCHEMA or row.get("scheduleSha256") != self.schedule_sha256 or row.get("scheduleOrdinal") != ordinal or not isinstance(work_id, str) or work_id != self._schedule_rows[ordinal].get("payloadWorkId") or not isinstance(cursor_start, int) or isinstance(cursor_start, bool) or not isinstance(cursor_end, int) or isinstance(cursor_end, bool) or cursor_start != prior_cursor_end or cursor_end < cursor_start or row.get("targetValuesRecorded") is not False ): raise ValueError("full payload work cursor ledger differs") prior_cursor_end = cursor_end return rows def _refresh_hash_prefix(self) -> None: ledger_stat = self.hash_ledger_path.stat() if ( ledger_stat.st_dev != self._hash_ledger_device or ledger_stat.st_ino != self._hash_ledger_inode or ledger_stat.st_size < self._hash_ledger_offset ): raise RuntimeError("full payload hash ledger identity changed") commit_frontier = _read_full_payload_hash_commit_frontier( self.hash_ledger_path, self.schedule_sha256, ) terminal = ( commit_frontier.durable_ledger_bytes if commit_frontier is not None else ledger_stat.st_size ) if terminal < self._hash_ledger_offset: raise RuntimeError( "full payload hash commit frontier moved behind the reader" ) with self.hash_ledger_path.open("rb") as handle: handle.seek(self._hash_ledger_offset) while handle.tell() < terminal: raw_line = handle.readline(terminal - handle.tell()) if not raw_line or not raw_line.endswith(b"\n"): break row = json.loads(raw_line) if not isinstance(row, dict): raise ValueError("full payload hash ledger row is malformed") work_id = row.get("payloadWorkId") if not isinstance(work_id, str): raise ValueError( "full payload hash ledger work identity is malformed" ) ordinal = self._schedule_ordinal_by_work_id.get(work_id) if ordinal is None or ordinal in self._hash_by_ordinal: raise ValueError("full payload hash ledger work identity differs") schedule_row = self._schedule_rows[ordinal] identity = row.get("sourceIdentity") observed_sha256 = row.get("observedSha256") relative_path = schedule_row.get("payloadRelativePath") if ( row.get("schema") != FULL_PAYLOAD_HASH_LEDGER_SCHEMA or row.get("scheduleSha256") != self.schedule_sha256 or row.get("payloadBytes") != schedule_row.get("payloadBytes") or row.get("payloadRelativePath") != relative_path or row.get("allPayloadBytesHashed") is not True or row.get("modelTrainingClaimed") is not False or row.get("rawDataDeletionAllowed") is not False or row.get("expectedSha256Matched") not in {None, True} or not isinstance(observed_sha256, str) or len(observed_sha256) != 64 or not isinstance(identity, dict) or not isinstance(relative_path, str) ): raise ValueError("full payload hash ledger row differs") payload_path = self.corpus_root / relative_path stat = payload_path.stat() if ( identity.get("device") != stat.st_dev or identity.get("inode") != stat.st_ino or identity.get("bytes") != stat.st_size or identity.get("mtimeNs") != stat.st_mtime_ns ): raise RuntimeError( "full payload changed after byte-hash admission: " f"{payload_path}" ) self._hash_by_ordinal[ordinal] = row self._hash_ledger_prefix_digest.update(raw_line) self._hash_ledger_chain_digest = hashlib.sha256( self._hash_ledger_chain_digest + raw_line ).digest() self._hash_ledger_rows_read += 1 self._hash_ledger_payload_bytes_read += int( row["payloadBytes"] ) self._hash_ledger_final_work_id = work_id self._hash_ledger_offset = handle.tell() if ( commit_frontier is not None and self._hash_ledger_offset == commit_frontier.durable_ledger_bytes and ( self._hash_ledger_rows_read != commit_frontier.durable_rows or self._hash_ledger_payload_bytes_read != commit_frontier.durable_payload_bytes or self._hash_ledger_prefix_digest.hexdigest() != commit_frontier.prefix_sha256 or self._hash_ledger_chain_digest.hex() != commit_frontier.prefix_chain_sha256 or self._hash_ledger_final_work_id != commit_frontier.final_payload_work_id or self._hash_ledger_rows_read - 1 != commit_frontier.final_sequence ) ): raise RuntimeError("full payload hash commit prefix differs") while self._hash_prefix_rows in self._hash_by_ordinal: self._hash_prefix_rows += 1 def _append_work_cursor_row(self) -> None: ordinal = self._current_schedule_ordinal cursor_end = self._stream_cursor schedule_row = self._schedule_rows[ordinal] hash_row = self._hash_by_ordinal[ordinal] if ( int(schedule_row["payloadBytes"]) > 0 and ( self._current_work_record_count < 1 or self._current_work_window_count < 1 ) ): raise RuntimeError( "non-empty full payload emitted no trainable token windows" ) row = { "schema": FULL_PAYLOAD_WORK_CURSOR_LEDGER_SCHEMA, "recordedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "scheduleSha256": self.schedule_sha256, "scheduleOrdinal": ordinal, "payloadWorkId": schedule_row["payloadWorkId"], "observedPayloadSha256": hash_row["observedSha256"], "cursorStart": self._current_work_cursor_start, "cursorEnd": cursor_end, "windowCount": self._current_work_window_count, "recordCount": self._current_work_record_count, "sourcePayloadBytesHashVerified": schedule_row["payloadBytes"], "payloadBytesReadThroughRecordAdapter": ( 0 if self._current_work_semantic_receipt_sha256s else schedule_row["payloadBytes"] ), "readerFamilies": sorted(self._current_work_reader_families), "semanticExportReceiptSha256s": sorted( self._current_work_semantic_receipt_sha256s ), "completeSemanticTransformConsumed": bool( self._current_work_semantic_receipt_sha256s ), "targetValuesRecorded": False, "rawDataDeletionAllowed": False, } if ordinal < len(self._work_cursor_rows): existing = self._work_cursor_rows[ordinal] compared_fields = ( "scheduleSha256", "scheduleOrdinal", "payloadWorkId", "observedPayloadSha256", "cursorStart", "cursorEnd", "windowCount", "recordCount", ) if any(existing.get(field) != row.get(field) for field in compared_fields): raise RuntimeError("full payload replay changed a work cursor row") return if ordinal != len(self._work_cursor_rows): raise RuntimeError("full payload work cursor ledger is not contiguous") with self.work_cursor_ledger_path.open("ab") as handle: handle.write(_json_bytes(row) + b"\n") handle.flush() os.fsync(handle.fileno()) self._work_cursor_rows.append(row) def _prepare_work(self) -> bool: self._refresh_hash_prefix() if self._current_schedule_ordinal >= len(self._schedule_rows): self._total_windows = self._stream_cursor self._blocked_on_hash = False return False if self._current_schedule_ordinal >= self._hash_prefix_rows: self._blocked_on_hash = True return False schedule_row = self._schedule_rows[self._current_schedule_ordinal] self._current_work_cursor_start = self._stream_cursor self._current_work_window_count = 0 self._current_work_record_count = 0 self._current_work_reader_families.clear() self._current_work_semantic_receipt_sha256s.clear() def counted_records() -> Iterator[FullPayloadTextRecord]: for record in iter_full_payload_text_records( self.corpus_root, schedule_row, scratch_root=self.scratch_root, ): self._current_work_record_count += 1 self._current_work_reader_families.add(record.reader_family) if record.semantic_export_receipt_sha256 is not None: self._current_work_semantic_receipt_sha256s.add( record.semantic_export_receipt_sha256 ) yield record self._current_window_iterator = iter_full_payload_token_windows( counted_records(), tokenizer=self.tokenizer, context_window_tokens=self.context_window_tokens, answer_tokens_per_window=self.answer_tokens_per_window, ) self._blocked_on_hash = False return True def _next_stream_row(self) -> tuple[int, dict[str, Any]] | None: while True: if self._current_window_iterator is None and not self._prepare_work(): return None iterator = self._current_window_iterator if iterator is None: raise RuntimeError("full payload work iterator was not initialized") try: row = next(iterator) except StopIteration: self._append_work_cursor_row() self._current_schedule_ordinal += 1 self._current_window_iterator = None continue cursor = self._stream_cursor self._stream_cursor += 1 self._current_work_window_count += 1 hash_row = self._hash_by_ordinal[self._current_schedule_ordinal] locator = str(row.get("payload_record_locator", "")) row["source_record_id"] = hashlib.sha256( ( f"{row.get('payload_work_id', '')}\x00{locator}\x00" f"{row.get('payload_record_ordinal', '')}" ).encode("utf-8") ).hexdigest() row["source_sha256"] = hash_row["observedSha256"] row["prompt_sha256"] = hashlib.sha256( json.dumps( row.get("prompt_ids", []), separators=(",", ":"), ).encode("utf-8") ).hexdigest() row["corpus_surface_family"] = "full_payload_sequential" row["full_payload_global_cursor"] = cursor return cursor, row def _reset_to_cursor(self, cursor: int) -> None: self._cache.clear() self._current_window_iterator = None self._total_windows = ( int(self._work_cursor_rows[-1]["cursorEnd"]) if len(self._work_cursor_rows) == len(self._schedule_rows) else None ) self._blocked_on_hash = False start_ordinal = 0 start_cursor = 0 for row in self._work_cursor_rows: cursor_end = int(row["cursorEnd"]) if cursor_end <= cursor: start_ordinal = int(row["scheduleOrdinal"]) + 1 start_cursor = cursor_end continue start_ordinal = int(row["scheduleOrdinal"]) start_cursor = int(row["cursorStart"]) break self._current_schedule_ordinal = start_ordinal self._stream_cursor = start_cursor self._positioned = True while self._stream_cursor < cursor: advanced = self._next_stream_row() if advanced is None: raise RuntimeError( "full payload cursor is beyond currently hash-verified data" ) def _ensure_cached(self, cursor: int, rows: int) -> int: if cursor < 0 or rows < 1: raise ValueError("full payload cache request geometry is invalid") if not self._positioned: self._reset_to_cursor(cursor) elif cursor < self._stream_cursor and cursor not in self._cache: self._reset_to_cursor(cursor) elif cursor > self._stream_cursor: while self._stream_cursor < cursor: advanced = self._next_stream_row() if advanced is None: raise RuntimeError( "full payload cursor is beyond currently hash-verified data" ) self._cache = { index: row for index, row in self._cache.items() if index >= cursor } wanted_end = cursor + rows while any(index not in self._cache for index in range(cursor, wanted_end)): advanced = self._next_stream_row() if advanced is None: break index, row = advanced self._cache[index] = row available = 0 while cursor + available in self._cache: available += 1 return available def proposal_window_limit(self, cursor: int) -> int: """Return a bounded, replayable proposal without crossing unavailable data.""" available = self._ensure_cached(cursor, self.maximum_proposal_rows + 1) proposal_rows = min(available, self.maximum_proposal_rows) if proposal_rows < 1: if self.training_complete(cursor): raise RuntimeError("full payload training schedule is complete") if self._blocked_on_hash: raise RuntimeError( "full payload training reached the durable byte-hash frontier" ) raise RuntimeError("full payload training produced no token windows") return proposal_rows def await_proposal_window(self, cursor: int) -> int: """Wait for append-only hash authority without ending the training epoch. This is an external-I/O boundary, not a model routing surface. The wait is deliberately uncapped: completion is owned by the immutable schedule, while progress is owned by newly fsynced rows on the already-bound hash ledger inode. Replacement, truncation, malformed rows, or payload drift continue to fail closed in ``_refresh_hash_prefix``. """ if cursor < 0: raise ValueError("full payload training cursor cannot be negative") while True: available = self._ensure_cached( cursor, self.maximum_proposal_rows + 1, ) proposal_rows = min(available, self.maximum_proposal_rows) if proposal_rows > 0: return proposal_rows if self.training_complete(cursor): return 0 if not self._blocked_on_hash: raise RuntimeError("full payload training produced no token windows") wait_started = time.monotonic() time.sleep(self.hash_frontier_poll_interval_seconds) self._hash_frontier_wait_seconds += time.monotonic() - wait_started self._hash_frontier_wait_count += 1 def batch_for_cursor(self, cursor: int) -> list[dict[str, Any]]: if self._ensure_cached(cursor, 1) < 1: raise IndexError("full payload training cursor is unavailable") return [self._cache[cursor]] def training_window_receipt( self, cursor_start: int, rows: int, ) -> dict[str, Any]: if rows < 1 or self._ensure_cached(cursor_start, rows) < rows: raise RuntimeError("full payload proposal is not fully materialized") digest = hashlib.sha256() work_ids: list[str] = [] first: dict[str, Any] | None = None last: dict[str, Any] | None = None for cursor in range(cursor_start, cursor_start + rows): row = self._cache[cursor] identity = { "cursor": cursor, "questionId": row.get("question_id"), "payloadWorkId": row.get("payload_work_id"), "recordLocator": row.get("payload_record_locator"), "recordOrdinal": row.get("payload_record_ordinal"), "tokenStart": row.get("payload_record_token_start"), "tokenEnd": row.get("payload_record_token_end"), "sourceSha256": row.get("source_sha256"), "readerFamily": row.get("payload_reader_family"), "semanticExportReceiptSha256": row.get( "semantic_export_receipt_sha256" ), } digest.update(_json_bytes(identity) + b"\n") work_id = str(row.get("payload_work_id", "")) if not work_ids or work_ids[-1] != work_id: work_ids.append(work_id) if first is None: first = identity last = identity return { "schema": FULL_PAYLOAD_TRAINING_WINDOW_RECEIPT_SCHEMA, "scheduleSha256": self.schedule_sha256, "cursorStart": cursor_start, "cursorEnd": cursor_start + rows, "rows": rows, "windowIdentitySha256": digest.hexdigest(), "payloadWorkIds": work_ids, "firstWindow": first, "lastWindow": last, "hashLedgerPath": str(self.hash_ledger_path), "hashVerifiedContiguousPayloadFiles": self._hash_prefix_rows, "assignedWindowConsumedAtCursorEnd": self.training_complete( cursor_start + rows ), "completeDatasetConsumedAtCursorEnd": self.training_complete( cursor_start + rows ), "targetValuesRecorded": False, "validationRowsObserved": 0, "targetEnteredForward": False, "rawDataDeletionAllowed": False, } def training_complete(self, cursor: int) -> bool: if cursor < 0: raise ValueError("full payload training cursor cannot be negative") return self._total_windows is not None and cursor >= self._total_windows def cursor_receipt(self, cursor: int) -> dict[str, Any]: if cursor < 0: raise ValueError("full payload training cursor cannot be negative") return { "schema": "nnf.resynthesis.full_payload_training_cursor.v1", "scheduleSha256": self.schedule_sha256, "retainedTokenWindows": cursor, "completedPayloadFiles": sum( int(row["cursorEnd"]) <= cursor for row in self._work_cursor_rows ), "scheduledPayloadFiles": len(self._schedule_rows), "hashVerifiedContiguousPayloadFiles": self._hash_prefix_rows, "assignedWindowConsumed": self.training_complete(cursor), "completeDatasetConsumed": self.training_complete(cursor), "totalTokenWindows": self._total_windows, "targetValuesRecorded": False, "rawDataDeletionAllowed": False, } def authority_receipt(self) -> dict[str, Any]: self._refresh_hash_prefix() return { "schema": "nnf.resynthesis.full_payload_sequential_training.v1", "schedulePath": str(self.schedule_path), "scheduleSha256": self.schedule_sha256, "scheduledPayloadFiles": len(self._schedule_rows), "hashLedgerPath": str(self.hash_ledger_path), "hashLedgerBytesRead": self._hash_ledger_offset, "hashLedgerObservedPrefixSha256": ( self._hash_ledger_prefix_digest.hexdigest() ), "hashVerifiedContiguousPayloadFiles": self._hash_prefix_rows, "hashFrontierWaitCount": self._hash_frontier_wait_count, "hashFrontierWaitSeconds": self._hash_frontier_wait_seconds, "appendOnlyHashLedgerFollowed": True, "hashFrontierWaitUncapped": True, "workCursorLedgerPath": str(self.work_cursor_ledger_path), "maximumProposalRows": self.maximum_proposal_rows, "contextWindowTokens": self.context_window_tokens, "answerTokensPerWindow": self.answer_tokens_per_window, "fullPayloadRecordsStreamed": True, "textProbeUsed": False, "completeDatasetClaimedTrained": False, "targetEnteredForward": False, "rawDataDeletionAllowed": False, } def default_full_payload_packed_collection_path( schedule_receipt_path: Path, ) -> Path: """Return the source-owned packed collection location used without a flag.""" return ( schedule_receipt_path.expanduser().resolve().parent / "full_payload_packed_tokens" / "collection.receipt.json" ) def resolve_full_payload_packed_collection_path( schedule_receipt_path: Path, ) -> Path: """Resolve an external packed store without changing model routing.""" configured = os.environ.get( "NNF_RESYNTHESIS_FULL_PAYLOAD_PACKED_COLLECTION", "", ).strip() return ( Path(configured).expanduser().resolve() if configured else default_full_payload_packed_collection_path(schedule_receipt_path) ) def _validated_full_payload_packed_sources( schedule_receipt_path: Path, corpus_root: Path, hash_ledger_path: Path, *, source_ids: Sequence[str] | None = None, payload_work_ids: Sequence[str] | None = None, ) -> tuple[dict[str, Any], Path, str, list[dict[str, Any]], list[dict[str, Any]]]: receipt_path = schedule_receipt_path.expanduser().resolve() root = corpus_root.expanduser().resolve() ledger_path = hash_ledger_path.expanduser().resolve() receipt = json.loads(receipt_path.read_text(encoding="utf-8")) if ( not isinstance(receipt, dict) or receipt.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA or receipt.get("passed") is not True ): raise ValueError("full payload packed source schedule did not pass") schedule_record = receipt.get("schedule") if not isinstance(schedule_record, dict): raise ValueError("full payload packed source schedule is absent") schedule_path = Path(str(schedule_record.get("path", ""))).expanduser().resolve() schedule_sha256 = str(schedule_record.get("sha256", "")) expected_rows = schedule_record.get("rows") if ( not schedule_path.is_file() or len(schedule_sha256) != 64 or file_sha256(schedule_path) != schedule_sha256 or not isinstance(expected_rows, int) or isinstance(expected_rows, bool) or expected_rows < 1 ): raise ValueError("full payload packed source schedule differs") persisted_schedule_rows = _read_jsonl(schedule_path) if len(persisted_schedule_rows) != expected_rows: raise ValueError("full payload packed source schedule row count differs") schedule_rows, _authority_upgrades = ( _enrich_full_payload_schedule_rows_from_inventory( receipt, persisted_schedule_rows, ) ) work_to_ordinal: dict[str, int] = {} scheduled_source_ids: set[str] = set() for ordinal, row in enumerate(schedule_rows): work_id = row.get("payloadWorkId") source_id = row.get("sourceId") if ( row.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_ENTRY_SCHEMA or row.get("scheduleOrdinal") != ordinal or not isinstance(work_id, str) or len(work_id) != 64 or work_id in work_to_ordinal or not isinstance(source_id, str) or not source_id or row.get("rightsDisposition") != "training_admissible" or row.get("targetEnteredForward") is not False ): raise ValueError("full payload packed source row is malformed") work_to_ordinal[work_id] = ordinal scheduled_source_ids.add(source_id) if source_ids is not None and payload_work_ids is not None: raise ValueError( "full payload packed source and work selections cannot be combined" ) if payload_work_ids is not None: selected_work_ids = tuple(payload_work_ids) if ( not selected_work_ids or any( not isinstance(work_id, str) or len(work_id) != 64 for work_id in selected_work_ids ) or len(set(selected_work_ids)) != len(selected_work_ids) or not set(selected_work_ids).issubset(work_to_ordinal) ): raise ValueError("full payload packed work selection differs") selected_work_set = set(selected_work_ids) selected_ordinals = tuple( ordinal for ordinal, row in enumerate(schedule_rows) if row["payloadWorkId"] in selected_work_set ) else: selected_source_ids = ( tuple(sorted(scheduled_source_ids)) if source_ids is None else tuple(source_ids) ) if ( not selected_source_ids or any( not isinstance(source_id, str) or not source_id for source_id in selected_source_ids ) or len(set(selected_source_ids)) != len(selected_source_ids) or not set(selected_source_ids).issubset(scheduled_source_ids) ): raise ValueError("full payload packed source selection differs") selected_source_set = set(selected_source_ids) selected_ordinals = tuple( ordinal for ordinal, row in enumerate(schedule_rows) if row["sourceId"] in selected_source_set ) selected_ordinal_set = set(selected_ordinals) if not ledger_path.is_file(): raise FileNotFoundError(f"full payload hash ledger is absent: {ledger_path}") durable_hash_rows, _durable_hash_frontier = ( _read_durable_full_payload_hash_rows( ledger_path, schedule_sha256, ) ) hash_by_ordinal: dict[int, dict[str, Any]] = {} proven_device_remaps: set[tuple[int, int]] = set() for row in durable_hash_rows: work_id = row.get("payloadWorkId") matched_ordinal = work_to_ordinal.get(str(work_id)) if matched_ordinal is None: raise ValueError("full payload packed hash work identity differs") if matched_ordinal not in selected_ordinal_set: continue if matched_ordinal in hash_by_ordinal: raise ValueError("full payload packed hash work identity differs") schedule_row = schedule_rows[matched_ordinal] relative_path = schedule_row.get("payloadRelativePath") observed_sha256 = row.get("observedSha256") identity = row.get("sourceIdentity") if ( row.get("schema") != FULL_PAYLOAD_HASH_LEDGER_SCHEMA or row.get("scheduleSha256") != schedule_sha256 or row.get("sourceId") != schedule_row.get("sourceId") or row.get("payloadBytes") != schedule_row.get("payloadBytes") or row.get("payloadRelativePath") != relative_path or row.get("allPayloadBytesHashed") is not True or row.get("modelTrainingClaimed") is not False or row.get("rawDataDeletionAllowed") is not False or row.get("expectedSha256Matched") not in {None, True} or not isinstance(relative_path, str) or not isinstance(observed_sha256, str) or len(observed_sha256) != 64 or not isinstance(identity, dict) ): raise ValueError("full payload packed hash row differs") payload_path = root / relative_path if not _full_payload_source_identity_matches( identity, payload_path, observed_sha256=observed_sha256, proven_device_remaps=proven_device_remaps, ): raise RuntimeError( f"full payload changed after byte-hash admission: {payload_path}" ) hash_by_ordinal[matched_ordinal] = row if set(hash_by_ordinal) != selected_ordinal_set: raise RuntimeError( "full payload packed build requires every selected hash identity" ) return ( receipt, schedule_path, schedule_sha256, [schedule_rows[index] for index in selected_ordinals], [hash_by_ordinal[index] for index in selected_ordinals], ) def _packed_schedule_authorities(row: Mapping[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: domain = row.get("domainAuthority") rights = row.get("rightsAuthority") source_id = row.get("sourceId") source_record_sha256 = row.get("sourceRecordSha256") if ( not isinstance(domain, dict) or not isinstance(rights, dict) or not isinstance(source_id, str) or not source_id or not isinstance(source_record_sha256, str) or len(source_record_sha256) != 64 or not isinstance(domain.get("domain"), str) or not domain["domain"] or not isinstance(domain.get("contentClaims"), list) or any( not isinstance(claim, str) or not claim for claim in domain["contentClaims"] ) or domain.get("derivation") == "legacy_full_payload_schedule_scope_v1" or rights.get("derivation") == "legacy_full_payload_schedule_row_v1" or rights.get("rightsDisposition") != "training_admissible" or not isinstance(rights.get("intendedUses"), list) or "model_training" not in set(rights.get("intendedUses", [])) or ( domain.get("sourceId") is not None and domain.get("sourceId") != source_id ) or ( rights.get("sourceId") is not None and rights.get("sourceId") != source_id ) or ( domain.get("sourceRecordSha256") is not None and domain.get("sourceRecordSha256") != source_record_sha256 ) or ( rights.get("sourceRecordSha256") is not None and rights.get("sourceRecordSha256") != source_record_sha256 ) ): raise ValueError("full payload packed domain or rights authority is malformed") return dict(domain), dict(rights) def _validated_full_payload_fastokens_authority( value: Mapping[str, Any], ) -> tuple[dict[str, Any], str, int]: """Validate the exact Fastokens/tokenizer bytes used to create a pack.""" record = dict(value) tokenizer_path_value = record.get("tokenizerPath") tokenizer_sha256 = record.get("tokenizerSha256") dependency_path_value = record.get("dependencyReceiptPath") dependency_sha256 = record.get("dependencyReceiptSha256") vocabulary_size = record.get("vocabularySize") backend_module = record.get("backendModule") backend_class = record.get("backendClass") fastokens_version = record.get("fastokensVersion") fastokens_commit = record.get("fastokensCommit") site_path_value = record.get("sitePath") source_archive_sha256 = record.get("sourceArchiveSha256") wheel_sha256 = record.get("wheelSha256") verified_installed_files = record.get("verifiedInstalledFiles") if ( record.get("schema") != "nnf.resynthesis.fastokens_boundary.v1" or record.get("backendOwner") != "Resynthesis" or record.get("fallbackAllowed") is not False or record.get("bpeIdentityPreserved") is not True or record.get("vgeRemainsDownstreamAuthority") is not True or not isinstance(tokenizer_path_value, str) or not tokenizer_path_value or not isinstance(tokenizer_sha256, str) or len(tokenizer_sha256) != 64 or not isinstance(dependency_path_value, str) or not dependency_path_value or not isinstance(dependency_sha256, str) or len(dependency_sha256) != 64 or not isinstance(backend_module, str) or not backend_module.startswith("fastokens.") or not isinstance(backend_class, str) or not backend_class or not isinstance(fastokens_version, str) or not fastokens_version or not isinstance(fastokens_commit, str) or re.fullmatch(r"[0-9a-f]{40}", fastokens_commit) is None or not isinstance(site_path_value, str) or not site_path_value or not isinstance(source_archive_sha256, str) or len(source_archive_sha256) != 64 or not isinstance(wheel_sha256, str) or len(wheel_sha256) != 64 or not isinstance(verified_installed_files, int) or isinstance(verified_installed_files, bool) or verified_installed_files < 1 or not isinstance(vocabulary_size, int) or isinstance(vocabulary_size, bool) or vocabulary_size < 1 ): raise ValueError("full payload packed Fastokens authority differs") tokenizer_path = Path(tokenizer_path_value).expanduser().resolve() dependency_path = Path(dependency_path_value).expanduser().resolve() site_path = Path(site_path_value).expanduser().resolve() if ( not tokenizer_path.is_file() or file_sha256(tokenizer_path) != tokenizer_sha256 or not dependency_path.is_file() or file_sha256(dependency_path) != dependency_sha256 or not site_path.is_dir() ): raise ValueError("full payload packed Fastokens authority bytes differ") dependency = json.loads(dependency_path.read_text(encoding="utf-8")) source_archive = ( dependency.get("sourceArchive") if isinstance(dependency, dict) else None ) wheel = dependency.get("wheel") if isinstance(dependency, dict) else None installed_files = ( dependency.get("installedFiles") if isinstance(dependency, dict) else None ) if ( not isinstance(dependency, dict) or dependency.get("schema") != "nnf.resynthesis.fastokens_dependency.v1" or dependency.get("name") != "fastokens" or dependency.get("version") != fastokens_version or dependency.get("upstreamCommit") != fastokens_commit or dependency.get("installedSite") != str(site_path) or dependency.get("buildPassed") is not True or dependency.get("activationAuthority") is not False or not isinstance(source_archive, dict) or source_archive.get("sha256") != source_archive_sha256 or not isinstance(wheel, dict) or wheel.get("sha256") != wheel_sha256 or not isinstance(installed_files, list) or len(installed_files) != verified_installed_files ): raise ValueError("full payload packed Fastokens dependency differs") for artifact, expected_sha256 in ( (source_archive, source_archive_sha256), (wheel, wheel_sha256), ): artifact_path_value = artifact.get("path") if ( not isinstance(artifact_path_value, str) or not artifact_path_value ): raise ValueError("full payload packed Fastokens artifact differs") artifact_path = Path(artifact_path_value).expanduser().resolve() if ( not artifact_path.is_file() or file_sha256(artifact_path) != expected_sha256 ): raise ValueError("full payload packed Fastokens artifact bytes differ") installed_paths: set[str] = set() for installed in installed_files: if not isinstance(installed, dict): raise ValueError("full payload packed Fastokens installed row differs") relative_value = installed.get("path") installed_sha256 = installed.get("sha256") if ( not isinstance(relative_value, str) or not relative_value or Path(relative_value).is_absolute() or ".." in Path(relative_value).parts or not isinstance(installed_sha256, str) or len(installed_sha256) != 64 or relative_value in installed_paths ): raise ValueError("full payload packed Fastokens installed row differs") installed_path = site_path / relative_value if ( not installed_path.is_file() or file_sha256(installed_path) != installed_sha256 ): raise ValueError( "full payload packed Fastokens installed bytes differ" ) installed_paths.add(relative_value) backend_relative_path = backend_module.replace(".", "/") + ".py" if backend_relative_path not in installed_paths: raise ValueError("full payload packed Fastokens backend module differs") authority_sha256 = _sha256_bytes(_json_bytes(record)) return record, authority_sha256, vocabulary_size def _write_packed_int32( handle: Any, values: FullPayloadTokenIdRow, *, vocabulary_size: int, ) -> int: if not isinstance(values, (list, array)) or not values: raise ValueError("full payload packed token row is empty") if any( not isinstance(value, int) or isinstance(value, bool) or value < 0 or value >= vocabulary_size for value in values ): raise ValueError("full payload packed token id is outside Fastokens vocabulary") # ``struct.pack(..., *values)`` expands every token into a Python call # argument. The Fastokens boundary has already produced a contiguous # integer row, so preserve it through the buffer protocol instead. packed = array("i", values) if packed.itemsize != 4: raise RuntimeError("host int array cannot represent packed int32 tokens") if sys.byteorder != "little": packed.byteswap() handle.write(packed.tobytes()) return len(values) def _write_full_payload_fastokens_int32_extent( handle: Any, rows: Sequence[FullPayloadTokenIdRow], *, vocabulary_size: int, ) -> int: """Pack one native Fastokens batch into one contiguous int32 extent.""" if not rows or vocabulary_size < 1 or any(not row for row in rows): raise ValueError("full payload packed Fastokens extent is empty") packed = array("i") try: for row in rows: packed.extend(row) except (OverflowError, TypeError) as error: raise ValueError( "full payload packed token id is not a signed int32" ) from error if packed.itemsize != 4 or not packed: raise RuntimeError( "host int array cannot represent packed int32 tokens" ) if sys.byteorder != "little": packed.byteswap() numpy = importlib.import_module("numpy") packed_values = numpy.frombuffer(packed, dtype="= vocabulary_size ): raise ValueError( "full payload packed token id is outside Fastokens vocabulary" ) payload = memoryview(packed).cast("B") try: written = handle.write(payload) if written != payload.nbytes: raise RuntimeError( "full payload packed Fastokens extent write is incomplete" ) finally: payload.release() return len(packed) def _write_full_payload_staging_extent( handle: Any, staging: io.BytesIO, ) -> int: """Issue one large write for an in-memory append-only sidecar extent.""" payload = staging.getbuffer() try: written = handle.write(payload) if not isinstance(written, int) or isinstance(written, bool): raise RuntimeError( "full payload packed sidecar extent write count differs" ) if written != payload.nbytes: raise RuntimeError( "full payload packed sidecar extent write is incomplete" ) return written finally: payload.release() def _full_payload_provenance_prefix(record: FullPayloadTextRecord) -> str: """Return the exact source-bound prefix used by every packed token row.""" return ( f"Source: {record.source_id}\n" f"Payload work: {record.payload_work_id}\n" f"Record: {record.locator}\n" "Content:\n" ) def _full_payload_source_record_sequence_sha256( records: Sequence[FullPayloadTextRecord], ) -> str: """Bind an ordered semantic-record sequence without host object metadata.""" if not records: raise ValueError("full payload training extent source sequence is empty") digest = hashlib.sha256( b"nnf.resynthesis.full_payload_training_extent_source_sequence.v2\x00" ) prior_ordinal: int | None = None for record in records: if ( not isinstance(record.ordinal, int) or isinstance(record.ordinal, bool) or record.ordinal < 0 or record.source_record_count != 1 or record.source_record_start_ordinal is not None or record.source_record_end_ordinal is not None or record.source_record_sequence_sha256 is not None or record.reader_degradation is not None or record.typed_numeric_bytes is not None or record.typed_numeric_encoding is not None ): raise ValueError( "full payload training extent source record is not raw" ) if prior_ordinal is not None and record.ordinal != prior_ordinal + 1: raise RuntimeError( "full payload training extent source ordinals are not contiguous" ) prior_ordinal = record.ordinal for value in ( record.payload_work_id, record.source_id, record.locator, record.reader_family, record.semantic_export_receipt_sha256 or "", record.text, ): encoded = value.encode("utf-8") digest.update(struct.pack(" FullPayloadTextRecord: """Create one exact ordered tokenizer extent from contiguous source rows.""" if not records: raise ValueError("full payload training extent is empty") first = records[0] if any( record.payload_work_id != first.payload_work_id or record.source_id != first.source_id or record.reader_family != first.reader_family or record.semantic_export_receipt_sha256 != first.semantic_export_receipt_sha256 for record in records ): raise RuntimeError( "full payload training extent crosses a source authority boundary" ) sequence_sha256 = _full_payload_source_record_sequence_sha256(records) end_ordinal = records[-1].ordinal + 1 return FullPayloadTextRecord( payload_work_id=first.payload_work_id, source_id=first.source_id, locator=( f"extent_v2:{first.ordinal}:{end_ordinal}:" f"{sequence_sha256}" ), ordinal=first.ordinal, # A single newline is an explicit learned document boundary. No source # text is stripped, normalized, or omitted; the sequence digest above # binds every original UTF-8 value and locator in exact order. text="\n".join(record.text for record in records), reader_family=first.reader_family, semantic_export_receipt_sha256=( first.semantic_export_receipt_sha256 ), source_record_count=len(records), source_record_start_ordinal=first.ordinal, source_record_end_ordinal=end_ordinal, source_record_sequence_sha256=sequence_sha256, ) def iter_full_payload_training_extents( records: Iterable[FullPayloadTextRecord], *, maximum_text_bytes: int = FULL_PAYLOAD_PACKED_TRAINING_EXTENT_BYTES, ) -> Iterator[FullPayloadTextRecord]: """Coalesce exact source rows into large restart-safe tokenizer extents.""" if ( not isinstance(maximum_text_bytes, int) or isinstance(maximum_text_bytes, bool) or maximum_text_bytes < 1 ): raise ValueError("full payload training extent byte frontier is invalid") pending: list[FullPayloadTextRecord] = [] pending_text_bytes = 0 def flush_pending() -> FullPayloadTextRecord | None: nonlocal pending nonlocal pending_text_bytes if not pending: return None extent = _coalesce_full_payload_training_extent(pending) pending = [] pending_text_bytes = 0 return extent for record in records: if record.reader_degradation is not None: extent = flush_pending() if extent is not None: yield extent yield record continue if record.typed_numeric_bytes is not None: if ( record.typed_numeric_encoding != "typed_numeric_hex_v1" or not record.typed_numeric_bytes ): raise ValueError( "full payload typed numeric record is malformed" ) extent = flush_pending() if extent is not None: yield extent yield record continue if record.typed_numeric_encoding is not None: raise ValueError( "full payload typed numeric encoding has no bytes" ) if ( record.source_record_count != 1 or record.source_record_start_ordinal is not None or record.source_record_end_ordinal is not None or record.source_record_sequence_sha256 is not None ): raise ValueError( "full payload training extent received an already-coalesced row" ) record_text_bytes = len(record.text.encode("utf-8")) separator_bytes = 1 if pending else 0 authority_differs = bool( pending and ( record.payload_work_id != pending[0].payload_work_id or record.source_id != pending[0].source_id or record.reader_family != pending[0].reader_family or record.semantic_export_receipt_sha256 != pending[0].semantic_export_receipt_sha256 ) ) if pending and ( authority_differs or pending_text_bytes + separator_bytes + record_text_bytes > maximum_text_bytes ): extent = flush_pending() if extent is None: raise RuntimeError( "full payload training extent flush lost pending rows" ) yield extent separator_bytes = 0 if pending and record.ordinal != pending[-1].ordinal + 1: raise RuntimeError( "full payload training extent source ordinals are not contiguous" ) pending.append(record) pending_text_bytes += separator_bytes + record_text_bytes extent = flush_pending() if extent is not None: yield extent def _full_payload_linux_allocator_trim_boundary() -> None: """Return released Fastokens cache pages at this explicit CPU/I/O boundary.""" if sys.platform != "linux": return allocator = ctypes.CDLL(None) malloc_trim = getattr(allocator, "malloc_trim", None) if not callable(malloc_trim): raise RuntimeError("Linux allocator malloc_trim boundary is absent") malloc_trim.argtypes = [ctypes.c_size_t] malloc_trim.restype = ctypes.c_int malloc_trim(0) def _reset_full_payload_fastokens_cache_boundary(tokenizer: Any) -> bool: """Reconstruct the Rust backend so its process-local SharedCache is empty.""" backend = getattr(tokenizer, "backend_tokenizer", None) if backend is None: # Explicit non-production tokenizer fixtures use the scalar boundary. # They own no Fastokens SharedCache, but their completed WorkID may # still release Python/glibc arenas at the same CPU/I/O boundary. _full_payload_linux_allocator_trim_boundary() return False get_state = getattr(backend, "__getstate__", None) set_state = getattr(backend, "__setstate__", None) if not callable(get_state) or not callable(set_state): raise TypeError("Fastokens backend pickle protocol is incomplete") # The process-worker tokenizer authority is immutable after initialization. # Serializing its multi-megabyte BPE JSON on every cache epoch was 21-43% # of the measured reset cost, so retain that exact state on the worker. state_attribute = "_nnf_full_payload_fastokens_reset_state" backend_state = getattr(tokenizer, state_attribute, None) if backend_state is None: backend_state = get_state() setattr(tokenizer, state_attribute, backend_state) set_state(backend_state) _full_payload_linux_allocator_trim_boundary() return True def _full_payload_fastokens_durable_progress_boundary( tokenizer: Any, *, durable_observed_records: int, durable_progress_frontier: int, durable_token_elements: int, durable_token_frontier: int, ) -> int: """Release Fastokens cache after bounded fsynced record/token growth. Fastokens' Rust ``SharedCache`` retains every distinct input in a worker. Record-dense sources such as OAS can therefore consume tens of GiB before a cache epoch completes. Reconstructing the compiled matcher at every 2,048 record durability commit costs substantially more than encoding that commit, so the shard writer resets only after bounded record or token growth. The boundary still runs exclusively after compressed streams, indexes, and progress JSON have all been fsynced; resume offsets remain unchanged. """ if ( isinstance(durable_observed_records, bool) or isinstance(durable_progress_frontier, bool) or isinstance(durable_token_elements, bool) or isinstance(durable_token_frontier, bool) or durable_observed_records < 0 or durable_progress_frontier < 0 or durable_token_elements < 0 or durable_token_frontier < 0 ): raise ValueError( "full payload Fastokens durable progress frontier is invalid" ) if ( durable_observed_records < durable_progress_frontier or durable_token_elements < durable_token_frontier ): raise RuntimeError( "full payload Fastokens durable progress frontier moved backward" ) record_growth = durable_observed_records - durable_progress_frontier token_growth = durable_token_elements - durable_token_frontier if ( record_growth < FULL_PAYLOAD_PACKED_FASTOKENS_CACHE_RECORDS and token_growth < FULL_PAYLOAD_PACKED_FASTOKENS_CACHE_TOKEN_ELEMENTS ): return durable_progress_frontier _reset_full_payload_fastokens_cache_boundary(tokenizer) return durable_observed_records def _full_payload_fastokens_batch_ids( tokenizer: Any, texts: Sequence[str], ) -> list[list[int]]: """Encode one external-I/O batch through the same Fastokens BPE.""" if not texts: raise ValueError("full payload Fastokens batch is empty") backend = getattr(tokenizer, "backend_tokenizer", None) encode_batch = getattr(backend, "encode_batch", None) if callable(encode_batch): encodings = encode_batch(list(texts), add_special_tokens=False) if len(encodings) != len(texts): raise RuntimeError("Fastokens batch result count differs") # Fastokens already materializes ``Encoding.ids`` as ``list[int]``. # Rewalking every token into a second Python list consumed 26-40% of # post-reset worker samples and did not add authority or validation; # `_write_packed_int32` validates the exact native rows before writing. token_rows = [cast(list[int], encoding.ids) for encoding in encodings] else: # Test and explicit non-production tokenizer boundaries retain the exact # scalar contract. Production pack authorities require Fastokens. token_rows = [ [ int(value) for value in tokenizer.encode( text, add_special_tokens=False, ) ] for text in texts ] if any(not row for row in token_rows): raise ValueError("full payload Fastokens batch emitted an empty token row") return token_rows FullPayloadTokenIdRow = list[int] | array[int] def _full_payload_typed_numeric_hex_token_ids( tokenizer: Any, values: bytes, ) -> array[int]: """Map exact numeric bytes to tokenizer-valid hex tokens vectorially.""" if not values: raise ValueError("full payload typed numeric bytes are empty") mapping_attribute = "_nnf_full_payload_hex_token_ids" cached_mapping = getattr(tokenizer, mapping_attribute, None) if cached_mapping is None: alphabet = "0123456789abcdef" encoded = [ tokenizer.encode(character, add_special_tokens=False) for character in alphabet ] if ( any(len(row) != 1 for row in encoded) or tokenizer.decode( [int(row[0]) for row in encoded], skip_special_tokens=False, ) != alphabet ): raise RuntimeError( "Fastokens cannot represent exact typed numeric hex digits" ) cached_mapping = tuple(int(row[0]) for row in encoded) setattr(tokenizer, mapping_attribute, cached_mapping) if ( not isinstance(cached_mapping, tuple) or len(cached_mapping) != 16 or any( not isinstance(token_id, int) or isinstance(token_id, bool) or token_id < 0 for token_id in cached_mapping ) ): raise RuntimeError("Fastokens typed numeric hex mapping differs") numpy = importlib.import_module("numpy") source = numpy.frombuffer(values, dtype=numpy.uint8) mapping = numpy.asarray(cached_mapping, dtype="> 4] mapped[1::2] = mapping[source & 15] result = array("i") result.frombytes(memoryview(mapped).cast("B")) if sys.byteorder != "little": result.byteswap() return result def _tokenize_full_payload_record_batch( tokenizer: Any, records: Sequence[FullPayloadTextRecord], ) -> list[ tuple[ FullPayloadTextRecord, FullPayloadTokenIdRow, FullPayloadTokenIdRow, ] ]: """Preserve v1 segmentation and isolate any cross-boundary BPE merge.""" if not records: raise ValueError("full payload packed record batch is empty") prefixes = [_full_payload_provenance_prefix(record) for record in records] token_rows = _full_payload_fastokens_batch_ids( tokenizer, ( *prefixes, *( prefix + record.text for prefix, record in zip(prefixes, records, strict=True) ), ), ) width = len(records) fallback_indexes = [ index for index in range(width) if ( len(token_rows[width + index]) <= len(token_rows[index]) or token_rows[width + index][: len(token_rows[index])] != token_rows[index] ) ] fallback_rows = ( _full_payload_fastokens_batch_ids( tokenizer, tuple(records[index].text for index in fallback_indexes), ) if fallback_indexes else [] ) fallback_by_index = dict(zip(fallback_indexes, fallback_rows, strict=True)) encoded: list[ tuple[ FullPayloadTextRecord, FullPayloadTokenIdRow, FullPayloadTokenIdRow, ] ] = [] for index, record in enumerate(records): prefix_ids = token_rows[index] combined_ids = token_rows[width + index] content_ids: FullPayloadTokenIdRow | None = fallback_by_index.get( index ) if content_ids is None: content_ids = combined_ids[len(prefix_ids) :] if record.typed_numeric_bytes is not None: if record.typed_numeric_encoding != "typed_numeric_hex_v1": raise RuntimeError( "full payload typed numeric encoding differs" ) numeric_ids = _full_payload_typed_numeric_hex_token_ids( tokenizer, record.typed_numeric_bytes, ) typed_content_ids = array("i", content_ids) typed_content_ids.extend(numeric_ids) content_ids = typed_content_ids elif record.typed_numeric_encoding is not None: raise RuntimeError( "full payload typed numeric encoding has no source bytes" ) encoded.append((record, prefix_ids, content_ids)) return encoded def _full_payload_packed_window_identity( record_identity_sha256: bytes, *, token_start: int, token_end: int, ) -> bytes: """Derive one exact loss-window identity from its sealed record identity.""" if ( len(record_identity_sha256) != hashlib.sha256().digest_size or token_start < 0 or token_end <= token_start ): raise ValueError("full payload packed window identity is invalid") return hashlib.sha256( b"nnf.resynthesis.full_payload_packed_window.v1\x00" + record_identity_sha256 + struct.pack(" tuple[int, int, int]: """Return exact row, prompt-token, and answer-token counts for one record.""" if ( prefix_tokens < 1 or content_tokens < 1 or context_window_tokens < 3 or answer_tokens_per_window < 1 or prefix_tokens >= context_window_tokens - 1 ): raise ValueError("full payload packed record geometry is invalid") answer_width = min( answer_tokens_per_window, context_window_tokens - prefix_tokens, ) window_count = (content_tokens + answer_width - 1) // answer_width prompt_tokens = 0 for token_start in range(0, content_tokens, answer_width): answer_length = min(answer_width, content_tokens - token_start) context_capacity = ( context_window_tokens - prefix_tokens - answer_length ) prompt_tokens += prefix_tokens + min(token_start, context_capacity) return window_count, prompt_tokens, content_tokens def _packed_artifact(path: Path) -> dict[str, Any]: stat = path.stat() return { "path": str(path), "bytes": stat.st_size, "sha256": file_sha256(path), "fileIdentity": { "device": stat.st_dev, "inode": stat.st_ino, "bytes": stat.st_size, "mtimeNs": stat.st_mtime_ns, "ctimeNs": stat.st_ctime_ns, }, } def _compress_full_payload_token_chunks( raw_tokens_path: Path, compressed_path: Path, chunk_index_path: Path, ) -> dict[str, Any]: """Seal one raw int32 token stream as independently decodable zstd frames.""" if ( not raw_tokens_path.is_file() or raw_tokens_path.stat().st_size < 1 or raw_tokens_path.stat().st_size % 4 != 0 ): raise RuntimeError("full payload raw token bytes are malformed") zstandard = importlib.import_module("zstandard") compressor = zstandard.ZstdCompressor( level=FULL_PAYLOAD_PACKED_TOKEN_ZSTD_LEVEL, write_checksum=True, write_content_size=True, ) decompressor = zstandard.ZstdDecompressor() compressed_temporary = compressed_path.with_name( f".{compressed_path.name}.{os.getpid()}.tmp" ) index_temporary = chunk_index_path.with_name( f".{chunk_index_path.name}.{os.getpid()}.tmp" ) compressed_temporary.unlink(missing_ok=True) index_temporary.unlink(missing_ok=True) raw_digest = hashlib.sha256() uncompressed_offset = 0 compressed_offset = 0 chunk_count = 0 with ( raw_tokens_path.open("rb") as raw_handle, compressed_temporary.open("wb") as compressed_handle, index_temporary.open("wb") as index_handle, ): while True: raw_chunk = raw_handle.read( FULL_PAYLOAD_PACKED_TOKEN_ZSTD_CHUNK_BYTES ) if not raw_chunk: break compressed_chunk = compressor.compress(raw_chunk) if ( decompressor.decompress( compressed_chunk, max_output_size=len(raw_chunk), ) != raw_chunk ): raise RuntimeError( "full payload compressed token chunk roundtrip differs" ) raw_digest.update(raw_chunk) compressed_handle.write(compressed_chunk) index_handle.write( FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.pack( compressed_offset, len(compressed_chunk), uncompressed_offset, len(raw_chunk), ) ) compressed_offset += len(compressed_chunk) uncompressed_offset += len(raw_chunk) chunk_count += 1 for handle in (compressed_handle, index_handle): handle.flush() os.fsync(handle.fileno()) if chunk_count < 1: raise RuntimeError("full payload token compression emitted no chunks") os.replace(compressed_temporary, compressed_path) os.replace(index_temporary, chunk_index_path) return { "schema": FULL_PAYLOAD_PACKED_TOKEN_ZSTD_SCHEMA, "codec": "zstd", "level": FULL_PAYLOAD_PACKED_TOKEN_ZSTD_LEVEL, "independentFrames": True, "checksumPerFrame": True, "chunkUncompressedBytes": FULL_PAYLOAD_PACKED_TOKEN_ZSTD_CHUNK_BYTES, "chunkIndexRecordBytes": ( FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.size ), "chunkCount": chunk_count, "uncompressedBytes": uncompressed_offset, "uncompressedSha256": raw_digest.hexdigest(), "compressed": _packed_artifact(compressed_path), "chunkIndex": _packed_artifact(chunk_index_path), "boundedRandomAccess": True, } def _compress_full_payload_metadata_chunks( raw_metadata_path: Path, compressed_path: Path, chunk_index_path: Path, ) -> dict[str, Any]: """Seal metadata bytes as independently decodable random-access frames.""" if ( not raw_metadata_path.is_file() or raw_metadata_path.stat().st_size < 1 ): raise RuntimeError("full payload raw metadata bytes are malformed") zstandard = importlib.import_module("zstandard") compressor = zstandard.ZstdCompressor( level=FULL_PAYLOAD_PACKED_METADATA_ZSTD_LEVEL, write_checksum=True, write_content_size=True, ) decompressor = zstandard.ZstdDecompressor() compressed_temporary = compressed_path.with_name( f".{compressed_path.name}.{os.getpid()}.tmp" ) index_temporary = chunk_index_path.with_name( f".{chunk_index_path.name}.{os.getpid()}.tmp" ) compressed_temporary.unlink(missing_ok=True) index_temporary.unlink(missing_ok=True) raw_digest = hashlib.sha256() uncompressed_offset = 0 compressed_offset = 0 chunk_count = 0 with ( raw_metadata_path.open("rb") as raw_handle, compressed_temporary.open("wb") as compressed_handle, index_temporary.open("wb") as index_handle, ): while True: raw_chunk = raw_handle.read( FULL_PAYLOAD_PACKED_METADATA_ZSTD_CHUNK_BYTES ) if not raw_chunk: break compressed_chunk = compressor.compress(raw_chunk) if ( decompressor.decompress( compressed_chunk, max_output_size=len(raw_chunk), ) != raw_chunk ): raise RuntimeError( "full payload compressed metadata chunk roundtrip differs" ) raw_digest.update(raw_chunk) compressed_handle.write(compressed_chunk) index_handle.write( FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.pack( compressed_offset, len(compressed_chunk), uncompressed_offset, len(raw_chunk), ) ) compressed_offset += len(compressed_chunk) uncompressed_offset += len(raw_chunk) chunk_count += 1 for handle in (compressed_handle, index_handle): handle.flush() os.fsync(handle.fileno()) if chunk_count < 1: raise RuntimeError("full payload metadata compression emitted no chunks") os.replace(compressed_temporary, compressed_path) os.replace(index_temporary, chunk_index_path) return { "schema": FULL_PAYLOAD_PACKED_METADATA_ZSTD_SCHEMA, "codec": "zstd", "level": FULL_PAYLOAD_PACKED_METADATA_ZSTD_LEVEL, "independentFrames": True, "checksumPerFrame": True, "chunkUncompressedBytes": FULL_PAYLOAD_PACKED_METADATA_ZSTD_CHUNK_BYTES, "chunkIndexRecordBytes": ( FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.size ), "chunkCount": chunk_count, "uncompressedBytes": uncompressed_offset, "compressedBytes": compressed_offset, "uncompressedSha256": raw_digest.hexdigest(), "compressed": _packed_artifact(compressed_path), "chunkIndex": _packed_artifact(chunk_index_path), "boundedRandomAccess": True, } def _append_full_payload_zstd_frames( raw_staging_path: Path, compressed_handle: Any, chunk_index_handle: Any, *, chunk_bytes: int, compression_level: int, index_struct: struct.Struct, compressed_offset: int, uncompressed_offset: int, raw_start_offset: int = 0, maximum_raw_bytes: int | None = None, ) -> tuple[int, int, int]: """Append one durable raw staging interval as indexed zstd frames. The progress receipt owns the returned byte/count frontier. A crash before that receipt is published may leave compressed/index tails, which the next resume truncates to the prior frontier. The raw staging interval is kept until after the receipt is atomically replaced, so no committed bytes can disappear between the two authorities. """ if ( chunk_bytes < 1 or compression_level < 1 or compressed_offset < 0 or uncompressed_offset < 0 or raw_start_offset < 0 or ( maximum_raw_bytes is not None and maximum_raw_bytes < 1 ) or compressed_handle.tell() != compressed_offset or chunk_index_handle.tell() % index_struct.size != 0 ): raise RuntimeError("full payload packed stream frontier differs") raw_bytes = ( raw_staging_path.stat().st_size if raw_staging_path.is_file() else 0 ) if raw_start_offset > raw_bytes: raise RuntimeError("full payload packed raw stream frontier differs") if raw_bytes == 0: return compressed_offset, uncompressed_offset, 0 zstandard = importlib.import_module("zstandard") compressor = zstandard.ZstdCompressor( level=compression_level, write_checksum=True, write_content_size=True, ) decompressor = zstandard.ZstdDecompressor() appended_chunks = 0 remaining_raw_bytes = ( raw_bytes - raw_start_offset if maximum_raw_bytes is None else min(maximum_raw_bytes, raw_bytes - raw_start_offset) ) with raw_staging_path.open("rb") as raw_handle: raw_handle.seek(raw_start_offset) while remaining_raw_bytes: raw_chunk = raw_handle.read( min(chunk_bytes, remaining_raw_bytes) ) if not raw_chunk: raise RuntimeError( "full payload packed raw stream ended before frontier" ) compressed_chunk = compressor.compress(raw_chunk) if ( decompressor.decompress( compressed_chunk, max_output_size=len(raw_chunk), ) != raw_chunk ): raise RuntimeError( "full payload packed stream compression roundtrip differs" ) compressed_handle.write(compressed_chunk) chunk_index_handle.write( index_struct.pack( compressed_offset, len(compressed_chunk), uncompressed_offset, len(raw_chunk), ) ) compressed_offset += len(compressed_chunk) uncompressed_offset += len(raw_chunk) appended_chunks += 1 remaining_raw_bytes -= len(raw_chunk) for handle in (compressed_handle, chunk_index_handle): handle.flush() os.fsync(handle.fileno()) return compressed_offset, uncompressed_offset, appended_chunks def _append_full_payload_zstd_buffer_frames( raw_payload: bytes | bytearray | memoryview, compressed_handle: Any, chunk_index_handle: Any, *, chunk_bytes: int, compression_level: int, index_struct: struct.Struct, compressed_offset: int, uncompressed_offset: int, ) -> tuple[int, int, int]: """Append one in-memory durability interval as indexed zstd frames. New packer intervals do not need a raw filesystem staging copy. The compressed/index files are fsynced before their frontier is published in the atomic progress receipt; a crash before publication leaves only a discardable tail that normal resume truncates to the prior receipt. The caller retains ``raw_payload`` until publication completes, so an in-process failure cannot lose the interval while attempting the durable append. """ if ( chunk_bytes < 1 or compression_level < 1 or compressed_offset < 0 or uncompressed_offset < 0 or compressed_handle.tell() != compressed_offset or chunk_index_handle.tell() % index_struct.size != 0 ): raise RuntimeError("full payload packed buffer frontier differs") payload_view = memoryview(raw_payload).cast("B") if payload_view.nbytes == 0: payload_view.release() return compressed_offset, uncompressed_offset, 0 zstandard = importlib.import_module("zstandard") compressor = zstandard.ZstdCompressor( level=compression_level, write_checksum=True, write_content_size=True, ) decompressor = zstandard.ZstdDecompressor() appended_chunks = 0 try: for chunk_start in range(0, payload_view.nbytes, chunk_bytes): raw_chunk = payload_view[ chunk_start:min(chunk_start + chunk_bytes, payload_view.nbytes) ] compressed_chunk = compressor.compress(raw_chunk) if ( decompressor.decompress( compressed_chunk, max_output_size=raw_chunk.nbytes, ) != raw_chunk ): raise RuntimeError( "full payload packed buffer compression roundtrip differs" ) compressed_handle.write(compressed_chunk) chunk_index_handle.write( index_struct.pack( compressed_offset, len(compressed_chunk), uncompressed_offset, raw_chunk.nbytes, ) ) compressed_offset += len(compressed_chunk) uncompressed_offset += raw_chunk.nbytes appended_chunks += 1 finally: payload_view.release() for handle in (compressed_handle, chunk_index_handle): handle.flush() os.fsync(handle.fileno()) return compressed_offset, uncompressed_offset, appended_chunks def _full_payload_zstd_stream_receipt( compressed_path: Path, chunk_index_path: Path, *, schema: str, chunk_bytes: int, compression_level: int, index_struct: struct.Struct, expected_uncompressed_bytes: int, ) -> dict[str, Any]: """Seal and verify one already-streamed compressed artifact.""" if ( expected_uncompressed_bytes < 1 or not compressed_path.is_file() or compressed_path.stat().st_size < 1 or not chunk_index_path.is_file() or chunk_index_path.stat().st_size < index_struct.size or chunk_index_path.stat().st_size % index_struct.size != 0 ): raise RuntimeError("full payload packed zstd stream is malformed") zstandard = importlib.import_module("zstandard") decompressor = zstandard.ZstdDecompressor() uncompressed_digest = hashlib.sha256() compressed_offset = 0 uncompressed_offset = 0 chunk_count = chunk_index_path.stat().st_size // index_struct.size with ( compressed_path.open("rb") as compressed_handle, chunk_index_path.open("rb") as index_handle, ): for _chunk_index in range(chunk_count): record = index_handle.read(index_struct.size) if len(record) != index_struct.size: raise RuntimeError( "full payload packed zstd stream index is truncated" ) ( observed_compressed_offset, compressed_width, observed_uncompressed_offset, uncompressed_width, ) = index_struct.unpack(record) if ( observed_compressed_offset != compressed_offset or observed_uncompressed_offset != uncompressed_offset or compressed_width < 1 or uncompressed_width < 1 or uncompressed_width > chunk_bytes ): raise RuntimeError( "full payload packed zstd stream index differs" ) compressed_handle.seek(compressed_offset) compressed_chunk = compressed_handle.read(compressed_width) if len(compressed_chunk) != compressed_width: raise RuntimeError( "full payload packed zstd stream frame is truncated" ) raw_chunk = decompressor.decompress( compressed_chunk, max_output_size=uncompressed_width, ) if len(raw_chunk) != uncompressed_width: raise RuntimeError( "full payload packed zstd stream frame differs" ) uncompressed_digest.update(raw_chunk) compressed_offset += compressed_width uncompressed_offset += uncompressed_width if ( compressed_offset != compressed_path.stat().st_size or uncompressed_offset != expected_uncompressed_bytes ): raise RuntimeError("full payload packed zstd stream geometry differs") return { "schema": schema, "codec": "zstd", "level": compression_level, "independentFrames": True, "checksumPerFrame": True, "chunkUncompressedBytes": chunk_bytes, "chunkIndexRecordBytes": index_struct.size, "chunkCount": chunk_count, "uncompressedBytes": uncompressed_offset, "compressedBytes": compressed_offset, "uncompressedSha256": uncompressed_digest.hexdigest(), "compressed": _packed_artifact(compressed_path), "chunkIndex": _packed_artifact(chunk_index_path), "boundedRandomAccess": True, "streamedAtDurableProgressBoundary": True, } def _validate_full_payload_zstd_progress_frontier( compressed_path: Path, chunk_index_path: Path, *, index_struct: struct.Struct, expected_compressed_bytes: int, expected_uncompressed_bytes: int, expected_chunk_count: int, ) -> None: """Validate an interrupted stream's exact last durable frame.""" if ( expected_compressed_bytes < 0 or expected_uncompressed_bytes < 0 or expected_chunk_count < 0 or (expected_chunk_count == 0) != ( expected_compressed_bytes == 0 and expected_uncompressed_bytes == 0 ) or compressed_path.stat().st_size != expected_compressed_bytes or chunk_index_path.stat().st_size != expected_chunk_count * index_struct.size ): raise RuntimeError("full payload packed zstd progress geometry differs") if expected_chunk_count == 0: return with ( compressed_path.open("rb") as compressed_handle, chunk_index_path.open("rb") as index_handle, ): index_handle.seek((expected_chunk_count - 1) * index_struct.size) record = index_handle.read(index_struct.size) if len(record) != index_struct.size: raise RuntimeError( "full payload packed zstd progress index is truncated" ) ( compressed_offset, compressed_width, uncompressed_offset, uncompressed_width, ) = index_struct.unpack(record) if ( compressed_offset + compressed_width != expected_compressed_bytes or uncompressed_offset + uncompressed_width != expected_uncompressed_bytes or compressed_width < 1 or uncompressed_width < 1 ): raise RuntimeError( "full payload packed zstd progress frontier differs" ) compressed_handle.seek(compressed_offset) compressed_chunk = compressed_handle.read(compressed_width) if len(compressed_chunk) != compressed_width: raise RuntimeError( "full payload packed zstd progress frame is truncated" ) zstandard = importlib.import_module("zstandard") decoded = zstandard.ZstdDecompressor().decompress( compressed_chunk, max_output_size=uncompressed_width, ) if len(decoded) != uncompressed_width: raise RuntimeError( "full payload packed zstd progress frame differs" ) @dataclass(frozen=True) class _FullPayloadCompactRecord: """One decoded compact external-I/O record.""" record_index: int token_offset: int prefix_length: int content_length: int window_start: int window_count: int record_ordinal: int locator: str reader_family: str semantic_export_receipt_sha256: str | None def _encode_full_payload_uvarint(value: int) -> bytes: """Encode one non-negative boundary integer canonically.""" if not isinstance(value, int) or isinstance(value, bool) or value < 0: raise ValueError("full payload compact record integer is invalid") encoded = bytearray() remaining = value while remaining >= 0x80: encoded.append((remaining & 0x7F) | 0x80) remaining >>= 7 encoded.append(remaining) return bytes(encoded) def _decode_full_payload_uvarint( payload: bytes, offset: int, ) -> tuple[int, int]: """Decode one canonical unsigned varint from a compact record frame.""" if offset < 0 or offset >= len(payload): raise RuntimeError("full payload compact record varint is absent") value = 0 shift = 0 cursor = offset while cursor < len(payload) and shift <= 63: byte = payload[cursor] cursor += 1 value |= (byte & 0x7F) << shift if byte < 0x80: if _encode_full_payload_uvarint(value) != payload[offset:cursor]: raise RuntimeError( "full payload compact record varint is noncanonical" ) return value, cursor shift += 7 raise RuntimeError("full payload compact record varint is malformed") def _encode_full_payload_compact_record( *, prefix_length: int, content_length: int, record_ordinal: int, locator: str, reader_family: str, semantic_export_receipt_sha256: str | None, ) -> bytes: """Encode only variable provenance; shard constants remain manifest-bound.""" locator_bytes = locator.encode("utf-8") reader_family_bytes = reader_family.encode("utf-8") if ( prefix_length < 1 or content_length < 1 or record_ordinal < 0 or not locator_bytes or not reader_family_bytes or ( semantic_export_receipt_sha256 is not None and re.fullmatch( r"[0-9a-f]{64}", semantic_export_receipt_sha256, ) is None ) ): raise RuntimeError("full payload compact record metadata differs") semantic_bytes = ( bytes.fromhex(semantic_export_receipt_sha256) if semantic_export_receipt_sha256 is not None else b"" ) return b"".join( ( _encode_full_payload_uvarint(prefix_length), _encode_full_payload_uvarint(content_length), _encode_full_payload_uvarint(record_ordinal), _encode_full_payload_uvarint(len(locator_bytes)), locator_bytes, _encode_full_payload_uvarint(len(reader_family_bytes)), reader_family_bytes, bytes((1 if semantic_bytes else 0,)), semantic_bytes, ) ) def _decode_full_payload_compact_record( payload: bytes, offset: int, ) -> tuple[tuple[int, int, int, str, str, str | None], int]: """Decode one exact variable-provenance record from a compact frame.""" prefix_length, cursor = _decode_full_payload_uvarint(payload, offset) content_length, cursor = _decode_full_payload_uvarint(payload, cursor) record_ordinal, cursor = _decode_full_payload_uvarint(payload, cursor) locator_length, cursor = _decode_full_payload_uvarint(payload, cursor) locator_end = cursor + locator_length if locator_length < 1 or locator_end > len(payload): raise RuntimeError("full payload compact record locator differs") locator = payload[cursor:locator_end].decode("utf-8") reader_length, cursor = _decode_full_payload_uvarint( payload, locator_end, ) reader_end = cursor + reader_length if reader_length < 1 or reader_end >= len(payload): raise RuntimeError("full payload compact record reader differs") reader_family = payload[cursor:reader_end].decode("utf-8") semantic_present = payload[reader_end] semantic_start = reader_end + 1 if semantic_present == 0: semantic_sha256 = None next_offset = semantic_start elif semantic_present == 1: next_offset = semantic_start + 32 if next_offset > len(payload): raise RuntimeError( "full payload compact record semantic receipt differs" ) semantic_sha256 = payload[semantic_start:next_offset].hex() else: raise RuntimeError("full payload compact record semantic marker differs") if prefix_length < 1 or content_length < 1: raise RuntimeError("full payload compact record token geometry differs") return ( ( prefix_length, content_length, record_ordinal, locator, reader_family, semantic_sha256, ), next_offset, ) def _encode_full_payload_compact_record_frame( records: Sequence[tuple[int, int, int, str, str, str | None]], ) -> bytes: """Dictionary-encode one bounded record frame before zstd sealing.""" if not records: raise ValueError("full payload compact record frame is empty") reader_families = list(dict.fromkeys(record[4] for record in records)) semantic_hashes: list[str] = list( dict.fromkeys( record[5] for record in records if record[5] is not None ) ) reader_indexes = { reader_family: index for index, reader_family in enumerate(reader_families) } semantic_indexes = { semantic_sha256: index + 1 for index, semantic_sha256 in enumerate(semantic_hashes) } payload = bytearray( _encode_full_payload_uvarint(len(reader_families)) ) for reader_family in reader_families: reader_bytes = reader_family.encode("utf-8") if not reader_bytes: raise RuntimeError( "full payload compact record reader differs" ) payload.extend(_encode_full_payload_uvarint(len(reader_bytes))) payload.extend(reader_bytes) payload.extend(_encode_full_payload_uvarint(len(semantic_hashes))) for dictionary_semantic_sha256 in semantic_hashes: if ( re.fullmatch( r"[0-9a-f]{64}", dictionary_semantic_sha256, ) is None ): raise RuntimeError( "full payload compact record semantic receipt differs" ) payload.extend(bytes.fromhex(dictionary_semantic_sha256)) previous_ordinal = 0 for index, record in enumerate(records): ( prefix_length, content_length, record_ordinal, locator, reader_family, record_semantic_sha256, ) = record ordinal_delta = ( record_ordinal if index == 0 else record_ordinal - previous_ordinal ) locator_bytes = locator.encode("utf-8") if ( prefix_length < 1 or content_length < 1 or ordinal_delta < 0 or not locator_bytes or reader_family not in reader_indexes ): raise RuntimeError( "full payload compact record frame metadata differs" ) for value in ( prefix_length, content_length, ordinal_delta, len(locator_bytes), ): payload.extend(_encode_full_payload_uvarint(value)) payload.extend(locator_bytes) payload.extend( _encode_full_payload_uvarint(reader_indexes[reader_family]) ) payload.extend( _encode_full_payload_uvarint( 0 if record_semantic_sha256 is None else semantic_indexes[record_semantic_sha256] ) ) previous_ordinal = record_ordinal return bytes(payload) def _decode_full_payload_compact_record_frame( payload: bytes, *, expected_record_count: int, ) -> tuple[tuple[int, int, int, str, str, str | None], ...]: """Decode one exact dictionary-bound compact record frame.""" reader_count, cursor = _decode_full_payload_uvarint(payload, 0) if reader_count < 1: raise RuntimeError("full payload compact record readers are absent") reader_families: list[str] = [] for _index in range(reader_count): reader_length, cursor = _decode_full_payload_uvarint( payload, cursor, ) reader_end = cursor + reader_length if reader_length < 1 or reader_end > len(payload): raise RuntimeError( "full payload compact record reader dictionary differs" ) reader_families.append(payload[cursor:reader_end].decode("utf-8")) cursor = reader_end semantic_count, cursor = _decode_full_payload_uvarint(payload, cursor) semantic_hashes: list[str] = [] for _index in range(semantic_count): semantic_end = cursor + 32 if semantic_end > len(payload): raise RuntimeError( "full payload compact record semantic dictionary differs" ) semantic_hashes.append(payload[cursor:semantic_end].hex()) cursor = semantic_end records: list[tuple[int, int, int, str, str, str | None]] = [] previous_ordinal = 0 for record_index in range(expected_record_count): prefix_length, cursor = _decode_full_payload_uvarint( payload, cursor, ) content_length, cursor = _decode_full_payload_uvarint( payload, cursor, ) ordinal_delta, cursor = _decode_full_payload_uvarint( payload, cursor, ) locator_length, cursor = _decode_full_payload_uvarint( payload, cursor, ) locator_end = cursor + locator_length if locator_length < 1 or locator_end > len(payload): raise RuntimeError( "full payload compact record locator differs" ) locator = payload[cursor:locator_end].decode("utf-8") reader_index, cursor = _decode_full_payload_uvarint( payload, locator_end, ) semantic_index, cursor = _decode_full_payload_uvarint( payload, cursor, ) if ( prefix_length < 1 or content_length < 1 or reader_index >= len(reader_families) or semantic_index > len(semantic_hashes) ): raise RuntimeError( "full payload compact record frame geometry differs" ) record_ordinal = ( ordinal_delta if record_index == 0 else previous_ordinal + ordinal_delta ) records.append( ( prefix_length, content_length, record_ordinal, locator, reader_families[reader_index], ( None if semantic_index == 0 else semantic_hashes[semantic_index - 1] ), ) ) previous_ordinal = record_ordinal if cursor != len(payload): raise RuntimeError("full payload compact record frame has a suffix") return tuple(records) def _seal_full_payload_packed_token_artifact( manifest_path: Path, manifest: Mapping[str, Any], *, raw_tokens_path: Path, compressed_path: Path, chunk_index_path: Path, ) -> dict[str, Any]: """Upgrade one complete raw-token manifest to exact chunked zstd storage.""" sealed = dict(manifest) existing_compression = sealed.get("tokenCompression") if isinstance(existing_compression, dict): if ( existing_compression.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_ZSTD_SCHEMA or existing_compression.get("uncompressedBytes") != int(sealed["tokenElements"]) * 4 or not _full_payload_packed_artifact_matches( existing_compression.get("compressed") ) or not _full_payload_packed_artifact_matches( existing_compression.get("chunkIndex") ) ): raise RuntimeError("full payload compressed token authority differs") raw_tokens_path.unlink(missing_ok=True) return sealed artifacts = sealed.get("artifacts") raw_artifact = artifacts.get("tokens") if isinstance(artifacts, dict) else None if ( not isinstance(artifacts, dict) or not isinstance(raw_artifact, dict) or not _full_payload_packed_artifact_matches(raw_artifact) or Path(str(raw_artifact["path"])).resolve() != raw_tokens_path.resolve() ): raise RuntimeError("full payload raw token authority differs") compression = _compress_full_payload_token_chunks( raw_tokens_path, compressed_path, chunk_index_path, ) if ( compression["uncompressedBytes"] != raw_artifact["bytes"] or compression["uncompressedSha256"] != raw_artifact["sha256"] ): raise RuntimeError("full payload compressed token source differs") sealed_artifacts = dict(artifacts) del sealed_artifacts["tokens"] sealed_artifacts["tokensZstd"] = compression["compressed"] sealed_artifacts["tokenChunkIndex"] = compression["chunkIndex"] sealed["artifacts"] = sealed_artifacts sealed["tokenCompression"] = compression sealed["rawTokenBytesRetained"] = False sealed["compressedTokenRandomAccess"] = True sealed.pop("shardAuthoritySha256", None) sealed["shardAuthoritySha256"] = _sha256_bytes(_json_bytes(sealed)) _atomic_json(manifest_path, sealed) raw_tokens_path.unlink() return sealed def _seal_full_payload_packed_metadata_artifact( manifest_path: Path, manifest: Mapping[str, Any], *, raw_metadata_path: Path, compressed_path: Path, chunk_index_path: Path, ) -> dict[str, Any]: """Upgrade one complete raw-metadata manifest to chunked zstd storage.""" sealed = dict(manifest) existing_compression = sealed.get("metadataCompression") if isinstance(existing_compression, dict): if ( existing_compression.get("schema") != FULL_PAYLOAD_PACKED_METADATA_ZSTD_SCHEMA or existing_compression.get("uncompressedBytes") != sealed.get("metadataUncompressedBytes") or existing_compression.get("compressedBytes") != sealed.get("metadataCompressedBytes") or not _full_payload_packed_artifact_matches( existing_compression.get("compressed") ) or not _full_payload_packed_artifact_matches( existing_compression.get("chunkIndex") ) ): raise RuntimeError("full payload compressed metadata authority differs") raw_metadata_path.unlink(missing_ok=True) return sealed artifacts = sealed.get("artifacts") raw_artifact = ( artifacts.get("metadata") if isinstance(artifacts, dict) else None ) if ( not isinstance(artifacts, dict) or not isinstance(raw_artifact, dict) or not _full_payload_packed_artifact_matches(raw_artifact) or Path(str(raw_artifact["path"])).resolve() != raw_metadata_path.resolve() ): raise RuntimeError("full payload raw metadata authority differs") compression = _compress_full_payload_metadata_chunks( raw_metadata_path, compressed_path, chunk_index_path, ) if ( compression["uncompressedBytes"] != raw_artifact["bytes"] or compression["uncompressedSha256"] != raw_artifact["sha256"] ): raise RuntimeError("full payload compressed metadata source differs") sealed_artifacts = dict(artifacts) del sealed_artifacts["metadata"] sealed_artifacts["metadataZstd"] = compression["compressed"] sealed_artifacts["metadataChunkIndex"] = compression["chunkIndex"] sealed["artifacts"] = sealed_artifacts sealed["metadataBytes"] = compression["uncompressedBytes"] sealed["metadataUncompressedBytes"] = compression["uncompressedBytes"] sealed["metadataCompressedBytes"] = compression["compressedBytes"] sealed["metadataCompression"] = compression sealed["rawMetadataBytesRetained"] = False sealed["compressedMetadataRandomAccess"] = True sealed["metadataStoredOncePerRecord"] = True sealed.pop("shardAuthoritySha256", None) sealed["shardAuthoritySha256"] = _sha256_bytes(_json_bytes(sealed)) _atomic_json(manifest_path, sealed) raw_metadata_path.unlink() return sealed def _full_payload_packed_artifact_matches(value: object) -> bool: """Verify a packed artifact without rehashing unchanged multi-GB files.""" if not isinstance(value, dict): return False path_value = value.get("path") byte_count = value.get("bytes") sha256 = value.get("sha256") identity = value.get("fileIdentity") if ( not isinstance(path_value, str) or not path_value or not isinstance(byte_count, int) or isinstance(byte_count, bool) or byte_count < 0 or not isinstance(sha256, str) or len(sha256) != 64 or not isinstance(identity, dict) ): return False path = Path(path_value).expanduser().resolve() if not path.is_file(): return False stat = path.stat() current_identity = { "device": stat.st_dev, "inode": stat.st_ino, "bytes": stat.st_size, "mtimeNs": stat.st_mtime_ns, "ctimeNs": stat.st_ctime_ns, } return bool( stat.st_size == byte_count and ( current_identity == identity or file_sha256(path) == sha256 ) ) def _full_payload_source_identity_matches( source_identity: object, source_path: Path, *, observed_sha256: str | None = None, proven_device_remaps: set[tuple[int, int]] | None = None, ) -> bool: """Verify the raw source identity without rereading its complete payload. Inode, byte, and nanosecond-mtime identity are always required. The device number admits directly only when unchanged: Linux assigns ``st_dev`` per mount, so a remount or reboot changes it without altering the payload. A drifted device is instead proven by re-hashing the payload against its admitted ledger hash; the hash ledger remains the content authority. A caller may share ``proven_device_remaps`` so one content proof blesses a uniform remount for every later file with the same old/new device pair. """ if not isinstance(source_identity, dict) or not source_path.is_file(): return False stat = source_path.stat() if ( source_identity.get("inode") != stat.st_ino or source_identity.get("bytes") != stat.st_size or source_identity.get("mtimeNs") != stat.st_mtime_ns ): return False identity_device = source_identity.get("device") if identity_device == stat.st_dev: return True if ( isinstance(identity_device, int) and not isinstance(identity_device, bool) and proven_device_remaps is not None and (identity_device, stat.st_dev) in proven_device_remaps ): return True if observed_sha256 and file_sha256(source_path) == observed_sha256: if ( isinstance(identity_device, int) and not isinstance(identity_device, bool) and proven_device_remaps is not None ): proven_device_remaps.add((identity_device, stat.st_dev)) return True return False def _full_payload_packed_proven_device_remaps( corpus_root: Path, schedule_rows: Sequence[Mapping[str, Any]], hash_rows: Sequence[Mapping[str, Any]], ) -> frozenset[tuple[int, int]]: """Carry parent-proven mount remaps into forked pack workers. This boundary runs only after ``_validated_full_payload_packed_sources`` has content-proved every selected source. Workers still require exact inode, byte, and nanosecond-mtime identity before admitting a remapped device. """ if len(schedule_rows) != len(hash_rows): raise ValueError("full payload packed remap rows differ") proven: set[tuple[int, int]] = set() for schedule_row, hash_row in zip(schedule_rows, hash_rows, strict=True): relative_path = schedule_row.get("payloadRelativePath") source_identity = hash_row.get("sourceIdentity") if ( not isinstance(relative_path, str) or not relative_path or not isinstance(source_identity, dict) ): raise ValueError("full payload packed remap identity differs") recorded_device = source_identity.get("device") if not isinstance(recorded_device, int) or isinstance( recorded_device, bool, ): raise ValueError("full payload packed remap device differs") current_device = (corpus_root / relative_path).stat().st_dev if recorded_device != current_device: proven.add((recorded_device, current_device)) return frozenset(proven) def _full_payload_packed_source_authority( *, source_id: str, schedule_sha256: str, schedule_rows: Sequence[Mapping[str, Any]], hash_rows: Sequence[Mapping[str, Any]], tokenizer_authority_sha256: str, context_window_tokens: int, answer_tokens_per_window: int, ) -> tuple[dict[str, Any], str]: """Bind one independently packable source to exact work/token authorities.""" work_rows: list[dict[str, Any]] = [] for schedule_row, hash_row in zip( schedule_rows, hash_rows, strict=True, ): domain_authority, rights_authority = _packed_schedule_authorities( schedule_row ) work_rows.append( { "scheduleOrdinal": schedule_row["scheduleOrdinal"], "payloadWorkId": schedule_row["payloadWorkId"], "payloadBytes": schedule_row["payloadBytes"], "observedPayloadSha256": hash_row["observedSha256"], "hashLedgerRowSha256": _sha256_bytes( _json_bytes(dict(hash_row)) ), "domainAuthoritySha256": _sha256_bytes( _json_bytes(domain_authority) ), "rightsAuthoritySha256": _sha256_bytes( _json_bytes(rights_authority) ), } ) authority = { "schema": "nnf.resynthesis.full_payload_packed_source_authority.v1", "sourceId": source_id, "scheduleSha256": schedule_sha256, "tokenizerAuthoritySha256": tokenizer_authority_sha256, "contextWindowTokens": context_window_tokens, "answerTokensPerWindow": answer_tokens_per_window, "payloadWork": work_rows, "payloadWorkCount": len(work_rows), "payloadBytes": sum(int(row["payloadBytes"]) for row in work_rows), } return authority, _sha256_bytes(_json_bytes(authority)) def _truncate_packed_artifact_to_durable_offset( path: Path, durable_bytes: int, ) -> None: """Discard torn append bytes beyond the last fsynced progress receipt.""" observed_bytes = path.stat().st_size if path.is_file() else 0 if observed_bytes < durable_bytes: raise RuntimeError( f"full payload packed shard {path.name} durable bytes are missing" ) if observed_bytes == durable_bytes: return with path.open("r+b") as handle: handle.truncate(durable_bytes) handle.flush() os.fsync(handle.fileno()) def _materialize_full_payload_packed_artifact( source_path: Path, target_path: Path, *, durable_bytes: int | None = None, expected_sha256: str | None = None, ) -> None: """Copy one immutable legacy artifact atomically without tokenization.""" source = source_path.expanduser().resolve() target = target_path.expanduser().resolve() source_bytes = source.stat().st_size if source.is_file() else -1 expected_bytes = source_bytes if durable_bytes is None else durable_bytes if expected_bytes < 0 or source_bytes < expected_bytes: raise RuntimeError("legacy packed artifact durable bytes are missing") if expected_sha256 is not None and ( len(expected_sha256) != 64 or file_sha256(source) != expected_sha256 ): raise RuntimeError("legacy packed artifact authority differs") if target.is_file(): if ( target.stat().st_size == expected_bytes and source_bytes == expected_bytes and file_sha256(target) == file_sha256(source) ): return raise RuntimeError("legacy packed artifact target already differs") target.parent.mkdir(parents=True, exist_ok=True) temporary = target.with_name(f".{target.name}.{os.getpid()}.tmp") temporary.unlink(missing_ok=True) if expected_bytes == source_bytes: try: os.link(source, temporary) except OSError: shutil.copyfile(source, temporary) else: remaining = expected_bytes with source.open("rb") as source_handle, temporary.open( "wb" ) as target_handle: while remaining: payload = source_handle.read(min(8 * 1024 * 1024, remaining)) if not payload: raise RuntimeError( "legacy packed artifact ended before durable bytes" ) target_handle.write(payload) remaining -= len(payload) target_handle.flush() os.fsync(target_handle.fileno()) os.replace(temporary, target) if expected_sha256 is not None and file_sha256(target) != expected_sha256: raise RuntimeError("legacy packed artifact authority differs") def _full_payload_packed_locator_geometry( locator_path: Path, *, record_count: int, context_window_tokens: int, answer_tokens_per_window: int, window_ends_path: Path, ) -> dict[str, int]: """Rebuild exact row/count geometry from record-oriented token locators.""" if ( record_count < 1 or not locator_path.is_file() or locator_path.stat().st_size != record_count * FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.size ): raise RuntimeError("legacy packed locator geometry differs") window_ends_path.parent.mkdir(parents=True, exist_ok=True) temporary = window_ends_path.with_name( f".{window_ends_path.name}.{os.getpid()}.tmp" ) temporary.unlink(missing_ok=True) rows = 0 prompt_tokens = 0 answer_tokens = 0 token_elements = 0 metadata_bytes = 0 with locator_path.open("rb") as locator_handle, temporary.open( "wb" ) as window_handle: locator_mmap = mmap.mmap( locator_handle.fileno(), 0, access=mmap.ACCESS_READ, ) try: for record_index in range(record_count): ( token_offset, prefix_tokens, content_tokens, metadata_offset, metadata_length, ) = FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.unpack_from( locator_mmap, record_index * FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.size, ) if ( token_offset != token_elements or metadata_offset != metadata_bytes or metadata_length < 1 ): raise RuntimeError( "legacy packed record offsets are not contiguous" ) ( record_rows, record_prompt_tokens, record_answer_tokens, ) = _full_payload_packed_record_training_geometry( prefix_tokens=prefix_tokens, content_tokens=content_tokens, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, ) rows += record_rows prompt_tokens += record_prompt_tokens answer_tokens += record_answer_tokens token_elements += prefix_tokens + content_tokens metadata_bytes += metadata_length window_handle.write(struct.pack(" dict[str, Path]: return { "tokens": root / f"{stem}.tokens.i32le", "tokensZstd": root / f"{stem}.tokens.i32le.zst", "tokenChunkIndex": root / f"{stem}.tokens.i32le.zst.index.u64le", "locators": root / f"{stem}.locators.u64le", "metadata": root / f"{stem}.metadata.jsonl", "metadataZstd": root / f"{stem}.metadata.jsonl.zst", "metadataChunkIndex": ( root / f"{stem}.metadata.jsonl.zst.index.u64le" ), "recordLocatorHashes": root / f"{stem}.record_locator.sha256", "windowEnds": root / f"{stem}.window_ends.u64le", "progress": root / f"{stem}.progress.json", "manifest": root / f"{stem}.manifest.json", } def _ceil_positive_ratio( value: int, numerator: int, denominator: int, ) -> int: """Scale one positive byte count without float precision loss.""" if ( isinstance(value, bool) or isinstance(numerator, bool) or isinstance(denominator, bool) or value < 0 or numerator < 0 or denominator < 1 ): raise ValueError("packed storage projection ratio is invalid") return (value * numerator + denominator - 1) // denominator def _full_payload_packed_incremental_batch_storage_projection() -> dict[str, int]: """Bound one worker's next streamed, resumable durability interval. The producer tokenizes at most one 8 MiB/128-record batch at a time and retains at most one 64 MiB interval (plus one tokenize-batch overshoot) in memory. It appends independently indexed zstd frames, fsyncs them and their sidecars, then publishes the progress receipt. The large record cap bounds pathological tiny-record sidecars; ordinary sources publish at 64 MiB. Admission therefore owns the same bounded interval as the writer instead of multiplying the complete source archive size by four bytes per token. A single semantic record may exceed the normal tokenize-batch byte bound. The writer measures that encoded record exactly and applies its incremental free-space guard before appending it, so the durable frontier remains resumable without pretending that the complete WorkID is one record. """ tokenize_batch_records = FULL_PAYLOAD_PACKED_PROGRESS_RECORDS tokenize_batch_source_bytes = FULL_PAYLOAD_PACKED_TOKENIZE_BATCH_BYTES tokenize_batch_token_bytes = 4 * ( tokenize_batch_source_bytes + tokenize_batch_records * FULL_PAYLOAD_PACKED_MEASUREMENT_PREFIX_TOKENS_PER_RECORD ) tokenize_batch_metadata_bytes = ( tokenize_batch_records * FULL_PAYLOAD_PACKED_MEASUREMENT_METADATA_BYTES_PER_RECORD ) tokenize_batch_index_bytes = ( tokenize_batch_records * FULL_PAYLOAD_PACKED_MEASUREMENT_INDEX_BYTES_PER_RECORD ) planned_append_bytes = ( tokenize_batch_token_bytes + tokenize_batch_metadata_bytes + tokenize_batch_index_bytes + tokenize_batch_source_bytes ) staged_raw_bytes_before_append = ( FULL_PAYLOAD_PACKED_DURABLE_PROGRESS_BYTES ) # This remains the writer's conservative free-space guard even though the # staged raw interval now lives in memory. It covers one compressed-copy # allowance, sidecars, and bounded zstd expansion without weakening # admission for incompressible input. persistent_append_bytes = planned_append_bytes temporary_bytes = staged_raw_bytes_before_append + planned_append_bytes return { "durableRecordUpperBound": ( FULL_PAYLOAD_PACKED_DURABLE_PROGRESS_RECORDS ), "durableRawStagingBytesUpperBound": ( staged_raw_bytes_before_append ), "tokenizeBatchRecordUpperBound": tokenize_batch_records, "tokenizeBatchSourceBytesUpperBound": ( tokenize_batch_source_bytes ), "tokenizeBatchRawTokenBytesUpperBound": ( tokenize_batch_token_bytes ), "tokenizeBatchRawMetadataBytesUpperBound": ( tokenize_batch_metadata_bytes ), "tokenizeBatchIndexBytesUpperBound": tokenize_batch_index_bytes, "plannedAppendBytesUpperBound": planned_append_bytes, "persistentAppendBytesUpperBound": persistent_append_bytes, "temporaryBytesUpperBound": temporary_bytes, } def _nearest_existing_directory(path: Path) -> Path: """Resolve the filesystem owner of a not-yet-created collection root.""" candidate = path.expanduser().resolve() while not candidate.exists(): if candidate == candidate.parent: raise FileNotFoundError( f"packed storage filesystem is unavailable: {path}" ) candidate = candidate.parent return candidate if candidate.is_dir() else candidate.parent def discover_passed_full_payload_schedule_union( authority_root: Path, ) -> tuple[dict[str, tuple[str, str, int]], dict[str, Any]]: """Discover the exact WorkID union of every passed schedule authority. The returned mapping is an internal comparison surface, not a serialized million-row receipt. Each WorkID is bound to a compact immutable identity digest, source ID, and payload byte count. Repeated schedule receipts are read once by schedule hash, and exact WorkID duplicates are counted once. """ root = authority_root.expanduser().resolve() if not root.is_dir(): raise FileNotFoundError( f"full payload authority root is absent: {root}" ) schedule_paths_by_sha256: dict[str, Path] = {} for receipt_path in sorted( root.rglob("full_payload_training_schedule.receipt.json") ): try: receipt = json.loads(receipt_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue schedule = receipt.get("schedule") if isinstance(receipt, dict) else None schedule_path_value = ( schedule.get("path") if isinstance(schedule, dict) else None ) schedule_sha256 = ( schedule.get("sha256") if isinstance(schedule, dict) else None ) if ( receipt.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA or receipt.get("passed") is not True or not isinstance(schedule_path_value, str) or not schedule_path_value or not isinstance(schedule_sha256, str) or len(schedule_sha256) != 64 ): continue schedule_path = Path(schedule_path_value).expanduser().resolve() if not schedule_path.is_file(): raise RuntimeError( "passed full payload schedule artifact is absent" ) prior_path = schedule_paths_by_sha256.get(schedule_sha256) if prior_path is not None and prior_path != schedule_path: if file_sha256(schedule_path) != schedule_sha256: raise RuntimeError( "passed full payload schedule artifact differs" ) continue schedule_paths_by_sha256[schedule_sha256] = schedule_path if not schedule_paths_by_sha256: raise RuntimeError("no passed full payload schedule authority was found") work_authority: dict[str, tuple[str, str, int]] = {} work_payload_identity: dict[str, tuple[str, object, int]] = {} source_ids: set[str] = set() duplicate_work_rows = 0 refreshed_record_contract_rows = 0 zero_byte_rows = 0 schedule_row_count = 0 for schedule_sha256, schedule_path in sorted( schedule_paths_by_sha256.items() ): if file_sha256(schedule_path) != schedule_sha256: raise RuntimeError("passed full payload schedule artifact differs") with schedule_path.open("r", encoding="utf-8") as handle: for line in handle: if not line.strip(): continue row = json.loads(line) work_id = row.get("payloadWorkId") source_id = row.get("sourceId") payload_bytes = row.get("payloadBytes") immutable = { "payloadWorkId": work_id, "sourceId": source_id, "sourceRecordSha256": row.get("sourceRecordSha256"), "payloadRelativePath": row.get("payloadRelativePath"), "payloadBytes": payload_bytes, } if ( row.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_ENTRY_SCHEMA or not isinstance(work_id, str) or len(work_id) != 64 or not isinstance(source_id, str) or not source_id or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or payload_bytes < 0 ): raise RuntimeError( "passed full payload schedule row is malformed" ) zero_byte_rows += int(payload_bytes == 0) identity_sha256 = _sha256_bytes(_json_bytes(immutable)) payload_identity = ( source_id, row.get("payloadRelativePath"), payload_bytes, ) prior = work_authority.get(work_id) if prior is not None: if work_payload_identity[work_id] != payload_identity: raise RuntimeError( "passed full payload WorkID authority conflicts" ) # The WorkID already binds source, path, bytes, and any # admitted content hash. A divergence limited to the # source record contract is a refresh annotation on the # identical payload, not an authority fork; the union is # a storage measurement surface, so the refresh deduplicates # with observability instead of failing closed. if prior[0] != identity_sha256: refreshed_record_contract_rows += 1 duplicate_work_rows += 1 continue work_authority[work_id] = ( identity_sha256, source_id, payload_bytes, ) work_payload_identity[work_id] = payload_identity source_ids.add(source_id) schedule_row_count += 1 union_digest = hashlib.sha256() for work_id in sorted(work_authority): identity_sha256, source_id, payload_bytes = work_authority[work_id] union_digest.update(work_id.encode("ascii")) union_digest.update(b"\x00") union_digest.update(identity_sha256.encode("ascii")) union_digest.update(b"\x00") union_digest.update(source_id.encode("utf-8")) union_digest.update(b"\x00") union_digest.update(str(payload_bytes).encode("ascii")) union_digest.update(b"\n") receipt = { "schema": "nnf.resynthesis.full_payload_passed_schedule_union.v1", "authorityRoot": str(root), "passedScheduleCount": len(schedule_paths_by_sha256), "uniquePayloadWorkCount": len(work_authority), "uniquePayloadBytes": sum( payload_bytes for _identity, _source_id, payload_bytes in work_authority.values() ), "sourceIds": sorted(source_ids), "sourceCount": len(source_ids), "duplicatePayloadWorkRowsDeduplicated": duplicate_work_rows, "refreshedRecordContractWorkRowsDeduplicated": ( refreshed_record_contract_rows ), "zeroBytePayloadWorkRows": zero_byte_rows, "scheduleRowsRead": schedule_row_count + duplicate_work_rows, "payloadWorkUnionSha256": union_digest.hexdigest(), "workIdsSerializedIntoReceipt": False, "targetEnteredForward": False, } return work_authority, receipt def _full_payload_packed_storage_sample( manifest_path: Path, *, scheduled_row: Mapping[str, Any], context_window_tokens: int, answer_tokens_per_window: int, ) -> dict[str, Any] | None: """Read exact final storage geometry from one compressed shard manifest.""" try: value = json.loads(manifest_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return None authority = ( value.pop("shardAuthoritySha256", None) if isinstance(value, dict) else None ) artifacts = value.get("artifacts") if isinstance(value, dict) else None token_compression = ( value.get("tokenCompression") if isinstance(value, dict) else None ) metadata_compression = ( value.get("metadataCompression") if isinstance(value, dict) else None ) record_storage = ( value.get("recordStorage") if isinstance(value, dict) else None ) compact_records = ( isinstance(value, dict) and value.get("packedStorageLayout") == FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT ) if ( not isinstance(value, dict) or value.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_SHARD_SCHEMA or value.get("passed") is not True or authority != _sha256_bytes(_json_bytes(value)) or value.get("payloadWorkId") != scheduled_row.get("payloadWorkId") or value.get("sourceId") != scheduled_row.get("sourceId") or value.get("sourceRecordSha256") != scheduled_row.get("sourceRecordSha256") or value.get("payloadRelativePath") != scheduled_row.get("payloadRelativePath") or value.get("payloadBytes") != scheduled_row.get("payloadBytes") or value.get("contextWindowTokens") != context_window_tokens or value.get("answerTokensPerWindow") != answer_tokens_per_window or value.get("rawTokenBytesRetained") is not False or value.get("rawMetadataBytesRetained") is not False or value.get("compressedTokenRandomAccess") is not True or value.get("compressedMetadataRandomAccess") is not True or not isinstance(artifacts, dict) or not isinstance(token_compression, dict) or token_compression.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_ZSTD_SCHEMA or ( compact_records and ( not isinstance(record_storage, dict) or record_storage.get("schema") != FULL_PAYLOAD_PACKED_COMPACT_RECORD_SCHEMA ) ) or ( not compact_records and ( not isinstance(metadata_compression, dict) or metadata_compression.get("schema") != FULL_PAYLOAD_PACKED_METADATA_ZSTD_SCHEMA ) ) ): return None persistent_bytes = manifest_path.stat().st_size for name in _full_payload_packed_required_artifact_names(value): artifact = artifacts.get(name) path_value = artifact.get("path") if isinstance(artifact, dict) else None byte_count = artifact.get("bytes") if isinstance(artifact, dict) else None if ( not isinstance(path_value, str) or not path_value or not isinstance(byte_count, int) or isinstance(byte_count, bool) or byte_count < 1 ): return None artifact_path = Path(path_value).expanduser().resolve() if not artifact_path.is_file() or artifact_path.stat().st_size != byte_count: return None persistent_bytes += byte_count progress_path = manifest_path.with_name( manifest_path.name.replace(".manifest.json", ".progress.json") ) if progress_path.is_file(): persistent_bytes += progress_path.stat().st_size raw_token_bytes = token_compression.get("uncompressedBytes") raw_metadata_bytes = ( record_storage.get("uncompressedBytes") if compact_records and isinstance(record_storage, dict) else ( metadata_compression.get("uncompressedBytes") if isinstance(metadata_compression, dict) else None ) ) record_count = value.get("recordCount") tokenizer_sha256 = value.get("tokenizerAuthoritySha256") if ( not isinstance(raw_token_bytes, int) or isinstance(raw_token_bytes, bool) or raw_token_bytes < 1 or not isinstance(raw_metadata_bytes, int) or isinstance(raw_metadata_bytes, bool) or raw_metadata_bytes < 1 or not isinstance(record_count, int) or isinstance(record_count, bool) or record_count < 1 or not isinstance(tokenizer_sha256, str) or len(tokenizer_sha256) != 64 ): return None value["shardAuthoritySha256"] = authority return { "manifestPath": str(manifest_path), "payloadWorkId": value["payloadWorkId"], "sourceId": value["sourceId"], "payloadBytes": value["payloadBytes"], "recordCount": record_count, "rawTokenBytes": raw_token_bytes, "rawMetadataBytes": raw_metadata_bytes, "persistentBytes": persistent_bytes, "tokenizerAuthoritySha256": tokenizer_sha256, "shardAuthoritySha256": authority, } def _full_payload_packed_legacy_partial_storage_sample( progress_path: Path, *, scheduled_row: Mapping[str, Any], context_window_tokens: int, answer_tokens_per_window: int, ) -> dict[str, Any] | None: """Measure one exact legacy raw frontier that must migrate on resume.""" try: progress = json.loads(progress_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return None work_id = scheduled_row.get("payloadWorkId") record_count = ( progress.get("recordCount") if isinstance(progress, dict) else None ) token_elements = ( progress.get("tokenElements") if isinstance(progress, dict) else None ) metadata_bytes = ( progress.get("metadataBytes") if isinstance(progress, dict) else None ) observed_records = ( progress.get("observedRecordCount") if isinstance(progress, dict) else None ) tokenizer_sha256 = ( progress.get("tokenizerAuthoritySha256") if isinstance(progress, dict) else None ) if ( not isinstance(progress, dict) or progress.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_PROGRESS_SCHEMA or progress.get("packedStorageLayout") is not None or progress.get("payloadWorkId") != work_id or progress.get("scheduleOrdinal") != scheduled_row.get("scheduleOrdinal") or progress.get("contextWindowTokens") != context_window_tokens or progress.get("answerTokensPerWindow") != answer_tokens_per_window or progress.get("targetEnteredForward") is not False or not isinstance(record_count, int) or isinstance(record_count, bool) or record_count < 1 or not isinstance(observed_records, int) or isinstance(observed_records, bool) or observed_records < record_count or not isinstance(token_elements, int) or isinstance(token_elements, bool) or token_elements < 1 or not isinstance(metadata_bytes, int) or isinstance(metadata_bytes, bool) or metadata_bytes < 1 or not isinstance(tokenizer_sha256, str) or len(tokenizer_sha256) != 64 ): return None stem = progress_path.name.removesuffix(".progress.json") paths = _full_payload_packed_paths(progress_path.parent, stem) durable_sizes = { "tokens": token_elements * 4, "metadata": metadata_bytes, "locators": ( record_count * FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.size ), "recordLocatorHashes": record_count * 32, "windowEnds": ( record_count * FULL_PAYLOAD_PACKED_TOKEN_WINDOW_INDEX_RECORD_BYTES ), } if any( not paths[name].is_file() or paths[name].stat().st_size < durable_bytes for name, durable_bytes in durable_sizes.items() ): return None migrated_token_bytes = progress.get( "legacyMigrationTokenUncompressedBytes", 0, ) migrated_metadata_bytes = progress.get( "legacyMigrationMetadataUncompressedBytes", 0, ) token_compressed_bytes = progress.get("tokenCompressedBytes", 0) metadata_compressed_bytes = progress.get("metadataCompressedBytes", 0) token_chunk_count = progress.get("tokenChunkCount", 0) metadata_chunk_count = progress.get("metadataChunkCount", 0) migration_values = ( migrated_token_bytes, migrated_metadata_bytes, token_compressed_bytes, metadata_compressed_bytes, token_chunk_count, metadata_chunk_count, ) if ( any( not isinstance(value, int) or isinstance(value, bool) or value < 0 for value in migration_values ) or migrated_token_bytes > durable_sizes["tokens"] or migrated_metadata_bytes > durable_sizes["metadata"] or ( migrated_token_bytes != durable_sizes["tokens"] and migrated_token_bytes % FULL_PAYLOAD_PACKED_TOKEN_ZSTD_CHUNK_BYTES != 0 ) or ( migrated_metadata_bytes != durable_sizes["metadata"] and migrated_metadata_bytes % FULL_PAYLOAD_PACKED_METADATA_ZSTD_CHUNK_BYTES != 0 ) or token_chunk_count != math.ceil( migrated_token_bytes / FULL_PAYLOAD_PACKED_TOKEN_ZSTD_CHUNK_BYTES ) or metadata_chunk_count != math.ceil( migrated_metadata_bytes / FULL_PAYLOAD_PACKED_METADATA_ZSTD_CHUNK_BYTES ) or (migrated_token_bytes == 0) != (token_compressed_bytes == 0) or (migrated_metadata_bytes == 0) != (metadata_compressed_bytes == 0) ): return None migrated_artifact_sizes = { "tokensZstd": token_compressed_bytes, "tokenChunkIndex": ( token_chunk_count * FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.size ), "metadataZstd": metadata_compressed_bytes, "metadataChunkIndex": ( metadata_chunk_count * FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.size ), } if any( durable_bytes > 0 and ( not paths[name].is_file() or paths[name].stat().st_size < durable_bytes ) for name, durable_bytes in migrated_artifact_sizes.items() ): return None raw_durable_bytes = durable_sizes["tokens"] + durable_sizes["metadata"] remaining_raw_bytes = ( durable_sizes["tokens"] - migrated_token_bytes + durable_sizes["metadata"] - migrated_metadata_bytes ) migration_index_bytes = ( ( math.ceil( durable_sizes["tokens"] / FULL_PAYLOAD_PACKED_TOKEN_ZSTD_CHUNK_BYTES ) - token_chunk_count ) * FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.size + ( math.ceil( durable_sizes["metadata"] / FULL_PAYLOAD_PACKED_METADATA_ZSTD_CHUNK_BYTES ) - metadata_chunk_count ) * FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.size ) # Match the writer's pre-mutation legacy migration fence exactly. The raw # frontier already occupies the filesystem and remains authoritative until # this compressed copy plus its indexes and progress receipt are durable. migration_copy_bytes = ( remaining_raw_bytes + math.ceil(remaining_raw_bytes / 16) + migration_index_bytes ) return { "progressPath": str(progress_path), "payloadWorkId": work_id, "sourceId": scheduled_row.get("sourceId"), "payloadBytes": scheduled_row.get("payloadBytes"), "recordCount": record_count, "rawDurableBytes": raw_durable_bytes, "migratedTokenBytes": migrated_token_bytes, "migratedMetadataBytes": migrated_metadata_bytes, "remainingMigrationRawBytes": remaining_raw_bytes, "migrationIndexBytes": migration_index_bytes, "migrationCopyBytes": migration_copy_bytes, "tokenizerAuthoritySha256": tokenizer_sha256, } def full_payload_packed_storage_admission( schedule_rows: Sequence[Mapping[str, Any]], output_root: Path, *, sample_roots: Sequence[Path], context_window_tokens: int, answer_tokens_per_window: int, source_output_roots: Mapping[str, Path] | None = None, universal_work_authority: Mapping[str, tuple[str, str, int]] | None = None, universal_union_receipt: Mapping[str, Any] | None = None, available_bytes_by_output_root: Mapping[Path, int] | None = None, worker_count: int = FULL_PAYLOAD_PACKED_MAX_WORKERS, measurement_only: bool = False, ) -> dict[str, Any]: """Fail closed unless per-source measured storage fits every filesystem.""" if ( not schedule_rows or isinstance(context_window_tokens, bool) or isinstance(answer_tokens_per_window, bool) or context_window_tokens < 3 or answer_tokens_per_window < 1 or answer_tokens_per_window >= context_window_tokens or isinstance(worker_count, bool) or worker_count < 1 or not isinstance(measurement_only, bool) or (measurement_only and len(schedule_rows) != 1) ): raise ValueError("full payload packed storage admission input is invalid") effective_worker_count = 1 if measurement_only else worker_count resolved_output = output_root.expanduser().resolve() configured_roots = { source_id: path.expanduser().resolve() for source_id, path in (source_output_roots or {}).items() } scheduled_by_work_id: dict[str, Mapping[str, Any]] = {} rows_by_source: dict[str, list[Mapping[str, Any]]] = {} for row in schedule_rows: work_id = row.get("payloadWorkId") source_id = row.get("sourceId") payload_bytes = row.get("payloadBytes") if ( not isinstance(work_id, str) or len(work_id) != 64 or work_id in scheduled_by_work_id or not isinstance(source_id, str) or not source_id or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or payload_bytes < 1 ): raise ValueError("full payload packed storage schedule differs") scheduled_by_work_id[work_id] = row rows_by_source.setdefault(source_id, []).append(row) if configured_roots and not set(rows_by_source).issubset(configured_roots): raise ValueError("full payload packed storage roots are incomplete") samples_by_work_id: dict[str, dict[str, Any]] = {} sample_paths_seen: set[Path] = set() for sample_root in sample_roots: resolved_sample_root = sample_root.expanduser().resolve() if not resolved_sample_root.is_dir(): continue for manifest_path in sorted(resolved_sample_root.rglob("*.manifest.json")): resolved_manifest = manifest_path.resolve() if resolved_manifest in sample_paths_seen: continue sample_paths_seen.add(resolved_manifest) try: candidate = json.loads( resolved_manifest.read_text(encoding="utf-8") ) except (OSError, json.JSONDecodeError): continue work_id = ( candidate.get("payloadWorkId") if isinstance(candidate, dict) else None ) if not isinstance(work_id, str): continue scheduled_row = scheduled_by_work_id.get(work_id) if scheduled_row is None: continue sample = _full_payload_packed_storage_sample( resolved_manifest, scheduled_row=scheduled_row, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, ) if sample is None: continue prior = samples_by_work_id.get(work_id) if prior is None or int(sample["persistentBytes"]) > int( prior["persistentBytes"] ): samples_by_work_id[work_id] = sample samples_by_source: dict[str, list[dict[str, Any]]] = {} for sample in samples_by_work_id.values(): samples_by_source.setdefault(str(sample["sourceId"]), []).append(sample) legacy_partials_by_work_id: dict[str, dict[str, Any]] = {} progress_paths_seen: set[Path] = set() for sample_root in sample_roots: resolved_sample_root = sample_root.expanduser().resolve() if not resolved_sample_root.is_dir(): continue for progress_path in sorted( resolved_sample_root.rglob("*.progress.json") ): resolved_progress = progress_path.resolve() if resolved_progress in progress_paths_seen: continue progress_paths_seen.add(resolved_progress) try: candidate = json.loads( resolved_progress.read_text(encoding="utf-8") ) except (OSError, json.JSONDecodeError): continue work_id = ( candidate.get("payloadWorkId") if isinstance(candidate, dict) else None ) if ( not isinstance(work_id, str) or work_id in samples_by_work_id ): continue scheduled_row = scheduled_by_work_id.get(work_id) if scheduled_row is None: continue partial = _full_payload_packed_legacy_partial_storage_sample( resolved_progress, scheduled_row=scheduled_row, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, ) if partial is None: continue prior = legacy_partials_by_work_id.get(work_id) if prior is None or int(partial["migrationCopyBytes"]) > int( prior["migrationCopyBytes"] ): legacy_partials_by_work_id[work_id] = partial legacy_partials_by_source: dict[str, list[dict[str, Any]]] = {} for partial in legacy_partials_by_work_id.values(): legacy_partials_by_source.setdefault( str(partial["sourceId"]), [], ).append(partial) tokenizer_authorities = { str(sample["tokenizerAuthoritySha256"]) for sample in ( *samples_by_work_id.values(), *legacy_partials_by_work_id.values(), ) } missing_sample_sources = sorted(set(rows_by_source) - set(samples_by_source)) bounded_measurement_sources = ( set(missing_sample_sources) if measurement_only else set() ) unmeasured_unbounded_sources = sorted( set(missing_sample_sources) - bounded_measurement_sources ) universal_missing_work_ids: set[str] = set() universal_extra_work_ids: set[str] = set() universal_coverage_passed = True if universal_work_authority is not None: selected_work_ids = set(scheduled_by_work_id) universal_work_ids = set(universal_work_authority) universal_missing_work_ids = universal_work_ids - selected_work_ids universal_extra_work_ids = selected_work_ids - universal_work_ids universal_coverage_passed = not ( universal_missing_work_ids or universal_extra_work_ids ) source_projections: list[dict[str, Any]] = [] filesystem_groups: dict[int, dict[str, Any]] = {} available_overrides = { path.expanduser().resolve(): byte_count for path, byte_count in (available_bytes_by_output_root or {}).items() } incremental_batch = ( _full_payload_packed_incremental_batch_storage_projection() ) for source_id, source_rows in sorted(rows_by_source.items()): target_root = configured_roots.get(source_id, resolved_output) filesystem_owner = _nearest_existing_directory(target_root) filesystem_device = filesystem_owner.stat().st_dev group = filesystem_groups.setdefault( filesystem_device, { "filesystemDevice": filesystem_device, "filesystemOwner": str(filesystem_owner), "outputRoots": set(), "sourceIds": [], "projectedPersistentBytes": 0, "existingReusablePersistentBytes": 0, "projectedRemainingPersistentBytes": 0, "temporaryCandidates": [], "legacyOversizedMigrationCandidates": [], "legacyBoundedMigrationCandidates": [], }, ) group["outputRoots"].add(str(target_root)) group["sourceIds"].append(source_id) source_payload_bytes = sum( int(row["payloadBytes"]) for row in source_rows ) source_samples = samples_by_source.get(source_id, []) existing_reusable_bytes = sum( int(sample["persistentBytes"]) for sample in source_samples if Path(str(sample["manifestPath"])).is_relative_to(target_root) ) source_legacy_partials = [ partial for partial in legacy_partials_by_source.get(source_id, []) if Path(str(partial["progressPath"])).is_relative_to(target_root) ] oversized_migration_bytes = max( ( int(partial["migrationCopyBytes"]) for partial in source_legacy_partials if int(partial["payloadBytes"]) > FULL_PAYLOAD_PACKED_MAX_INFLIGHT_PAYLOAD_BYTES ), default=0, ) bounded_migration_bytes = sum( sorted( ( int(partial["migrationCopyBytes"]) for partial in source_legacy_partials if int(partial["payloadBytes"]) <= FULL_PAYLOAD_PACKED_MAX_INFLIGHT_PAYLOAD_BYTES ), reverse=True, )[:effective_worker_count] ) legacy_migration_copy_bytes = max( oversized_migration_bytes, bounded_migration_bytes, ) group["existingReusablePersistentBytes"] += existing_reusable_bytes group["legacyOversizedMigrationCandidates"].extend( int(partial["migrationCopyBytes"]) for partial in source_legacy_partials if int(partial["payloadBytes"]) > FULL_PAYLOAD_PACKED_MAX_INFLIGHT_PAYLOAD_BYTES ) group["legacyBoundedMigrationCandidates"].extend( int(partial["migrationCopyBytes"]) for partial in source_legacy_partials if int(partial["payloadBytes"]) <= FULL_PAYLOAD_PACKED_MAX_INFLIGHT_PAYLOAD_BYTES ) if not source_samples: if source_id in bounded_measurement_sources: # Admit only the next durable record batch. The shard writer # applies the same exact staged-raw plus compressed-copy fence # represented here, so measurement can establish real density # without treating the complete WorkID as one in-memory record. persistent_upper_bound = int( incremental_batch["persistentAppendBytesUpperBound"] ) temporary_upper_bound = int( incremental_batch["temporaryBytesUpperBound"] ) group["projectedPersistentBytes"] += persistent_upper_bound group["projectedRemainingPersistentBytes"] += ( persistent_upper_bound ) group["temporaryCandidates"].append(temporary_upper_bound) source_projections.append( { "sourceId": source_id, "outputRoot": str(target_root), "scheduledPayloadWorkCount": 1, "scheduledPayloadBytes": source_payload_bytes, "sampledCompleteWorkCount": 0, "boundedFirstWorkMeasurement": True, "recordUpperBound": incremental_batch[ "durableRecordUpperBound" ], "rawTokenBytesUpperBound": incremental_batch[ "tokenizeBatchRawTokenBytesUpperBound" ], "rawMetadataBytesUpperBound": incremental_batch[ "tokenizeBatchRawMetadataBytesUpperBound" ], "indexBytesUpperBound": incremental_batch[ "tokenizeBatchIndexBytesUpperBound" ], "incrementalBatchStorage": dict( incremental_batch ), "projectedPersistentBytes": persistent_upper_bound, "projectedRemainingPersistentBytes": ( persistent_upper_bound ), "projectedMaximumSingleWorkTemporaryBytes": ( temporary_upper_bound ), "projectionScope": "next_durable_record_batch", "incrementalStorageGuardRequired": True, "completeWorkIdSizeDefinesIncrementalStorage": False, "oversizedRecordUsesExactRuntimeStorageGuard": True, "fullSourceProjectionDeferredUntilMeasured": True, "measured": False, } ) continue source_projections.append( { "sourceId": source_id, "outputRoot": str(target_root), "scheduledPayloadWorkCount": len(source_rows), "scheduledPayloadBytes": source_payload_bytes, "sampledCompleteWorkCount": 0, "measured": False, } ) continue sample_payload_bytes = sum( int(sample["payloadBytes"]) for sample in source_samples ) sample_persistent_bytes = sum( int(sample["persistentBytes"]) for sample in source_samples ) sample_raw_token_bytes = sum( int(sample["rawTokenBytes"]) for sample in source_samples ) sample_raw_metadata_bytes = sum( int(sample["rawMetadataBytes"]) for sample in source_samples ) sample_record_count = sum( int(sample["recordCount"]) for sample in source_samples ) if sample_persistent_bytes > ( sample_payload_bytes * FULL_PAYLOAD_PACKED_CREDIBLE_SAMPLE_DENSITY ): # The measured sample's fixed per-record overheads dominate its # tiny payload, so a payload-ratio projection of the whole source # is not credible. Admit one bounded durability interval per # concurrent worker; complete WorkID size is diagnostic because # the producer streams records rather than materializing an # archive as one raw-token allocation. largest_work_bytes = max( int(row["payloadBytes"]) for row in source_rows ) concurrent_incremental_workers = min( effective_worker_count, len(source_rows), ) persistent_per_worker = int( incremental_batch["persistentAppendBytesUpperBound"] ) temporary_per_worker = int( incremental_batch["temporaryBytesUpperBound"] ) persistent_upper_bound = ( persistent_per_worker * concurrent_incremental_workers ) group["projectedPersistentBytes"] += persistent_upper_bound group["projectedRemainingPersistentBytes"] += ( persistent_upper_bound ) group["temporaryCandidates"].extend( [temporary_per_worker] * concurrent_incremental_workers ) source_projections.append( { "sourceId": source_id, "outputRoot": str(target_root), "scheduledPayloadWorkCount": len(source_rows), "scheduledPayloadBytes": source_payload_bytes, "sampledCompleteWorkCount": len(source_samples), "sampledPayloadBytes": sample_payload_bytes, "sampledRecordCount": sample_record_count, "sampledPersistentBytes": sample_persistent_bytes, "sampleDensityCredible": False, "boundedFirstWorkMeasurement": False, "recordUpperBound": incremental_batch[ "durableRecordUpperBound" ], "largestScheduledWorkBytes": largest_work_bytes, "rawTokenBytesUpperBound": incremental_batch[ "tokenizeBatchRawTokenBytesUpperBound" ], "rawMetadataBytesUpperBound": incremental_batch[ "tokenizeBatchRawMetadataBytesUpperBound" ], "indexBytesUpperBound": incremental_batch[ "tokenizeBatchIndexBytesUpperBound" ], "incrementalBatchStorage": dict(incremental_batch), "concurrentIncrementalWorkerCount": ( concurrent_incremental_workers ), "projectedPersistentBytes": persistent_upper_bound, "projectedRemainingPersistentBytes": ( persistent_upper_bound ), "projectedMaximumSingleWorkTemporaryBytes": ( temporary_per_worker ), "existingReusablePersistentBytes": ( existing_reusable_bytes ), "legacyPartialWorkCount": len(source_legacy_partials), "projectedLegacyMigrationCopyBytes": ( legacy_migration_copy_bytes ), "projectionScope": "next_durable_record_batch", "incrementalStorageGuardRequired": True, "completeWorkIdSizeDefinesIncrementalStorage": False, "oversizedRecordUsesExactRuntimeStorageGuard": True, "fullSourceProjectionDeferredUntilCredibleSample": True, "measured": True, } ) continue projected_persistent_bytes = _ceil_positive_ratio( source_payload_bytes, sample_persistent_bytes, sample_payload_bytes, ) existing_reusable_bytes = sum( int(sample["persistentBytes"]) for sample in source_samples if Path(str(sample["manifestPath"])).is_relative_to(target_root) ) remaining_persistent_bytes = max( 0, projected_persistent_bytes - existing_reusable_bytes, ) temporary_ratio_bytes = ( sample_raw_token_bytes + sample_raw_metadata_bytes + sample_persistent_bytes ) temporary_candidates = [ _ceil_positive_ratio( int(row["payloadBytes"]), temporary_ratio_bytes, sample_payload_bytes, ) for row in source_rows ] concurrent_temporary_estimate = sum( sorted(temporary_candidates, reverse=True)[:effective_worker_count] ) full_source_projection_bytes = ( remaining_persistent_bytes + concurrent_temporary_estimate + legacy_migration_copy_bytes ) full_source_required_estimate = ( full_source_projection_bytes + _ceil_positive_ratio( full_source_projection_bytes, 1, FULL_PAYLOAD_PACKED_STORAGE_HEADROOM_DIVISOR, ) + FULL_PAYLOAD_PACKED_STORAGE_MINIMUM_RESERVE_BYTES ) full_source_free_bytes = ( available_overrides[target_root] if target_root in available_overrides else shutil.disk_usage(target_root).free ) if full_source_required_estimate > full_source_free_bytes: # The credible measured projection of the remaining source does # not fit the current free space (the disk filled since the # source's samples were sealed). Refusing outright would strand # the remaining payload forever even though the guarded shard # writer rechecks exact free space before every durable batch # append and stops at the last fsynced batch. Admit one bounded # durability interval per concurrent worker instead. The complete # WorkID size remains diagnostic and cannot inflate that interval. largest_work_bytes = max( int(row["payloadBytes"]) for row in source_rows ) concurrent_incremental_workers = min( effective_worker_count, len(source_rows), ) persistent_per_worker = int( incremental_batch["persistentAppendBytesUpperBound"] ) temporary_per_worker = int( incremental_batch["temporaryBytesUpperBound"] ) persistent_upper_bound = ( persistent_per_worker * concurrent_incremental_workers ) group["projectedPersistentBytes"] += persistent_upper_bound group["projectedRemainingPersistentBytes"] += ( persistent_upper_bound ) group["temporaryCandidates"].extend( [temporary_per_worker] * concurrent_incremental_workers ) source_projections.append( { "sourceId": source_id, "outputRoot": str(target_root), "scheduledPayloadWorkCount": len(source_rows), "scheduledPayloadBytes": source_payload_bytes, "sampledCompleteWorkCount": len(source_samples), "sampledPayloadBytes": sample_payload_bytes, "sampledRecordCount": sample_record_count, "sampledPersistentBytes": sample_persistent_bytes, "sampleDensityCredible": True, "boundedFirstWorkMeasurement": False, "recordUpperBound": incremental_batch[ "durableRecordUpperBound" ], "largestScheduledWorkBytes": largest_work_bytes, "rawTokenBytesUpperBound": incremental_batch[ "tokenizeBatchRawTokenBytesUpperBound" ], "rawMetadataBytesUpperBound": incremental_batch[ "tokenizeBatchRawMetadataBytesUpperBound" ], "indexBytesUpperBound": incremental_batch[ "tokenizeBatchIndexBytesUpperBound" ], "incrementalBatchStorage": dict(incremental_batch), "concurrentIncrementalWorkerCount": ( concurrent_incremental_workers ), "projectedPersistentBytes": persistent_upper_bound, "projectedRemainingPersistentBytes": ( persistent_upper_bound ), "projectedMaximumSingleWorkTemporaryBytes": ( temporary_per_worker ), "existingReusablePersistentBytes": ( existing_reusable_bytes ), "legacyPartialWorkCount": len(source_legacy_partials), "projectedLegacyMigrationCopyBytes": ( legacy_migration_copy_bytes ), "projectionScope": "next_durable_record_batch", "incrementalStorageGuardRequired": True, "completeWorkIdSizeDefinesIncrementalStorage": False, "oversizedRecordUsesExactRuntimeStorageGuard": True, "fullSourceProjectionExceededFreeBytes": True, "measured": True, } ) continue group["projectedPersistentBytes"] += projected_persistent_bytes group["projectedRemainingPersistentBytes"] += ( remaining_persistent_bytes ) group["temporaryCandidates"].extend(temporary_candidates) source_projections.append( { "sourceId": source_id, "outputRoot": str(target_root), "scheduledPayloadWorkCount": len(source_rows), "scheduledPayloadBytes": source_payload_bytes, "sampledCompleteWorkCount": len(source_samples), "sampledPayloadBytes": sample_payload_bytes, "sampledRecordCount": sample_record_count, "sampledRawTokenBytes": sample_raw_token_bytes, "sampledRawMetadataBytes": sample_raw_metadata_bytes, "sampledPersistentBytes": sample_persistent_bytes, "projectedPersistentBytes": projected_persistent_bytes, "existingReusablePersistentBytes": existing_reusable_bytes, "projectedRemainingPersistentBytes": ( remaining_persistent_bytes ), "projectedMaximumSingleWorkTemporaryBytes": max( temporary_candidates ), "legacyPartialWorkCount": len(source_legacy_partials), "projectedLegacyMigrationCopyBytes": ( legacy_migration_copy_bytes ), "measured": True, } ) filesystem_receipts: list[dict[str, Any]] = [] for filesystem_device, group in sorted(filesystem_groups.items()): output_roots = sorted(group["outputRoots"]) override_values = { available_overrides[Path(output_root)] for output_root in output_roots if Path(output_root) in available_overrides } if len(override_values) > 1: raise ValueError( "packed storage available-byte overrides disagree on one filesystem" ) free_bytes = ( next(iter(override_values)) if override_values else shutil.disk_usage(Path(str(group["filesystemOwner"]))).free ) candidates = sorted( (int(value) for value in group["temporaryCandidates"]), reverse=True, ) concurrent_temporary_bytes = sum( candidates[:effective_worker_count] ) oversized_migration_candidates = tuple( int(value) for value in group["legacyOversizedMigrationCandidates"] ) bounded_migration_candidates = sorted( ( int(value) for value in group["legacyBoundedMigrationCandidates"] ), reverse=True, ) # The executor admits at most one payload larger than its 1 GiB # pending-byte budget, irrespective of source. Smaller legacy # frontiers can migrate up to the worker count concurrently. legacy_migration_copy_bytes = max( max(oversized_migration_candidates, default=0), sum(bounded_migration_candidates[:effective_worker_count]), ) projection_before_headroom = ( int(group["projectedRemainingPersistentBytes"]) + concurrent_temporary_bytes + legacy_migration_copy_bytes ) safety_headroom_bytes = _ceil_positive_ratio( projection_before_headroom, 1, FULL_PAYLOAD_PACKED_STORAGE_HEADROOM_DIVISOR, ) required_free_bytes = ( projection_before_headroom + safety_headroom_bytes + FULL_PAYLOAD_PACKED_STORAGE_MINIMUM_RESERVE_BYTES ) missing_on_filesystem = sorted( set(group["sourceIds"]) & set(unmeasured_unbounded_sources) ) filesystem_receipts.append( { "filesystemDevice": filesystem_device, "filesystemOwner": group["filesystemOwner"], "outputRoots": output_roots, "sourceIds": sorted(group["sourceIds"]), "projectedPersistentBytes": group[ "projectedPersistentBytes" ], "existingReusablePersistentBytes": group[ "existingReusablePersistentBytes" ], "projectedRemainingPersistentBytes": group[ "projectedRemainingPersistentBytes" ], "maximumConcurrentPackingWorkers": effective_worker_count, "projectedConcurrentRawTemporaryBytes": ( concurrent_temporary_bytes ), "projectedLegacyMigrationCopyBytes": ( legacy_migration_copy_bytes ), "safetyHeadroomBytes": safety_headroom_bytes, "safetyHeadroomBasis": ( "ceil_remaining_persistent_plus_concurrent_raw_plus_" "legacy_migration_divided_by_16" ), "minimumFilesystemReserveBytes": ( FULL_PAYLOAD_PACKED_STORAGE_MINIMUM_RESERVE_BYTES ), "requiredFreeBytes": required_free_bytes, "freeBytesAtAdmission": free_bytes, "freeBytesAfterProjectedAdmission": free_bytes - required_free_bytes, "missingMeasuredSourceIds": missing_on_filesystem, "passed": ( not missing_on_filesystem and free_bytes >= required_free_bytes ), } ) tokenizer_authority_consistent = len(tokenizer_authorities) <= 1 passed = bool( not unmeasured_unbounded_sources and tokenizer_authority_consistent and universal_coverage_passed and filesystem_receipts and all(receipt["passed"] is True for receipt in filesystem_receipts) ) receipt = { "schema": FULL_PAYLOAD_PACKED_STORAGE_ADMISSION_SCHEMA, "passed": passed, "contextWindowTokens": context_window_tokens, "answerTokensPerWindow": answer_tokens_per_window, "scheduledPayloadWorkCount": len(schedule_rows), "scheduledPayloadWorkIdsSha256": _sha256_bytes( _json_bytes(sorted(scheduled_by_work_id)) ), "scheduledPayloadBytes": sum( int(row["payloadBytes"]) for row in schedule_rows ), "scheduledSourceIds": sorted(rows_by_source), "scheduledSourceCount": len(rows_by_source), "sampleRoots": sorted( str(path.expanduser().resolve()) for path in sample_roots ), "sampledCompleteWorkCount": len(samples_by_work_id), "sampledSourceCount": len(samples_by_source), "missingMeasuredSourceIds": missing_sample_sources, "unmeasuredUnboundedSourceIds": unmeasured_unbounded_sources, "allSelectedSourcesMeasured": not missing_sample_sources, "boundedFirstWorkMeasurement": measurement_only, "boundedMeasurementSourceIds": sorted(bounded_measurement_sources), "broadPackingAllowed": not measurement_only, "sampleTokenizerAuthoritySha256s": sorted(tokenizer_authorities), "sampleTokenizerAuthorityConsistent": tokenizer_authority_consistent, "sourceProjections": source_projections, "filesystems": filesystem_receipts, "universalPassedAuthorityUnionRequired": ( universal_work_authority is not None ), "universalPassedAuthorityUnion": ( dict(universal_union_receipt) if universal_union_receipt is not None else None ), "universalCoveragePassed": universal_coverage_passed, "universalMissingPayloadWorkCount": len( universal_missing_work_ids ), "universalExtraPayloadWorkCount": len(universal_extra_work_ids), "globalAverageProjectionUsed": False, "perSourceRecordDensityMeasured": True, "concurrentRawTemporaryBytesIncluded": True, "rawPayloadDeletionAllowed": False, "tokenizerInvoked": False, "rawPayloadReadersInvoked": False, "targetEnteredForward": False, } receipt["storageAdmissionSha256"] = _sha256_bytes(_json_bytes(receipt)) return receipt def _validated_full_payload_packed_storage_admission_receipt( receipt_path: Path, ) -> dict[str, Any]: """Validate one immutable pre-tokenizer storage admission authority.""" path = receipt_path.expanduser().resolve() loaded = json.loads(path.read_text(encoding="utf-8")) authority = ( loaded.pop("storageAdmissionSha256", None) if isinstance(loaded, dict) else None ) if ( not isinstance(loaded, dict) or loaded.get("schema") != FULL_PAYLOAD_PACKED_STORAGE_ADMISSION_SCHEMA or loaded.get("passed") is not True or not isinstance(authority, str) or len(authority) != 64 or authority != _sha256_bytes(_json_bytes(loaded)) ): raise RuntimeError( "full payload packed storage admission authority differs" ) loaded["storageAdmissionSha256"] = authority return loaded def _validate_full_payload_packed_admission_source_roots( admission: Mapping[str, Any], source_roots: Mapping[str, Path], ) -> None: """Bind every selected source root to its admitted filesystem owner.""" expected_roots = { source_id: path.expanduser().resolve() for source_id, path in source_roots.items() } projections = admission.get("sourceProjections") filesystems = admission.get("filesystems") if ( not expected_roots or not isinstance(projections, list) or not isinstance(filesystems, list) or not filesystems ): raise RuntimeError( "full payload packed admission source roots are absent" ) projection_roots: dict[str, Path] = {} for projection in projections: source_id = ( projection.get("sourceId") if isinstance(projection, dict) else None ) output_root = ( projection.get("outputRoot") if isinstance(projection, dict) else None ) if ( not isinstance(source_id, str) or source_id not in expected_roots or source_id in projection_roots or not isinstance(output_root, str) or not output_root ): raise RuntimeError( "full payload packed admission source projection differs" ) projection_roots[source_id] = Path( output_root ).expanduser().resolve() if projection_roots != expected_roots: raise RuntimeError( "full payload packed admission source output root differs" ) expected_sources_by_device: dict[int, set[str]] = {} expected_roots_by_device: dict[int, set[Path]] = {} for source_id, output_root in expected_roots.items(): owner = _nearest_existing_directory(output_root) device = int(owner.stat().st_dev) expected_sources_by_device.setdefault(device, set()).add(source_id) expected_roots_by_device.setdefault(device, set()).add(output_root) observed_devices: set[int] = set() for filesystem in filesystems: if not isinstance(filesystem, dict): raise RuntimeError( "full payload packed admission filesystem differs" ) observed_device = filesystem.get("filesystemDevice") owner_value = filesystem.get("filesystemOwner") output_root_values = filesystem.get("outputRoots") source_ids = filesystem.get("sourceIds") if ( not isinstance(observed_device, int) or isinstance(observed_device, bool) or observed_device in observed_devices or observed_device not in expected_sources_by_device or not isinstance(owner_value, str) or not owner_value or not isinstance(output_root_values, list) or not isinstance(source_ids, list) or filesystem.get("passed") is not True ): raise RuntimeError( "full payload packed admission filesystem differs" ) recorded_owner = Path(owner_value).expanduser().resolve() observed_roots = { Path(str(value)).expanduser().resolve() for value in output_root_values if isinstance(value, str) and value } observed_sources = { str(value) for value in source_ids if isinstance(value, str) and value } if ( len(observed_roots) != len(output_root_values) or len(observed_sources) != len(source_ids) or not recorded_owner.is_dir() or int(recorded_owner.stat().st_dev) != observed_device or observed_roots != expected_roots_by_device[observed_device] or observed_sources != expected_sources_by_device[observed_device] ): raise RuntimeError( "full payload packed admission filesystem ownership differs" ) observed_devices.add(observed_device) if observed_devices != set(expected_sources_by_device): raise RuntimeError( "full payload packed admission filesystem coverage differs" ) def _validated_legacy_full_payload_packed_manifest( manifest_path: Path, *, schedule_row: Mapping[str, Any], hash_row: Mapping[str, Any], tokenizer_authority_sha256: str, context_window_tokens: int, answer_tokens_per_window: int, ) -> dict[str, Any]: """Validate one complete record-oriented legacy shard before adoption.""" value = json.loads(manifest_path.read_text(encoding="utf-8")) authority = ( value.pop("shardAuthoritySha256", None) if isinstance(value, dict) else None ) artifacts = value.get("artifacts") if isinstance(value, dict) else None required = ( "locators", "recordLocatorHashes", "windowEnds", ) has_compact_records = ( isinstance(artifacts, dict) and set(artifacts) == { "tokensZstd", "tokenChunkIndex", "recordsZstd", "recordChunkIndex", } and isinstance(value, dict) and value.get("packedStorageLayout") == FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT and isinstance(value.get("recordStorage"), dict) and value["recordStorage"].get("schema") == FULL_PAYLOAD_PACKED_COMPACT_RECORD_SCHEMA and value["recordStorage"].get("compressed") == artifacts.get("recordsZstd") and value["recordStorage"].get("chunkIndex") == artifacts.get("recordChunkIndex") ) has_raw_tokens = isinstance(artifacts, dict) and isinstance( artifacts.get("tokens"), dict, ) has_compressed_tokens = ( isinstance(artifacts, dict) and isinstance(artifacts.get("tokensZstd"), dict) and isinstance(artifacts.get("tokenChunkIndex"), dict) and isinstance(value, dict) and isinstance(value.get("tokenCompression"), dict) and value["tokenCompression"].get("compressed") == artifacts.get("tokensZstd") and value["tokenCompression"].get("chunkIndex") == artifacts.get("tokenChunkIndex") ) has_raw_metadata = isinstance(artifacts, dict) and isinstance( artifacts.get("metadata"), dict, ) has_compressed_metadata = ( isinstance(artifacts, dict) and isinstance(artifacts.get("metadataZstd"), dict) and isinstance(artifacts.get("metadataChunkIndex"), dict) and isinstance(value, dict) and isinstance(value.get("metadataCompression"), dict) and value["metadataCompression"].get("compressed") == artifacts.get("metadataZstd") and value["metadataCompression"].get("chunkIndex") == artifacts.get("metadataChunkIndex") ) token_compression = ( value.get("tokenCompression") if isinstance(value, dict) else None ) metadata_compression = ( value.get("metadataCompression") if isinstance(value, dict) else None ) token_elements = value.get("tokenElements") if isinstance(value, dict) else None record_count = value.get("recordCount") if isinstance(value, dict) else None observed_record_count = ( value.get("observedRecordCount", record_count) if isinstance(value, dict) else None ) token_boundary_contract = ( value.get( "tokenBoundaryContract", FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT, ) if isinstance(value, dict) else None ) if ( not isinstance(value, dict) or value.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_SHARD_SCHEMA or value.get("passed") is not True or authority != _sha256_bytes(_json_bytes(value)) or value.get("payloadWorkId") != schedule_row.get("payloadWorkId") or value.get("sourceId") != schedule_row.get("sourceId") or value.get("sourceRecordSha256") != schedule_row.get("sourceRecordSha256") or value.get("payloadRelativePath") != schedule_row.get("payloadRelativePath") or value.get("payloadBytes") != schedule_row.get("payloadBytes") or value.get("observedPayloadSha256") != hash_row.get("observedSha256") or value.get("tokenizerAuthoritySha256") != tokenizer_authority_sha256 or value.get("contextWindowTokens") != context_window_tokens or value.get("answerTokensPerWindow") != answer_tokens_per_window or value.get("tokensStoredOncePerRecord") is not True or value.get("targetValuesRecorded") is not False or value.get("targetEnteredForward") is not False or not isinstance(record_count, int) or isinstance(record_count, bool) or record_count < 1 or not isinstance(observed_record_count, int) or isinstance(observed_record_count, bool) or observed_record_count < record_count or token_boundary_contract not in FULL_PAYLOAD_PACKED_SUPPORTED_TOKEN_BOUNDARY_CONTRACTS or ( token_boundary_contract == FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT and observed_record_count != record_count ) or not isinstance(artifacts, dict) or has_raw_tokens == has_compressed_tokens or ( not has_compact_records and has_raw_metadata == has_compressed_metadata ) or ( has_compact_records and ( not has_compressed_tokens or value.get("recordLocatorHashesPersisted") is not False or value.get("recordIdentityDerivedAtRead") is not True or value.get("recordWindowEndIndexPersisted") is not False or value.get("recordWindowGeometryEmbedded") is not True or value["recordStorage"].get("recordCount") != record_count or value["recordStorage"].get("rows") != value.get("rows") or value["recordStorage"].get("tokenElements") != token_elements or value["recordStorage"].get("targetEnteredForward") is not False or any( not _full_payload_packed_artifact_matches( artifacts.get(name) ) for name in ( "recordsZstd", "recordChunkIndex", ) ) ) ) or ( has_compressed_tokens and ( not isinstance(token_compression, dict) or not isinstance(token_elements, int) or isinstance(token_elements, bool) or token_elements < 1 or token_compression.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_ZSTD_SCHEMA or token_compression.get("boundedRandomAccess") is not True or value.get("rawTokenBytesRetained") is not False or value.get("compressedTokenRandomAccess") is not True or token_compression.get("uncompressedBytes") != token_elements * 4 ) ) or ( has_compressed_metadata and ( not isinstance(metadata_compression, dict) or metadata_compression.get("schema") != FULL_PAYLOAD_PACKED_METADATA_ZSTD_SCHEMA or metadata_compression.get("boundedRandomAccess") is not True or value.get("rawMetadataBytesRetained") is not False or value.get("compressedMetadataRandomAccess") is not True or metadata_compression.get("uncompressedBytes") != value.get("metadataUncompressedBytes") or metadata_compression.get("compressedBytes") != artifacts["metadataZstd"].get("bytes") ) ) or ( not has_compact_records and any( not _full_payload_packed_artifact_matches( artifacts.get(name) ) for name in required ) ) or ( has_raw_tokens and not _full_payload_packed_artifact_matches(artifacts.get("tokens")) ) or ( has_compressed_tokens and ( not _full_payload_packed_artifact_matches( artifacts.get("tokensZstd") ) or not _full_payload_packed_artifact_matches( artifacts.get("tokenChunkIndex") ) ) ) or ( has_raw_metadata and not _full_payload_packed_artifact_matches( artifacts.get("metadata") ) ) or ( has_compressed_metadata and ( not _full_payload_packed_artifact_matches( artifacts.get("metadataZstd") ) or not _full_payload_packed_artifact_matches( artifacts.get("metadataChunkIndex") ) ) ) ): raise RuntimeError("legacy full payload packed shard authority differs") value["shardAuthoritySha256"] = authority return value def _adopt_complete_legacy_full_payload_packed_shard( *, legacy_manifest_path: Path, target_root: Path, schedule_receipt_path: Path, schedule_path: Path, schedule_sha256: str, schedule_row: dict[str, Any], hash_row: dict[str, Any], tokenizer_authority: dict[str, Any], tokenizer_authority_sha256: str, context_window_tokens: int, answer_tokens_per_window: int, ) -> dict[str, Any]: """Adopt and compress one complete legacy shard without raw parsing.""" ordinal = int(schedule_row["scheduleOrdinal"]) work_id = str(schedule_row["payloadWorkId"]) stem = f"{ordinal:08d}_{work_id[:16]}" target = _full_payload_packed_paths(target_root, stem) if target["manifest"].is_file(): current_value = json.loads( target["manifest"].read_text(encoding="utf-8") ) if not isinstance(current_value, dict): raise RuntimeError( "current packed shard conflicts with legacy adoption" ) current = _upgrade_full_payload_packed_manifest_authority( current_value ) if ( current.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_SHARD_SCHEMA or current.get("passed") is not True or current.get("scheduleSha256") != schedule_sha256 or current.get("scheduleOrdinal") != ordinal or current.get("payloadWorkId") != work_id or current.get("sourceId") != schedule_row.get("sourceId") or current.get("observedPayloadSha256") != hash_row.get("observedSha256") or current.get("tokenizerAuthoritySha256") != tokenizer_authority_sha256 or current.get("contextWindowTokens") != context_window_tokens or current.get("answerTokensPerWindow") != answer_tokens_per_window ): raise RuntimeError("current packed shard conflicts with legacy adoption") token_sealed = _seal_full_payload_packed_token_artifact( target["manifest"], current, raw_tokens_path=target["tokens"], compressed_path=target["tokensZstd"], chunk_index_path=target["tokenChunkIndex"], ) return _seal_full_payload_packed_metadata_artifact( target["manifest"], token_sealed, raw_metadata_path=target["metadata"], compressed_path=target["metadataZstd"], chunk_index_path=target["metadataChunkIndex"], ) legacy = _validated_legacy_full_payload_packed_manifest( legacy_manifest_path, schedule_row=schedule_row, hash_row=hash_row, tokenizer_authority_sha256=tokenizer_authority_sha256, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, ) legacy_artifacts = legacy["artifacts"] record_count = int(legacy["recordCount"]) target_root.mkdir(parents=True, exist_ok=True) for name in ("locators", "recordLocatorHashes"): _materialize_full_payload_packed_artifact( Path(str(legacy_artifacts[name]["path"])), target[name], expected_sha256=str(legacy_artifacts[name]["sha256"]), ) if "metadata" in legacy_artifacts: _materialize_full_payload_packed_artifact( Path(str(legacy_artifacts["metadata"]["path"])), target["metadata"], expected_sha256=str(legacy_artifacts["metadata"]["sha256"]), ) else: for source_name, target_name in ( ("metadataZstd", "metadataZstd"), ("metadataChunkIndex", "metadataChunkIndex"), ): _materialize_full_payload_packed_artifact( Path(str(legacy_artifacts[source_name]["path"])), target[target_name], expected_sha256=str(legacy_artifacts[source_name]["sha256"]), ) geometry = _full_payload_packed_locator_geometry( target["locators"], record_count=record_count, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, window_ends_path=target["windowEnds"], ) if ( geometry["rows"] != legacy.get("rows") or geometry["tokenElements"] != legacy.get("tokenElements") or target["recordLocatorHashes"].stat().st_size != record_count * 32 or _packed_artifact(target["windowEnds"])["sha256"] != legacy_artifacts["windowEnds"]["sha256"] ): raise RuntimeError("legacy full payload packed shard geometry differs") if "tokens" in legacy_artifacts: compression = _compress_full_payload_token_chunks( Path(str(legacy_artifacts["tokens"]["path"])), target["tokensZstd"], target["tokenChunkIndex"], ) expected_token_sha256 = legacy_artifacts["tokens"]["sha256"] else: for source_name, target_name in ( ("tokensZstd", "tokensZstd"), ("tokenChunkIndex", "tokenChunkIndex"), ): _materialize_full_payload_packed_artifact( Path(str(legacy_artifacts[source_name]["path"])), target[target_name], expected_sha256=str(legacy_artifacts[source_name]["sha256"]), ) compression = dict(legacy["tokenCompression"]) compression["compressed"] = _packed_artifact(target["tokensZstd"]) compression["chunkIndex"] = _packed_artifact(target["tokenChunkIndex"]) expected_token_sha256 = compression.get("uncompressedSha256") if ( compression.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_ZSTD_SCHEMA or compression.get("boundedRandomAccess") is not True or compression["uncompressedBytes"] != geometry["tokenElements"] * 4 or compression["uncompressedSha256"] != expected_token_sha256 ): raise RuntimeError("legacy packed token compression authority differs") if "metadata" in legacy_artifacts: if target["metadata"].stat().st_size != geometry["metadataBytes"]: raise RuntimeError("legacy full payload packed metadata geometry differs") metadata_compression = _compress_full_payload_metadata_chunks( target["metadata"], target["metadataZstd"], target["metadataChunkIndex"], ) expected_metadata_sha256 = legacy_artifacts["metadata"]["sha256"] else: metadata_compression = dict(legacy["metadataCompression"]) metadata_compression["compressed"] = _packed_artifact( target["metadataZstd"] ) metadata_compression["chunkIndex"] = _packed_artifact( target["metadataChunkIndex"] ) expected_metadata_sha256 = metadata_compression.get( "uncompressedSha256" ) if ( metadata_compression.get("schema") != FULL_PAYLOAD_PACKED_METADATA_ZSTD_SCHEMA or metadata_compression.get("boundedRandomAccess") is not True or metadata_compression["uncompressedBytes"] != geometry["metadataBytes"] or metadata_compression["uncompressedSha256"] != expected_metadata_sha256 or metadata_compression.get("compressedBytes") != target["metadataZstd"].stat().st_size ): raise RuntimeError("legacy packed metadata compression authority differs") progress_path = legacy_manifest_path.with_name( legacy_manifest_path.name.replace(".manifest.json", ".progress.json") ) legacy_progress = ( json.loads(progress_path.read_text(encoding="utf-8")) if progress_path.is_file() else {} ) domain_authority, rights_authority = _packed_schedule_authorities(schedule_row) progress = { "schema": FULL_PAYLOAD_PACKED_TOKEN_PROGRESS_SCHEMA, "scheduleSha256": schedule_sha256, "scheduleOrdinal": ordinal, "payloadWorkId": work_id, "tokenizerAuthoritySha256": tokenizer_authority_sha256, "contextWindowTokens": context_window_tokens, "answerTokensPerWindow": answer_tokens_per_window, "tokenBoundaryContract": ( FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT ), **geometry, "recordCount": record_count, "observedRecordCount": record_count, "readerFamilies": list(legacy_progress.get("readerFamilies", [])), "semanticExportReceiptSha256s": list( legacy_progress.get("semanticExportReceiptSha256s", []) ), "complete": True, "targetValuesRecorded": False, "targetEnteredForward": False, "adoptedWithoutRetokenization": True, } _atomic_json(target["progress"], progress) manifest = { "schema": FULL_PAYLOAD_PACKED_TOKEN_SHARD_SCHEMA, "passed": True, "scheduleReceipt": _packed_artifact(schedule_receipt_path), "schedule": _packed_artifact(schedule_path), "scheduleSha256": schedule_sha256, "scheduleOrdinal": ordinal, "payloadWorkId": work_id, "sourceId": schedule_row["sourceId"], "sourceRecordSha256": schedule_row["sourceRecordSha256"], "payloadRelativePath": schedule_row["payloadRelativePath"], "payloadBytes": schedule_row["payloadBytes"], "observedPayloadSha256": hash_row["observedSha256"], "sourceIdentity": hash_row["sourceIdentity"], "hashLedgerRowSha256": _sha256_bytes(_json_bytes(hash_row)), "domainAuthority": domain_authority, "domainAuthoritySha256": _sha256_bytes(_json_bytes(domain_authority)), "rightsAuthority": rights_authority, "rightsAuthoritySha256": _sha256_bytes(_json_bytes(rights_authority)), "tokenizerAuthority": dict(tokenizer_authority), "tokenizerAuthoritySha256": tokenizer_authority_sha256, "contextWindowTokens": context_window_tokens, "answerTokensPerWindow": answer_tokens_per_window, "tokenBoundaryContract": ( FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT ), **geometry, "recordCount": record_count, "observedRecordCount": record_count, "trainingTokenElements": ( geometry["promptTokenElements"] + geometry["answerTokenElements"] ), "artifacts": { "tokensZstd": compression["compressed"], "tokenChunkIndex": compression["chunkIndex"], "locators": _packed_artifact(target["locators"]), "metadataZstd": metadata_compression["compressed"], "metadataChunkIndex": metadata_compression["chunkIndex"], "recordLocatorHashes": _packed_artifact( target["recordLocatorHashes"] ), "windowEnds": _packed_artifact(target["windowEnds"]), }, "tokenCompression": compression, "metadataCompression": metadata_compression, "metadataUncompressedBytes": metadata_compression["uncompressedBytes"], "metadataCompressedBytes": metadata_compression["compressedBytes"], "rawTokenBytesRetained": False, "rawMetadataBytesRetained": False, "compressedTokenRandomAccess": True, "compressedMetadataRandomAccess": True, "recordLocatorHashesPersisted": True, "windowLocatorHashesPersisted": False, "windowIdentityDerivedAtReceipt": True, "recordWindowEndIndexPersisted": True, "tokensStoredOncePerRecord": True, "metadataStoredOncePerRecord": True, "adoptedWithoutRetokenization": True, "legacyShardManifest": _packed_artifact(legacy_manifest_path), "targetValuesRecorded": False, "targetEnteredForward": False, "mmapReadable": True, } manifest["shardAuthoritySha256"] = _sha256_bytes(_json_bytes(manifest)) _atomic_json(target["manifest"], manifest) target["metadata"].unlink(missing_ok=True) return manifest def adopt_legacy_full_payload_packed_shards( legacy_shard_root: Path, schedule_receipt_path: Path, corpus_root: Path, hash_ledger_path: Path, output_receipt_path: Path, *, context_window_tokens: int, answer_tokens_per_window: int, source_ids: Sequence[str] | None = None, payload_work_ids: Sequence[str] | None = None, source_output_roots: Mapping[str, Path] | None = None, ) -> dict[str, Any]: """Adopt complete flat shards into source objects without tokenization. An explicit ``payload_work_ids`` selection is a canonical reconciliation boundary: every selected WorkID must already have one complete, sealed legacy manifest. This is intentionally stricter than source-wide legacy adoption, whose incomplete work may remain available for a later manual resume. Legacy manifests outside an explicit canonical selection are not adopted and cannot silently expand the resulting collection. """ legacy_root = legacy_shard_root.expanduser().resolve() if not legacy_root.is_dir(): raise FileNotFoundError(f"legacy packed shard root is absent: {legacy_root}") ( _schedule_receipt, schedule_path, schedule_sha256, schedule_rows, hash_rows, ) = _validated_full_payload_packed_sources( schedule_receipt_path, corpus_root, hash_ledger_path, source_ids=source_ids, payload_work_ids=payload_work_ids, ) canonical_work_id_reconciliation = payload_work_ids is not None selected_work_ids = { str(schedule_row["payloadWorkId"]) for schedule_row in schedule_rows } if len(selected_work_ids) != len(schedule_rows): raise RuntimeError("legacy packed selected work identity differs") legacy_manifest_paths = sorted(legacy_root.glob("*.manifest.json")) if not legacy_manifest_paths: raise RuntimeError("legacy packed shard root has no complete manifests") legacy_manifest_by_work_id: dict[str, Path] = {} legacy_manifest_outside_selection_count = 0 for legacy_manifest_path in legacy_manifest_paths: candidate = json.loads( legacy_manifest_path.read_text(encoding="utf-8") ) candidate_work_id = ( candidate.get("payloadWorkId") if isinstance(candidate, dict) else None ) if not isinstance(candidate_work_id, str) or len(candidate_work_id) != 64: continue if ( canonical_work_id_reconciliation and candidate_work_id not in selected_work_ids ): legacy_manifest_outside_selection_count += 1 continue if candidate_work_id in legacy_manifest_by_work_id: raise RuntimeError( "legacy packed shard root has duplicate completed work identity" ) legacy_manifest_by_work_id[candidate_work_id] = legacy_manifest_path if canonical_work_id_reconciliation and set(legacy_manifest_by_work_id) != ( selected_work_ids ): raise RuntimeError("canonical legacy packed work coverage differs") first_legacy_path = ( legacy_manifest_by_work_id[str(schedule_rows[0]["payloadWorkId"])] if canonical_work_id_reconciliation else legacy_manifest_paths[0] ) first_legacy = json.loads(first_legacy_path.read_text(encoding="utf-8")) tokenizer_authority = ( first_legacy.get("tokenizerAuthority") if isinstance(first_legacy, dict) else None ) if not isinstance(tokenizer_authority, dict): raise RuntimeError("legacy packed Fastokens authority is absent") ( tokenizer_record, tokenizer_authority_sha256, _vocabulary_size, ) = _validated_full_payload_fastokens_authority(tokenizer_authority) if canonical_work_id_reconciliation: for index, schedule_row in enumerate(schedule_rows): work_id = str(schedule_row["payloadWorkId"]) try: _validated_legacy_full_payload_packed_manifest( legacy_manifest_by_work_id[work_id], schedule_row=schedule_row, hash_row=hash_rows[index], tokenizer_authority_sha256=tokenizer_authority_sha256, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, ) except RuntimeError as error: raise RuntimeError( "canonical legacy packed shard authority differs: " f"payloadWorkId={work_id}" ) from error selected_source_ids = tuple( dict.fromkeys(str(row["sourceId"]) for row in schedule_rows) ) configured_roots = ( { source_id: path.expanduser().resolve() for source_id, path in source_output_roots.items() } if source_output_roots is not None else {} ) if configured_roots and not set(selected_source_ids).issubset(configured_roots): raise ValueError("legacy packed source output roots are incomplete") output_path = output_receipt_path.expanduser().resolve() source_authority_by_id: dict[str, tuple[dict[str, Any], str]] = {} target_root_by_source: dict[str, Path] = {} for source_id in selected_source_ids: positions = tuple( index for index, row in enumerate(schedule_rows) if row["sourceId"] == source_id ) source_authority = _full_payload_packed_source_authority( source_id=source_id, schedule_sha256=schedule_sha256, schedule_rows=[schedule_rows[index] for index in positions], hash_rows=[hash_rows[index] for index in positions], tokenizer_authority_sha256=tokenizer_authority_sha256, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, ) source_authority_by_id[source_id] = source_authority target_root_by_source[source_id] = ( configured_roots.get(source_id, output_path.parent) / "objects" / source_authority[1] / "work_shards" ) def adopt(index: int) -> dict[str, Any]: schedule_row = schedule_rows[index] ordinal = int(schedule_row["scheduleOrdinal"]) work_id = str(schedule_row["payloadWorkId"]) legacy_manifest = legacy_manifest_by_work_id.get(work_id) if legacy_manifest is None: if canonical_work_id_reconciliation: raise RuntimeError("canonical legacy packed work coverage differs") return { "scheduleOrdinal": ordinal, "payloadWorkId": work_id, "sourceId": schedule_row["sourceId"], "status": "partial_or_missing_requires_manual_resume", } try: _validated_legacy_full_payload_packed_manifest( legacy_manifest, schedule_row=schedule_row, hash_row=hash_rows[index], tokenizer_authority_sha256=tokenizer_authority_sha256, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, ) except RuntimeError as error: if canonical_work_id_reconciliation: raise RuntimeError( "canonical legacy packed shard authority differs: " f"payloadWorkId={work_id}" ) from error return { "scheduleOrdinal": ordinal, "payloadWorkId": work_id, "sourceId": schedule_row["sourceId"], "status": "completed_object_identity_mismatch_unadopted", } manifest = _adopt_complete_legacy_full_payload_packed_shard( legacy_manifest_path=legacy_manifest, target_root=target_root_by_source[str(schedule_row["sourceId"])], schedule_receipt_path=schedule_receipt_path.expanduser().resolve(), schedule_path=schedule_path, schedule_sha256=schedule_sha256, schedule_row=schedule_row, hash_row=hash_rows[index], tokenizer_authority=tokenizer_record, tokenizer_authority_sha256=tokenizer_authority_sha256, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, ) target_stem = f"{ordinal:08d}_{work_id[:16]}" return { "scheduleOrdinal": ordinal, "payloadWorkId": work_id, "sourceId": schedule_row["sourceId"], "status": "adopted_without_retokenization", "manifest": _packed_artifact( target_root_by_source[str(schedule_row["sourceId"])] / f"{target_stem}.manifest.json" ), "shardAuthoritySha256": manifest["shardAuthoritySha256"], "domainAuthoritySha256": manifest[ "domainAuthoritySha256" ], "rightsAuthoritySha256": manifest[ "rightsAuthoritySha256" ], "rows": manifest["rows"], "tokenElements": manifest["tokenElements"], } worker_count = min( len(schedule_rows), 4, ) with ThreadPoolExecutor( max_workers=worker_count, thread_name_prefix="nnf-full-payload-adopt", ) as executor: futures = [executor.submit(adopt, index) for index in range(len(schedule_rows))] adopted = [future.result() for future in futures] incomplete = [ row for row in adopted if row["status"] != "adopted_without_retokenization" ] if canonical_work_id_reconciliation and ( incomplete or { str(row["payloadWorkId"]) for row in adopted } != selected_work_ids ): raise RuntimeError("canonical legacy packed work coverage differs") receipt = { "schema": "nnf.resynthesis.full_payload_packed_legacy_adoption.v1", "passed": True, "legacyShardRoot": str(legacy_root), "scheduleReceipt": _packed_artifact( schedule_receipt_path.expanduser().resolve() ), "scheduleSha256": schedule_sha256, "hashLedger": _packed_artifact(hash_ledger_path.expanduser().resolve()), "tokenizerAuthority": tokenizer_record, "tokenizerAuthoritySha256": tokenizer_authority_sha256, "contextWindowTokens": context_window_tokens, "answerTokensPerWindow": answer_tokens_per_window, "selectedSourceIds": list(selected_source_ids), "sourceAuthorities": { source_id: authority[1] for source_id, authority in source_authority_by_id.items() }, "payloadWork": adopted, "payloadWorkCount": len(adopted), "adoptedCompleteCount": len(adopted) - len(incomplete), "manualResumeRequiredCount": len(incomplete), "allSelectedPayloadWorkTrainingReady": not incomplete, "rawPayloadReadersInvoked": False, "tokenizerInvoked": False, "retokenizationPerformed": False, "maximumCompressionWorkers": worker_count, "targetEnteredForward": False, } if canonical_work_id_reconciliation: receipt.update( { "selectedPayloadWorkIdsSha256": _sha256_bytes( _json_bytes(sorted(selected_work_ids)) ), "canonicalWorkIdReconciliation": True, "canonicalWorkIdExactCoverage": not incomplete, "legacyManifestOutsideCanonicalSelectionCount": ( legacy_manifest_outside_selection_count ), } ) receipt["adoptionAuthoritySha256"] = _sha256_bytes(_json_bytes(receipt)) if output_path.is_file(): existing = json.loads(output_path.read_text(encoding="utf-8")) if existing != receipt: raise RuntimeError("legacy packed adoption receipt already differs") return dict(existing) _atomic_json(output_path, receipt) return receipt def reconcile_canonical_full_payload_packed_legacy_shards( legacy_shard_root: Path, schedule_receipt_path: Path, corpus_root: Path, hash_ledger_path: Path, output_receipt_path: Path, *, canonical_payload_work_ids: Sequence[str], context_window_tokens: int, answer_tokens_per_window: int, source_output_roots: Mapping[str, Path] | None = None, ) -> dict[str, Any]: """Rebind an exact canonical WorkID set to sealed legacy packed shards. This is the narrow bridge used by federated membership: it accepts only the WorkIDs named by the canonical authority, verifies the old sealed shard against the current source/hash/tokenizer geometry, and publishes no work beyond that set. It never opens raw source payloads or invokes a tokenizer. """ return adopt_legacy_full_payload_packed_shards( legacy_shard_root, schedule_receipt_path, corpus_root, hash_ledger_path, output_receipt_path, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, payload_work_ids=canonical_payload_work_ids, source_output_roots=source_output_roots, ) def _full_payload_packed_collection_semantic_projection( value: Mapping[str, Any], ) -> dict[str, Any]: """Remove only artifact identities changed by raw-to-zstd migration.""" projected = dict(value) projected.pop("collectionAuthoritySha256", None) projected.pop("compressedTokenRandomAccess", None) projected.setdefault( "tokenBoundaryContract", FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT, ) admission = projected.get("storageAdmission") if isinstance(admission, dict): # Admission receipt identity and self-hash cover a free-space # snapshot taken at one instant; they are not collection semantics. projected["storageAdmission"] = { key: field for key, field in admission.items() if key not in {"receipt", "storageAdmissionSha256"} } raw_shards = projected.get("shards") if isinstance(raw_shards, list): projected["shards"] = [ { key: field for key, field in shard.items() if key not in {"manifest", "shardAuthoritySha256"} } if isinstance(shard, dict) else shard for shard in raw_shards ] return projected def _full_payload_packed_shard_stem( schedule_row: Mapping[str, Any], ) -> str: """Return the layout-independent exact identity prefix for one work item.""" ordinal = schedule_row.get("scheduleOrdinal") work_id = schedule_row.get("payloadWorkId") if ( not isinstance(ordinal, int) or isinstance(ordinal, bool) or ordinal < 0 or not isinstance(work_id, str) or len(work_id) != 64 ): raise ValueError("full payload packed shard identity is malformed") return f"{ordinal:08d}_{work_id[:16]}" def _full_payload_packed_layout_has_durable_evidence( shard_root: Path, schedule_row: Mapping[str, Any], *, schedule_sha256: str, ) -> bool: """Detect one exact shard without treating an empty lock as work.""" stem = _full_payload_packed_shard_stem(schedule_row) manifest_path = shard_root / f"{stem}.manifest.json" progress_path = shard_root / f"{stem}.progress.json" for path in (manifest_path, progress_path): if not path.is_file(): continue value = json.loads(path.read_text(encoding="utf-8")) if ( not isinstance(value, dict) or value.get("scheduleSha256") != schedule_sha256 or value.get("scheduleOrdinal") != schedule_row["scheduleOrdinal"] or value.get("payloadWorkId") != schedule_row["payloadWorkId"] ): raise RuntimeError( "full payload packed layout identity differs: " f"{path}" ) return True return any( (shard_root / f"{stem}{suffix}").is_file() for suffix in ( ".tokens.i32le", ".tokens.i32le.zst", ".tokens.i32le.zst.index.u64le", ".locators.u64le", ".metadata.jsonl", ".record_locator.sha256", ".window_ends.u64le", ) ) def _full_payload_packed_layout_token_boundary_evidence( shard_root: Path, schedule_row: Mapping[str, Any], *, schedule_sha256: str, ) -> str | None: """Return one existing shard's immutable token-boundary authority.""" stem = _full_payload_packed_shard_stem(schedule_row) observed: set[str] = set() for path in ( shard_root / f"{stem}.manifest.json", shard_root / f"{stem}.progress.json", ): if not path.is_file(): continue value = json.loads(path.read_text(encoding="utf-8")) if ( not isinstance(value, dict) or value.get("scheduleSha256") != schedule_sha256 or value.get("scheduleOrdinal") != schedule_row["scheduleOrdinal"] or value.get("payloadWorkId") != schedule_row["payloadWorkId"] ): raise RuntimeError( "full payload packed layout identity differs: " f"{path}" ) boundary = value.get( "tokenBoundaryContract", FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT, ) if ( boundary not in FULL_PAYLOAD_PACKED_SUPPORTED_TOKEN_BOUNDARY_CONTRACTS ): raise RuntimeError( "full payload packed layout token boundary differs: " f"{path}" ) observed.add(str(boundary)) if len(observed) > 1: raise RuntimeError( "full payload packed shard token-boundary evidence conflicts" ) if observed: return next(iter(observed)) if _full_payload_packed_layout_has_durable_evidence( shard_root, schedule_row, schedule_sha256=schedule_sha256, ): # Pre-contract raw sidecars are v1 by construction. return FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT return None def _full_payload_packed_manifest_owns_artifacts( manifest_path: Path, manifest: Mapping[str, Any], ) -> bool: """Require every recorded shard artifact to live beside its manifest.""" artifacts = manifest.get("artifacts") if not isinstance(artifacts, dict): return False manifest_root = manifest_path.parent.resolve() return all( isinstance(artifact, dict) and isinstance(artifact.get("path"), str) and Path(str(artifact["path"])).resolve().is_relative_to(manifest_root) for artifact in artifacts.values() ) def _full_payload_packed_required_artifact_names( manifest: Mapping[str, Any], ) -> set[str]: """Return the exact versioned artifact family for one packed shard.""" if ( manifest.get("packedStorageLayout") == FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT ): return { "tokensZstd", "tokenChunkIndex", "recordsZstd", "recordChunkIndex", } return { "tokensZstd", "tokenChunkIndex", "locators", "metadataZstd", "metadataChunkIndex", "recordLocatorHashes", "windowEnds", } def _full_payload_packed_manifest_content_identity_sha256( manifest: Mapping[str, Any], ) -> str: """Hash authoritative shard content while ignoring replica locations.""" def normalize(value: Any) -> Any: if isinstance(value, Mapping): artifact_record = ( isinstance(value.get("path"), str) and isinstance(value.get("bytes"), int) and not isinstance(value.get("bytes"), bool) and isinstance(value.get("sha256"), str) ) return { str(key): normalize(item) for key, item in value.items() if key != "shardAuthoritySha256" and not (artifact_record and key == "path") # Filesystem identity proves that the artifact at a path is # still the sealed byte sequence. It is deliberately not a # semantic part of those bytes: an exact replica can have a # different device, inode, and timestamps after a copy or # remount. The artifact byte count and SHA-256 remain in the # normalized identity and continue to fail closed on content # drift. and not (artifact_record and key == "fileIdentity") } if isinstance(value, list): return [normalize(item) for item in value] return value return _sha256_bytes(_json_bytes(normalize(manifest))) def _full_payload_packed_cross_layout_duplicate_matches( legacy_manifest_path: Path, scoped_manifest_path: Path, ) -> bool: """Recognize one exact raw-to-compressed ownership handoff. A mutable-source transition can finish the same admitted work in the old flat layout and the source-owned layout concurrently. Both manifests must remain self-sealed and artifact-local, and the compressed shard must bind the exact uncompressed token digest from the raw shard. """ legacy_value = json.loads(legacy_manifest_path.read_text(encoding="utf-8")) scoped_value = json.loads(scoped_manifest_path.read_text(encoding="utf-8")) if ( not isinstance(legacy_value, dict) or not isinstance(scoped_value, dict) or not _full_payload_packed_manifest_owns_artifacts( legacy_manifest_path, legacy_value, ) or not _full_payload_packed_manifest_owns_artifacts( scoped_manifest_path, scoped_value, ) ): return False try: legacy = _upgrade_full_payload_packed_manifest_authority(legacy_value) scoped = _upgrade_full_payload_packed_manifest_authority(scoped_value) except RuntimeError: return False legacy_artifacts = legacy.get("artifacts") scoped_artifacts = scoped.get("artifacts") compression = scoped.get("tokenCompression") metadata_compression = scoped.get("metadataCompression") if ( not isinstance(legacy_artifacts, dict) or not isinstance(scoped_artifacts, dict) or not isinstance(compression, dict) or not isinstance(metadata_compression, dict) or not isinstance(legacy_artifacts.get("tokens"), dict) or not isinstance(legacy_artifacts.get("metadata"), dict) or "tokens" in scoped_artifacts or "metadata" in scoped_artifacts or scoped.get("rawTokenBytesRetained") is not False or scoped.get("rawMetadataBytesRetained") is not False or scoped.get("compressedTokenRandomAccess") is not True or scoped.get("compressedMetadataRandomAccess") is not True or compression.get("uncompressedSha256") != legacy_artifacts["tokens"].get("sha256") or compression.get("uncompressedBytes") != legacy_artifacts["tokens"].get("bytes") or metadata_compression.get("uncompressedSha256") != legacy_artifacts["metadata"].get("sha256") or metadata_compression.get("uncompressedBytes") != legacy_artifacts["metadata"].get("bytes") ): return False identity_fields = ( "scheduleSha256", "scheduleOrdinal", "payloadWorkId", "sourceId", "sourceRecordSha256", "payloadRelativePath", "payloadBytes", "observedPayloadSha256", "hashLedgerRowSha256", "domainAuthoritySha256", "rightsAuthoritySha256", "tokenizerAuthoritySha256", "contextWindowTokens", "answerTokensPerWindow", "recordCount", "observedRecordCount", "rows", "tokenElements", "promptTokenElements", "answerTokenElements", "trainingTokenElements", "targetValuesRecorded", "targetEnteredForward", ) if any(legacy.get(field) != scoped.get(field) for field in identity_fields): return False for name in ( "locators", "recordLocatorHashes", "windowEnds", ): legacy_artifact = legacy_artifacts.get(name) scoped_artifact = scoped_artifacts.get(name) if ( not isinstance(legacy_artifact, dict) or not isinstance(scoped_artifact, dict) or legacy_artifact.get("sha256") != scoped_artifact.get("sha256") or legacy_artifact.get("bytes") != scoped_artifact.get("bytes") ): return False return True def _full_payload_packed_durable_token_accounting( locator_path: Path, *, record_count: int, token_elements: int, rows: int, context_window_tokens: int, answer_tokens_per_window: int, ) -> tuple[int, int]: """Recover prompt/answer totals from sealed record locators only.""" if ( record_count < 0 or token_elements < 0 or rows < 0 or locator_path.stat().st_size != record_count * FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.size ): raise RuntimeError("full payload packed durable locator geometry differs") prompt_elements = 0 answer_elements = 0 observed_rows = 0 expected_token_offset = 0 with locator_path.open("rb") as locator_file: locator_mmap = mmap.mmap( locator_file.fileno(), 0, access=mmap.ACCESS_READ, ) try: for record_index in range(record_count): ( token_offset, prefix_length, content_length, _metadata_offset, _metadata_length, ) = FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.unpack_from( locator_mmap, record_index * FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.size, ) if ( token_offset != expected_token_offset or prefix_length >= context_window_tokens - 1 or content_length < 1 ): raise RuntimeError( "full payload packed durable token locator differs" ) answer_width = min( answer_tokens_per_window, context_window_tokens - prefix_length, ) window_count = ( content_length + answer_width - 1 ) // answer_width full_windows = max(0, window_count - 1) full_context_capacity = ( context_window_tokens - prefix_length - answer_width ) uncapped_full_windows = min( full_windows, full_context_capacity // answer_width + 1, ) full_context_elements = ( answer_width * uncapped_full_windows * (uncapped_full_windows - 1) // 2 + ( full_windows - uncapped_full_windows ) * full_context_capacity ) final_start = full_windows * answer_width final_answer_length = content_length - final_start final_context_capacity = ( context_window_tokens - prefix_length - final_answer_length ) prompt_elements += ( window_count * prefix_length + full_context_elements + min(final_start, final_context_capacity) ) answer_elements += content_length observed_rows += window_count expected_token_offset += prefix_length + content_length finally: locator_mmap.close() if expected_token_offset != token_elements or observed_rows != rows: raise RuntimeError("full payload packed durable token accounting differs") return prompt_elements, answer_elements def _full_payload_compressed_metadata_slice( compressed_mmap: mmap.mmap, chunk_index_mmap: mmap.mmap, *, chunk_count: int, byte_start: int, byte_end: int, cache: dict[int, bytes], ) -> bytes: """Read one legacy metadata record through bounded zstd-frame reuse.""" if byte_start < 0 or byte_end <= byte_start: raise RuntimeError("full payload packed metadata slice is invalid") pieces: list[bytes] = [] cursor = byte_start zstandard = importlib.import_module("zstandard") while cursor < byte_end: lower = 0 upper = chunk_count selected = -1 while lower < upper: middle = (lower + upper) // 2 ( _compressed_offset, _compressed_bytes, uncompressed_offset, uncompressed_bytes, ) = FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.unpack_from( chunk_index_mmap, middle * FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.size, ) if cursor < uncompressed_offset: upper = middle elif cursor >= uncompressed_offset + uncompressed_bytes: lower = middle + 1 else: selected = middle break if selected < 0: raise RuntimeError( "full payload packed metadata byte offset is unavailable" ) ( compressed_offset, compressed_bytes, uncompressed_offset, uncompressed_bytes, ) = FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.unpack_from( chunk_index_mmap, selected * FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.size, ) decoded = cache.get(selected) if decoded is None: compressed = compressed_mmap[ compressed_offset : compressed_offset + compressed_bytes ] decoded = zstandard.ZstdDecompressor().decompress( compressed, max_output_size=uncompressed_bytes, ) if len(decoded) != uncompressed_bytes: raise RuntimeError( "full payload packed metadata chunk decode differs" ) if len(cache) >= 2: cache.pop(next(iter(cache))) cache[selected] = decoded take_end = min(byte_end, uncompressed_offset + uncompressed_bytes) pieces.append( decoded[ cursor - uncompressed_offset : take_end - uncompressed_offset ] ) cursor = take_end return b"".join(pieces) def _compact_full_payload_packed_record_storage( manifest_path: Path, manifest: Mapping[str, Any], ) -> dict[str, Any]: """Seal one newly completed v1 shard into compact v2 record storage. Tokens are never decoded or rewritten. The conversion reads only the already-sealed per-record locator/provenance authority and derives the same record identity that the v1 writer persisted. """ if ( manifest.get("packedStorageLayout") == FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT ): return dict(manifest) artifacts = manifest.get("artifacts") metadata_compression = manifest.get("metadataCompression") required_legacy = { "tokensZstd", "tokenChunkIndex", "locators", "metadataZstd", "metadataChunkIndex", "recordLocatorHashes", "windowEnds", } record_count = manifest.get("recordCount") token_elements = manifest.get("tokenElements") rows = manifest.get("rows") if ( manifest.get("packedStorageLayout") != FULL_PAYLOAD_PACKED_STREAM_STORAGE_LAYOUT or not isinstance(artifacts, dict) or set(artifacts) != required_legacy or not isinstance(metadata_compression, dict) or not isinstance(record_count, int) or isinstance(record_count, bool) or record_count < 1 or not isinstance(token_elements, int) or isinstance(token_elements, bool) or token_elements < 1 or not isinstance(rows, int) or isinstance(rows, bool) or rows < 1 ): raise RuntimeError("full payload compact record source differs") for artifact in artifacts.values(): if not _full_payload_packed_artifact_matches(artifact): raise RuntimeError("full payload compact record artifact differs") locators_path = Path(str(artifacts["locators"]["path"])).resolve() metadata_path = Path(str(artifacts["metadataZstd"]["path"])).resolve() metadata_index_path = Path( str(artifacts["metadataChunkIndex"]["path"]) ).resolve() record_hashes_path = Path( str(artifacts["recordLocatorHashes"]["path"]) ).resolve() window_ends_path = Path(str(artifacts["windowEnds"]["path"])).resolve() metadata_chunk_count = metadata_compression.get("chunkCount") if ( locators_path.stat().st_size != record_count * FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.size or record_hashes_path.stat().st_size != record_count * 32 or window_ends_path.stat().st_size != record_count * FULL_PAYLOAD_PACKED_TOKEN_WINDOW_INDEX_RECORD_BYTES or not isinstance(metadata_chunk_count, int) or isinstance(metadata_chunk_count, bool) or metadata_chunk_count < 1 or metadata_index_path.stat().st_size != metadata_chunk_count * FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.size ): raise RuntimeError("full payload compact record geometry differs") stem = manifest_path.name.removesuffix(".manifest.json") records_path = manifest_path.with_name(f"{stem}.records.v2.zst") record_index_path = manifest_path.with_name( f"{stem}.records.v2.zst.index.u64le" ) records_temporary = records_path.with_name( f".{records_path.name}.{os.getpid()}.tmp" ) index_temporary = record_index_path.with_name( f".{record_index_path.name}.{os.getpid()}.tmp" ) records_temporary.unlink(missing_ok=True) index_temporary.unlink(missing_ok=True) zstandard = importlib.import_module("zstandard") compressor = zstandard.ZstdCompressor( level=FULL_PAYLOAD_PACKED_COMPACT_RECORD_ZSTD_LEVEL, write_checksum=True, write_content_size=True, ) compressed_offset = 0 raw_record_bytes = 0 chunk_count = 0 expected_token_offset = 0 expected_metadata_offset = 0 expected_window_end = 0 raw_digest = hashlib.sha256() with ( locators_path.open("rb") as locator_file, metadata_path.open("rb") as metadata_file, metadata_index_path.open("rb") as metadata_index_file, record_hashes_path.open("rb") as record_hash_file, window_ends_path.open("rb") as window_ends_file, records_temporary.open("wb") as records_handle, index_temporary.open("wb") as index_handle, ): locator_mmap = mmap.mmap( locator_file.fileno(), 0, access=mmap.ACCESS_READ, ) metadata_mmap = mmap.mmap( metadata_file.fileno(), 0, access=mmap.ACCESS_READ, ) metadata_index_mmap = mmap.mmap( metadata_index_file.fileno(), 0, access=mmap.ACCESS_READ, ) record_hash_mmap = mmap.mmap( record_hash_file.fileno(), 0, access=mmap.ACCESS_READ, ) window_ends_mmap = mmap.mmap( window_ends_file.fileno(), 0, access=mmap.ACCESS_READ, ) metadata_cache: dict[int, bytes] = {} try: for frame_record_start in range( 0, record_count, FULL_PAYLOAD_PACKED_COMPACT_RECORD_CHUNK_RECORDS, ): frame_record_end = min( record_count, frame_record_start + FULL_PAYLOAD_PACKED_COMPACT_RECORD_CHUNK_RECORDS, ) frame_token_start = expected_token_offset frame_window_start = expected_window_end frame_records: list[ tuple[int, int, int, str, str, str | None] ] = [] for record_index in range( frame_record_start, frame_record_end, ): ( token_offset, prefix_length, content_length, metadata_offset, metadata_length, ) = FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.unpack_from( locator_mmap, record_index * FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.size, ) metadata_payload = ( _full_payload_compressed_metadata_slice( metadata_mmap, metadata_index_mmap, chunk_count=metadata_chunk_count, byte_start=metadata_offset, byte_end=metadata_offset + metadata_length, cache=metadata_cache, ) ) metadata_value = json.loads(metadata_payload) if not isinstance(metadata_value, dict): raise RuntimeError( "full payload compact record metadata differs" ) record_ordinal = metadata_value.get( "payload_record_ordinal" ) locator = metadata_value.get("payload_record_locator") reader_family = metadata_value.get( "payload_reader_family" ) semantic_sha256 = metadata_value.get( "semantic_export_receipt_sha256" ) record_identity = { "payloadWorkId": manifest["payloadWorkId"], "recordLocator": locator, "recordOrdinal": record_ordinal, "sourceSha256": manifest["observedPayloadSha256"], } expected_record_hash = hashlib.sha256( _json_bytes(record_identity) ).digest() recorded_record_hash = record_hash_mmap[ record_index * 32 : (record_index + 1) * 32 ] window_count, _prompt_tokens, _answer_tokens = ( _full_payload_packed_record_training_geometry( prefix_tokens=prefix_length, content_tokens=content_length, context_window_tokens=int( manifest["contextWindowTokens"] ), answer_tokens_per_window=int( manifest["answerTokensPerWindow"] ), ) ) recorded_window_end = struct.unpack_from( " dict[str, Any]: """Manually migrate one sealed WorkID under its exact writer lock. Callers must re-seal any collection/cohort receipt that embeds the prior manifest artifact before resuming training. The old v1 reader remains supported, so migration can proceed WorkID by WorkID without retokenizing or changing token/window identities. """ resolved_manifest = manifest_path.expanduser().resolve() if not resolved_manifest.is_file(): raise FileNotFoundError( f"full payload packed manifest is absent: {resolved_manifest}" ) stem = resolved_manifest.name.removesuffix(".manifest.json") lock_path = resolved_manifest.with_name(f"{stem}.lock") lock_handle = lock_path.open("a+b") fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX) try: value = json.loads(resolved_manifest.read_text(encoding="utf-8")) if not isinstance(value, dict): raise RuntimeError("full payload packed manifest differs") validated = _upgrade_full_payload_packed_manifest_authority(value) if ( validated.get("packedStorageLayout") == FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT ): return validated return _compact_full_payload_packed_record_storage( resolved_manifest, validated, ) finally: fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN) lock_handle.close() def _upgrade_full_payload_packed_manifest_authority( manifest: Mapping[str, Any], ) -> dict[str, Any]: """Normalize a complete legacy raw shard without re-tokenizing it.""" upgraded = dict(manifest) recorded_authority = upgraded.pop("shardAuthoritySha256", None) if ( not isinstance(recorded_authority, str) or len(recorded_authority) != 64 or recorded_authority != _sha256_bytes(_json_bytes(upgraded)) ): raise RuntimeError("full payload packed shard authority differs") artifacts = upgraded.get("artifacts") if not isinstance(artifacts, dict) or any( not _full_payload_packed_artifact_matches(artifact) for artifact in artifacts.values() ): raise RuntimeError("full payload packed shard artifact differs") legacy_window_hash = artifacts.get("windowLocatorHashes") if legacy_window_hash is not None: if upgraded.get("windowLocatorHashesPersisted") is not True: raise RuntimeError( "full payload legacy window-hash authority differs" ) artifacts = dict(artifacts) artifacts.pop("windowLocatorHashes") upgraded["artifacts"] = artifacts record_count = upgraded.get("recordCount") observed_record_count = upgraded.get( "observedRecordCount", record_count, ) token_elements = upgraded.get("tokenElements") rows = upgraded.get("rows") context_window_tokens = upgraded.get("contextWindowTokens") answer_tokens_per_window = upgraded.get("answerTokensPerWindow") token_boundary_contract = upgraded.get( "tokenBoundaryContract", FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT, ) compact_records = ( upgraded.get("packedStorageLayout") == FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT ) if ( not isinstance(record_count, int) or isinstance(record_count, bool) or record_count < 1 or not isinstance(observed_record_count, int) or isinstance(observed_record_count, bool) or observed_record_count < record_count or not isinstance(token_elements, int) or isinstance(token_elements, bool) or token_elements < 1 or not isinstance(rows, int) or isinstance(rows, bool) or rows < 1 or not isinstance(context_window_tokens, int) or isinstance(context_window_tokens, bool) or not isinstance(answer_tokens_per_window, int) or isinstance(answer_tokens_per_window, bool) or token_boundary_contract not in FULL_PAYLOAD_PACKED_SUPPORTED_TOKEN_BOUNDARY_CONTRACTS or ( not compact_records and not isinstance(artifacts.get("locators"), dict) ) ): raise RuntimeError("full payload packed shard geometry differs") prompt_elements = upgraded.get("promptTokenElements") answer_elements = upgraded.get("answerTokenElements") if ( not isinstance(prompt_elements, int) or isinstance(prompt_elements, bool) or not isinstance(answer_elements, int) or isinstance(answer_elements, bool) ): prompt_elements, answer_elements = ( _full_payload_packed_durable_token_accounting( Path(str(artifacts["locators"]["path"])).resolve(), record_count=record_count, token_elements=token_elements, rows=rows, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, ) ) upgraded.update( { "tokenBoundaryContract": token_boundary_contract, "observedRecordCount": observed_record_count, "promptTokenElements": prompt_elements, "answerTokenElements": answer_elements, "trainingTokenElements": prompt_elements + answer_elements, "recordLocatorHashesPersisted": not compact_records, "windowLocatorHashesPersisted": False, "windowIdentityDerivedAtReceipt": True, "recordWindowEndIndexPersisted": not compact_records, "tokensStoredOncePerRecord": True, "targetValuesRecorded": False, "targetEnteredForward": False, "mmapReadable": True, } ) if compact_records: upgraded.update( { "recordIdentityDerivedAtRead": True, "recordWindowGeometryEmbedded": True, } ) upgraded["shardAuthoritySha256"] = _sha256_bytes( _json_bytes(upgraded) ) return upgraded def _build_full_payload_packed_shard( *, schedule_receipt_path: Path, schedule_path: Path, schedule_sha256: str, schedule_row: dict[str, Any], hash_row: dict[str, Any], corpus_root: Path, shard_root: Path, tokenizer: Any, tokenizer_authority: Mapping[str, Any], tokenizer_vocabulary_size: int, context_window_tokens: int, answer_tokens_per_window: int, scratch_root: Path | None, token_boundary_contract: str = ( FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT ), proven_device_remaps: set[tuple[int, int]] | None = None, ) -> dict[str, Any]: if ( token_boundary_contract not in FULL_PAYLOAD_PACKED_SUPPORTED_TOKEN_BOUNDARY_CONTRACTS ): raise ValueError("full payload packed token boundary is unsupported") preferred_token_chunk_bytes = ( min( FULL_PAYLOAD_PACKED_EXTENT_TOKEN_ZSTD_CHUNK_BYTES, FULL_PAYLOAD_PACKED_TOKEN_ZSTD_CHUNK_BYTES, ) if token_boundary_contract == FULL_PAYLOAD_PACKED_EXTENT_TOKEN_BOUNDARY_CONTRACT else FULL_PAYLOAD_PACKED_TOKEN_ZSTD_CHUNK_BYTES ) ordinal = int(schedule_row["scheduleOrdinal"]) work_id = str(schedule_row["payloadWorkId"]) stem = f"{ordinal:08d}_{work_id[:16]}" paths = { "tokens": shard_root / f"{stem}.tokens.i32le", "tokensZstd": shard_root / f"{stem}.tokens.i32le.zst", "tokenChunkIndex": shard_root / f"{stem}.tokens.i32le.zst.index.u64le", "locators": shard_root / f"{stem}.locators.u64le", "metadata": shard_root / f"{stem}.metadata.jsonl", "metadataZstd": shard_root / f"{stem}.metadata.jsonl.zst", "metadataChunkIndex": ( shard_root / f"{stem}.metadata.jsonl.zst.index.u64le" ), "recordLocatorHashes": shard_root / f"{stem}.record_locator.sha256", "windowEnds": shard_root / f"{stem}.window_ends.u64le", "progress": shard_root / f"{stem}.progress.json", "manifest": shard_root / f"{stem}.manifest.json", "readerDegradationReceipt": ( shard_root / f"{stem}.reader_degradation.receipt.json" ), "lock": shard_root / f"{stem}.lock", } shard_root.mkdir(parents=True, exist_ok=True) domain_authority, rights_authority = _packed_schedule_authorities(schedule_row) tokenizer_sha256 = _sha256_bytes(_json_bytes(dict(tokenizer_authority))) source_path = corpus_root / str(schedule_row["payloadRelativePath"]) if not _full_payload_source_identity_matches( hash_row.get("sourceIdentity"), source_path, observed_sha256=hash_row.get("observedSha256"), proven_device_remaps=proven_device_remaps, ): raise RuntimeError("full payload packed source identity differs before build") lock_handle = paths["lock"].open("a+b") fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX) try: if paths["manifest"].is_file(): existing_value = json.loads( paths["manifest"].read_text(encoding="utf-8") ) if not isinstance(existing_value, dict): raise RuntimeError("full payload packed shard manifest differs") existing = _upgrade_full_payload_packed_manifest_authority( existing_value ) if ( existing.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_SHARD_SCHEMA or existing.get("passed") is not True or existing.get("scheduleSha256") != schedule_sha256 or existing.get("scheduleOrdinal") != ordinal or existing.get("payloadWorkId") != work_id or existing.get("tokenizerAuthoritySha256") != tokenizer_sha256 or existing.get("contextWindowTokens") != context_window_tokens or existing.get("answerTokensPerWindow") != answer_tokens_per_window or existing.get("tokenBoundaryContract") != token_boundary_contract ): raise RuntimeError("full payload packed shard manifest differs") if existing != existing_value: _atomic_json(paths["manifest"], existing) if ( existing.get("packedStorageLayout") == FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT ): if ( set(existing.get("artifacts", {})) != _full_payload_packed_required_artifact_names(existing) or not isinstance(existing.get("recordStorage"), dict) or existing["recordStorage"].get("schema") != FULL_PAYLOAD_PACKED_COMPACT_RECORD_SCHEMA or existing.get("recordLocatorHashesPersisted") is not False or existing.get("recordIdentityDerivedAtRead") is not True or existing.get("targetEnteredForward") is not False ): raise RuntimeError( "full payload packed compact shard differs" ) return existing token_sealed = _seal_full_payload_packed_token_artifact( paths["manifest"], existing, raw_tokens_path=paths["tokens"], compressed_path=paths["tokensZstd"], chunk_index_path=paths["tokenChunkIndex"], ) return _seal_full_payload_packed_metadata_artifact( paths["manifest"], token_sealed, raw_metadata_path=paths["metadata"], compressed_path=paths["metadataZstd"], chunk_index_path=paths["metadataChunkIndex"], ) progress: dict[str, Any] = { "schema": FULL_PAYLOAD_PACKED_TOKEN_PROGRESS_SCHEMA, "scheduleSha256": schedule_sha256, "scheduleOrdinal": ordinal, "payloadWorkId": work_id, "tokenizerAuthoritySha256": tokenizer_sha256, "contextWindowTokens": context_window_tokens, "answerTokensPerWindow": answer_tokens_per_window, "tokenBoundaryContract": token_boundary_contract, "rows": 0, "tokenElements": 0, "promptTokenElements": 0, "answerTokenElements": 0, "metadataBytes": 0, "recordCount": 0, "observedRecordCount": 0, "readerFamilies": [], "semanticExportReceiptSha256s": [], "readerDegradations": [], "readerDegradationCount": 0, "readerDegradationRetryCount": 0, "semanticDecodePassed": True, "trainingKnowledgeReady": True, "packedStorageLayout": ( FULL_PAYLOAD_PACKED_STREAM_STORAGE_LAYOUT ), "tokenCompressedBytes": 0, "tokenChunkCount": 0, "tokenChunkUncompressedBytes": ( preferred_token_chunk_bytes ), "metadataCompressedBytes": 0, "metadataChunkCount": 0, "complete": False, "targetValuesRecorded": False, "targetEnteredForward": False, } if paths["progress"].is_file(): loaded = json.loads(paths["progress"].read_text(encoding="utf-8")) if ( isinstance(loaded, dict) and loaded.get("tokenBoundaryContract") is None ): loaded["tokenBoundaryContract"] = ( FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT ) _atomic_json(paths["progress"], loaded) if ( isinstance(loaded, dict) and loaded.get("tokenChunkUncompressedBytes") is None ): loaded["tokenChunkUncompressedBytes"] = ( FULL_PAYLOAD_PACKED_TOKEN_ZSTD_CHUNK_BYTES if loaded.get("tokenChunkCount", 0) != 0 else preferred_token_chunk_bytes ) _atomic_json(paths["progress"], loaded) if isinstance(loaded, dict): loaded.setdefault("readerDegradations", []) loaded.setdefault("readerDegradationCount", 0) loaded.setdefault("readerDegradationRetryCount", 0) loaded.setdefault("semanticDecodePassed", True) loaded.setdefault("trainingKnowledgeReady", True) if ( token_boundary_contract == FULL_PAYLOAD_PACKED_EXTENT_TOKEN_BOUNDARY_CONTRACT ): progress["tokenChunkUncompressedBytes"] = ( loaded.get("tokenChunkUncompressedBytes") ) compared = ( "schema", "scheduleSha256", "scheduleOrdinal", "payloadWorkId", "tokenizerAuthoritySha256", "contextWindowTokens", "answerTokensPerWindow", "tokenBoundaryContract", "tokenChunkUncompressedBytes", ) if not isinstance(loaded, dict) or any( loaded.get(field) != progress.get(field) for field in compared ): raise RuntimeError("full payload packed shard progress differs") retry_count = loaded.get("readerDegradationRetryCount") degradation_count = loaded.get("readerDegradationCount") if ( not isinstance(retry_count, int) or isinstance(retry_count, bool) or retry_count < 0 ): raise RuntimeError( "full payload packed reader degradation retry count differs" ) if ( isinstance(degradation_count, int) and not isinstance(degradation_count, bool) and degradation_count > 0 and loaded.get("complete") is False ): prior_receipt = loaded.get("readerDegradationReceipt") prior_receipt_sha256 = ( prior_receipt.get("sha256") if isinstance(prior_receipt, dict) else None ) progress.update( { "readerDegradationRetryCount": retry_count + 1, "lastReaderDegradationCount": degradation_count, "lastReaderDegradationReceiptSha256": ( prior_receipt_sha256 ), } ) # A degradation is not a durable training cursor. Replaying # this one shard lets an improved reader replace prior # diagnostic rows without disturbing any sealed sibling. paths["readerDegradationReceipt"].unlink(missing_ok=True) _fsync_directory(paths["readerDegradationReceipt"].parent) _atomic_json(paths["progress"], progress) _fsync_directory(paths["progress"].parent) else: progress = loaded else: _atomic_json(paths["progress"], progress) token_chunk_bytes = progress.get( "tokenChunkUncompressedBytes" ) if ( not isinstance(token_chunk_bytes, int) or isinstance(token_chunk_bytes, bool) or token_chunk_bytes not in { FULL_PAYLOAD_PACKED_TOKEN_ZSTD_CHUNK_BYTES, FULL_PAYLOAD_PACKED_EXTENT_TOKEN_ZSTD_CHUNK_BYTES, } or ( token_boundary_contract == FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT and token_chunk_bytes != FULL_PAYLOAD_PACKED_TOKEN_ZSTD_CHUNK_BYTES ) ): raise RuntimeError( "full payload packed token chunk geometry differs" ) completed_rows = int(progress["rows"]) token_elements = int(progress["tokenElements"]) metadata_bytes = int(progress["metadataBytes"]) record_count = int(progress.get("recordCount", 0)) recorded_prompt_elements = progress.get("promptTokenElements") recorded_answer_elements = progress.get("answerTokenElements") if ( isinstance(recorded_prompt_elements, int) and not isinstance(recorded_prompt_elements, bool) and isinstance(recorded_answer_elements, int) and not isinstance(recorded_answer_elements, bool) ): prompt_token_elements = recorded_prompt_elements answer_token_elements = recorded_answer_elements elif record_count: prompt_token_elements, answer_token_elements = ( _full_payload_packed_durable_token_accounting( paths["locators"], record_count=record_count, token_elements=token_elements, rows=completed_rows, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, ) ) else: prompt_token_elements = 0 answer_token_elements = 0 durable_observed_records = int( progress.get("observedRecordCount", record_count) ) if durable_observed_records < record_count: raise RuntimeError( "full payload packed observed-record cursor precedes packed records" ) fastokens_durable_progress_frontier = durable_observed_records fastokens_durable_token_frontier = token_elements common_expected_sizes = { "locators": record_count * FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.size, "recordLocatorHashes": record_count * 32, "windowEnds": record_count * FULL_PAYLOAD_PACKED_TOKEN_WINDOW_INDEX_RECORD_BYTES, } for name, size in common_expected_sizes.items(): _truncate_packed_artifact_to_durable_offset(paths[name], size) storage_layout = progress.get("packedStorageLayout") if storage_layout not in { None, FULL_PAYLOAD_PACKED_STREAM_STORAGE_LAYOUT, }: raise RuntimeError( "full payload packed progress storage layout differs" ) # JSON receipt values are validated below before either migration or # append. Declare their post-boundary type once so resumable nested # durability helpers never inherit the boundary's ``Any`` type. token_compressed_bytes: int token_chunk_count: int metadata_compressed_bytes: int metadata_chunk_count: int if storage_layout is None: # Legacy partial shards own cumulative raw streams. Convert only # their fsynced frontier, under the existing per-WorkID lock, then # continue with bounded staging files. No tokenizer or source # reader is invoked for the already durable bytes. migration_token_bytes = token_elements * 4 token_compressed_bytes = progress.get( "tokenCompressedBytes", 0, ) token_chunk_count = progress.get("tokenChunkCount", 0) token_uncompressed_bytes = progress.get( "legacyMigrationTokenUncompressedBytes", 0, ) metadata_compressed_bytes = progress.get( "metadataCompressedBytes", 0, ) metadata_chunk_count = progress.get( "metadataChunkCount", 0, ) metadata_uncompressed_bytes = progress.get( "legacyMigrationMetadataUncompressedBytes", 0, ) migration_frontier_values = ( token_compressed_bytes, token_chunk_count, token_uncompressed_bytes, metadata_compressed_bytes, metadata_chunk_count, metadata_uncompressed_bytes, ) if any( not isinstance(value, int) or isinstance(value, bool) or value < 0 for value in migration_frontier_values ): raise RuntimeError( "full payload packed legacy migration frontier differs" ) if ( token_uncompressed_bytes > migration_token_bytes or metadata_uncompressed_bytes > metadata_bytes or ( token_uncompressed_bytes != migration_token_bytes and token_uncompressed_bytes % token_chunk_bytes != 0 ) or ( metadata_uncompressed_bytes != metadata_bytes and metadata_uncompressed_bytes % FULL_PAYLOAD_PACKED_METADATA_ZSTD_CHUNK_BYTES != 0 ) or token_chunk_count != math.ceil( token_uncompressed_bytes / token_chunk_bytes ) or metadata_chunk_count != math.ceil( metadata_uncompressed_bytes / FULL_PAYLOAD_PACKED_METADATA_ZSTD_CHUNK_BYTES ) or (token_uncompressed_bytes == 0) != (token_compressed_bytes == 0) or (metadata_uncompressed_bytes == 0) != (metadata_compressed_bytes == 0) ): raise RuntimeError( "full payload packed legacy migration geometry differs" ) remaining_migration_bytes = ( migration_token_bytes - token_uncompressed_bytes + metadata_bytes - metadata_uncompressed_bytes ) remaining_index_bytes = ( ( math.ceil( migration_token_bytes / token_chunk_bytes ) - token_chunk_count ) * FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.size + ( math.ceil( metadata_bytes / FULL_PAYLOAD_PACKED_METADATA_ZSTD_CHUNK_BYTES ) - metadata_chunk_count ) * FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.size ) # A legacy migration cannot reclaim its raw durable prefix until # the compressed frontier and progress receipt are both durable. # Admit the remaining worst-case copy before mutating a prior # compressed tail. The extra one-sixteenth is deliberately larger # than zstd frame overhead, so incompressible input fails before # mutation rather than filling the filesystem mid-seal. migration_required_free_bytes = ( FULL_PAYLOAD_PACKED_STORAGE_MINIMUM_RESERVE_BYTES + remaining_migration_bytes + math.ceil(remaining_migration_bytes / 16) + remaining_index_bytes ) if ( shutil.disk_usage(shard_root).free < migration_required_free_bytes ): raise RuntimeError( "full payload packed legacy migration storage admission " "failed before compressed frontier mutation" ) _truncate_packed_artifact_to_durable_offset( paths["tokens"], migration_token_bytes, ) _truncate_packed_artifact_to_durable_offset( paths["metadata"], metadata_bytes, ) for path, durable_bytes in ( (paths["tokensZstd"], token_compressed_bytes), ( paths["tokenChunkIndex"], token_chunk_count * FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.size, ), (paths["metadataZstd"], metadata_compressed_bytes), ( paths["metadataChunkIndex"], metadata_chunk_count * FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.size, ), ): _truncate_packed_artifact_to_durable_offset( path, durable_bytes, ) if not path.is_file(): with path.open("ab") as empty_handle: empty_handle.flush() os.fsync(empty_handle.fileno()) _validate_full_payload_zstd_progress_frontier( paths["tokensZstd"], paths["tokenChunkIndex"], index_struct=FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT, expected_compressed_bytes=token_compressed_bytes, expected_uncompressed_bytes=token_uncompressed_bytes, expected_chunk_count=token_chunk_count, ) _validate_full_payload_zstd_progress_frontier( paths["metadataZstd"], paths["metadataChunkIndex"], index_struct=FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT, expected_compressed_bytes=metadata_compressed_bytes, expected_uncompressed_bytes=metadata_uncompressed_bytes, expected_chunk_count=metadata_chunk_count, ) with ( paths["tokensZstd"].open("r+b") as migration_token_handle, paths["tokenChunkIndex"].open("r+b") as migration_token_index, paths["metadataZstd"].open("r+b") as migration_metadata_handle, paths["metadataChunkIndex"].open("r+b") as migration_metadata_index, ): migration_token_handle.seek(token_compressed_bytes) migration_token_index.seek( token_chunk_count * FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.size ) migration_metadata_handle.seek(metadata_compressed_bytes) migration_metadata_index.seek( metadata_chunk_count * FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.size ) while token_uncompressed_bytes < migration_token_bytes: ( token_compressed_bytes, token_uncompressed_bytes, appended_token_chunks, ) = _append_full_payload_zstd_frames( paths["tokens"], migration_token_handle, migration_token_index, chunk_bytes=token_chunk_bytes, compression_level=FULL_PAYLOAD_PACKED_TOKEN_ZSTD_LEVEL, index_struct=( FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT ), compressed_offset=token_compressed_bytes, uncompressed_offset=token_uncompressed_bytes, raw_start_offset=token_uncompressed_bytes, maximum_raw_bytes=( FULL_PAYLOAD_PACKED_LEGACY_MIGRATION_CHECKPOINT_BYTES ), ) token_chunk_count += appended_token_chunks progress.update( { "tokenCompressedBytes": token_compressed_bytes, "tokenChunkCount": token_chunk_count, "legacyMigrationTokenUncompressedBytes": ( token_uncompressed_bytes ), "metadataCompressedBytes": ( metadata_compressed_bytes ), "metadataChunkCount": metadata_chunk_count, "legacyMigrationMetadataUncompressedBytes": ( metadata_uncompressed_bytes ), } ) _atomic_json(paths["progress"], progress) _fsync_directory(paths["progress"].parent) while metadata_uncompressed_bytes < metadata_bytes: ( metadata_compressed_bytes, metadata_uncompressed_bytes, appended_metadata_chunks, ) = _append_full_payload_zstd_frames( paths["metadata"], migration_metadata_handle, migration_metadata_index, chunk_bytes=( FULL_PAYLOAD_PACKED_METADATA_ZSTD_CHUNK_BYTES ), compression_level=( FULL_PAYLOAD_PACKED_METADATA_ZSTD_LEVEL ), index_struct=( FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT ), compressed_offset=metadata_compressed_bytes, uncompressed_offset=metadata_uncompressed_bytes, raw_start_offset=metadata_uncompressed_bytes, maximum_raw_bytes=( FULL_PAYLOAD_PACKED_LEGACY_MIGRATION_CHECKPOINT_BYTES ), ) metadata_chunk_count += appended_metadata_chunks progress.update( { "tokenCompressedBytes": token_compressed_bytes, "tokenChunkCount": token_chunk_count, "legacyMigrationTokenUncompressedBytes": ( token_uncompressed_bytes ), "metadataCompressedBytes": ( metadata_compressed_bytes ), "metadataChunkCount": metadata_chunk_count, "legacyMigrationMetadataUncompressedBytes": ( metadata_uncompressed_bytes ), } ) _atomic_json(paths["progress"], progress) _fsync_directory(paths["progress"].parent) if ( token_uncompressed_bytes != token_elements * 4 or metadata_uncompressed_bytes != metadata_bytes ): raise RuntimeError( "full payload packed legacy stream migration differs" ) progress.update( { "packedStorageLayout": ( FULL_PAYLOAD_PACKED_STREAM_STORAGE_LAYOUT ), "tokenCompressedBytes": token_compressed_bytes, "tokenChunkCount": token_chunk_count, "metadataCompressedBytes": metadata_compressed_bytes, "metadataChunkCount": metadata_chunk_count, "legacyMigrationRawTruncationPending": True, } ) _atomic_json(paths["progress"], progress) _fsync_directory(paths["progress"].parent) storage_layout = FULL_PAYLOAD_PACKED_STREAM_STORAGE_LAYOUT raw_truncation_pending = progress.get( "legacyMigrationRawTruncationPending" ) if raw_truncation_pending is not None and raw_truncation_pending is not True: raise RuntimeError( "full payload packed legacy raw truncation authority differs" ) if raw_truncation_pending is True: for staging_path in (paths["tokens"], paths["metadata"]): with staging_path.open("a+b") as staging_handle: staging_handle.seek(0) staging_handle.truncate(0) staging_handle.flush() os.fsync(staging_handle.fileno()) progress.pop("legacyMigrationRawTruncationPending", None) progress.pop("legacyMigrationTokenUncompressedBytes", None) progress.pop("legacyMigrationMetadataUncompressedBytes", None) _atomic_json(paths["progress"], progress) _fsync_directory(paths["progress"].parent) raw_token_compressed_bytes = progress.get("tokenCompressedBytes") raw_token_chunk_count = progress.get("tokenChunkCount") raw_metadata_compressed_bytes = progress.get( "metadataCompressedBytes" ) raw_metadata_chunk_count = progress.get("metadataChunkCount") if ( not isinstance(raw_token_compressed_bytes, int) or isinstance(raw_token_compressed_bytes, bool) or raw_token_compressed_bytes < 0 or not isinstance(raw_token_chunk_count, int) or isinstance(raw_token_chunk_count, bool) or raw_token_chunk_count < 0 or not isinstance(raw_metadata_compressed_bytes, int) or isinstance(raw_metadata_compressed_bytes, bool) or raw_metadata_compressed_bytes < 0 or not isinstance(raw_metadata_chunk_count, int) or isinstance(raw_metadata_chunk_count, bool) or raw_metadata_chunk_count < 0 or (token_elements == 0) != ( raw_token_compressed_bytes == 0 and raw_token_chunk_count == 0 ) or (metadata_bytes == 0) != ( raw_metadata_compressed_bytes == 0 and raw_metadata_chunk_count == 0 ) ): raise RuntimeError( "full payload packed compressed progress frontier differs" ) token_compressed_bytes = raw_token_compressed_bytes token_chunk_count = raw_token_chunk_count metadata_compressed_bytes = raw_metadata_compressed_bytes metadata_chunk_count = raw_metadata_chunk_count # ``struct.Struct.size`` is untyped at the Python I/O boundary. Preserve # its runtime-derived geometry while giving the nested durability # helpers an explicit integer contract. token_index_record_bytes: int = ( FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.size ) metadata_index_record_bytes: int = ( FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.size ) token_locator_record_bytes: int = ( FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.size ) for path, durable_bytes in ( (paths["tokensZstd"], token_compressed_bytes), ( paths["tokenChunkIndex"], token_chunk_count * token_index_record_bytes, ), (paths["metadataZstd"], metadata_compressed_bytes), ( paths["metadataChunkIndex"], metadata_chunk_count * metadata_index_record_bytes, ), ): _truncate_packed_artifact_to_durable_offset(path, durable_bytes) if not path.is_file(): with path.open("ab") as empty_handle: empty_handle.flush() os.fsync(empty_handle.fileno()) _validate_full_payload_zstd_progress_frontier( paths["tokensZstd"], paths["tokenChunkIndex"], index_struct=FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT, expected_compressed_bytes=token_compressed_bytes, expected_uncompressed_bytes=token_elements * 4, expected_chunk_count=token_chunk_count, ) _validate_full_payload_zstd_progress_frontier( paths["metadataZstd"], paths["metadataChunkIndex"], index_struct=FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT, expected_compressed_bytes=metadata_compressed_bytes, expected_uncompressed_bytes=metadata_bytes, expected_chunk_count=metadata_chunk_count, ) for staging_path in (paths["tokens"], paths["metadata"]): _truncate_packed_artifact_to_durable_offset(staging_path, 0) durable_record_count = record_count durable_packed_bytes = ( token_compressed_bytes + token_chunk_count * token_index_record_bytes + metadata_compressed_bytes + metadata_chunk_count * metadata_index_record_bytes + sum(common_expected_sizes.values()) ) reader_families = set(progress.get("readerFamilies", [])) semantic_hashes = set(progress.get("semanticExportReceiptSha256s", [])) reader_degradations = progress.get("readerDegradations", []) if ( not isinstance(reader_degradations, list) or progress.get("readerDegradationCount", 0) != len(reader_degradations) or any( not isinstance(degradation, dict) or degradation.get("schema") != FULL_PAYLOAD_READER_DEGRADATION_SCHEMA or degradation.get("semanticDecodePassed") is not False or degradation.get("trainingKnowledgeReady") is not False for degradation in reader_degradations ) ): raise RuntimeError( "full payload packed reader degradation progress differs" ) reader_degradations = [ dict(degradation) for degradation in reader_degradations ] # Start the reader at the exact durable source-record frontier. Native # matrix/object adapters seek directly; stream adapters discard only # their already-committed prefix. No completed record is tokenized or # appended twice after a deliberate resume. observed_records = durable_observed_records token_staging = io.BytesIO() metadata_staging = io.BytesIO() locator_staging = io.BytesIO() record_hash_staging = io.BytesIO() window_ends_staging = io.BytesIO() with ( paths["tokensZstd"].open("a+b") as token_compressed_handle, paths["tokenChunkIndex"].open("a+b") as token_index_handle, paths["locators"].open("ab") as locators_handle, paths["metadataZstd"].open("a+b") as metadata_compressed_handle, paths["metadataChunkIndex"].open("a+b") as metadata_index_handle, paths["recordLocatorHashes"].open("ab") as record_hash_handle, paths["windowEnds"].open("ab") as window_ends_handle, ): for handle in ( token_compressed_handle, token_index_handle, metadata_compressed_handle, metadata_index_handle, ): handle.seek(0, os.SEEK_END) def staged_payload_bytes() -> int: return token_staging.tell() + metadata_staging.tell() def staged_total_bytes() -> int: return ( staged_payload_bytes() + locator_staging.tell() + record_hash_staging.tell() + window_ends_staging.tell() ) def packed_artifact_bytes() -> int: return ( token_compressed_bytes + token_chunk_count * token_index_record_bytes + metadata_compressed_bytes + metadata_chunk_count * metadata_index_record_bytes + record_count * token_locator_record_bytes + record_count * 32 + record_count * FULL_PAYLOAD_PACKED_TOKEN_WINDOW_INDEX_RECORD_BYTES + staged_payload_bytes() ) def publish_durable_progress() -> bool: nonlocal durable_observed_records nonlocal durable_packed_bytes nonlocal durable_record_count nonlocal fastokens_durable_progress_frontier nonlocal fastokens_durable_token_frontier nonlocal metadata_chunk_count nonlocal metadata_compressed_bytes nonlocal token_chunk_count nonlocal token_compressed_bytes if ( record_count == durable_record_count and staged_total_bytes() == 0 ): return False if record_count < durable_record_count: raise RuntimeError( "full payload packed durable frontier moved backward" ) pending_record_count = record_count - durable_record_count expected_sidecar_sizes = ( pending_record_count * token_locator_record_bytes, pending_record_count * 32, pending_record_count * FULL_PAYLOAD_PACKED_TOKEN_WINDOW_INDEX_RECORD_BYTES, ) observed_sidecar_sizes = ( locator_staging.tell(), record_hash_staging.tell(), window_ends_staging.tell(), ) if observed_sidecar_sizes != expected_sidecar_sizes: raise RuntimeError( "full payload packed sidecar staging geometry differs" ) for handle, staging_handle in ( (locators_handle, locator_staging), (record_hash_handle, record_hash_staging), (window_ends_handle, window_ends_staging), ): _write_full_payload_staging_extent( handle, staging_handle, ) for handle in ( locators_handle, record_hash_handle, window_ends_handle, ): handle.flush() os.fsync(handle.fileno()) ( token_compressed_bytes, token_uncompressed_bytes, added_token_chunks, ) = _append_full_payload_zstd_buffer_frames( token_staging.getbuffer(), token_compressed_handle, token_index_handle, chunk_bytes=token_chunk_bytes, compression_level=FULL_PAYLOAD_PACKED_TOKEN_ZSTD_LEVEL, index_struct=FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT, compressed_offset=token_compressed_bytes, uncompressed_offset=( int(progress["tokenElements"]) * 4 ), ) ( metadata_compressed_bytes, metadata_uncompressed_bytes, added_metadata_chunks, ) = _append_full_payload_zstd_buffer_frames( metadata_staging.getbuffer(), metadata_compressed_handle, metadata_index_handle, chunk_bytes=( FULL_PAYLOAD_PACKED_METADATA_ZSTD_CHUNK_BYTES ), compression_level=FULL_PAYLOAD_PACKED_METADATA_ZSTD_LEVEL, index_struct=( FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT ), compressed_offset=metadata_compressed_bytes, uncompressed_offset=int(progress["metadataBytes"]), ) token_chunk_count += added_token_chunks metadata_chunk_count += added_metadata_chunks if ( token_uncompressed_bytes != token_elements * 4 or metadata_uncompressed_bytes != metadata_bytes ): raise RuntimeError( "full payload packed compressed durable frontier differs" ) progress.update( { "rows": completed_rows, "tokenElements": token_elements, "promptTokenElements": prompt_token_elements, "answerTokenElements": answer_token_elements, "metadataBytes": metadata_bytes, "recordCount": record_count, "observedRecordCount": observed_records, "readerFamilies": sorted(reader_families), "semanticExportReceiptSha256s": sorted( semantic_hashes ), "packedStorageLayout": ( FULL_PAYLOAD_PACKED_STREAM_STORAGE_LAYOUT ), "tokenCompressedBytes": token_compressed_bytes, "tokenChunkCount": token_chunk_count, "metadataCompressedBytes": ( metadata_compressed_bytes ), "metadataChunkCount": metadata_chunk_count, } ) _atomic_json(paths["progress"], progress) _fsync_directory(paths["progress"].parent) for staging_handle in ( token_staging, metadata_staging, locator_staging, record_hash_staging, window_ends_staging, ): staging_handle.seek(0) staging_handle.truncate(0) durable_record_count = record_count durable_observed_records = observed_records durable_packed_bytes = packed_artifact_bytes() next_fastokens_progress_frontier = ( _full_payload_fastokens_durable_progress_boundary( tokenizer, durable_observed_records=durable_observed_records, durable_progress_frontier=( fastokens_durable_progress_frontier ), durable_token_elements=token_elements, durable_token_frontier=( fastokens_durable_token_frontier ), ) ) if ( next_fastokens_progress_frontier != fastokens_durable_progress_frontier ): fastokens_durable_token_frontier = token_elements fastokens_durable_progress_frontier = ( next_fastokens_progress_frontier ) return True def persist_record_batch( records: Sequence[FullPayloadTextRecord], ) -> None: nonlocal completed_rows nonlocal token_elements nonlocal prompt_token_elements nonlocal answer_token_elements nonlocal metadata_bytes nonlocal record_count encoded_batch = _tokenize_full_payload_record_batch( tokenizer, records, ) planned_append_bytes = sum( 4 * (len(prefix_ids) + len(content_ids)) + FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.size + 32 + FULL_PAYLOAD_PACKED_TOKEN_WINDOW_INDEX_RECORD_BYTES + FULL_PAYLOAD_PACKED_MEASUREMENT_METADATA_BYTES_PER_RECORD + len(record.text.encode("utf-8")) for record, prefix_ids, content_ids in encoded_batch ) available_storage_bytes = shutil.disk_usage(shard_root).free required_storage_bytes = ( FULL_PAYLOAD_PACKED_STORAGE_MINIMUM_RESERVE_BYTES + staged_total_bytes() + planned_append_bytes * 2 ) if available_storage_bytes < required_storage_bytes: raise RuntimeError( "full payload packed incremental storage admission " "failed before durable batch append" ) # Authorities are constant for the whole source shard; hash them # once instead of recomputing for every record (previously this # was 2 json.dumps + 2 sha256 per row across the entire source). rights_authority_sha256 = _sha256_bytes(_json_bytes(rights_authority)) domain_authority_sha256 = _sha256_bytes(_json_bytes(domain_authority)) native_fastokens_batch = callable( getattr( getattr(tokenizer, "backend_tokenizer", None), "encode_batch", None, ) ) if native_fastokens_batch: packed_batch_elements = ( _write_full_payload_fastokens_int32_extent( token_staging, tuple( token_row for _record, prefix_ids, content_ids in encoded_batch for token_row in (prefix_ids, content_ids) ), vocabulary_size=tokenizer_vocabulary_size, ) ) expected_batch_elements = sum( len(prefix_ids) + len(content_ids) for _record, prefix_ids, content_ids in encoded_batch ) if packed_batch_elements != expected_batch_elements: raise RuntimeError( "full payload packed Fastokens batch geometry differs" ) for record, prefix_ids, content_ids in encoded_batch: ( window_count, record_prompt_tokens, record_answer_tokens, ) = _full_payload_packed_record_training_geometry( prefix_tokens=len(prefix_ids), content_tokens=len(content_ids), context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, ) source_record_id = hashlib.sha256( ( f"{work_id}\x00{record.locator}\x00" f"{record.ordinal}" ).encode("utf-8") ).hexdigest() metadata = { "schema": FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_SCHEMA, "source_id": record.source_id, "payload_work_id": work_id, "payload_record_locator": record.locator, "payload_record_ordinal": record.ordinal, "source_record_count": record.source_record_count, "source_record_start_ordinal": ( record.source_record_start_ordinal ), "source_record_end_ordinal": ( record.source_record_end_ordinal ), "source_record_sequence_sha256": ( record.source_record_sequence_sha256 ), "payload_record_token_count": len(content_ids), "payload_reader_family": record.reader_family, "typed_numeric_encoding": ( record.typed_numeric_encoding ), "typed_numeric_value_bytes": ( len(record.typed_numeric_bytes) if record.typed_numeric_bytes is not None else None ), "semantic_export_receipt_sha256": ( record.semantic_export_receipt_sha256 ), "source_record_id": source_record_id, "source_sha256": hash_row["observedSha256"], "corpus_surface_family": "full_payload_packed", "generalization_axis": "task_family", "generalization_group": work_id, "rights_disposition": "training_admissible", "rights_authority_sha256": rights_authority_sha256, "domain_authority_sha256": domain_authority_sha256, "target_values_recorded": False, "target_entered_forward": False, "task_intent_targets_entered_forward": False, "model_scores_observed": False, } metadata_payload = _json_bytes(metadata) + b"\n" record_identity = { "payloadWorkId": work_id, "recordLocator": record.locator, "recordOrdinal": record.ordinal, "sourceSha256": hash_row["observedSha256"], } record_hash = hashlib.sha256( _json_bytes(record_identity) ).digest() row_token_offset = token_elements if native_fastokens_batch: prefix_length = len(prefix_ids) content_length = len(content_ids) else: prefix_length = _write_packed_int32( token_staging, prefix_ids, vocabulary_size=tokenizer_vocabulary_size, ) content_length = _write_packed_int32( token_staging, content_ids, vocabulary_size=tokenizer_vocabulary_size, ) locator_staging.write( FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.pack( row_token_offset, prefix_length, content_length, metadata_bytes, len(metadata_payload), ) ) metadata_staging.write(metadata_payload) record_hash_staging.write(record_hash) completed_rows += window_count window_ends_staging.write( struct.pack("= FULL_PAYLOAD_PACKED_DURABLE_PROGRESS_BYTES ): publish_durable_progress() elif ( record_count - durable_record_count >= FULL_PAYLOAD_PACKED_DURABLE_PROGRESS_RECORDS ): publish_durable_progress() pending_records: list[FullPayloadTextRecord] = [] pending_record_bytes = 0 source_records: Iterable[FullPayloadTextRecord] = ( iter_full_payload_text_records( corpus_root, schedule_row, scratch_root=scratch_root, start_record_ordinal=durable_observed_records, ) ) if ( token_boundary_contract == FULL_PAYLOAD_PACKED_EXTENT_TOKEN_BOUNDARY_CONTRACT ): source_records = iter_full_payload_training_extents( source_records, maximum_text_bytes=min( FULL_PAYLOAD_PACKED_TRAINING_EXTENT_BYTES, answer_tokens_per_window, ), ) for record in source_records: source_record_count = record.source_record_count if ( not isinstance(source_record_count, int) or isinstance(source_record_count, bool) or source_record_count < 1 ): raise RuntimeError( "full payload packed source-record count differs" ) next_observed_records = ( observed_records + source_record_count ) if next_observed_records <= durable_observed_records: observed_records = next_observed_records continue if observed_records < durable_observed_records: raise RuntimeError( "full payload packed durable source cursor splits " "a training extent" ) observed_records = next_observed_records if record.reader_degradation is not None: # The degradation itself is not durable until its # authority is written below. Publish every preceding # semantic record at the prior observed cursor first. observed_records -= source_record_count try: if pending_records: persist_record_batch(pending_records) pending_records.clear() pending_record_bytes = 0 publish_durable_progress() finally: observed_records += source_record_count degradation = dict(record.reader_degradation) recorded_authority = degradation.pop( "readerDegradationAuthoritySha256", None, ) if ( degradation.get("schema") != FULL_PAYLOAD_READER_DEGRADATION_SCHEMA or degradation.get("trainingKnowledgeReady") is not False or degradation.get("semanticDecodePassed") is not False or degradation.get("rawBytesPreserved") is not True or degradation.get("reacquisitionRequired") is not True or recorded_authority != _sha256_bytes(_json_bytes(degradation)) ): raise RuntimeError( "full payload packed reader degradation differs" ) degradation.update( { "payloadWorkId": work_id, "sourceId": schedule_row["sourceId"], "sourcePayloadPath": str(source_path), "sourcePayloadBytes": schedule_row[ "payloadBytes" ], "sourcePayloadSha256": hash_row[ "observedSha256" ], "recordLocator": record.locator, "recordOrdinal": record.ordinal, } ) degradation["readerDegradationAuthoritySha256"] = ( _sha256_bytes(_json_bytes(degradation)) ) reader_degradations.append(degradation) progress.update( { "observedRecordCount": observed_records, "readerDegradations": reader_degradations, "readerDegradationCount": len( reader_degradations ), "semanticDecodePassed": False, "trainingKnowledgeReady": False, "complete": False, } ) _atomic_json(paths["progress"], progress) _fsync_directory(paths["progress"].parent) continue record_text_bytes = len(record.text.encode("utf-8")) if ( pending_records and pending_record_bytes + record_text_bytes > FULL_PAYLOAD_PACKED_TOKENIZE_BATCH_BYTES ): persist_record_batch(pending_records) pending_records.clear() pending_record_bytes = 0 pending_records.append(record) pending_record_bytes += record_text_bytes if ( len(pending_records) == FULL_PAYLOAD_PACKED_PROGRESS_RECORDS or pending_record_bytes >= FULL_PAYLOAD_PACKED_TOKENIZE_BATCH_BYTES ): persist_record_batch(pending_records) pending_records.clear() pending_record_bytes = 0 if pending_records: persist_record_batch(pending_records) publish_durable_progress() if reader_degradations: degradation_receipt = { "schema": ( FULL_PAYLOAD_READER_DEGRADATION_RECEIPT_SCHEMA ), "passed": False, "payloadWorkId": work_id, "sourceId": schedule_row["sourceId"], "sourcePayloadPath": str(source_path), "sourcePayloadBytes": schedule_row["payloadBytes"], "sourcePayloadSha256": hash_row["observedSha256"], "degradations": reader_degradations, "readerDegradationCount": len(reader_degradations), "semanticDecodePassed": False, "trainingKnowledgeReady": False, "rawBytesPreserved": True, "rawBytesEncodedIntoTraining": False, "reacquisitionRequired": True, "validRecordsPackedButCollectionUnsealed": ( record_count > 0 ), "targetEnteredForward": False, } degradation_receipt["readerDegradationReceiptSha256"] = ( _sha256_bytes(_json_bytes(degradation_receipt)) ) _atomic_json( paths["readerDegradationReceipt"], degradation_receipt, ) progress.update( { "readerDegradationReceipt": _packed_artifact( paths["readerDegradationReceipt"] ), "semanticDecodePassed": False, "trainingKnowledgeReady": False, "complete": False, } ) _atomic_json(paths["progress"], progress) raise RuntimeError( "full payload semantic reader degradation requires " "reacquisition before training; " f"receipt={paths['readerDegradationReceipt']}" ) if ( completed_rows < 1 or observed_records < 1 or observed_records < durable_observed_records ): raise RuntimeError("full payload packed shard emitted no token windows") if not _full_payload_source_identity_matches( hash_row.get("sourceIdentity"), source_path, observed_sha256=hash_row.get("observedSha256"), proven_device_remaps=proven_device_remaps, ): raise RuntimeError("full payload packed source changed during build") progress.update( { "rows": completed_rows, "tokenElements": token_elements, "promptTokenElements": prompt_token_elements, "answerTokenElements": answer_token_elements, "metadataBytes": metadata_bytes, "recordCount": record_count, "observedRecordCount": observed_records, "readerFamilies": sorted(reader_families), "semanticExportReceiptSha256s": sorted(semantic_hashes), "complete": True, } ) _atomic_json(paths["progress"], progress) _fsync_directory(paths["progress"].parent) token_compression = _full_payload_zstd_stream_receipt( paths["tokensZstd"], paths["tokenChunkIndex"], schema=FULL_PAYLOAD_PACKED_TOKEN_ZSTD_SCHEMA, chunk_bytes=token_chunk_bytes, compression_level=FULL_PAYLOAD_PACKED_TOKEN_ZSTD_LEVEL, index_struct=FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT, expected_uncompressed_bytes=token_elements * 4, ) metadata_compression = _full_payload_zstd_stream_receipt( paths["metadataZstd"], paths["metadataChunkIndex"], schema=FULL_PAYLOAD_PACKED_METADATA_ZSTD_SCHEMA, chunk_bytes=FULL_PAYLOAD_PACKED_METADATA_ZSTD_CHUNK_BYTES, compression_level=FULL_PAYLOAD_PACKED_METADATA_ZSTD_LEVEL, index_struct=FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT, expected_uncompressed_bytes=metadata_bytes, ) artifacts = { "tokensZstd": token_compression["compressed"], "tokenChunkIndex": token_compression["chunkIndex"], "locators": _packed_artifact(paths["locators"]), "metadataZstd": metadata_compression["compressed"], "metadataChunkIndex": metadata_compression["chunkIndex"], "recordLocatorHashes": _packed_artifact( paths["recordLocatorHashes"] ), "windowEnds": _packed_artifact(paths["windowEnds"]), } manifest = { "schema": FULL_PAYLOAD_PACKED_TOKEN_SHARD_SCHEMA, "passed": True, "scheduleReceipt": _packed_artifact(schedule_receipt_path), "schedule": _packed_artifact(schedule_path), "scheduleSha256": schedule_sha256, "scheduleOrdinal": ordinal, "payloadWorkId": work_id, "sourceId": schedule_row["sourceId"], "sourceRecordSha256": schedule_row["sourceRecordSha256"], "payloadRelativePath": schedule_row["payloadRelativePath"], "payloadBytes": schedule_row["payloadBytes"], "observedPayloadSha256": hash_row["observedSha256"], "sourceIdentity": hash_row["sourceIdentity"], "hashLedgerRowSha256": _sha256_bytes(_json_bytes(hash_row)), "domainAuthority": domain_authority, "domainAuthoritySha256": _sha256_bytes(_json_bytes(domain_authority)), "rightsAuthority": rights_authority, "rightsAuthoritySha256": _sha256_bytes(_json_bytes(rights_authority)), "tokenizerAuthority": dict(tokenizer_authority), "tokenizerAuthoritySha256": tokenizer_sha256, "contextWindowTokens": context_window_tokens, "answerTokensPerWindow": answer_tokens_per_window, "tokenBoundaryContract": token_boundary_contract, "rows": completed_rows, "recordCount": record_count, "observedRecordCount": observed_records, "tokenElements": token_elements, "promptTokenElements": prompt_token_elements, "answerTokenElements": answer_token_elements, "trainingTokenElements": ( prompt_token_elements + answer_token_elements ), "metadataBytes": metadata_bytes, "metadataUncompressedBytes": metadata_bytes, "metadataCompressedBytes": metadata_compression[ "compressedBytes" ], "artifacts": artifacts, "tokenCompression": token_compression, "metadataCompression": metadata_compression, "packedStorageLayout": ( FULL_PAYLOAD_PACKED_STREAM_STORAGE_LAYOUT ), "rawTokenBytesRetained": False, "compressedTokenRandomAccess": True, "rawMetadataBytesRetained": False, "compressedMetadataRandomAccess": True, "metadataStoredOncePerRecord": True, "recordLocatorHashesPersisted": True, "windowLocatorHashesPersisted": False, "windowIdentityDerivedAtReceipt": True, "recordWindowEndIndexPersisted": True, "tokensStoredOncePerRecord": True, "targetValuesRecorded": False, "targetEnteredForward": False, "mmapReadable": True, } manifest["shardAuthoritySha256"] = _sha256_bytes(_json_bytes(manifest)) _atomic_json(paths["manifest"], manifest) compact_manifest = _compact_full_payload_packed_record_storage( paths["manifest"], manifest, ) paths["tokens"].unlink(missing_ok=True) paths["metadata"].unlink(missing_ok=True) return compact_manifest finally: fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN) lock_handle.close() _FULL_PAYLOAD_PACKED_PROCESS_TOKENIZER: Any | None = None def _initialize_full_payload_packed_process_worker( serialized_tokenizer: bytes, ) -> None: """Give each forked shard worker its own tokenizer executor instance.""" global _FULL_PAYLOAD_PACKED_PROCESS_TOKENIZER tokenizer = pickle.loads(serialized_tokenizer) backend = getattr(tokenizer, "backend_tokenizer", None) if not callable(getattr(tokenizer, "encode", None)) and not callable( getattr(backend, "encode_batch", None) ): raise TypeError("full payload packed process tokenizer is not executable") _FULL_PAYLOAD_PACKED_PROCESS_TOKENIZER = tokenizer def _build_full_payload_packed_shard_process( job: _FullPayloadPackedShardJob, ) -> _FullPayloadPackedShardResult: """Run one shard through its existing lock/progress authority in a process.""" tokenizer = _FULL_PAYLOAD_PACKED_PROCESS_TOKENIZER if tokenizer is None: raise RuntimeError("full payload packed process tokenizer is absent") ordinal = int(job.schedule_row["scheduleOrdinal"]) work_id = str(job.schedule_row["payloadWorkId"]) manifest_path = ( job.shard_root / f"{ordinal:08d}_{work_id[:16]}.manifest.json" ) manifest_was_present = manifest_path.is_file() manifest = _build_full_payload_packed_shard( schedule_receipt_path=job.schedule_receipt_path, schedule_path=job.schedule_path, schedule_sha256=job.schedule_sha256, schedule_row=job.schedule_row, hash_row=job.hash_row, corpus_root=job.corpus_root, shard_root=job.shard_root, tokenizer=tokenizer, tokenizer_authority=job.tokenizer_authority, tokenizer_vocabulary_size=job.tokenizer_vocabulary_size, context_window_tokens=job.context_window_tokens, answer_tokens_per_window=job.answer_tokens_per_window, scratch_root=job.scratch_root, token_boundary_contract=job.token_boundary_contract, proven_device_remaps=set(job.proven_device_remaps), ) if not manifest_was_present: # Process workers are reused for later WorkIDs. A newly sealed manifest # proves this WorkID is complete before its source cache is discarded. # Pre-existing manifests still pass full validation in the builder, but # they did not populate this process cache and must not recompile BPE. _reset_full_payload_fastokens_cache_boundary(tokenizer) return _FullPayloadPackedShardResult( worker_pid=os.getpid(), manifest=manifest, ) def _run_full_payload_packed_shard_process_jobs( tokenizer: Any, jobs: Sequence[_FullPayloadPackedShardJob], *, worker_count: int, ) -> list[_FullPayloadPackedShardResult]: """Execute memory-bounded shard batches outside the GIL.""" if not jobs or worker_count < 1 or worker_count > len(jobs): raise ValueError("full payload packed process geometry is invalid") payload_bytes_by_ordinal: list[int] = [] for job in jobs: payload_bytes = job.schedule_row.get("payloadBytes") if ( not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or payload_bytes < 1 ): raise ValueError("full payload packed process payload bytes differ") payload_bytes_by_ordinal.append(payload_bytes) serialized_tokenizer = pickle.dumps( tokenizer, protocol=pickle.HIGHEST_PROTOCOL, ) process_context = multiprocessing.get_context("fork") ordered_results: list[_FullPayloadPackedShardResult | None] = [ None ] * len(jobs) submission_order = sorted( range(len(jobs)), key=lambda ordinal: ( payload_bytes_by_ordinal[ordinal] > FULL_PAYLOAD_PACKED_MAX_INFLIGHT_PAYLOAD_BYTES, ordinal, ), ) executor = ProcessPoolExecutor( max_workers=worker_count, mp_context=process_context, initializer=_initialize_full_payload_packed_process_worker, initargs=(serialized_tokenizer,), ) pending: dict[ Future[_FullPayloadPackedShardResult], tuple[int, int], ] = {} next_submission = 0 pending_payload_bytes = 0 maximum_pending = min(len(jobs), worker_count * 2) def submit_next() -> None: nonlocal next_submission nonlocal pending_payload_bytes ordinal = submission_order[next_submission] payload_bytes = payload_bytes_by_ordinal[ordinal] future = executor.submit( _build_full_payload_packed_shard_process, jobs[ordinal], ) pending[future] = (ordinal, payload_bytes) pending_payload_bytes += payload_bytes next_submission += 1 def next_job_fits() -> bool: if next_submission >= len(jobs) or len(pending) >= maximum_pending: return False if not pending: # One work must make progress even when its compressed input alone # exceeds the normal concurrent budget. return True ordinal = submission_order[next_submission] return ( pending_payload_bytes + payload_bytes_by_ordinal[ordinal] <= FULL_PAYLOAD_PACKED_MAX_INFLIGHT_PAYLOAD_BYTES ) try: while next_job_fits(): submit_next() while pending: completed, _ = wait( tuple(pending), return_when=FIRST_COMPLETED, ) for future in completed: ordinal, payload_bytes = pending.pop(future) pending_payload_bytes -= payload_bytes # Child exceptions propagate before any collection receipt is # sealed. A completed shard remains safe under its own lock and # immutable manifest authority. ordered_results[ordinal] = future.result() while next_job_fits(): submit_next() except BaseException: for future in pending: future.cancel() executor.shutdown(wait=True, cancel_futures=True) raise executor.shutdown(wait=True, cancel_futures=False) if any(result is None for result in ordered_results): raise RuntimeError("full payload packed process results are incomplete") return [ result for result in ordered_results if result is not None ] def build_full_payload_packed_token_collection( schedule_receipt_path: Path, corpus_root: Path, hash_ledger_path: Path, output_receipt_path: Path, *, tokenizer: Any, context_window_tokens: int, answer_tokens_per_window: int, scratch_root: Path | None = None, tokenizer_authority: Mapping[str, Any] | None = None, source_ids: Sequence[str] | None = None, payload_work_ids: Sequence[str] | None = None, source_output_roots: Mapping[str, Path] | None = None, storage_admission_receipt_path: Path | None = None, ) -> dict[str, Any]: """Persist admitted full-payload tokens as resumable compressed mmap shards. The builder is an explicit CPU/I/O job. It never starts training, never loads a model onto a GPU, and publishes a collection only after every selected source work item is sealed. """ if ( isinstance(context_window_tokens, bool) or isinstance(answer_tokens_per_window, bool) or context_window_tokens < 3 or answer_tokens_per_window < 1 or answer_tokens_per_window >= context_window_tokens ): raise ValueError("full payload packed token geometry is invalid") ( _schedule_receipt, schedule_path, schedule_sha256, schedule_rows, hash_rows, ) = _validated_full_payload_packed_sources( schedule_receipt_path, corpus_root, hash_ledger_path, source_ids=source_ids, payload_work_ids=payload_work_ids, ) resolved_corpus_root = corpus_root.expanduser().resolve() proven_device_remaps = _full_payload_packed_proven_device_remaps( resolved_corpus_root, schedule_rows, hash_rows, ) output_path = output_receipt_path.expanduser().resolve() output_path.parent.mkdir(parents=True, exist_ok=True) selected_source_ids = tuple( dict.fromkeys(str(row["sourceId"]) for row in schedule_rows) ) configured_roots = ( { source_id: path.expanduser().resolve() for source_id, path in source_output_roots.items() } if source_output_roots is not None else {} ) if configured_roots and not set(selected_source_ids).issubset(configured_roots): raise ValueError("full payload packed source output roots are incomplete") actual_source_roots = { source_id: configured_roots.get(source_id, output_path.parent) for source_id in selected_source_ids } storage_admission_record: dict[str, Any] | None = None if storage_admission_receipt_path is not None: resolved_admission_path = ( storage_admission_receipt_path.expanduser().resolve() ) admission = ( _validated_full_payload_packed_storage_admission_receipt( resolved_admission_path ) ) admission_authority = admission["storageAdmissionSha256"] if ( admission.get("contextWindowTokens") != context_window_tokens or admission.get("answerTokensPerWindow") != answer_tokens_per_window or admission.get("scheduledPayloadWorkCount") != len(schedule_rows) or admission.get("scheduledPayloadWorkIdsSha256") != _sha256_bytes( _json_bytes( sorted(str(row["payloadWorkId"]) for row in schedule_rows) ) ) or admission.get("scheduledSourceIds") != sorted(selected_source_ids) ): raise RuntimeError( "full payload packed storage admission authority differs" ) _validate_full_payload_packed_admission_source_roots( admission, actual_source_roots, ) storage_admission_record = { "receipt": _packed_artifact(resolved_admission_path), "storageAdmissionSha256": admission_authority, "scheduledPayloadWorkCount": len(schedule_rows), "scheduledPayloadWorkIdsSha256": admission[ "scheduledPayloadWorkIdsSha256" ], "scheduledSourceIds": list(sorted(selected_source_ids)), "sourceOutputRoots": { source_id: str(actual_source_roots[source_id]) for source_id in sorted(actual_source_roots) }, "passed": True, } if tokenizer_authority is None: from resynthesis.tokenizer_backend import tokenizer_boundary_receipt tokenizer_authority = tokenizer_boundary_receipt(tokenizer) ( tokenizer_record, tokenizer_sha256, tokenizer_vocabulary_size, ) = _validated_full_payload_fastokens_authority(tokenizer_authority) source_authority_by_id: dict[str, tuple[dict[str, Any], str]] = {} shard_root_by_source: dict[str, Path] = {} for source_id in selected_source_ids: positions = tuple( index for index, row in enumerate(schedule_rows) if row["sourceId"] == source_id ) source_authority_pair = _full_payload_packed_source_authority( source_id=source_id, schedule_sha256=schedule_sha256, schedule_rows=[schedule_rows[index] for index in positions], hash_rows=[hash_rows[index] for index in positions], tokenizer_authority_sha256=tokenizer_sha256, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, ) source_authority_by_id[source_id] = source_authority_pair storage_root = actual_source_roots[source_id] shard_root_by_source[source_id] = ( storage_root / "objects" / source_authority_pair[1] / "work_shards" ) legacy_shard_root = output_path.parent / "shards" candidate_shard_roots = { legacy_shard_root, *shard_root_by_source.values(), } manifest_owner_by_work_id: dict[str, Path] = {} reconciled_scoped_work_ids: set[str] = set() for candidate_root in candidate_shard_roots: if not candidate_root.is_dir(): continue for manifest_path in candidate_root.glob("*.manifest.json"): manifest_value = json.loads( manifest_path.read_text(encoding="utf-8") ) if ( not isinstance(manifest_value, dict) or manifest_value.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_SHARD_SCHEMA or not isinstance( manifest_value.get("payloadWorkId"), str, ) ): raise RuntimeError( "full payload packed layout manifest is malformed: " f"{manifest_path}" ) manifest_work_id = str(manifest_value["payloadWorkId"]) prior_owner = manifest_owner_by_work_id.get(manifest_work_id) if prior_owner is not None and prior_owner != manifest_path: owner_paths = (prior_owner, manifest_path) legacy_owner = next( ( path for path in owner_paths if path.parent.resolve() == legacy_shard_root.resolve() ), None, ) scoped_owner = next( ( path for path in owner_paths if path.parent.resolve() != legacy_shard_root.resolve() ), None, ) if ( legacy_owner is not None and scoped_owner is not None and _full_payload_packed_cross_layout_duplicate_matches( legacy_owner, scoped_owner, ) ): reconciled_scoped_work_ids.add(manifest_work_id) manifest_owner_by_work_id[manifest_work_id] = scoped_owner continue raise RuntimeError( "full payload packed duplicate work ownership differs: " f"payloadWorkId={manifest_work_id} " f"first={prior_owner} second={manifest_path}" ) manifest_owner_by_work_id[manifest_work_id] = manifest_path shard_root_by_ordinal: dict[int, Path] = {} for schedule_row in schedule_rows: source_id = str(schedule_row["sourceId"]) scoped_root = shard_root_by_source[source_id] legacy_has_work = _full_payload_packed_layout_has_durable_evidence( legacy_shard_root, schedule_row, schedule_sha256=schedule_sha256, ) scoped_has_work = _full_payload_packed_layout_has_durable_evidence( scoped_root, schedule_row, schedule_sha256=schedule_sha256, ) if legacy_has_work and scoped_has_work: if ( str(schedule_row["payloadWorkId"]) not in reconciled_scoped_work_ids ): raise RuntimeError( "full payload packed duplicate layout evidence differs: " f"payloadWorkId={schedule_row['payloadWorkId']} " f"legacy={legacy_shard_root} scoped={scoped_root}" ) shard_root_by_ordinal[int(schedule_row["scheduleOrdinal"])] = ( scoped_root ) continue shard_root_by_ordinal[int(schedule_row["scheduleOrdinal"])] = ( legacy_shard_root if legacy_has_work else scoped_root ) boundary_evidence: set[str] = set() for schedule_row in schedule_rows: source_id = str(schedule_row["sourceId"]) for candidate_root in { legacy_shard_root, shard_root_by_source[source_id], }: observed_boundary = ( _full_payload_packed_layout_token_boundary_evidence( candidate_root, schedule_row, schedule_sha256=schedule_sha256, ) ) if observed_boundary is not None: boundary_evidence.add(observed_boundary) if len(boundary_evidence) > 1: raise RuntimeError( "full payload packed selected collection mixes immutable " "token-boundary authorities" ) token_boundary_contract = ( next(iter(boundary_evidence)) if boundary_evidence else FULL_PAYLOAD_PACKED_EXTENT_TOKEN_BOUNDARY_CONTRACT ) resolved_scratch_root = ( scratch_root.expanduser().resolve() if scratch_root is not None else None ) worker_count = min( len(schedule_rows), FULL_PAYLOAD_PACKED_MAX_WORKERS, ) jobs = [ _FullPayloadPackedShardJob( schedule_receipt_path=schedule_receipt_path.expanduser().resolve(), schedule_path=schedule_path, schedule_sha256=schedule_sha256, schedule_row=schedule_rows[ordinal], hash_row=hash_rows[ordinal], corpus_root=resolved_corpus_root, shard_root=shard_root_by_ordinal[ int(schedule_rows[ordinal]["scheduleOrdinal"]) ], tokenizer_authority=dict(tokenizer_record), tokenizer_vocabulary_size=tokenizer_vocabulary_size, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, token_boundary_contract=token_boundary_contract, scratch_root=( resolved_scratch_root / source_authority_by_id[ str(schedule_rows[ordinal]["sourceId"]) ][1] / f"{int(schedule_rows[ordinal]['scheduleOrdinal']):08d}" if resolved_scratch_root is not None else None ), proven_device_remaps=proven_device_remaps, ) for ordinal in range(len(schedule_rows)) ] process_results = _run_full_payload_packed_shard_process_jobs( tokenizer, jobs, worker_count=worker_count, ) manifests = [result.manifest for result in process_results] if [ int(manifest["scheduleOrdinal"]) for manifest in manifests ] != [ int(row["scheduleOrdinal"]) for row in schedule_rows ]: raise RuntimeError("full payload packed process result order differs") cursor_end = 0 shard_records: list[dict[str, Any]] = [] for manifest_row in manifests: cursor_start = cursor_end cursor_end += int(manifest_row["rows"]) shard_records.append( { "scheduleOrdinal": manifest_row["scheduleOrdinal"], "payloadWorkId": manifest_row["payloadWorkId"], "sourceId": manifest_row["sourceId"], "domainAuthoritySha256": manifest_row["domainAuthoritySha256"], "rightsAuthoritySha256": manifest_row["rightsAuthoritySha256"], "cursorStart": cursor_start, "cursorEnd": cursor_end, "rows": manifest_row["rows"], "manifest": _packed_artifact( shard_root_by_ordinal[ int(manifest_row["scheduleOrdinal"]) ] / ( f"{int(manifest_row['scheduleOrdinal']):08d}_" f"{str(manifest_row['payloadWorkId'])[:16]}.manifest.json" ) ), "shardAuthoritySha256": manifest_row[ "shardAuthoritySha256" ], } ) if cursor_end < 1: raise RuntimeError("full payload packed collection is empty") source_records: list[dict[str, Any]] = [] for source_id in selected_source_ids: source_manifests = [ manifest_row for manifest_row in manifests if manifest_row["sourceId"] == source_id ] source_authority_record, source_authority_sha256 = ( source_authority_by_id[source_id] ) source_receipt = { "schema": FULL_PAYLOAD_PACKED_TOKEN_SOURCE_SCHEMA, "passed": True, "sourceAuthority": source_authority_record, "sourceAuthoritySha256": source_authority_sha256, "sourceId": source_id, "scheduleSha256": schedule_sha256, "tokenizerAuthoritySha256": tokenizer_sha256, "tokenBoundaryContract": token_boundary_contract, "payloadWorkCount": len(source_manifests), "rows": sum(int(row["rows"]) for row in source_manifests), "recordCount": sum(int(row["recordCount"]) for row in source_manifests), "observedRecordCount": sum( int(row["observedRecordCount"]) for row in source_manifests ), "tokenElements": sum( int(row["tokenElements"]) for row in source_manifests ), "promptTokenElements": sum( int(row["promptTokenElements"]) for row in source_manifests ), "answerTokenElements": sum( int(row["answerTokenElements"]) for row in source_manifests ), "allSelectedPayloadWorkComplete": True, "targetEnteredForward": False, } source_receipt["sourceReceiptSha256"] = _sha256_bytes( _json_bytes(source_receipt) ) source_receipt_path = ( shard_root_by_source[source_id].parent / "source.receipt.json" ) if source_receipt_path.is_file(): existing_source = json.loads( source_receipt_path.read_text(encoding="utf-8") ) if existing_source != source_receipt: raise RuntimeError( "full payload packed source receipt already differs" ) else: _atomic_json(source_receipt_path, source_receipt) source_records.append( { "sourceId": source_id, "sourceAuthoritySha256": source_authority_sha256, "receipt": _packed_artifact(source_receipt_path), "payloadWorkCount": source_receipt["payloadWorkCount"], "rows": source_receipt["rows"], "tokenElements": source_receipt["tokenElements"], } ) collection = { "schema": FULL_PAYLOAD_PACKED_TOKEN_COLLECTION_SCHEMA, "passed": True, "scheduleReceipt": _packed_artifact( schedule_receipt_path.expanduser().resolve() ), "schedule": _packed_artifact(schedule_path), "scheduleSha256": schedule_sha256, "corpusRoot": str(corpus_root.expanduser().resolve()), "hashLedger": _packed_artifact(hash_ledger_path.expanduser().resolve()), "tokenizerAuthority": tokenizer_record, "tokenizerAuthoritySha256": tokenizer_sha256, "tokenizerVocabularySize": tokenizer_vocabulary_size, "contextWindowTokens": context_window_tokens, "answerTokensPerWindow": answer_tokens_per_window, "tokenBoundaryContract": token_boundary_contract, "selectedSourceIds": list(selected_source_ids), "sources": source_records, "sourceCount": len(source_records), "shards": shard_records, "shardCount": len(shard_records), "rows": cursor_end, "recordCount": sum(int(row["recordCount"]) for row in manifests), "observedRecordCount": sum( int(row["observedRecordCount"]) for row in manifests ), "tokenElements": sum(int(row["tokenElements"]) for row in manifests), "promptTokenElements": sum( int(row["promptTokenElements"]) for row in manifests ), "answerTokenElements": sum( int(row["answerTokenElements"]) for row in manifests ), "globalCursorStart": 0, "globalCursorEnd": cursor_end, "exactDisjointShardWindows": True, "mmapReadable": True, "compressedTokenRandomAccess": True, "rawPayloadParsedOnlyDuringBuild": True, "targetValuesRecorded": False, "targetEnteredForward": False, } if storage_admission_record is not None: collection["storageAdmission"] = storage_admission_record collection["collectionAuthoritySha256"] = _sha256_bytes( _json_bytes(collection) ) if output_path.is_file(): existing = json.loads(output_path.read_text(encoding="utf-8")) if not isinstance(existing, dict): raise RuntimeError("full payload packed collection receipt is malformed") if existing == collection: return dict(existing) if ( existing.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_COLLECTION_SCHEMA or existing.get("passed") is not True or _full_payload_packed_collection_semantic_projection(existing) != _full_payload_packed_collection_semantic_projection( collection ) ): raise RuntimeError("full payload packed collection receipt differs") existing_admission = existing.get("storageAdmission") collection_admission = collection.get("storageAdmission") if ( isinstance(existing_admission, dict) and isinstance(collection_admission, dict) and existing_admission.get("storageAdmissionSha256") != collection_admission.get("storageAdmissionSha256") ): # The semantic collection is identical, but the earlier build is # the authority of record other receipts may already bind. # Preserve its receipt rather than rewriting an equivalent # collection under a fresh admission snapshot. return dict(existing) _atomic_json(output_path, collection) return collection _atomic_json(output_path, collection) return collection def seal_completed_full_payload_packed_token_cohort( schedule_receipt_path: Path, hash_ledger_path: Path, packed_root: Path, output_receipt_path: Path, *, prior_cohort_receipt_paths: Sequence[Path] = (), ) -> dict[str, Any]: """Seal newly completed packed WorkIDs without waiting for the full source. This boundary never opens raw payloads and never invokes a tokenizer. A cohort contains only independently self-sealed compressed token manifests whose schedule/hash identities are already durable. Prior cohorts are explicit exclusion authorities so later accepted unions can reject replay. """ receipt_path = schedule_receipt_path.expanduser().resolve() ledger_path = hash_ledger_path.expanduser().resolve() root = packed_root.expanduser().resolve() output_path = output_receipt_path.expanduser().resolve() if not receipt_path.is_file() or not ledger_path.is_file() or not root.is_dir(): raise FileNotFoundError( "full payload packed cohort schedule, ledger, or packed root is absent" ) schedule_receipt = json.loads(receipt_path.read_text(encoding="utf-8")) schedule_record = ( schedule_receipt.get("schedule") if isinstance(schedule_receipt, dict) else None ) if ( not isinstance(schedule_receipt, dict) or schedule_receipt.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA or schedule_receipt.get("passed") is not True or not isinstance(schedule_record, dict) ): raise ValueError("full payload packed cohort schedule did not pass") schedule_path = Path( str(schedule_record.get("path", "")) ).expanduser().resolve() schedule_sha256 = schedule_record.get("sha256") scheduled_rows = schedule_record.get("rows") if ( not schedule_path.is_file() or not isinstance(schedule_sha256, str) or len(schedule_sha256) != 64 or file_sha256(schedule_path) != schedule_sha256 or not isinstance(scheduled_rows, int) or isinstance(scheduled_rows, bool) or scheduled_rows < 1 ): raise ValueError("full payload packed cohort schedule differs") schedule_rows = _read_jsonl(schedule_path) if len(schedule_rows) != scheduled_rows: raise ValueError("full payload packed cohort schedule row count differs") schedule_by_work_id: dict[str, dict[str, Any]] = {} for ordinal, row in enumerate(schedule_rows): work_id = row.get("payloadWorkId") if ( row.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_ENTRY_SCHEMA or row.get("scheduleOrdinal") != ordinal or not isinstance(work_id, str) or len(work_id) != 64 or work_id in schedule_by_work_id or row.get("rightsDisposition") != "training_admissible" or row.get("targetEnteredForward") is not False ): raise ValueError("full payload packed cohort schedule row differs") schedule_by_work_id[work_id] = row durable_hash_rows, _durable_hash_frontier = ( _read_durable_full_payload_hash_rows( ledger_path, schedule_sha256, ) ) hash_by_work_id: dict[str, dict[str, Any]] = {} for row in durable_hash_rows: work_id = row.get("payloadWorkId") if work_id not in schedule_by_work_id: raise ValueError("full payload packed cohort hash WorkID differs") schedule_row = schedule_by_work_id[str(work_id)] if ( row.get("schema") != FULL_PAYLOAD_HASH_LEDGER_SCHEMA or row.get("scheduleSha256") != schedule_sha256 or row.get("sourceId") != schedule_row.get("sourceId") or row.get("payloadRelativePath") != schedule_row.get("payloadRelativePath") or row.get("payloadBytes") != schedule_row.get("payloadBytes") or row.get("allPayloadBytesHashed") is not True or row.get("expectedSha256Matched") not in {None, True} or not isinstance(row.get("observedSha256"), str) or len(str(row["observedSha256"])) != 64 or work_id in hash_by_work_id ): raise ValueError("full payload packed cohort hash row differs") hash_by_work_id[str(work_id)] = row prior_work_ids: set[str] = set() prior_records: list[dict[str, Any]] = [] prior_tokenizer_sha256: str | None = None prior_context_window: int | None = None prior_answer_window: int | None = None prior_token_boundary_contract: str | None = None visited_prior_paths: set[Path] = set() visiting_prior_paths: set[Path] = set() prior_authorities: set[str] = set() prior_closure_by_path: dict[Path, frozenset[str]] = {} prior_binding_by_path: dict[Path, tuple[str, int, str]] = {} def visit_prior_cohort( raw_prior_path: Path, *, expected_record: dict[str, Any] | None = None, ) -> frozenset[str]: """Validate and flatten one complete prior-cohort lineage DAG.""" nonlocal prior_tokenizer_sha256 nonlocal prior_context_window nonlocal prior_answer_window nonlocal prior_token_boundary_contract prior_path = raw_prior_path.expanduser().resolve() if expected_record is not None: expected_receipt = expected_record.get("receipt") if ( not isinstance(expected_receipt, dict) or not _full_payload_packed_artifact_matches(expected_receipt) or Path(str(expected_receipt.get("path", ""))) .expanduser() .resolve() != prior_path or expected_receipt.get("sha256") != file_sha256(prior_path) ): raise RuntimeError( "prior full payload packed cohort lineage receipt differs" ) if prior_path in visiting_prior_paths: raise RuntimeError("prior full payload packed cohort lineage cycles") if prior_path in visited_prior_paths: if expected_record is not None: authority, work_count, work_ids_sha256 = ( prior_binding_by_path[prior_path] ) if ( expected_record.get("collectionAuthoritySha256") != authority or expected_record.get("payloadWorkCount") != work_count or expected_record.get("payloadWorkIdsSha256") != work_ids_sha256 ): raise RuntimeError( "prior full payload packed cohort lineage binding differs" ) return prior_closure_by_path[prior_path] prior = json.loads(prior_path.read_text(encoding="utf-8")) prior_authority = ( prior.pop("collectionAuthoritySha256", None) if isinstance(prior, dict) else None ) prior_ids = prior.get("payloadWorkIds") if isinstance(prior, dict) else None nested_records = ( prior.get("priorCohorts") if isinstance(prior, dict) else None ) if ( not isinstance(prior, dict) or prior.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_COHORT_SCHEMA or prior.get("passed") is not True or not isinstance(prior_authority, str) or len(prior_authority) != 64 or prior_authority != _sha256_bytes(_json_bytes(prior)) or prior.get("scheduleSha256") != schedule_sha256 or prior.get("completeSourceScheduleClaimed") is not False or prior.get("globalDatasetTrainingClaimed") is not False or prior.get("targetEnteredForward") is not False or not isinstance(prior_ids, list) or not prior_ids or len(prior_ids) != len(set(prior_ids)) or any( not isinstance(work_id, str) or len(work_id) != 64 or work_id not in schedule_by_work_id for work_id in prior_ids ) or prior.get("payloadWorkIdsSha256") != _sha256_bytes(_json_bytes({"payloadWorkIds": prior_ids})) or not isinstance(nested_records, list) or prior.get("priorCohortCount") != len(nested_records) ): raise RuntimeError("prior full payload packed cohort authority differs") if expected_record is not None and ( expected_record.get("collectionAuthoritySha256") != prior_authority or expected_record.get("payloadWorkCount") != len(prior_ids) or expected_record.get("payloadWorkIdsSha256") != prior["payloadWorkIdsSha256"] ): raise RuntimeError( "prior full payload packed cohort lineage binding differs" ) if prior_authority in prior_authorities: raise RuntimeError( "prior full payload packed cohort authority is duplicated" ) visiting_prior_paths.add(prior_path) lineage_ids: set[str] = set() for nested_record in nested_records: if not isinstance(nested_record, dict): raise RuntimeError( "prior full payload packed cohort lineage is malformed" ) nested_receipt = nested_record.get("receipt") if not isinstance(nested_receipt, dict): raise RuntimeError( "prior full payload packed cohort lineage is malformed" ) nested_path = Path( str(nested_receipt.get("path", "")) ).expanduser().resolve() nested_ids = visit_prior_cohort( nested_path, expected_record=nested_record, ) lineage_ids.update(nested_ids) own_ids = {str(work_id) for work_id in prior_ids} if ( lineage_ids.intersection(own_ids) or prior_work_ids.intersection(own_ids) ): raise RuntimeError("prior full payload packed cohort authority differs") lineage_ids.update(own_ids) if ( prior.get("lineagePayloadWorkCount") != len(lineage_ids) or prior.get("lineagePayloadWorkIdsSha256") != _sha256_bytes(_json_bytes(sorted(lineage_ids))) ): raise RuntimeError( "prior full payload packed cohort lineage authority differs" ) prior_collection_tokenizer_sha256 = prior.get( "tokenizerAuthoritySha256" ) context_window = prior.get("contextWindowTokens") answer_window = prior.get("answerTokensPerWindow") prior_boundary = prior.get( "tokenBoundaryContract", FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT, ) if ( not isinstance(prior_collection_tokenizer_sha256, str) or len(prior_collection_tokenizer_sha256) != 64 or not isinstance(context_window, int) or isinstance(context_window, bool) or context_window < 3 or not isinstance(answer_window, int) or isinstance(answer_window, bool) or answer_window < 1 or answer_window >= context_window or prior_boundary not in FULL_PAYLOAD_PACKED_SUPPORTED_TOKEN_BOUNDARY_CONTRACTS or ( prior_tokenizer_sha256 is not None and prior_tokenizer_sha256 != prior_collection_tokenizer_sha256 ) or ( prior_context_window is not None and prior_context_window != context_window ) or ( prior_answer_window is not None and prior_answer_window != answer_window ) or ( prior_token_boundary_contract is not None and prior_token_boundary_contract != prior_boundary ) ): raise RuntimeError("prior full payload packed cohort geometry differs") prior_tokenizer_sha256 = prior_collection_tokenizer_sha256 prior_context_window = context_window prior_answer_window = answer_window prior_token_boundary_contract = str(prior_boundary) prior_work_ids.update(own_ids) prior_authorities.add(prior_authority) prior["collectionAuthoritySha256"] = prior_authority prior_records.append( { "receipt": _packed_artifact(prior_path), "collectionAuthoritySha256": prior_authority, "payloadWorkCount": len(prior_ids), "payloadWorkIdsSha256": prior["payloadWorkIdsSha256"], } ) visiting_prior_paths.remove(prior_path) visited_prior_paths.add(prior_path) closure = frozenset(lineage_ids) prior_closure_by_path[prior_path] = closure prior_binding_by_path[prior_path] = ( prior_authority, len(prior_ids), str(prior["payloadWorkIdsSha256"]), ) return closure for raw_prior_path in prior_cohort_receipt_paths: visit_prior_cohort(raw_prior_path) candidate_paths: dict[str, list[Path]] = {} scanned_manifest_count = 0 for manifest_path in sorted(root.rglob("*.manifest.json"), key=str): scanned_manifest_count += 1 try: value = json.loads(manifest_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue if ( not isinstance(value, dict) or value.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_SHARD_SCHEMA or value.get("passed") is not True or value.get("scheduleSha256") != schedule_sha256 ): continue work_id = value.get("payloadWorkId") if not isinstance(work_id, str) or work_id not in schedule_by_work_id: raise RuntimeError("full payload packed cohort manifest WorkID differs") candidate_paths.setdefault(work_id, []).append(manifest_path) candidate_work_ids = [ work_id for work_id in candidate_paths if work_id not in prior_work_ids ] candidate_work_ids.sort( key=lambda work_id: int(schedule_by_work_id[work_id]["scheduleOrdinal"]) ) manifests: list[tuple[Path, dict[str, Any], dict[str, Any]]] = [] tokenizer_record: dict[str, Any] | None = None tokenizer_sha256: str | None = None tokenizer_vocabulary_size: int | None = None context_window_tokens: int | None = None answer_tokens_per_window: int | None = None token_boundary_contract: str | None = None ready_candidates: dict[str, list[dict[str, Any]]] = {} skipped_non_training_ready_manifest_count = 0 for work_id in candidate_work_ids: hash_row = hash_by_work_id.get(work_id) for manifest_path in candidate_paths[work_id]: try: raw_manifest = json.loads( manifest_path.read_text(encoding="utf-8") ) manifest_tokenizer = ( raw_manifest.get("tokenizerAuthority") if isinstance(raw_manifest, dict) else None ) if not isinstance(manifest_tokenizer, dict): raise RuntimeError( "full payload packed cohort tokenizer is absent" ) ( observed_tokenizer_record, observed_tokenizer_sha256, observed_vocabulary_size, ) = _validated_full_payload_fastokens_authority( manifest_tokenizer ) observed_context = raw_manifest.get("contextWindowTokens") observed_answer = raw_manifest.get("answerTokensPerWindow") if ( not isinstance(observed_context, int) or isinstance(observed_context, bool) or observed_context < 3 or not isinstance(observed_answer, int) or isinstance(observed_answer, bool) or observed_answer < 1 or observed_answer >= observed_context or hash_row is None ): raise RuntimeError( "full payload packed cohort manifest is not training ready" ) manifest = _validated_legacy_full_payload_packed_manifest( manifest_path, schedule_row=schedule_by_work_id[work_id], hash_row=hash_row, tokenizer_authority_sha256=observed_tokenizer_sha256, context_window_tokens=observed_context, answer_tokens_per_window=observed_answer, ) if ( manifest.get("scheduleOrdinal") != schedule_by_work_id[work_id]["scheduleOrdinal"] or manifest.get("hashLedgerRowSha256") != _sha256_bytes(_json_bytes(hash_row)) or set(manifest.get("artifacts", {})) != _full_payload_packed_required_artifact_names( manifest ) or manifest.get("rawTokenBytesRetained") is not False or manifest.get("rawMetadataBytesRetained") is not False or manifest.get("compressedTokenRandomAccess") is not True or manifest.get("compressedMetadataRandomAccess") is not True or not _full_payload_packed_manifest_owns_artifacts( manifest_path, manifest, ) ): raise RuntimeError( "full payload packed cohort manifest is not training ready" ) except ( OSError, json.JSONDecodeError, KeyError, RuntimeError, TypeError, ValueError, ): skipped_non_training_ready_manifest_count += 1 continue ready_candidates.setdefault(work_id, []).append( { "path": manifest_path, "manifest": manifest, "tokenizerRecord": observed_tokenizer_record, "tokenizerSha256": observed_tokenizer_sha256, "tokenizerVocabularySize": observed_vocabulary_size, "contextWindowTokens": observed_context, "answerTokensPerWindow": observed_answer, "contentIdentitySha256": ( _full_payload_packed_manifest_content_identity_sha256( manifest ) ), } ) selected_candidates: list[dict[str, Any]] = [] deduplicated_exact_manifest_count = 0 for work_id in candidate_work_ids: candidates = ready_candidates.get(work_id, []) if not candidates: continue content_identities = { str(candidate["contentIdentitySha256"]) for candidate in candidates } if len(content_identities) != 1: raise RuntimeError( "full payload packed cohort has conflicting duplicate " "manifest authority" ) deduplicated_exact_manifest_count += len(candidates) - 1 selected_candidates.append(candidates[0]) if not selected_candidates: raise RuntimeError("no newly completed full payload packed cohort exists") for candidate in selected_candidates: manifest_path = cast(Path, candidate["path"]) manifest = cast(dict[str, Any], candidate["manifest"]) observed_tokenizer_record = cast( dict[str, Any], candidate["tokenizerRecord"], ) observed_tokenizer_sha256 = str(candidate["tokenizerSha256"]) observed_vocabulary_size = int( candidate["tokenizerVocabularySize"] ) observed_context = int(candidate["contextWindowTokens"]) observed_answer = int(candidate["answerTokensPerWindow"]) observed_boundary = manifest.get( "tokenBoundaryContract", FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT, ) if ( ( tokenizer_sha256 is not None and tokenizer_sha256 != observed_tokenizer_sha256 ) or ( context_window_tokens is not None and context_window_tokens != observed_context ) or ( answer_tokens_per_window is not None and answer_tokens_per_window != observed_answer ) or ( prior_tokenizer_sha256 is not None and prior_tokenizer_sha256 != observed_tokenizer_sha256 ) or ( prior_context_window is not None and prior_context_window != observed_context ) or ( prior_answer_window is not None and prior_answer_window != observed_answer ) or observed_boundary not in FULL_PAYLOAD_PACKED_SUPPORTED_TOKEN_BOUNDARY_CONTRACTS or ( token_boundary_contract is not None and token_boundary_contract != observed_boundary ) or ( prior_token_boundary_contract is not None and prior_token_boundary_contract != observed_boundary ) ): raise RuntimeError("full payload packed cohort geometry differs") tokenizer_record = observed_tokenizer_record tokenizer_sha256 = observed_tokenizer_sha256 tokenizer_vocabulary_size = observed_vocabulary_size context_window_tokens = observed_context answer_tokens_per_window = observed_answer token_boundary_contract = str(observed_boundary) work_id = str(manifest["payloadWorkId"]) manifests.append( (manifest_path, manifest, schedule_by_work_id[work_id]) ) if ( tokenizer_record is None or tokenizer_sha256 is None or tokenizer_vocabulary_size is None or context_window_tokens is None or answer_tokens_per_window is None or token_boundary_contract is None ): raise RuntimeError("full payload packed cohort geometry disappeared") cursor_end = 0 shard_records: list[dict[str, Any]] = [] for local_ordinal, (manifest_path, manifest, schedule_row) in enumerate( manifests ): cursor_start = cursor_end cursor_end += int(manifest["rows"]) shard_records.append( { "scheduleOrdinal": local_ordinal, "sourceScheduleOrdinal": schedule_row["scheduleOrdinal"], "payloadWorkId": manifest["payloadWorkId"], "sourceId": manifest["sourceId"], "domainAuthoritySha256": manifest[ "domainAuthoritySha256" ], "rightsAuthoritySha256": manifest[ "rightsAuthoritySha256" ], "cursorStart": cursor_start, "cursorEnd": cursor_end, "rows": manifest["rows"], "manifest": _packed_artifact(manifest_path), "shardAuthoritySha256": manifest[ "shardAuthoritySha256" ], } ) payload_work_ids = [ str(record["payloadWorkId"]) for record in shard_records ] lineage_work_ids = sorted(prior_work_ids.union(payload_work_ids)) selected_payload_bytes = sum( int(schedule_by_work_id[work_id]["payloadBytes"]) for work_id in payload_work_ids ) selected_hash_row_sha256s = [ _sha256_bytes(_json_bytes(hash_by_work_id[work_id])) for work_id in payload_work_ids ] cohort = { "schema": FULL_PAYLOAD_PACKED_TOKEN_COHORT_SCHEMA, "passed": True, "scheduleReceipt": _packed_artifact(receipt_path), "schedule": _packed_artifact(schedule_path), "scheduleSha256": schedule_sha256, "hashLedger": { "path": str(ledger_path), "selectedRows": len(selected_hash_row_sha256s), "selectedRowSha256sSha256": _sha256_bytes( _json_bytes(selected_hash_row_sha256s) ), "appendMayContinueAfterCohortSeal": True, }, "packedRoot": str(root), "tokenizerAuthority": tokenizer_record, "tokenizerAuthoritySha256": tokenizer_sha256, "tokenizerVocabularySize": tokenizer_vocabulary_size, "contextWindowTokens": context_window_tokens, "answerTokensPerWindow": answer_tokens_per_window, "tokenBoundaryContract": token_boundary_contract, "selectedSourceIds": sorted( {str(manifest["sourceId"]) for _, manifest, _ in manifests} ), "shards": shard_records, "shardCount": len(shard_records), "rows": cursor_end, "recordCount": sum( int(manifest["recordCount"]) for _, manifest, _ in manifests ), "observedRecordCount": sum( int(manifest["observedRecordCount"]) for _, manifest, _ in manifests ), "tokenElements": sum( int(manifest["tokenElements"]) for _, manifest, _ in manifests ), "promptTokenElements": sum( int(manifest["promptTokenElements"]) for _, manifest, _ in manifests ), "answerTokenElements": sum( int(manifest["answerTokenElements"]) for _, manifest, _ in manifests ), "selectedPayloadWorkCount": len(payload_work_ids), "selectedPayloadBytes": selected_payload_bytes, "payloadWorkIds": payload_work_ids, "payloadWorkIdsSha256": _sha256_bytes( _json_bytes({"payloadWorkIds": payload_work_ids}) ), "sourceSchedulePayloadFileCount": len(schedule_rows), "sourceSchedulePayloadBytes": sum( int(row["payloadBytes"]) for row in schedule_rows ), "priorCohorts": prior_records, "priorCohortCount": len(prior_records), "priorPayloadWorkCount": len(prior_work_ids), "lineagePayloadWorkCount": len(lineage_work_ids), "lineagePayloadWorkIdsSha256": _sha256_bytes( _json_bytes(lineage_work_ids) ), "scannedPackedManifestCount": scanned_manifest_count, "candidateManifestCount": sum( len(candidate_paths[work_id]) for work_id in candidate_work_ids ), "skippedNonTrainingReadyManifestCount": ( skipped_non_training_ready_manifest_count ), "deduplicatedExactManifestCount": ( deduplicated_exact_manifest_count ), "globalCursorStart": 0, "globalCursorEnd": cursor_end, "exactDisjointShardWindows": True, "completeSelectedCohortSealed": True, "completeSourceScheduleClaimed": False, "completePayloadCollectionClaimed": False, "globalDatasetTrainingClaimed": False, "mmapReadable": True, "compressedTokenRandomAccess": True, "rawPayloadParsedOnlyDuringBuild": True, "rawPayloadRequiredAtTraining": False, "rawPayloadReadersInvoked": False, "tokenizerInvoked": False, "targetValuesRecorded": False, "targetEnteredForward": False, "promotionEligible": False, "rawDataDeletionAllowed": False, } cohort["collectionAuthoritySha256"] = _sha256_bytes( _json_bytes(cohort) ) output_path.parent.mkdir(parents=True, exist_ok=True) if output_path.is_file(): existing = json.loads(output_path.read_text(encoding="utf-8")) if existing != cohort: raise RuntimeError( "full payload packed cohort receipt already differs" ) return dict(existing) _atomic_json(output_path, cohort) return cohort def _validated_full_payload_packed_ready_component( receipt_path: Path, ) -> tuple[dict[str, Any], dict[str, Any]]: """Validate one sealed packed authority using metadata artifacts only.""" resolved = receipt_path.expanduser().resolve() if not resolved.is_file(): raise FileNotFoundError("full payload packed-ready component is absent") loaded_value = json.loads(resolved.read_text(encoding="utf-8")) recorded_authority = ( loaded_value.pop("collectionAuthoritySha256", None) if isinstance(loaded_value, dict) else None ) schema = ( loaded_value.get("schema") if isinstance(loaded_value, dict) else None ) incremental_cohort = schema == FULL_PAYLOAD_PACKED_TOKEN_COHORT_SCHEMA if ( not isinstance(loaded_value, dict) or schema not in { FULL_PAYLOAD_PACKED_TOKEN_COLLECTION_SCHEMA, FULL_PAYLOAD_PACKED_TOKEN_COHORT_SCHEMA, } or loaded_value.get("passed") is not True or not isinstance(recorded_authority, str) or len(recorded_authority) != 64 or recorded_authority != _sha256_bytes(_json_bytes(loaded_value)) or loaded_value.get("exactDisjointShardWindows") is not True or loaded_value.get("mmapReadable") is not True or loaded_value.get("compressedTokenRandomAccess") is not True or loaded_value.get("rawPayloadParsedOnlyDuringBuild") is not True or loaded_value.get("rawPayloadRequiredAtTraining") not in {None, False} or loaded_value.get("targetValuesRecorded") is not False or loaded_value.get("targetEnteredForward") is not False or ( incremental_cohort and ( loaded_value.get("completeSelectedCohortSealed") is not True or loaded_value.get("completeSourceScheduleClaimed") is not False or loaded_value.get("completePayloadCollectionClaimed") is not False or loaded_value.get("globalDatasetTrainingClaimed") is not False or loaded_value.get("promotionEligible") is not False ) ) ): raise ValueError("full payload packed-ready component authority differs") schedule_receipt = loaded_value.get("scheduleReceipt") schedule = loaded_value.get("schedule") hash_ledger = loaded_value.get("hashLedger") if ( not _full_payload_packed_artifact_matches(schedule_receipt) or not _full_payload_packed_artifact_matches(schedule) or ( not incremental_cohort and not _full_payload_packed_artifact_matches(hash_ledger) ) or ( incremental_cohort and ( not isinstance(hash_ledger, dict) or type(hash_ledger.get("selectedRows")) is not int or hash_ledger.get("appendMayContinueAfterCohortSeal") is not True ) ) ): raise ValueError("full payload packed-ready component binding differs") boundary_contract = loaded_value.get("tokenBoundaryContract") if boundary_contract is None: boundary_contract = FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT loaded_value["tokenBoundaryContract"] = boundary_contract if ( boundary_contract not in FULL_PAYLOAD_PACKED_SUPPORTED_TOKEN_BOUNDARY_CONTRACTS ): raise ValueError("full payload packed-ready token boundary differs") tokenizer_authority = loaded_value.get("tokenizerAuthority") if not isinstance(tokenizer_authority, dict): raise ValueError("full payload packed-ready tokenizer authority is absent") ( tokenizer_record, tokenizer_sha256, tokenizer_vocabulary_size, ) = _validated_full_payload_fastokens_authority(tokenizer_authority) context_window_tokens = loaded_value.get("contextWindowTokens") answer_tokens_per_window = loaded_value.get("answerTokensPerWindow") schedule_sha256 = loaded_value.get("scheduleSha256") rows = loaded_value.get("rows") shards = loaded_value.get("shards") if ( loaded_value.get("tokenizerAuthoritySha256") != tokenizer_sha256 or loaded_value.get("tokenizerVocabularySize") != tokenizer_vocabulary_size or not isinstance(context_window_tokens, int) or isinstance(context_window_tokens, bool) or context_window_tokens < 3 or not isinstance(answer_tokens_per_window, int) or isinstance(answer_tokens_per_window, bool) or answer_tokens_per_window < 1 or answer_tokens_per_window >= context_window_tokens or not isinstance(schedule_sha256, str) or len(schedule_sha256) != 64 or type(rows) is not int or rows < 1 or loaded_value.get("globalCursorStart") != 0 or loaded_value.get("globalCursorEnd") != rows or not isinstance(shards, list) or not shards or loaded_value.get("shardCount") != len(shards) ): raise ValueError("full payload packed-ready component geometry differs") work_ids: list[str] = [] expected_cursor = 0 prior_source_ordinal = -1 record_count = 0 token_elements = 0 prompt_token_elements = 0 answer_token_elements = 0 source_bytes = 0 for ordinal, raw_shard in enumerate(shards): if not isinstance(raw_shard, dict): raise ValueError("full payload packed-ready shard is malformed") schedule_ordinal = raw_shard.get("scheduleOrdinal") work_id = raw_shard.get("payloadWorkId") source_id = raw_shard.get("sourceId") cursor_start = raw_shard.get("cursorStart") cursor_end = raw_shard.get("cursorEnd") shard_rows = raw_shard.get("rows") shard_authority = raw_shard.get("shardAuthoritySha256") source_ordinal = raw_shard.get("sourceScheduleOrdinal") manifest_record = raw_shard.get("manifest") if ( not isinstance(work_id, str) or len(work_id) != 64 or work_id in work_ids or not isinstance(source_id, str) or not source_id or type(cursor_start) is not int or type(cursor_end) is not int or type(shard_rows) is not int or cursor_start != expected_cursor or cursor_end <= cursor_start or shard_rows != cursor_end - cursor_start or not isinstance(shard_authority, str) or len(shard_authority) != 64 or not _full_payload_packed_artifact_matches(manifest_record) or ( incremental_cohort and ( type(schedule_ordinal) is not int or schedule_ordinal != ordinal or type(source_ordinal) is not int or source_ordinal <= prior_source_ordinal ) ) or ( not incremental_cohort and ( type(schedule_ordinal) is not int or schedule_ordinal <= prior_source_ordinal or source_ordinal is not None ) ) ): raise ValueError("full payload packed-ready shard authority differs") assert isinstance(manifest_record, dict) manifest_path = Path( str(manifest_record["path"]) ).expanduser().resolve() manifest_value = json.loads(manifest_path.read_text(encoding="utf-8")) manifest_authority = ( manifest_value.pop("shardAuthoritySha256", None) if isinstance(manifest_value, dict) else None ) manifest_boundary = ( manifest_value.get("tokenBoundaryContract") if isinstance(manifest_value, dict) else None ) if manifest_boundary is None: manifest_boundary = FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT manifest_metrics = ( ( manifest_value.get("recordCount"), manifest_value.get("tokenElements"), manifest_value.get("promptTokenElements"), manifest_value.get("answerTokenElements"), manifest_value.get("payloadBytes"), ) if isinstance(manifest_value, dict) else () ) if ( not isinstance(manifest_value, dict) or manifest_value.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_SHARD_SCHEMA or manifest_value.get("passed") is not True or not isinstance(manifest_authority, str) or manifest_authority != _sha256_bytes(_json_bytes(manifest_value)) or manifest_authority != shard_authority or manifest_value.get("payloadWorkId") != work_id or manifest_value.get("sourceId") != source_id or manifest_value.get("scheduleOrdinal") != ( source_ordinal if incremental_cohort else schedule_ordinal ) or manifest_value.get("scheduleSha256") != schedule_sha256 or manifest_value.get("rows") != shard_rows or manifest_value.get("tokenizerAuthoritySha256") != tokenizer_sha256 or manifest_value.get("contextWindowTokens") != context_window_tokens or manifest_value.get("answerTokensPerWindow") != answer_tokens_per_window or manifest_boundary != boundary_contract or manifest_value.get("compressedTokenRandomAccess") is not True or manifest_value.get("compressedMetadataRandomAccess") is not True or manifest_value.get("rawTokenBytesRetained") is not False or manifest_value.get("rawMetadataBytesRetained") is not False or len(manifest_metrics) != 5 or any(type(value) is not int or value < 0 for value in manifest_metrics) ): raise ValueError("full payload packed-ready shard manifest differs") ( shard_record_count, shard_token_elements, shard_prompt_token_elements, shard_answer_token_elements, shard_source_bytes, ) = cast(tuple[int, int, int, int, int], manifest_metrics) work_ids.append(work_id) expected_cursor = cursor_end record_count += shard_record_count token_elements += shard_token_elements prompt_token_elements += shard_prompt_token_elements answer_token_elements += shard_answer_token_elements source_bytes += shard_source_bytes canonical_schedule_ordinal = ( source_ordinal if incremental_cohort else schedule_ordinal ) assert isinstance(canonical_schedule_ordinal, int) prior_source_ordinal = canonical_schedule_ordinal if ( expected_cursor != rows or loaded_value.get("recordCount") != record_count or loaded_value.get("tokenElements") != token_elements or loaded_value.get("promptTokenElements") != prompt_token_elements or loaded_value.get("answerTokenElements") != answer_token_elements or ( incremental_cohort and ( loaded_value.get("selectedPayloadWorkCount") != len(work_ids) or loaded_value.get("selectedPayloadBytes") != source_bytes or loaded_value.get("payloadWorkIds") != work_ids or loaded_value.get("payloadWorkIdsSha256") != _sha256_bytes(_json_bytes({"payloadWorkIds": work_ids})) ) ) ): raise ValueError("full payload packed-ready component denominator differs") loaded_value["collectionAuthoritySha256"] = recorded_authority summary = { "componentId": recorded_authority, "collectionSchema": schema, "collectionReceipt": _packed_artifact(resolved), "collectionAuthoritySha256": recorded_authority, "scheduleReceipt": schedule_receipt, "schedule": schedule, "scheduleSha256": schedule_sha256, "hashLedger": hash_ledger, "corpusRoot": loaded_value.get("corpusRoot"), "tokenizerAuthority": tokenizer_record, "tokenizerAuthoritySha256": tokenizer_sha256, "tokenizerVocabularySize": tokenizer_vocabulary_size, "contextWindowTokens": context_window_tokens, "answerTokensPerWindow": answer_tokens_per_window, "tokenBoundaryContract": boundary_contract, "payloadWorkIds": work_ids, "payloadWorkIdsSha256": _sha256_bytes( _json_bytes({"payloadWorkIds": work_ids}) ), "payloadWorkCount": len(work_ids), "rows": rows, "recordCount": record_count, "tokenElements": token_elements, "promptTokenElements": prompt_token_elements, "answerTokenElements": answer_token_elements, "sourceBytes": source_bytes, } return loaded_value, summary def _full_payload_packed_ready_discovery_roots( search_roots: Sequence[Path], ) -> tuple[Path, ...]: """Resolve only namespaces that can contain sealed packed authorities.""" discovered: set[Path] = set() patterns = ( "attempt*/nnf_resynthesis_full_payload_packed*", "nnf_resynthesis_full_payload_packed*", "attempt*/nnf_resynthesis/corpus_derived/*full_payload_packed*", "nnf_resynthesis/corpus_derived/*full_payload_packed*", ) for raw_root in search_roots: root = raw_root.expanduser().resolve() if ( root.is_dir() and "quarantine" not in {part.casefold() for part in root.parts} and ( root.name.startswith("nnf_resynthesis_full_payload_packed") or ( "full_payload_packed" in root.name and root.parent.name == "corpus_derived" ) ) ): discovered.add(root) try: discovered.update( candidate.resolve() for pattern in patterns for candidate in root.glob(pattern) if candidate.is_dir() and "quarantine" not in {part.casefold() for part in candidate.parts} ) except OSError: continue return tuple(sorted(discovered, key=str)) def _full_payload_packed_ready_receipt_paths( search_roots: Sequence[Path], ) -> tuple[Path, ...]: """Discover passed-receipt candidates without opening token payloads.""" paths: set[Path] = set() for root in _full_payload_packed_ready_discovery_roots(search_roots): try: paths.update( path.resolve() for pattern in ( "**/collection.receipt.json", "**/*cohort*.json", ) for path in root.glob(pattern) if path.is_file() and "quarantine" not in {part.casefold() for part in path.parts} ) except OSError: continue return tuple(sorted(paths, key=str)) def _full_payload_packed_manifest_core_identity( manifest: Mapping[str, Any], ) -> dict[str, Any]: """Return the byte/geometry proof shared by legacy and enriched packs. An enriched repack is allowed to improve the domain, rights, boundary, and record-metadata authorities. It is the same trainable token surface only when the raw payload hash, token artifacts, tokenizer, schedule, geometry, and all token/row counts remain exact. """ artifacts = manifest.get("artifacts") token_artifacts = { name: value for name, value in cast( Mapping[str, Any], artifacts if isinstance(artifacts, Mapping) else {}, ).items() if name in {"tokensZstd", "tokenChunkIndex"} } def semantic_artifact(value: Any) -> Any: if isinstance(value, Mapping): artifact = ( isinstance(value.get("path"), str) and isinstance(value.get("bytes"), int) and not isinstance(value.get("bytes"), bool) and isinstance(value.get("sha256"), str) ) return { str(key): semantic_artifact(child) for key, child in value.items() if not (artifact and key in {"path", "fileIdentity"}) } if isinstance(value, list): return [semantic_artifact(child) for child in value] return value scalar_names = ( "schema", "passed", "payloadWorkId", "sourceId", "payloadRelativePath", "payloadBytes", "observedPayloadSha256", "scheduleOrdinal", "scheduleSha256", "hashLedgerRowSha256", "tokenizerAuthoritySha256", "contextWindowTokens", "answerTokensPerWindow", "rows", "recordCount", "observedRecordCount", "tokenElements", "promptTokenElements", "answerTokenElements", "trainingTokenElements", "rawTokenBytesRetained", "compressedTokenRandomAccess", "tokensStoredOncePerRecord", "targetValuesRecorded", "targetEnteredForward", ) return { "scalars": {name: manifest.get(name) for name in scalar_names}, "tokenArtifacts": semantic_artifact(token_artifacts), "tokenCompression": semantic_artifact( manifest.get("tokenCompression") ), } def _full_payload_packed_manifest_authority_rank( manifest: Mapping[str, Any], ) -> int: """Rank only the proven legacy-to-hash-bound authority transition.""" domain = manifest.get("domainAuthority") rights = manifest.get("rightsAuthority") boundary = manifest.get("tokenBoundaryContract") enriched = bool( isinstance(domain, Mapping) and domain.get("derivation") == "hash_bound_admitted_inventory_content_claims_v1" and isinstance(domain.get("sourceRecordSha256"), str) and isinstance(rights, Mapping) and rights.get("derivation") == "hash_bound_admitted_inventory_rights_v1" and rights.get("rightsDisposition") == "training_admissible" and isinstance(rights.get("sourceRecordSha256"), str) and boundary in FULL_PAYLOAD_PACKED_SUPPORTED_TOKEN_BOUNDARY_CONTRACTS ) legacy = bool( isinstance(domain, Mapping) and domain.get("derivation") == "legacy_full_payload_schedule_scope_v1" and isinstance(rights, Mapping) and rights.get("derivation") == "legacy_full_payload_schedule_row_v1" and boundary in { None, FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT, } ) if enriched: return 2 if legacy: return 1 return 0 def _select_full_payload_packed_manifest_authority( candidates: Sequence[dict[str, Any]], ) -> dict[str, Any]: """Select one exact replica or one proven enriched authority upgrade.""" by_content: dict[str, list[dict[str, Any]]] = {} for candidate in candidates: manifest = cast(Mapping[str, Any], candidate["manifest"]) identity = _full_payload_packed_manifest_content_identity_sha256( manifest ) by_content.setdefault(identity, []).append(candidate) if len(by_content) == 1: return min( candidates, key=lambda candidate: ( -_full_payload_packed_manifest_authority_rank( cast(Mapping[str, Any], candidate["manifest"]) ), str(candidate["componentPath"]), str(candidate["manifestPath"]), ), ) core_identities = { _sha256_bytes( _json_bytes( _full_payload_packed_manifest_core_identity( cast(Mapping[str, Any], candidate["manifest"]) ) ) ) for candidate in candidates } ranked = [ ( _full_payload_packed_manifest_authority_rank( cast(Mapping[str, Any], candidate["manifest"]) ), candidate, ) for candidate in candidates ] top_rank = max(rank for rank, _candidate in ranked) top_identities = { _full_payload_packed_manifest_content_identity_sha256( cast(Mapping[str, Any], candidate["manifest"]) ) for rank, candidate in ranked if rank == top_rank } if ( len(core_identities) != 1 or top_rank != 2 or len(top_identities) != 1 or any(rank not in {1, 2} for rank, _candidate in ranked) ): raise RuntimeError( "full payload packed WorkID has unresolved content authorities" ) enriched = [ candidate for rank, candidate in ranked if rank == top_rank ] return min( enriched, key=lambda candidate: ( str(candidate["componentPath"]), str(candidate["manifestPath"]), ), ) def _full_payload_packed_ready_excluded_work_ids( receipt_paths: Sequence[Path], ) -> set[str]: """Load exact WorkID exclusions from prior immutable training receipts.""" excluded: set[str] = set() for raw_path in receipt_paths: path = raw_path.expanduser().resolve() value = json.loads(path.read_text(encoding="utf-8")) if ( isinstance(value, dict) and value.get("schema") == FULL_PAYLOAD_FEDERATED_PACKED_TOKEN_COLLECTION_SCHEMA ): collection, _authority, _components = ( _validated_federated_full_payload_packed_collection( path, validate_native_federation=False, ) ) work_ids = collection.get("payloadWorkIds") else: _component, summary = ( _validated_full_payload_packed_ready_component(path) ) work_ids = summary.get("payloadWorkIds") if ( not isinstance(work_ids, list) or any( not isinstance(work_id, str) or len(work_id) != 64 for work_id in work_ids ) ): raise RuntimeError( "full payload packed exclusion WorkID authority differs" ) excluded.update(cast(list[str], work_ids)) return excluded def _full_payload_packed_ready_exclusion_receipt_records( receipt_paths: Sequence[Path], ) -> list[dict[str, Any]]: """Bind every prior ownership exclusion to exact immutable receipt bytes.""" records: list[dict[str, Any]] = [] for path in sorted( { raw_path.expanduser().resolve() for raw_path in receipt_paths }, key=str, ): before = path.stat() encoded = path.read_bytes() work_ids = sorted( _full_payload_packed_ready_excluded_work_ids((path,)) ) after = path.stat() if ( before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns, ) != ( after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns, ): raise RuntimeError( "full payload packed exclusion receipt changed during " "validation" ) records.append( { "schema": ( "nnf.resynthesis." "full_payload_packed_ready_exclusion_receipt.v1" ), "path": str(path), "bytes": len(encoded), "sha256": _sha256_bytes(encoded), "payloadWorkCount": len(work_ids), "payloadWorkIdsSha256": _sha256_bytes( _json_bytes(work_ids) ), } ) return records def _write_full_payload_packed_ready_component_view( output_root: Path, *, source_component: Mapping[str, Any], selected: Sequence[dict[str, Any]], excluded_work_ids_sha256: str, ) -> Path: """Seal one immutable, artifact-only view of selected source shards.""" ordered = sorted( selected, key=lambda candidate: int( cast(Mapping[str, Any], candidate["manifest"])[ "scheduleOrdinal" ] ), ) cursor = 0 shards: list[dict[str, Any]] = [] manifests: list[Mapping[str, Any]] = [] for local_ordinal, candidate in enumerate(ordered): manifest = cast(Mapping[str, Any], candidate["manifest"]) source_shard = cast(Mapping[str, Any], candidate["shard"]) rows = int(manifest["rows"]) manifests.append(manifest) shards.append( { "scheduleOrdinal": local_ordinal, "sourceScheduleOrdinal": int(manifest["scheduleOrdinal"]), "payloadWorkId": manifest["payloadWorkId"], "sourceId": manifest["sourceId"], "domainAuthoritySha256": manifest.get( "domainAuthoritySha256" ), "rightsAuthoritySha256": manifest.get( "rightsAuthoritySha256" ), "cursorStart": cursor, "cursorEnd": cursor + rows, "rows": rows, "manifest": source_shard["manifest"], "shardAuthoritySha256": manifest[ "shardAuthoritySha256" ], } ) cursor += rows work_ids = [str(manifest["payloadWorkId"]) for manifest in manifests] source_schedule_file_count = source_component.get( "sourceSchedulePayloadFileCount", source_component.get("selectedPayloadWorkCount", len(work_ids)), ) source_schedule_bytes = source_component.get( "sourceSchedulePayloadBytes", source_component.get( "selectedPayloadBytes", sum(int(manifest["payloadBytes"]) for manifest in manifests), ), ) hash_row_sha256s = [ str(manifest["hashLedgerRowSha256"]) for manifest in manifests ] view: dict[str, Any] = { "schema": FULL_PAYLOAD_PACKED_TOKEN_COHORT_SCHEMA, "passed": True, "scheduleReceipt": source_component["scheduleReceipt"], "schedule": source_component["schedule"], "scheduleSha256": source_component["scheduleSha256"], "hashLedger": { "path": str( cast(Mapping[str, Any], source_component["hashLedger"]).get( "path", "", ) ), "selectedRows": len(work_ids), "selectedRowSha256sSha256": _sha256_bytes( _json_bytes(hash_row_sha256s) ), "appendMayContinueAfterCohortSeal": True, }, "packedRoot": source_component.get("packedRoot"), "tokenizerAuthority": source_component["tokenizerAuthority"], "tokenizerAuthoritySha256": source_component[ "tokenizerAuthoritySha256" ], "tokenizerVocabularySize": source_component[ "tokenizerVocabularySize" ], "contextWindowTokens": source_component["contextWindowTokens"], "answerTokensPerWindow": source_component[ "answerTokensPerWindow" ], "tokenBoundaryContract": source_component.get( "tokenBoundaryContract", FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT, ), "selectedSourceIds": sorted( {str(manifest["sourceId"]) for manifest in manifests} ), "shards": shards, "shardCount": len(shards), "rows": cursor, "recordCount": sum( int(manifest["recordCount"]) for manifest in manifests ), "observedRecordCount": sum( int(manifest["observedRecordCount"]) for manifest in manifests ), "tokenElements": sum( int(manifest["tokenElements"]) for manifest in manifests ), "promptTokenElements": sum( int(manifest["promptTokenElements"]) for manifest in manifests ), "answerTokenElements": sum( int(manifest["answerTokenElements"]) for manifest in manifests ), "selectedPayloadWorkCount": len(work_ids), "selectedPayloadBytes": sum( int(manifest["payloadBytes"]) for manifest in manifests ), "payloadWorkIds": work_ids, "payloadWorkIdsSha256": _sha256_bytes( _json_bytes({"payloadWorkIds": work_ids}) ), "sourceSchedulePayloadFileCount": source_schedule_file_count, "sourceSchedulePayloadBytes": source_schedule_bytes, "priorCohorts": [], "priorCohortCount": 0, "priorPayloadWorkCount": 0, "lineagePayloadWorkCount": len(work_ids), "lineagePayloadWorkIdsSha256": _sha256_bytes( _json_bytes(sorted(work_ids)) ), "excludedPriorPayloadWorkIdsSha256": excluded_work_ids_sha256, "globalCursorStart": 0, "globalCursorEnd": cursor, "exactDisjointShardWindows": True, "completeSelectedCohortSealed": True, "completeSourceScheduleClaimed": False, "completePayloadCollectionClaimed": False, "globalDatasetTrainingClaimed": False, "mmapReadable": True, "compressedTokenRandomAccess": True, "rawPayloadParsedOnlyDuringBuild": True, "rawPayloadRequiredAtTraining": False, "rawPayloadReadersInvoked": False, "tokenizerInvoked": False, "targetValuesRecorded": False, "targetEnteredForward": False, "promotionEligible": False, "rawDataDeletionAllowed": False, } view["collectionAuthoritySha256"] = _sha256_bytes(_json_bytes(view)) authority = str(view["collectionAuthoritySha256"]) path = output_root / "components" / f"{authority}.receipt.json" path.parent.mkdir(parents=True, exist_ok=True) if path.is_file(): existing = json.loads(path.read_text(encoding="utf-8")) if existing != view: raise RuntimeError( "full payload packed component view already differs" ) else: _atomic_json(path, view) _validated_full_payload_packed_ready_component(path) return path def compose_full_payload_packed_ready_federation_snapshot( current_pointer_path: Path, search_roots: Sequence[Path], *, exclusion_receipt_paths: Sequence[Path] = (), ) -> dict[str, Any]: """Publish an immutable all-root snapshot and atomically move its locator. Running learners open the snapshot named by the locator once and remain bound to those exact bytes. A later refresh may advance only the small locator; newly sealed WorkIDs therefore enter the next descendant cohort without mutating an in-flight schedule or replaying prior WorkIDs. """ pointer_path = current_pointer_path.expanduser().resolve() pointer_path.parent.mkdir(parents=True, exist_ok=True) lock_path = pointer_path.with_name(pointer_path.name + ".lock") with lock_path.open("a+b") as lock_stream: fcntl.flock(lock_stream.fileno(), fcntl.LOCK_EX) existing_pointer: dict[str, Any] | None = None carried_exclusion_paths: list[Path] = [] existing_exclusion_records_present = False if pointer_path.is_file(): loaded_pointer = json.loads( pointer_path.read_text(encoding="utf-8") ) if not isinstance(loaded_pointer, dict): raise RuntimeError( "full payload packed current pointer is malformed" ) loaded_pointer_authority = loaded_pointer.get( "pointerAuthoritySha256" ) if ( loaded_pointer.get("schema") != ( "nnf.resynthesis." "full_payload_packed_ready_federation_current.v1" ) or loaded_pointer.get("passed") is not True or not isinstance(loaded_pointer_authority, str) or loaded_pointer_authority != _sha256_bytes( _json_bytes( { key: value for key, value in loaded_pointer.items() if key != "pointerAuthoritySha256" } ) ) ): raise RuntimeError( "full payload packed current pointer authority differs" ) existing_pointer = loaded_pointer existing_records = existing_pointer.get( "exclusionReceipts" ) if existing_records is not None: existing_exclusion_records_present = True if ( not isinstance(existing_records, list) or any( not isinstance(record, dict) or record.get("schema") != ( "nnf.resynthesis." "full_payload_packed_ready_" "exclusion_receipt.v1" ) or not isinstance(record.get("path"), str) or not record["path"] for record in existing_records ) ): raise RuntimeError( "full payload packed current exclusion records differ" ) carried_exclusion_paths.extend( Path(str(record["path"])).expanduser().resolve() for record in existing_records ) observed_existing_records = ( _full_payload_packed_ready_exclusion_receipt_records( carried_exclusion_paths ) ) if observed_existing_records != existing_records: raise RuntimeError( "full payload packed current exclusion receipt " "bytes differ" ) effective_exclusion_paths = tuple( sorted( { *carried_exclusion_paths, *( path.expanduser().resolve() for path in exclusion_receipt_paths ), }, key=str, ) ) candidate_paths = _full_payload_packed_ready_receipt_paths( search_roots ) candidates_by_work_id: dict[str, list[dict[str, Any]]] = {} component_values: dict[Path, dict[str, Any]] = {} invalid_receipt_count = 0 for component_path in candidate_paths: try: component, _summary = ( _validated_full_payload_packed_ready_component( component_path ) ) except ( FileNotFoundError, OSError, KeyError, RuntimeError, TypeError, ValueError, json.JSONDecodeError, ): invalid_receipt_count += 1 continue component_values[component_path] = component for shard in cast(list[dict[str, Any]], component["shards"]): manifest_path = Path( str( cast(Mapping[str, Any], shard["manifest"])["path"] ) ).expanduser().resolve() manifest = json.loads( manifest_path.read_text(encoding="utf-8") ) work_id = str(shard["payloadWorkId"]) candidates_by_work_id.setdefault(work_id, []).append( { "componentPath": component_path, "component": component, "shard": shard, "manifestPath": manifest_path, "manifest": manifest, } ) exclusion_records = ( _full_payload_packed_ready_exclusion_receipt_records( effective_exclusion_paths ) ) excluded_work_ids = _full_payload_packed_ready_excluded_work_ids( effective_exclusion_paths ) excluded_sha256 = _sha256_bytes( _json_bytes(sorted(excluded_work_ids)) ) if ( existing_pointer is not None and not existing_exclusion_records_present and existing_pointer.get("excludedPriorPayloadWorkCount", 0) not in {None, 0} and ( existing_pointer.get("excludedPriorPayloadWorkCount") != len(excluded_work_ids) or existing_pointer.get( "excludedPriorPayloadWorkIdsSha256" ) != excluded_sha256 ) ): raise RuntimeError( "legacy full payload packed pointer requires its exact " "prior exclusion receipts before it can advance" ) selected_by_component: dict[Path, list[dict[str, Any]]] = {} exact_replica_work_ids = 0 upgraded_work_ids = 0 unresolved_work_ids: list[str] = [] for work_id in sorted(candidates_by_work_id): if work_id in excluded_work_ids: continue candidates = candidates_by_work_id[work_id] identities = { _full_payload_packed_manifest_content_identity_sha256( cast(Mapping[str, Any], candidate["manifest"]) ) for candidate in candidates } try: selected = _select_full_payload_packed_manifest_authority( candidates ) except RuntimeError: unresolved_work_ids.append(work_id) continue if len(candidates) > 1 and len(identities) == 1: exact_replica_work_ids += 1 elif len(identities) > 1: upgraded_work_ids += 1 selected_by_component.setdefault( cast(Path, selected["componentPath"]), [], ).append(selected) if unresolved_work_ids: raise RuntimeError( "full payload packed snapshot has unresolved WorkID " f"authorities: {len(unresolved_work_ids)}" ) selected_work_ids = { str(candidate["shard"]["payloadWorkId"]) for selected in selected_by_component.values() for candidate in selected } discovered_unexcluded_work_ids = ( set(candidates_by_work_id) - excluded_work_ids ) if ( not selected_work_ids or selected_work_ids != discovered_unexcluded_work_ids ): raise RuntimeError( "full payload packed snapshot WorkID coverage is incomplete" ) component_paths = [ _write_full_payload_packed_ready_component_view( pointer_path.parent, source_component=component_values[component_path], selected=selected, excluded_work_ids_sha256=excluded_sha256, ) for component_path, selected in sorted( selected_by_component.items(), key=lambda item: str(item[0]), ) ] component_authorities = [ str( json.loads(path.read_text(encoding="utf-8"))[ "collectionAuthoritySha256" ] ) for path in component_paths ] snapshot_key = _sha256_bytes( _json_bytes( { "componentAuthoritySha256s": sorted( component_authorities ), "excludedPayloadWorkIdsSha256": excluded_sha256, } ) ) snapshot_path = ( pointer_path.parent / "snapshots" / f"{snapshot_key}.receipt.json" ) snapshot = compose_full_payload_packed_ready_federation( snapshot_path, component_paths, ) snapshot_bytes = snapshot_path.read_bytes() pointer: dict[str, Any] = { "schema": ( "nnf.resynthesis." "full_payload_packed_ready_federation_current.v1" ), "passed": True, "snapshot": { "path": str(snapshot_path), "bytes": len(snapshot_bytes), "sha256": hashlib.sha256(snapshot_bytes).hexdigest(), }, "federationAuthoritySha256": snapshot[ "federationAuthoritySha256" ], "collectionAuthoritySha256": snapshot[ "collectionAuthoritySha256" ], "payloadWorkCount": snapshot["payloadWorkCount"], "packedWindowCount": snapshot["packedWindowCount"], "tokenElements": snapshot["tokenElements"], "componentCount": snapshot["componentCount"], "discoveredCandidateReceiptCount": len(candidate_paths), "invalidCandidateReceiptCountExcluded": invalid_receipt_count, "exactReplicaWorkIdCountReconciled": exact_replica_work_ids, "authorityUpgradeWorkIdCountReconciled": upgraded_work_ids, "exclusionReceiptCount": len(exclusion_records), "exclusionReceipts": exclusion_records, "excludedPriorPayloadWorkCount": len(excluded_work_ids), "excludedPriorPayloadWorkIdsSha256": excluded_sha256, "exclusionSetMonotonic": True, "priorExcludedWorkIdsMayReenter": False, "snapshotImmutable": True, "runningLearnerSnapshotMutationAllowed": False, "newlySealedWorkIdsEnterNextSnapshot": True, "targetEnteredForward": False, } pointer["pointerAuthoritySha256"] = _sha256_bytes( _json_bytes(pointer) ) if pointer_path.is_file(): existing = json.loads( pointer_path.read_text(encoding="utf-8") ) if isinstance(existing, dict) and existing == pointer: return existing _atomic_json(pointer_path, pointer) return pointer def compose_full_payload_packed_ready_federation( output_receipt_path: Path, component_receipt_paths: Sequence[Path], ) -> dict[str, Any]: """Compose disjoint sealed packed authorities without copying token data.""" def without_transient_mount_identity(value: object) -> object: """Ignore only mount-local device numbers for idempotence checks.""" def normalize(item: object, in_file_identity: bool = False) -> object: if isinstance(item, dict): return { str(key): normalize(child, str(key) == "fileIdentity") for key, child in item.items() if not (in_file_identity and str(key) == "device") and str(key) not in { "collectionAuthoritySha256", "federationAuthoritySha256", "federatedScheduleSha256", } } if isinstance(item, list): return [normalize(child, False) for child in item] return item return normalize(value) if not component_receipt_paths: raise ValueError( "full payload packed-ready federation requires a component" ) output_path = output_receipt_path.expanduser().resolve() validated = [ _validated_full_payload_packed_ready_component(path) for path in component_receipt_paths ] summaries = sorted( (summary for _loaded, summary in validated), key=lambda summary: str(summary["componentId"]), ) component_ids = [str(summary["componentId"]) for summary in summaries] if len(component_ids) != len(set(component_ids)): raise RuntimeError( "full payload packed-ready component authority overlaps" ) tokenizer_sha256s = { str(summary["tokenizerAuthoritySha256"]) for summary in summaries } vocabularies = { int(summary["tokenizerVocabularySize"]) for summary in summaries } context_windows = { int(summary["contextWindowTokens"]) for summary in summaries } answer_windows = { int(summary["answerTokensPerWindow"]) for summary in summaries } boundary_contracts = { str(summary["tokenBoundaryContract"]) for summary in summaries } if ( len(tokenizer_sha256s) != 1 or len(vocabularies) != 1 or len(context_windows) != 1 or len(answer_windows) != 1 or len(boundary_contracts) != 1 ): raise RuntimeError( "full payload packed-ready tokenizer or geometry differs" ) observed_work_ids: set[str] = set() ordered_work_ids: list[str] = [] components: list[dict[str, Any]] = [] global_cursor = 0 for component_index, summary in enumerate(summaries): work_ids = cast(list[str], summary["payloadWorkIds"]) overlap = observed_work_ids.intersection(work_ids) if overlap: raise RuntimeError( "full payload packed-ready WorkID ownership overlaps or conflicts" ) observed_work_ids.update(work_ids) ordered_work_ids.extend(work_ids) component_rows = int(summary["rows"]) cursor_start = global_cursor global_cursor += component_rows components.append( { "componentIndex": component_index, "componentId": summary["componentId"], "componentAuthoritySha256": summary["componentId"], "collectionSchema": summary["collectionSchema"], "collectionReceipt": summary["collectionReceipt"], "collectionAuthoritySha256": summary[ "collectionAuthoritySha256" ], "scheduleReceipt": summary["scheduleReceipt"], "schedule": summary["schedule"], "scheduleSha256": summary["scheduleSha256"], "hashLedger": summary["hashLedger"], "corpusRoot": summary["corpusRoot"], "selectedPayloadWorkCount": len(work_ids), "selectedPayloadWorkIdsSha256": summary[ "payloadWorkIdsSha256" ], "componentCursorStart": 0, "componentCursorEnd": component_rows, "globalCursorStart": cursor_start, "globalCursorEnd": global_cursor, "rows": component_rows, "recordCount": summary["recordCount"], "tokenElements": summary["tokenElements"], "promptTokenElements": summary["promptTokenElements"], "answerTokenElements": summary["answerTokenElements"], "sourceBytes": summary["sourceBytes"], } ) tokenizer_authority = cast( dict[str, Any], summaries[0]["tokenizerAuthority"], ) tokenizer_sha256 = next(iter(tokenizer_sha256s)) tokenizer_vocabulary_size = next(iter(vocabularies)) context_window_tokens = next(iter(context_windows)) answer_tokens_per_window = next(iter(answer_windows)) boundary_contract = next(iter(boundary_contracts)) payload_work_ids_sha256 = _sha256_bytes( _json_bytes({"payloadWorkIds": ordered_work_ids}) ) record_count = sum(int(component["recordCount"]) for component in components) token_elements = sum( int(component["tokenElements"]) for component in components ) prompt_token_elements = sum( int(component["promptTokenElements"]) for component in components ) answer_token_elements = sum( int(component["answerTokenElements"]) for component in components ) source_bytes = sum(int(component["sourceBytes"]) for component in components) federation_authority = _sha256_bytes( _json_bytes( { "federationMode": FULL_PAYLOAD_PACKED_READY_FEDERATION_MODE, "componentAuthoritySha256s": component_ids, "payloadWorkIdsSha256": payload_work_ids_sha256, "payloadWorkCount": len(ordered_work_ids), "rows": global_cursor, "recordCount": record_count, "tokenElements": token_elements, "promptTokenElements": prompt_token_elements, "answerTokenElements": answer_token_elements, "sourceBytes": source_bytes, "tokenizerAuthoritySha256": tokenizer_sha256, "tokenizerVocabularySize": tokenizer_vocabulary_size, "contextWindowTokens": context_window_tokens, "answerTokensPerWindow": answer_tokens_per_window, "tokenBoundaryContract": boundary_contract, } ) ) collection = { "schema": FULL_PAYLOAD_FEDERATED_PACKED_TOKEN_COLLECTION_SCHEMA, "federationMode": FULL_PAYLOAD_PACKED_READY_FEDERATION_MODE, "passed": True, "metadataOnlyFederation": True, "federationAuthoritySha256": federation_authority, "federatedScheduleSha256": federation_authority, "tokenizerAuthority": tokenizer_authority, "tokenizerAuthoritySha256": tokenizer_sha256, "tokenizerVocabularySize": tokenizer_vocabulary_size, "contextWindowTokens": context_window_tokens, "answerTokensPerWindow": answer_tokens_per_window, "tokenBoundaryContract": boundary_contract, "components": components, "componentCount": len(components), "componentAuthoritySha256s": component_ids, "payloadWorkIds": ordered_work_ids, "payloadWorkIdsSha256": payload_work_ids_sha256, "payloadWorkCount": len(ordered_work_ids), "rows": global_cursor, "packedWindowCount": global_cursor, "recordCount": record_count, "tokenElements": token_elements, "promptTokenElements": prompt_token_elements, "answerTokenElements": answer_token_elements, "sourceBytes": source_bytes, "globalCursorStart": 0, "globalCursorEnd": global_cursor, "canonicalPayloadFileCount": len(ordered_work_ids), "canonicalPayloadBytes": source_bytes, "exactDisjointComponentWindows": True, "exactDisjointPayloadWorkIds": True, "rawPayloadCopied": False, "rawPayloadRequiredAtTraining": False, "rawPayloadReadersInvoked": False, "tokenizerInvoked": False, "completeSourceScheduleClaimed": False, "globalDatasetTrainingClaimed": False, "mmapReadable": True, "compressedTokenRandomAccess": True, "targetValuesRecorded": False, "targetEnteredForward": False, "promotionEligible": False, "rawDataDeletionAllowed": False, } collection["collectionAuthoritySha256"] = _sha256_bytes( _json_bytes(collection) ) output_path.parent.mkdir(parents=True, exist_ok=True) if output_path.is_file(): existing = json.loads(output_path.read_text(encoding="utf-8")) if existing != collection: if without_transient_mount_identity(existing) != ( without_transient_mount_identity(collection) ): raise RuntimeError( "full payload packed-ready federation already differs" ) return dict(existing) _atomic_json(output_path, collection) return collection def build_federated_full_payload_packed_token_collection( federation_authority_receipt_path: Path, output_receipt_path: Path, *, tokenizer: Any, context_window_tokens: int, answer_tokens_per_window: int, tokenizer_authority: Mapping[str, Any] | None = None, component_output_roots: Mapping[str, Path] | None = None, ) -> dict[str, Any]: """Pack a multiroot federation through native source-root readers. This runs the established per-root builder for the canonical membership of each component. The resulting collection is a small cursor map over component collections; raw payloads are neither copied nor mounted through a synthetic root. """ if ( isinstance(context_window_tokens, bool) or isinstance(answer_tokens_per_window, bool) or context_window_tokens < 3 or answer_tokens_per_window < 1 or answer_tokens_per_window >= context_window_tokens ): raise ValueError("full payload federated packed geometry is invalid") ( federation, membership_rows, validated_components, ) = _validated_full_payload_multiroot_authority( federation_authority_receipt_path, validate_native_components=True, ) if tokenizer_authority is None: from resynthesis.tokenizer_backend import tokenizer_boundary_receipt tokenizer_authority = tokenizer_boundary_receipt(tokenizer) ( tokenizer_record, tokenizer_sha256, tokenizer_vocabulary_size, ) = _validated_full_payload_fastokens_authority(tokenizer_authority) output_path = output_receipt_path.expanduser().resolve() configured_roots = ( { component_id: root.expanduser().resolve() for component_id, root in component_output_roots.items() } if component_output_roots is not None else {} ) canonical_work_ids_by_component: dict[str, list[str]] = {} for membership in membership_rows: canonical = membership["canonical"] component_id = str(canonical["componentId"]) canonical_work_ids_by_component.setdefault(component_id, []).append( str(canonical["payloadWorkId"]) ) if configured_roots and not set(canonical_work_ids_by_component).issubset( configured_roots ): raise ValueError("full payload federated component output roots are incomplete") cursor_end = 0 packed_components: list[dict[str, Any]] = [] for component, _schedule_rows, _hash_rows in validated_components: component_id = str(component["componentId"]) selected_work_ids = tuple( sorted(canonical_work_ids_by_component.get(component_id, ())) ) if not selected_work_ids: continue # A federated child receipt is federation-owned metadata, while the # source-scoped compressed shards may already live in a separately # admitted pack root. Keeping those locations distinct lets the # federation reuse durable per-source objects instead of tokenizing a # second copy merely to publish its cursor map. component_collection_root = output_path.parent / "components" / component_id component_storage_root = configured_roots.get( component_id, component_collection_root, ) component_collection_root.mkdir(parents=True, exist_ok=True) component_storage_root.mkdir(parents=True, exist_ok=True) schedule_receipt = component["scheduleReceipt"] hash_ledger = component["hashLedger"] component_collection_path = component_collection_root / "collection.receipt.json" source_ids = { str(row["canonical"]["sourceId"]) for row in membership_rows if str(row["canonical"]["componentId"]) == component_id } native_collection = build_full_payload_packed_token_collection( Path(str(schedule_receipt["path"])), Path(str(component["corpusRoot"])), Path(str(hash_ledger["path"])), component_collection_path, tokenizer=tokenizer, context_window_tokens=context_window_tokens, answer_tokens_per_window=answer_tokens_per_window, tokenizer_authority=tokenizer_record, payload_work_ids=selected_work_ids, source_output_roots={source_id: component_storage_root for source_id in source_ids}, ) if ( native_collection.get("tokenizerAuthoritySha256") != tokenizer_sha256 or native_collection.get("contextWindowTokens") != context_window_tokens or native_collection.get("answerTokensPerWindow") != answer_tokens_per_window or native_collection.get("selectedSourceIds") is None ): raise RuntimeError("full payload federated component pack differs") local_rows = native_collection.get("rows") if ( not isinstance(local_rows, int) or isinstance(local_rows, bool) or local_rows < 1 ): raise RuntimeError("full payload federated component has no token rows") cursor_start = cursor_end cursor_end += local_rows packed_components.append( { "componentId": component_id, "componentAuthoritySha256": component_id, "corpusRoot": component["corpusRoot"], "scheduleReceipt": component["scheduleReceipt"], "hashLedger": component["hashLedger"], "selectedPayloadWorkIdsSha256": _sha256_bytes( _json_bytes(list(selected_work_ids)) ), "selectedPayloadWorkCount": len(selected_work_ids), "collectionReceipt": _packed_artifact(component_collection_path), "collectionAuthoritySha256": native_collection[ "collectionAuthoritySha256" ], "componentCursorStart": 0, "componentCursorEnd": local_rows, "globalCursorStart": cursor_start, "globalCursorEnd": cursor_end, } ) if cursor_end < 1 or not packed_components: raise RuntimeError("full payload federated packed collection is empty") federation_authority = federation["federationAuthoritySha256"] collection = { "schema": FULL_PAYLOAD_FEDERATED_PACKED_TOKEN_COLLECTION_SCHEMA, "passed": True, "federationReceipt": _packed_artifact( federation_authority_receipt_path.expanduser().resolve() ), "federationAuthoritySha256": federation_authority, "federatedScheduleSha256": federation_authority, "tokenizerAuthority": tokenizer_record, "tokenizerAuthoritySha256": tokenizer_sha256, "tokenizerVocabularySize": tokenizer_vocabulary_size, "contextWindowTokens": context_window_tokens, "answerTokensPerWindow": answer_tokens_per_window, "tokenBoundaryContract": ( FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT ), "components": packed_components, "componentCount": len(packed_components), "rows": cursor_end, "globalCursorStart": 0, "globalCursorEnd": cursor_end, "canonicalPayloadFileCount": federation["canonicalPayloadFileCount"], "canonicalPayloadBytes": federation["canonicalPayloadBytes"], "exactDisjointComponentWindows": True, "rawPayloadCopied": False, "rawPayloadRequiredAtTraining": False, "mmapReadable": True, "compressedTokenRandomAccess": True, "targetValuesRecorded": False, "targetEnteredForward": False, } collection["collectionAuthoritySha256"] = _sha256_bytes(_json_bytes(collection)) if output_path.is_file(): existing = json.loads(output_path.read_text(encoding="utf-8")) if not isinstance(existing, dict): raise RuntimeError( "full payload federated packed collection is malformed" ) if existing != collection: raise RuntimeError("full payload federated packed collection differs") return existing _atomic_json(output_path, collection) return collection def _validated_federated_full_payload_packed_collection( collection_receipt_path: Path, *, tokenizer_authority: Mapping[str, Any] | None = None, validate_native_federation: bool, ) -> tuple[dict[str, Any], dict[str, Any], list[dict[str, Any]]]: """Validate one lazy federated token collection without synthesizing a root. A federation reader is allowed to run after raw source disks have been detached, so its normal validation is artifact-only. Build and planning callers request the stronger native-root fence. In both cases every child collection must cover exactly the canonical membership work IDs for its component; a child cannot silently add a duplicate or omit an admitted payload. """ receipt_path = collection_receipt_path.expanduser().resolve() if not receipt_path.is_file(): raise FileNotFoundError("full payload federated collection is absent") loaded = json.loads(receipt_path.read_text(encoding="utf-8")) recorded_authority = ( loaded.pop("collectionAuthoritySha256", None) if isinstance(loaded, dict) else None ) if ( not isinstance(loaded, dict) or loaded.get("schema") != FULL_PAYLOAD_FEDERATED_PACKED_TOKEN_COLLECTION_SCHEMA or loaded.get("passed") is not True or not isinstance(recorded_authority, str) or len(recorded_authority) != 64 or recorded_authority != _sha256_bytes(_json_bytes(loaded)) or loaded.get("exactDisjointComponentWindows") is not True or loaded.get("rawPayloadCopied") is not False or loaded.get("rawPayloadRequiredAtTraining") is not False or loaded.get("mmapReadable") is not True or loaded.get("compressedTokenRandomAccess") is not True or loaded.get("targetEnteredForward") is not False ): raise ValueError("full payload federated packed authority differs") packed_ready_mode = ( loaded.get("federationMode") == FULL_PAYLOAD_PACKED_READY_FEDERATION_MODE ) if packed_ready_mode: if validate_native_federation: raise ValueError( "packed-ready federation has no native multiroot authority" ) component_values = loaded.get("components") if ( loaded.get("metadataOnlyFederation") is not True or loaded.get("exactDisjointPayloadWorkIds") is not True or loaded.get("rawPayloadReadersInvoked") is not False or loaded.get("tokenizerInvoked") is not False or loaded.get("completeSourceScheduleClaimed") is not False or loaded.get("globalDatasetTrainingClaimed") is not False or loaded.get("promotionEligible") is not False or not isinstance(component_values, list) or not component_values or loaded.get("componentCount") != len(component_values) ): raise ValueError( "full payload packed-ready federation authority differs" ) summaries: list[dict[str, Any]] = [] stored_components: list[dict[str, Any]] = [] for component in component_values: collection_record = ( component.get("collectionReceipt") if isinstance(component, dict) else None ) if ( not isinstance(component, dict) or not _full_payload_packed_artifact_matches(collection_record) ): raise ValueError( "full payload packed-ready federation component differs" ) assert isinstance(collection_record, dict) _child, summary = _validated_full_payload_packed_ready_component( Path(str(collection_record["path"])) ) summaries.append(summary) stored_components.append(component) sorted_pairs = sorted( zip(stored_components, summaries), key=lambda pair: str(pair[1]["componentId"]), ) sorted_summaries = [summary for _stored, summary in sorted_pairs] component_ids = [ str(summary["componentId"]) for summary in sorted_summaries ] if ( len(component_ids) != len(set(component_ids)) or loaded.get("componentAuthoritySha256s") != component_ids ): raise RuntimeError( "full payload packed-ready component authority overlaps" ) expected_components: list[dict[str, Any]] = [] observed_work_ids: set[str] = set() ordered_work_ids: list[str] = [] global_cursor = 0 for component_index, (stored_component, summary) in enumerate( sorted_pairs ): work_ids = cast(list[str], summary["payloadWorkIds"]) if observed_work_ids.intersection(work_ids): raise RuntimeError( "full payload packed-ready WorkID ownership overlaps or conflicts" ) observed_work_ids.update(work_ids) ordered_work_ids.extend(work_ids) component_rows = int(summary["rows"]) cursor_start = global_cursor global_cursor += component_rows expected_components.append( { "componentIndex": component_index, "componentId": summary["componentId"], "componentAuthoritySha256": summary["componentId"], "collectionSchema": summary["collectionSchema"], # The sealed artifact record is content-proven above by # _full_payload_packed_artifact_matches (sha256 fallback). # A fresh stat snapshot would falsely reject the sealed # receipt after a remount changes the device number. "collectionReceipt": stored_component[ "collectionReceipt" ], "collectionAuthoritySha256": summary[ "collectionAuthoritySha256" ], "scheduleReceipt": summary["scheduleReceipt"], "schedule": summary["schedule"], "scheduleSha256": summary["scheduleSha256"], "hashLedger": summary["hashLedger"], "corpusRoot": summary["corpusRoot"], "selectedPayloadWorkCount": len(work_ids), "selectedPayloadWorkIdsSha256": summary[ "payloadWorkIdsSha256" ], "componentCursorStart": 0, "componentCursorEnd": component_rows, "globalCursorStart": cursor_start, "globalCursorEnd": global_cursor, "rows": component_rows, "recordCount": summary["recordCount"], "tokenElements": summary["tokenElements"], "promptTokenElements": summary["promptTokenElements"], "answerTokenElements": summary["answerTokenElements"], "sourceBytes": summary["sourceBytes"], } ) if component_values != expected_components: raise RuntimeError( "full payload packed-ready component cursor mapping differs" ) tokenizer_sha256 = str( sorted_summaries[0]["tokenizerAuthoritySha256"] ) tokenizer_vocabulary_size = int( sorted_summaries[0]["tokenizerVocabularySize"] ) context_window_tokens = int( sorted_summaries[0]["contextWindowTokens"] ) answer_tokens_per_window = int( sorted_summaries[0]["answerTokensPerWindow"] ) boundary_contract = str( sorted_summaries[0]["tokenBoundaryContract"] ) if any( summary["tokenizerAuthoritySha256"] != tokenizer_sha256 or summary["tokenizerVocabularySize"] != tokenizer_vocabulary_size or summary["contextWindowTokens"] != context_window_tokens or summary["answerTokensPerWindow"] != answer_tokens_per_window or summary["tokenBoundaryContract"] != boundary_contract for summary in sorted_summaries ): raise RuntimeError( "full payload packed-ready tokenizer or geometry differs" ) payload_work_ids_sha256 = _sha256_bytes( _json_bytes({"payloadWorkIds": ordered_work_ids}) ) record_count = sum( int(component["recordCount"]) for component in expected_components ) token_elements = sum( int(component["tokenElements"]) for component in expected_components ) prompt_token_elements = sum( int(component["promptTokenElements"]) for component in expected_components ) answer_token_elements = sum( int(component["answerTokenElements"]) for component in expected_components ) source_bytes = sum( int(component["sourceBytes"]) for component in expected_components ) federation_authority = _sha256_bytes( _json_bytes( { "federationMode": FULL_PAYLOAD_PACKED_READY_FEDERATION_MODE, "componentAuthoritySha256s": component_ids, "payloadWorkIdsSha256": payload_work_ids_sha256, "payloadWorkCount": len(ordered_work_ids), "rows": global_cursor, "recordCount": record_count, "tokenElements": token_elements, "promptTokenElements": prompt_token_elements, "answerTokenElements": answer_token_elements, "sourceBytes": source_bytes, "tokenizerAuthoritySha256": tokenizer_sha256, "tokenizerVocabularySize": tokenizer_vocabulary_size, "contextWindowTokens": context_window_tokens, "answerTokensPerWindow": answer_tokens_per_window, "tokenBoundaryContract": boundary_contract, } ) ) if ( loaded.get("federationAuthoritySha256") != federation_authority or loaded.get("federatedScheduleSha256") != federation_authority or loaded.get("payloadWorkIds") != ordered_work_ids or loaded.get("payloadWorkIdsSha256") != payload_work_ids_sha256 or loaded.get("payloadWorkCount") != len(ordered_work_ids) or loaded.get("rows") != global_cursor or loaded.get("packedWindowCount") != global_cursor or loaded.get("recordCount") != record_count or loaded.get("tokenElements") != token_elements or loaded.get("promptTokenElements") != prompt_token_elements or loaded.get("answerTokenElements") != answer_token_elements or loaded.get("sourceBytes") != source_bytes or loaded.get("canonicalPayloadFileCount") != len(ordered_work_ids) or loaded.get("canonicalPayloadBytes") != source_bytes or loaded.get("globalCursorStart") != 0 or loaded.get("globalCursorEnd") != global_cursor or loaded.get("tokenizerAuthoritySha256") != tokenizer_sha256 or loaded.get("tokenizerVocabularySize") != tokenizer_vocabulary_size or loaded.get("contextWindowTokens") != context_window_tokens or loaded.get("answerTokensPerWindow") != answer_tokens_per_window or loaded.get("tokenBoundaryContract") != boundary_contract ): raise RuntimeError( "full payload packed-ready federation denominator differs" ) stored_tokenizer_authority = loaded.get("tokenizerAuthority") if not isinstance(stored_tokenizer_authority, dict): raise ValueError( "full payload packed-ready Fastokens authority is absent" ) ( _stored_tokenizer_record, stored_tokenizer_sha256, _stored_vocabulary_size, ) = _validated_full_payload_fastokens_authority( stored_tokenizer_authority ) if stored_tokenizer_sha256 != tokenizer_sha256: raise RuntimeError( "full payload packed-ready Fastokens authority differs" ) if tokenizer_authority is not None: ( _runtime_tokenizer_record, runtime_tokenizer_sha256, _runtime_vocabulary_size, ) = _validated_full_payload_fastokens_authority( tokenizer_authority ) if runtime_tokenizer_sha256 != tokenizer_sha256: raise RuntimeError( "full payload packed-ready runtime Fastokens authority differs" ) loaded["collectionAuthoritySha256"] = recorded_authority ready_federation = { "schema": ( "nnf.resynthesis.full_payload_packed_ready_federation_authority.v1" ), "federationMode": FULL_PAYLOAD_PACKED_READY_FEDERATION_MODE, "federationAuthoritySha256": federation_authority, "componentAuthoritySha256s": component_ids, "payloadWorkIdsSha256": payload_work_ids_sha256, "payloadWorkCount": len(ordered_work_ids), "rows": global_cursor, "metadataOnlyFederation": True, } return loaded, ready_federation, expected_components federation_record = loaded.get("federationReceipt") if ( not isinstance(federation_record, dict) or not _full_payload_packed_artifact_matches(federation_record) ): raise RuntimeError("full payload federated authority receipt differs") federation, membership_rows, _native_components = ( _validated_full_payload_multiroot_authority( Path(str(federation_record["path"])), validate_native_components=validate_native_federation, ) ) native_federation_authority = federation.get("federationAuthoritySha256") if ( not isinstance(native_federation_authority, str) or native_federation_authority != loaded.get("federationAuthoritySha256") or native_federation_authority != loaded.get("federatedScheduleSha256") or loaded.get("canonicalPayloadFileCount") != federation.get("canonicalPayloadFileCount") or loaded.get("canonicalPayloadBytes") != federation.get("canonicalPayloadBytes") ): raise RuntimeError("full payload federated collection binding differs") native_boundary_contract = loaded.get("tokenBoundaryContract") if native_boundary_contract is None: loaded["tokenBoundaryContract"] = ( FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT ) elif native_boundary_contract != FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT: raise RuntimeError( "full payload federated token boundary contract differs" ) stored_tokenizer_authority = loaded.get("tokenizerAuthority") if not isinstance(stored_tokenizer_authority, dict): raise ValueError("full payload federated Fastokens authority is absent") ( _stored_tokenizer_record, stored_tokenizer_sha256, _stored_vocabulary_size, ) = _validated_full_payload_fastokens_authority(stored_tokenizer_authority) if loaded.get("tokenizerAuthoritySha256") != stored_tokenizer_sha256: raise RuntimeError("full payload federated Fastokens authority differs") if tokenizer_authority is not None: ( _runtime_tokenizer_record, runtime_tokenizer_sha256, _runtime_vocabulary_size, ) = _validated_full_payload_fastokens_authority(tokenizer_authority) if runtime_tokenizer_sha256 != stored_tokenizer_sha256: raise RuntimeError( "full payload federated runtime Fastokens authority differs" ) rows = loaded.get("rows") native_context_window_tokens = loaded.get("contextWindowTokens") native_answer_tokens_per_window = loaded.get("answerTokensPerWindow") if ( not isinstance(rows, int) or isinstance(rows, bool) or rows < 1 or loaded.get("globalCursorStart") != 0 or loaded.get("globalCursorEnd") != rows or not isinstance(native_context_window_tokens, int) or isinstance(native_context_window_tokens, bool) or native_context_window_tokens < 3 or not isinstance(native_answer_tokens_per_window, int) or isinstance(native_answer_tokens_per_window, bool) or native_answer_tokens_per_window < 1 or native_answer_tokens_per_window >= native_context_window_tokens ): raise ValueError("full payload federated packed geometry differs") federation_components = federation.get("components") if not isinstance(federation_components, list): raise ValueError("full payload federated source components are absent") federation_component_by_id = { str(component.get("componentId", "")): component for component in federation_components if isinstance(component, dict) } if len(federation_component_by_id) != len(federation_components): raise ValueError("full payload federated source component differs") canonical_work_ids_by_component: dict[str, set[str]] = {} for membership in membership_rows: canonical = membership.get("canonical") if not isinstance(canonical, dict): raise ValueError("full payload federated canonical membership differs") component_id = canonical.get("componentId") work_id = canonical.get("payloadWorkId") if ( not isinstance(component_id, str) or component_id not in federation_component_by_id or not isinstance(work_id, str) or len(work_id) != 64 ): raise ValueError("full payload federated canonical work differs") canonical_work_ids_by_component.setdefault(component_id, set()).add(work_id) component_values = loaded.get("components") if ( not isinstance(component_values, list) or not component_values or loaded.get("componentCount") != len(component_values) or len(component_values) != len(canonical_work_ids_by_component) ): raise ValueError("full payload federated components are absent") validated_components: list[dict[str, Any]] = [] observed_component_ids: set[str] = set() prior_end = 0 for raw_component in component_values: if not isinstance(raw_component, dict): raise ValueError("full payload federated component is malformed") component = dict(raw_component) component_id = component.get("componentId") start = component.get("globalCursorStart") end = component.get("globalCursorEnd") source_component = federation_component_by_id.get(str(component_id)) collection_record = component.get("collectionReceipt") expected_work_ids = canonical_work_ids_by_component.get(str(component_id)) if ( not isinstance(component_id, str) or component_id in observed_component_ids or source_component is None or not expected_work_ids or not isinstance(start, int) or isinstance(start, bool) or not isinstance(end, int) or isinstance(end, bool) or start != prior_end or end <= start or component.get("componentCursorStart") != 0 or component.get("componentCursorEnd") != end - start or component.get("componentAuthoritySha256") != component_id or component.get("corpusRoot") != source_component.get("corpusRoot") or component.get("scheduleReceipt") != source_component.get("scheduleReceipt") or component.get("hashLedger") != source_component.get("hashLedger") or component.get("selectedPayloadWorkCount") != len(expected_work_ids) or component.get("selectedPayloadWorkIdsSha256") != _sha256_bytes(_json_bytes(sorted(expected_work_ids))) or not isinstance(collection_record, dict) or not _full_payload_packed_artifact_matches(collection_record) ): raise ValueError("full payload federated component cursor differs") child_path = Path(str(collection_record["path"])).expanduser().resolve() child = json.loads(child_path.read_text(encoding="utf-8")) if not isinstance(child, dict): raise RuntimeError("full payload federated child collection differs") child_authority = ( child.pop("collectionAuthoritySha256", None) ) child_shards = child.get("shards") if not isinstance(child_shards, list): raise RuntimeError("full payload federated child collection differs") child_work_ids = { shard.get("payloadWorkId") for shard in child_shards if isinstance(shard, dict) and isinstance(shard.get("payloadWorkId"), str) } if ( child.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_COLLECTION_SCHEMA or child.get("passed") is not True or not isinstance(child_authority, str) or child_authority != _sha256_bytes(_json_bytes(child)) or child_authority != component.get("collectionAuthoritySha256") or child.get("scheduleReceipt") != source_component.get("scheduleReceipt") or child.get("hashLedger") != source_component.get("hashLedger") or child.get("scheduleSha256") != source_component.get("scheduleSha256") or child.get("corpusRoot") != source_component.get("corpusRoot") or child.get("rows") != end - start or child.get("tokenizerAuthoritySha256") != stored_tokenizer_sha256 or child.get("contextWindowTokens") != native_context_window_tokens or child.get("answerTokensPerWindow") != native_answer_tokens_per_window or len(child_work_ids) != len(child_shards) or child_work_ids != expected_work_ids ): raise RuntimeError("full payload federated child collection differs") child_boundary_contract = child.get("tokenBoundaryContract") if child_boundary_contract is None: child["tokenBoundaryContract"] = ( FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT ) elif ( child_boundary_contract not in FULL_PAYLOAD_PACKED_SUPPORTED_TOKEN_BOUNDARY_CONTRACTS ): raise RuntimeError( "full payload federated child token boundary contract differs" ) child["collectionAuthoritySha256"] = child_authority observed_component_ids.add(component_id) validated_components.append(component) prior_end = end if prior_end != rows or observed_component_ids != set(canonical_work_ids_by_component): raise ValueError("full payload federated cursor coverage differs") loaded["collectionAuthoritySha256"] = recorded_authority return loaded, federation, validated_components @dataclass class _FullPayloadPackedShard: """One open mmap shard with bounded token and metadata frame reuse.""" cursor_start: int cursor_end: int manifest: dict[str, Any] compressed_file: Any compressed_mmap: Any token_chunk_index_file: Any token_chunk_index_mmap: Any token_chunk_count: int locator_file: Any locator_mmap: Any metadata_compressed_file: Any metadata_compressed_mmap: Any metadata_chunk_index_file: Any metadata_chunk_index_mmap: Any metadata_chunk_count: int metadata_uncompressed_bytes: int record_hash_file: Any record_hash_mmap: Any window_ends_file: Any window_ends_mmap: Any token_chunk_cache: dict[int, Any] token_chunk_last_used: dict[int, int] metadata_chunk_cache: dict[int, bytes] metadata_chunk_last_used: dict[int, int] metadata_cache: dict[int, dict[str, Any]] metadata_last_used: dict[int, int] compact_record_file: Any | None = None compact_record_mmap: mmap.mmap | None = None compact_record_index_file: Any | None = None compact_record_index_mmap: mmap.mmap | None = None compact_record_chunk_count: int = 0 compact_record_chunk_cache: dict[ int, tuple[_FullPayloadCompactRecord, ...], ] | None = None compact_record_chunk_last_used: dict[int, int] | None = None compact_record_by_index: dict[int, _FullPayloadCompactRecord] | None = None compact_record_use_counter: int = 0 token_chunk_use_counter: int = 0 metadata_chunk_use_counter: int = 0 metadata_use_counter: int = 0 single_window_geometry: PackedTokenShardGeometryPacket | None = None _MAXIMUM_DECOMPRESSED_TOKEN_CHUNKS = 2 _MAXIMUM_DECOMPRESSED_METADATA_CHUNKS = 2 _MAXIMUM_METADATA_RECORDS = 128 _MAXIMUM_DECOMPRESSED_COMPACT_RECORD_CHUNKS = 2 def close(self) -> None: for mapped in ( self.compressed_mmap, self.token_chunk_index_mmap, self.locator_mmap, self.metadata_compressed_mmap, self.metadata_chunk_index_mmap, self.record_hash_mmap, self.window_ends_mmap, self.compact_record_mmap, self.compact_record_index_mmap, ): if mapped is not None: mapped.close() for handle in ( self.compressed_file, self.token_chunk_index_file, self.locator_file, self.metadata_compressed_file, self.metadata_chunk_index_file, self.record_hash_file, self.window_ends_file, self.compact_record_file, self.compact_record_index_file, ): if handle is not None: handle.close() self.token_chunk_cache = {} self.token_chunk_last_used = {} self.metadata_chunk_cache = {} self.metadata_chunk_last_used = {} self.metadata_cache = {} self.metadata_last_used = {} self.compact_record_chunk_cache = {} self.compact_record_chunk_last_used = {} self.compact_record_by_index = {} self.single_window_geometry = None def _token_chunk_bounds(self, index: int) -> tuple[int, int, int, int]: if index < 0 or index >= self.token_chunk_count: raise RuntimeError("full payload packed token chunk index differs") return FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.unpack_from( self.token_chunk_index_mmap, index * FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.size, ) def _token_chunk_for_byte_offset(self, byte_offset: int) -> int: lower = 0 upper = self.token_chunk_count while lower < upper: middle = (lower + upper) // 2 _compressed_offset, _compressed_bytes, start, width = ( self._token_chunk_bounds(middle) ) if byte_offset < start: upper = middle elif byte_offset >= start + width: lower = middle + 1 else: return middle raise RuntimeError("full payload packed token byte offset is unavailable") def _token_chunk(self, index: int) -> Any: cached = self.token_chunk_cache.get(index) if cached is not None: self.token_chunk_use_counter += 1 self.token_chunk_last_used[index] = self.token_chunk_use_counter return cached token_chunk = self._decode_token_chunk_boundary(index) if ( len(self.token_chunk_cache) >= self._MAXIMUM_DECOMPRESSED_TOKEN_CHUNKS ): retired = min( self.token_chunk_last_used, key=lambda candidate: self.token_chunk_last_used[candidate], ) self.token_chunk_cache.pop(retired) self.token_chunk_last_used.pop(retired, None) self.token_chunk_cache[index] = token_chunk self.token_chunk_use_counter += 1 self.token_chunk_last_used[index] = self.token_chunk_use_counter return token_chunk def _decode_token_chunk_boundary(self, index: int) -> Any: """Decode one independent token frame without mutating shared cache.""" compressed_offset, compressed_bytes, _start, uncompressed_bytes = ( self._token_chunk_bounds(index) ) compressed = memoryview(self.compressed_mmap)[ compressed_offset : compressed_offset + compressed_bytes ] import torch import zstandard decoded = zstandard.ZstdDecompressor().decompress( compressed, max_output_size=uncompressed_bytes, ) if len(decoded) != uncompressed_bytes or uncompressed_bytes % 4: raise RuntimeError("full payload packed token chunk decode differs") token_chunk = torch.frombuffer(bytearray(decoded), dtype=torch.int32) if token_chunk.numel() * 4 != uncompressed_bytes: raise RuntimeError("full payload packed token chunk geometry differs") return token_chunk def _decode_token_chunk_range_boundary( self, chunk_indexes: tuple[int, ...], ) -> tuple[Any, int]: """Decode independent frames natively into one contiguous token arena. ``python-zstandard`` owns the parallel frame decode. Python performs one linear copy from its output segments into the final writable arena; it never launches one Python future per frame and never concatenates tensors. The returned byte offset binds the arena to the immutable chunk-index coordinate system. """ if len(chunk_indexes) < 2 or any( following != prior + 1 for prior, following in zip( chunk_indexes, chunk_indexes[1:], ) ): raise RuntimeError( "full payload packed token chunk range is not contiguous" ) compressed_frames: list[bytes | bytearray | memoryview] = [] decompressed_sizes = array("Q") first_uncompressed_offset: int | None = None expected_uncompressed_offset: int | None = None for index in chunk_indexes: ( compressed_offset, compressed_bytes, uncompressed_offset, uncompressed_bytes, ) = self._token_chunk_bounds(index) if ( uncompressed_bytes < 1 or uncompressed_bytes % 4 or ( expected_uncompressed_offset is not None and uncompressed_offset != expected_uncompressed_offset ) ): raise RuntimeError( "full payload packed token chunk range geometry differs" ) if first_uncompressed_offset is None: first_uncompressed_offset = uncompressed_offset expected_uncompressed_offset = ( uncompressed_offset + uncompressed_bytes ) compressed_frames.append( memoryview(self.compressed_mmap)[ compressed_offset : compressed_offset + compressed_bytes ] ) decompressed_sizes.append(uncompressed_bytes) import torch import zstandard decoded_segments: Any = ( zstandard.ZstdDecompressor().multi_decompress_to_buffer( compressed_frames, decompressed_sizes=memoryview(decompressed_sizes), threads=min(len(compressed_frames), 8), ) ) expected_bytes = sum(decompressed_sizes) if ( first_uncompressed_offset is None or len(decoded_segments) != len(compressed_frames) or decoded_segments.size() != expected_bytes ): raise RuntimeError( "full payload packed token parallel decode frontier differs" ) arena = bytearray(expected_bytes) arena_view = memoryview(arena) copied_bytes = 0 for segment_index in range(len(decompressed_sizes)): segment = decoded_segments[segment_index] expected_segment_bytes = decompressed_sizes[segment_index] if len(segment) != expected_segment_bytes: raise RuntimeError( "full payload packed token decoded segment differs" ) segment_end = copied_bytes + expected_segment_bytes arena_view[copied_bytes:segment_end] = segment copied_bytes = segment_end if copied_bytes != expected_bytes: raise RuntimeError( "full payload packed token contiguous arena differs" ) token_arena_t = torch.frombuffer(arena, dtype=torch.int32) if token_arena_t.numel() * 4 != expected_bytes: raise RuntimeError( "full payload packed token contiguous tensor differs" ) return token_arena_t, first_uncompressed_offset def token_slice(self, token_start: int, token_end: int) -> Any: if ( token_start < 0 or token_end <= token_start or token_end > int(self.manifest["tokenElements"]) ): raise RuntimeError("full payload packed token slice is invalid") byte_start = token_start * 4 byte_end = token_end * 4 first_chunk_index = self._token_chunk_for_byte_offset(byte_start) final_chunk_index = self._token_chunk_for_byte_offset(byte_end - 1) if first_chunk_index == final_chunk_index: ( _compressed_offset, _compressed_bytes, chunk_start, _chunk_width, ) = self._token_chunk_bounds(first_chunk_index) chunk = self._token_chunk(first_chunk_index) return chunk[ (byte_start - chunk_start) // 4 : (byte_end - chunk_start) // 4 ] token_arena_t, arena_byte_start = ( self._decode_token_chunk_range_boundary( tuple(range(first_chunk_index, final_chunk_index + 1)) ) ) return token_arena_t[ (byte_start - arena_byte_start) // 4 : (byte_end - arena_byte_start) // 4 ] def copy_token_range_into( self, destination_t: Any, *, destination_start: int, token_start: int, token_end: int, ) -> int: """Copy a sealed token extent directly into one packet arena. This external-I/O boundary deliberately avoids constructing chunk tensors or concatenating per-row tensors. A multi-frame extent is decoded in parallel and each immutable decoded segment is copied directly into its final packet offset. It never passes through the temporary contiguous arena required by ``token_slice`` callers. """ import torch if ( not isinstance(destination_t, torch.Tensor) or destination_t.device.type != "cpu" or destination_t.dtype != torch.int32 or destination_t.ndim != 1 or not destination_t.is_contiguous() ): raise RuntimeError( "full payload packed token destination differs" ) if ( isinstance(destination_start, bool) or isinstance(token_start, bool) or isinstance(token_end, bool) or destination_start < 0 or token_start < 0 or token_end <= token_start or token_end > int(self.manifest["tokenElements"]) or destination_start + token_end - token_start > destination_t.numel() ): raise RuntimeError("full payload packed token copy range is invalid") copied_tokens = token_end - token_start byte_start = token_start * 4 byte_end = token_end * 4 first_chunk_index = self._token_chunk_for_byte_offset(byte_start) final_chunk_index = self._token_chunk_for_byte_offset(byte_end - 1) if first_chunk_index == final_chunk_index: ( _compressed_offset, _compressed_bytes, chunk_start, _chunk_width, ) = self._token_chunk_bounds(first_chunk_index) source_t = self._token_chunk(first_chunk_index)[ (byte_start - chunk_start) // 4 : (byte_end - chunk_start) // 4 ] destination_cursor = destination_start + copied_tokens destination_t[destination_start:destination_cursor].copy_(source_t) else: # ``BufferWithSegmentsCollection`` intentionally exposes separate # decoded buffers rather than one contiguous buffer. Copying those # buffers into a bytearray and then copying that arena into the # packet doubled host-memory traffic for every cross-frame span. # NumPy writes into the CPU tensor's existing storage, so each # selected decoded byte is copied exactly once. import numpy as np import zstandard compressed_frames: list[bytes | bytearray | memoryview] = [] decompressed_sizes = array("Q") chunk_bounds: list[tuple[int, int]] = [] expected_uncompressed_offset: int | None = None for chunk_index in range( first_chunk_index, final_chunk_index + 1, ): ( compressed_offset, compressed_bytes, uncompressed_offset, uncompressed_bytes, ) = self._token_chunk_bounds(chunk_index) if ( uncompressed_bytes < 1 or uncompressed_bytes % 4 or ( expected_uncompressed_offset is not None and uncompressed_offset != expected_uncompressed_offset ) ): raise RuntimeError( "full payload packed token copy geometry differs" ) expected_uncompressed_offset = ( uncompressed_offset + uncompressed_bytes ) compressed_frames.append( memoryview(self.compressed_mmap)[ compressed_offset : compressed_offset + compressed_bytes ] ) decompressed_sizes.append(uncompressed_bytes) chunk_bounds.append( (uncompressed_offset, uncompressed_bytes) ) decoded_segments: Any = ( zstandard.ZstdDecompressor().multi_decompress_to_buffer( compressed_frames, decompressed_sizes=memoryview(decompressed_sizes), threads=min(len(compressed_frames), 8), ) ) if ( len(decoded_segments) != len(chunk_bounds) or decoded_segments.size() != sum(decompressed_sizes) ): raise RuntimeError( "full payload packed token parallel copy decode differs" ) destination_array = destination_t.numpy() destination_cursor = destination_start for segment_index, ( uncompressed_offset, uncompressed_bytes, ) in enumerate(chunk_bounds): selected_byte_start = max( byte_start, uncompressed_offset, ) selected_byte_end = min( byte_end, uncompressed_offset + uncompressed_bytes, ) if selected_byte_end <= selected_byte_start: continue segment_byte_start = ( selected_byte_start - uncompressed_offset ) selected_byte_count = ( selected_byte_end - selected_byte_start ) if ( segment_byte_start % 4 or selected_byte_count % 4 ): raise RuntimeError( "full payload packed token selected bytes differ" ) source_array = np.frombuffer( memoryview(decoded_segments[segment_index]), dtype=" tuple[PackedTokenPhysicalSourceRange, ...]: """Bind one selected token extent to physical frames and packet bytes.""" artifacts = self.manifest.get("artifacts") token_artifact = ( artifacts.get("tokensZstd") if isinstance(artifacts, dict) else None ) object_path_value = ( token_artifact.get("path") if isinstance(token_artifact, dict) else None ) object_sha256 = ( token_artifact.get("sha256") if isinstance(token_artifact, dict) else None ) object_byte_count = ( token_artifact.get("bytes") if isinstance(token_artifact, dict) else None ) if ( not isinstance(object_path_value, str) or not isinstance(object_sha256, str) or not isinstance(object_byte_count, int) or isinstance(object_byte_count, bool) or object_byte_count < 1 or token_start < 0 or token_end <= token_start or token_end > int(self.manifest["tokenElements"]) or packet_byte_start < 0 ): raise RuntimeError( "full payload packed physical source authority differs" ) object_path = str(Path(object_path_value).resolve()) source_byte_cursor = token_start * 4 source_byte_end = token_end * 4 packet_byte_cursor = packet_byte_start ranges: list[PackedTokenPhysicalSourceRange] = [] while source_byte_cursor < source_byte_end: chunk_index = self._token_chunk_for_byte_offset( source_byte_cursor ) ( compressed_offset, compressed_bytes, uncompressed_offset, uncompressed_bytes, ) = self._token_chunk_bounds(chunk_index) selected_end = min( source_byte_end, uncompressed_offset + uncompressed_bytes, ) selected_bytes = selected_end - source_byte_cursor ranges.append( PackedTokenPhysicalSourceRange( component_sha256=component_sha256, payload_work_id=str(self.manifest["payloadWorkId"]), object_path=object_path, object_sha256=object_sha256, object_byte_count=object_byte_count, object_compressed_byte_start=compressed_offset, object_compressed_byte_end=( compressed_offset + compressed_bytes ), source_uncompressed_byte_start=source_byte_cursor, source_uncompressed_byte_end=selected_end, packet_byte_start=packet_byte_cursor, packet_byte_end=packet_byte_cursor + selected_bytes, global_cursor=global_cursor, ) ) source_byte_cursor = selected_end packet_byte_cursor += selected_bytes if packet_byte_cursor != packet_byte_start + (token_end - token_start) * 4: raise RuntimeError( "full payload packed physical packet frontier differs" ) return tuple(ranges) def single_window_physical_source_ranges_boundary( self, *, geometry: PackedTokenShardGeometryPacket, row_start: int, row_end: int, component_sha256: str, global_cursor_start: int, ) -> tuple[PackedTokenPhysicalSourceRange, ...]: """Vector-read V2 row offsets into exact physical frame provenance.""" if ( row_start < 0 or row_end <= row_start or row_end > len(geometry) or global_cursor_start < 0 ): raise ValueError( "full payload single-window provenance geometry differs" ) artifacts = self.manifest.get("artifacts") token_artifact = ( artifacts.get("tokensZstd") if isinstance(artifacts, dict) else None ) if not isinstance(token_artifact, dict): raise RuntimeError( "full payload single-window token artifact differs" ) object_path_value = token_artifact.get("path") object_sha256 = token_artifact.get("sha256") object_byte_count = token_artifact.get("bytes") if ( not isinstance(object_path_value, str) or not isinstance(object_sha256, str) or not isinstance(object_byte_count, int) or isinstance(object_byte_count, bool) or object_byte_count < 1 ): raise RuntimeError( "full payload single-window token authority differs" ) object_path = str(Path(object_path_value).resolve()) payload_work_id = str(self.manifest["payloadWorkId"]) token_offsets = tuple( int(value) for value in geometry.token_offsets_t[ row_start : row_end + 1 ].tolist() ) packet_byte_cursor = 0 ranges: list[PackedTokenPhysicalSourceRange] = [] chunk_index = self._token_chunk_for_byte_offset( token_offsets[0] * 4 ) for local_row_index, ( token_start, token_end, ) in enumerate( zip(token_offsets, token_offsets[1:]) ): source_byte_cursor = token_start * 4 source_byte_end = token_end * 4 global_cursor = global_cursor_start + local_row_index while source_byte_cursor < source_byte_end: ( compressed_offset, compressed_bytes, uncompressed_offset, uncompressed_bytes, ) = self._token_chunk_bounds(chunk_index) chunk_end = uncompressed_offset + uncompressed_bytes if source_byte_cursor < uncompressed_offset: raise RuntimeError( "full payload single-window provenance gap differs" ) if source_byte_cursor >= chunk_end: chunk_index += 1 if chunk_index >= self.token_chunk_count: raise RuntimeError( "full payload single-window token frame exhausted" ) continue selected_end = min(source_byte_end, chunk_end) selected_bytes = selected_end - source_byte_cursor ranges.append( PackedTokenPhysicalSourceRange( component_sha256=component_sha256, payload_work_id=payload_work_id, object_path=object_path, object_sha256=object_sha256, object_byte_count=object_byte_count, object_compressed_byte_start=compressed_offset, object_compressed_byte_end=( compressed_offset + compressed_bytes ), source_uncompressed_byte_start=source_byte_cursor, source_uncompressed_byte_end=selected_end, packet_byte_start=packet_byte_cursor, packet_byte_end=( packet_byte_cursor + selected_bytes ), global_cursor=global_cursor, ) ) source_byte_cursor = selected_end packet_byte_cursor += selected_bytes if ( not ranges or packet_byte_cursor != (token_offsets[-1] - token_offsets[0]) * 4 ): raise RuntimeError( "full payload single-window provenance frontier differs" ) return tuple(ranges) def _metadata_chunk_bounds(self, index: int) -> tuple[int, int, int, int]: if index < 0 or index >= self.metadata_chunk_count: raise RuntimeError("full payload packed metadata chunk index differs") return FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.unpack_from( self.metadata_chunk_index_mmap, index * FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.size, ) def _metadata_chunk_for_byte_offset(self, byte_offset: int) -> int: lower = 0 upper = self.metadata_chunk_count while lower < upper: middle = (lower + upper) // 2 _compressed_offset, _compressed_bytes, start, width = ( self._metadata_chunk_bounds(middle) ) if byte_offset < start: upper = middle elif byte_offset >= start + width: lower = middle + 1 else: return middle raise RuntimeError("full payload packed metadata byte offset is unavailable") def _metadata_chunk(self, index: int) -> bytes: cached = self.metadata_chunk_cache.get(index) if cached is not None: self.metadata_chunk_use_counter += 1 self.metadata_chunk_last_used[index] = self.metadata_chunk_use_counter return cached if ( len(self.metadata_chunk_cache) >= self._MAXIMUM_DECOMPRESSED_METADATA_CHUNKS ): retired = min( self.metadata_chunk_last_used, key=lambda candidate: self.metadata_chunk_last_used[candidate], ) self.metadata_chunk_cache.pop(retired) self.metadata_chunk_last_used.pop(retired, None) compressed_offset, compressed_bytes, _start, uncompressed_bytes = ( self._metadata_chunk_bounds(index) ) compressed = self.metadata_compressed_mmap[ compressed_offset : compressed_offset + compressed_bytes ] import zstandard decoded = zstandard.ZstdDecompressor().decompress( compressed, max_output_size=uncompressed_bytes, ) if len(decoded) != uncompressed_bytes: raise RuntimeError("full payload packed metadata chunk decode differs") self.metadata_chunk_cache[index] = decoded self.metadata_chunk_use_counter += 1 self.metadata_chunk_last_used[index] = self.metadata_chunk_use_counter return decoded def metadata_slice(self, byte_start: int, byte_end: int) -> bytes: if ( byte_start < 0 or byte_end <= byte_start or byte_end > self.metadata_uncompressed_bytes ): raise RuntimeError("full payload packed metadata slice is invalid") pieces: list[bytes] = [] byte_cursor = byte_start while byte_cursor < byte_end: chunk_index = self._metadata_chunk_for_byte_offset(byte_cursor) _compressed_offset, _compressed_bytes, chunk_start, chunk_width = ( self._metadata_chunk_bounds(chunk_index) ) take_end = min(byte_end, chunk_start + chunk_width) chunk = self._metadata_chunk(chunk_index) pieces.append( chunk[ byte_cursor - chunk_start : take_end - chunk_start ] ) byte_cursor = take_end return b"".join(pieces) def metadata_record( self, record_index: int, metadata_offset: int, metadata_length: int, ) -> dict[str, Any]: """Parse one sealed record once across its contiguous training windows.""" if ( self.manifest.get("packedStorageLayout") == FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT ): record = self.compact_record(record_index) work_id = str(self.manifest["payloadWorkId"]) locator = record.locator record_ordinal = record.record_ordinal source_record_id = hashlib.sha256( ( f"{work_id}\x00{locator}\x00{record_ordinal}" ).encode("utf-8") ).hexdigest() return { "schema": FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_SCHEMA, "source_id": self.manifest["sourceId"], "payload_work_id": work_id, "payload_record_locator": locator, "payload_record_ordinal": record_ordinal, "payload_record_token_count": record.content_length, "payload_reader_family": record.reader_family, "semantic_export_receipt_sha256": ( record.semantic_export_receipt_sha256 ), "source_record_id": source_record_id, "source_sha256": self.manifest["observedPayloadSha256"], "corpus_surface_family": "full_payload_packed", "generalization_axis": "task_family", "generalization_group": work_id, "rights_disposition": "training_admissible", "rights_authority_sha256": self.manifest[ "rightsAuthoritySha256" ], "domain_authority_sha256": self.manifest[ "domainAuthoritySha256" ], "target_values_recorded": False, "target_entered_forward": False, "task_intent_targets_entered_forward": False, "model_scores_observed": False, } cached = self.metadata_cache.get(record_index) if cached is not None: self.metadata_use_counter += 1 self.metadata_last_used[record_index] = self.metadata_use_counter return dict(cached) if ( record_index < 0 or metadata_offset < 0 or metadata_length < 1 or metadata_offset + metadata_length > self.metadata_uncompressed_bytes ): raise RuntimeError("full payload packed metadata range differs") if len(self.metadata_cache) >= self._MAXIMUM_METADATA_RECORDS: retired = min( self.metadata_last_used, key=lambda candidate: self.metadata_last_used[candidate], ) self.metadata_cache.pop(retired) self.metadata_last_used.pop(retired, None) parsed = json.loads( self.metadata_slice( metadata_offset, metadata_offset + metadata_length, ) ) if ( not isinstance(parsed, dict) or parsed.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_SCHEMA or parsed.get("target_entered_forward") is not False or parsed.get("target_values_recorded") is not False or "target_ids" in parsed or "input_ids" in parsed or "prompt_ids" in parsed or "answer_ids" in parsed ): raise RuntimeError("full payload packed metadata differs") base = dict(parsed) self.metadata_cache[record_index] = base self.metadata_use_counter += 1 self.metadata_last_used[record_index] = self.metadata_use_counter return dict(base) def _compact_record_chunk_bounds( self, index: int, ) -> tuple[int, int, int, int, int, int, int, int]: if ( index < 0 or index >= self.compact_record_chunk_count or self.compact_record_index_mmap is None ): raise RuntimeError( "full payload packed compact record index differs" ) return FULL_PAYLOAD_PACKED_COMPACT_RECORD_INDEX_STRUCT.unpack_from( self.compact_record_index_mmap, index * FULL_PAYLOAD_PACKED_COMPACT_RECORD_INDEX_STRUCT.size, ) def _compact_record_chunk( self, index: int, ) -> tuple[_FullPayloadCompactRecord, ...]: cache = self.compact_record_chunk_cache last_used = self.compact_record_chunk_last_used record_by_index = self.compact_record_by_index if ( cache is None or last_used is None or record_by_index is None or self.compact_record_mmap is None ): raise RuntimeError( "full payload packed compact record storage is absent" ) cached = cache.get(index) if cached is not None: self.compact_record_use_counter += 1 last_used[index] = self.compact_record_use_counter return cached if ( len(cache) >= self._MAXIMUM_DECOMPRESSED_COMPACT_RECORD_CHUNKS ): retired = min(last_used, key=lambda candidate: last_used[candidate]) for record in cache.pop(retired): record_by_index.pop(record.record_index, None) last_used.pop(retired, None) ( compressed_offset, compressed_bytes, frame_record_start, frame_record_count, frame_token_start, frame_token_count, frame_window_start, frame_window_count, ) = self._compact_record_chunk_bounds(index) compressed = self.compact_record_mmap[ compressed_offset : compressed_offset + compressed_bytes ] zstandard = importlib.import_module("zstandard") payload = zstandard.ZstdDecompressor().decompress(compressed) records: list[_FullPayloadCompactRecord] = [] token_offset = frame_token_start window_start = frame_window_start decoded_records = _decode_full_payload_compact_record_frame( payload, expected_record_count=frame_record_count, ) for ordinal, decoded in enumerate(decoded_records): ( prefix_length, content_length, record_ordinal, locator, reader_family, semantic_sha256, ) = decoded window_count, _prompt_tokens, _answer_tokens = ( _full_payload_packed_record_training_geometry( prefix_tokens=prefix_length, content_tokens=content_length, context_window_tokens=int( self.manifest["contextWindowTokens"] ), answer_tokens_per_window=int( self.manifest["answerTokensPerWindow"] ), ) ) record = _FullPayloadCompactRecord( record_index=frame_record_start + ordinal, token_offset=token_offset, prefix_length=prefix_length, content_length=content_length, window_start=window_start, window_count=window_count, record_ordinal=record_ordinal, locator=locator, reader_family=reader_family, semantic_export_receipt_sha256=semantic_sha256, ) records.append(record) record_by_index[record.record_index] = record token_offset += prefix_length + content_length window_start += window_count if ( token_offset != frame_token_start + frame_token_count or window_start != frame_window_start + frame_window_count ): raise RuntimeError( "full payload packed compact record frame geometry differs" ) result = tuple(records) cache[index] = result self.compact_record_use_counter += 1 last_used[index] = self.compact_record_use_counter return result def compact_record(self, record_index: int) -> _FullPayloadCompactRecord: record_by_index = self.compact_record_by_index if record_by_index is None or record_index < 0: raise RuntimeError( "full payload packed compact record is unavailable" ) cached = record_by_index.get(record_index) if cached is not None: return cached lower = 0 upper = self.compact_record_chunk_count while lower < upper: middle = (lower + upper) // 2 ( _compressed_offset, _compressed_bytes, frame_record_start, frame_record_count, _frame_token_start, _frame_token_count, _frame_window_start, _frame_window_count, ) = self._compact_record_chunk_bounds(middle) if record_index < frame_record_start: upper = middle elif record_index >= frame_record_start + frame_record_count: lower = middle + 1 else: records = self._compact_record_chunk(middle) return records[record_index - frame_record_start] raise RuntimeError("full payload packed compact record is unavailable") def compact_record_for_window( self, row_index: int, ) -> tuple[_FullPayloadCompactRecord, int]: lower = 0 upper = self.compact_record_chunk_count while lower < upper: middle = (lower + upper) // 2 ( _compressed_offset, _compressed_bytes, _frame_record_start, _frame_record_count, _frame_token_start, _frame_token_count, frame_window_start, frame_window_count, ) = self._compact_record_chunk_bounds(middle) if row_index < frame_window_start: upper = middle elif row_index >= frame_window_start + frame_window_count: lower = middle + 1 else: records = self._compact_record_chunk(middle) record_lower = 0 record_upper = len(records) while record_lower < record_upper: record_middle = (record_lower + record_upper) // 2 record = records[record_middle] if row_index < record.window_start: record_upper = record_middle elif ( row_index >= record.window_start + record.window_count ): record_lower = record_middle + 1 else: return record, row_index - record.window_start break raise RuntimeError("full payload packed compact window is unavailable") def single_window_geometry_boundary( self, ) -> PackedTokenShardGeometryPacket | None: """Cache exact one-window record geometry as tensor-native columns. V2 training extents normally fit one target window. Their stored prefix+content records are therefore already the exact contiguous transfer arena; decoding their immutable compact geometry once avoids repeating Python record lookup for every transaction. """ cached = self.single_window_geometry if cached is not None: return cached if ( self.manifest.get("packedStorageLayout") != FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT or self.manifest.get("rows") != self.manifest.get("recordCount") ): return None import torch record_count = int(self.manifest["recordCount"]) token_offsets = [0] prompt_lengths: list[int] = [] prompt_digests = bytearray() window_digests = bytearray() expected_token_offset = 0 for record_index in range(record_count): record = self.compact_record(record_index) if ( record.record_index != record_index or record.window_start != record_index or record.window_count != 1 or record.token_offset != expected_token_offset ): return None record_hash = self.record_identity_sha256(record_index) token_end = record.prefix_length + record.content_length expected_token_offset += token_end token_offsets.append(expected_token_offset) prompt_lengths.append(record.prefix_length) prompt_digests.extend( hashlib.sha256( b"nnf.resynthesis.full_payload_packed_prompt.v1\x00" + record_hash + struct.pack( " bytes: if ( self.manifest.get("packedStorageLayout") == FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT ): record = self.compact_record(record_index) identity = { "payloadWorkId": self.manifest["payloadWorkId"], "recordLocator": record.locator, "recordOrdinal": record.record_ordinal, "sourceSha256": self.manifest["observedPayloadSha256"], } return hashlib.sha256(_json_bytes(identity)).digest() return bytes( self.record_hash_mmap[ record_index * 32 : (record_index + 1) * 32 ] ) def _packed_identity_rows_from_packet_boundary( packet: PackedTokenBatchPacket, *, global_cursor_field: str, ) -> tuple[PackedTokenTrainingRow, ...]: """Expose minimal row identity views over one already-built token packet. This external-I/O adapter exists only for compatibility consumers that still inspect row authority fields. It never reopens source records, parses metadata, decodes token frames, or copies token spans. Compute continues to use ``packet`` as the sole token payload. """ if global_cursor_field not in { "full_payload_global_cursor", "full_payload_federated_global_cursor", }: raise ValueError("packed token global cursor field differs") prompt_digest_bytes = packet.prompt_sha256_t.numpy().tobytes() window_digest_bytes = packet.window_sha256_t.numpy().tobytes() authority_bytes = packet.authority_sha256_t.numpy().tobytes() authority_rows = tuple( ( authority_bytes[ authority_index * 64 : authority_index * 64 + 32 ].hex(), authority_bytes[ authority_index * 64 + 32 : (authority_index + 1) * 64 ].hex(), ) for authority_index in range(packet.authority_sha256_t.shape[0]) ) authority_indices = tuple( int(value) for value in packet.row_authority_index_t.tolist() ) global_cursor_start = int(packet.global_cursor_start_t[0]) identity_rows: list[PackedTokenTrainingRow] = [] for row_index, authority_index in enumerate(authority_indices): if authority_index < 0 or authority_index >= len(authority_rows): raise RuntimeError( "packed token identity authority index differs" ) prompt_sha256 = prompt_digest_bytes[ row_index * 32 : (row_index + 1) * 32 ].hex() window_sha256 = window_digest_bytes[ row_index * 32 : (row_index + 1) * 32 ].hex() component_sha256, payload_work_id = authority_rows[authority_index] prompt_ids_t = packet.prompt_ids_boundary(row_index) identity_rows.append( PackedTokenTrainingRow( { "full_payload_component_id": component_sha256, "payload_work_id": payload_work_id, "prompt_sha256": prompt_sha256, "window_sha256": window_sha256, global_cursor_field: global_cursor_start + row_index, "target_entered_forward": False, "task_intent_targets_entered_forward": False, }, prompt_ids=prompt_ids_t, answer_ids=packet.answer_ids_boundary(row_index), corrected_prompt_ids=prompt_ids_t, task_intent_targets=( packet.task_intent_targets_boundary(row_index) ), source_row_sha256=prompt_sha256, batch_packet=packet, batch_packet_row_index=row_index, ) ) if len(identity_rows) != len(packet): raise RuntimeError("packed token identity row frontier differs") return tuple(identity_rows) class FullPayloadPackedTrainingBatches: """Bounded-mmap cursor over one sealed full-payload component collection. This is an external-I/O boundary. It opens at most four source-file shards at a time, reuses immutable mmap pages across contiguous CUDA-wave rows, and derives every loss-window identity from the sealed record identity rather than persisting one duplicate hash per 32-token window. """ # Cursor rows are complete teacher-forced token sequences. Group them # into the same optimizer/CUDA range-packet width as the sealed JSONL # path; the CUDA capacity planner remains responsible for splitting that # range into device-resident waves. bulk_sequence_rows_per_update = 2_048 _MAXIMUM_OPEN_SHARDS = 4 def __init__( self, collection_receipt_path: Path, *, corpus_root: Path | None = None, schedule_receipt_path: Path | None = None, hash_ledger_path: Path | None = None, tokenizer_authority: Mapping[str, Any] | None = None, maximum_proposal_rows: int, assigned_cursor_start: int = 0, assigned_cursor_end: int | None = None, ) -> None: if ( isinstance(maximum_proposal_rows, bool) or maximum_proposal_rows < 1 ): raise ValueError("full payload packed proposal width must be positive") self.collection_receipt_path = ( collection_receipt_path.expanduser().resolve() ) self.corpus_root = ( corpus_root.expanduser().resolve() if corpus_root is not None else None ) self.schedule_receipt_path = ( schedule_receipt_path.expanduser().resolve() if schedule_receipt_path is not None else None ) self.hash_ledger_path = ( hash_ledger_path.expanduser().resolve() if hash_ledger_path is not None else None ) collection_value = json.loads( self.collection_receipt_path.read_text(encoding="utf-8") ) recorded_authority = ( collection_value.pop("collectionAuthoritySha256", None) if isinstance(collection_value, dict) else None ) collection_schema = ( collection_value.get("schema") if isinstance(collection_value, dict) else None ) incremental_cohort = ( collection_schema == FULL_PAYLOAD_PACKED_TOKEN_COHORT_SCHEMA ) if ( not isinstance(collection_value, dict) or collection_schema not in { FULL_PAYLOAD_PACKED_TOKEN_COLLECTION_SCHEMA, FULL_PAYLOAD_PACKED_TOKEN_COHORT_SCHEMA, } or collection_value.get("passed") is not True or not isinstance(recorded_authority, str) or len(recorded_authority) != 64 or recorded_authority != _sha256_bytes(_json_bytes(collection_value)) or collection_value.get("exactDisjointShardWindows") is not True or collection_value.get("targetEnteredForward") is not False or ( incremental_cohort and ( collection_value.get("completeSelectedCohortSealed") is not True or collection_value.get("completeSourceScheduleClaimed") is not False or collection_value.get("completePayloadCollectionClaimed") is not False or collection_value.get("globalDatasetTrainingClaimed") is not False or collection_value.get("promotionEligible") is not False ) ) ): raise ValueError("full payload packed collection authority differs") collection_boundary_contract = collection_value.get( "tokenBoundaryContract" ) if collection_boundary_contract is None: collection_value["tokenBoundaryContract"] = ( FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT ) elif ( collection_boundary_contract not in FULL_PAYLOAD_PACKED_SUPPORTED_TOKEN_BOUNDARY_CONTRACTS ): raise ValueError( "full payload packed token boundary contract differs" ) self.incremental_cohort = incremental_cohort storage_admission = collection_value.get("storageAdmission") if storage_admission is not None: admission_receipt = ( storage_admission.get("receipt") if isinstance(storage_admission, dict) else None ) if ( not isinstance(admission_receipt, dict) or not _full_payload_packed_artifact_matches( admission_receipt ) ): raise ValueError( "full payload packed storage admission link differs" ) validated_admission = ( _validated_full_payload_packed_storage_admission_receipt( Path(str(admission_receipt["path"])) ) ) if ( storage_admission.get("passed") is not True or storage_admission.get("storageAdmissionSha256") != validated_admission.get("storageAdmissionSha256") or storage_admission.get("scheduledPayloadWorkCount") != validated_admission.get("scheduledPayloadWorkCount") or storage_admission.get( "scheduledPayloadWorkIdsSha256" ) != validated_admission.get( "scheduledPayloadWorkIdsSha256" ) or storage_admission.get("scheduledSourceIds") != validated_admission.get("scheduledSourceIds") ): raise ValueError( "full payload packed storage admission authority differs" ) stored_tokenizer_authority = collection_value.get("tokenizerAuthority") if not isinstance(stored_tokenizer_authority, dict): raise ValueError("full payload packed tokenizer authority is absent") ( _stored_tokenizer_record, stored_tokenizer_sha256, _stored_vocabulary_size, ) = _validated_full_payload_fastokens_authority( stored_tokenizer_authority ) if ( collection_value.get("tokenizerAuthoritySha256") != stored_tokenizer_sha256 ): raise ValueError("full payload packed tokenizer authority differs") if tokenizer_authority is not None: ( _runtime_tokenizer_record, runtime_tokenizer_sha256, _runtime_vocabulary_size, ) = _validated_full_payload_fastokens_authority( tokenizer_authority ) if runtime_tokenizer_sha256 != stored_tokenizer_sha256: raise RuntimeError( "full payload packed runtime Fastokens authority differs" ) self._validate_component_binding(collection_value) collection_value["collectionAuthoritySha256"] = recorded_authority self.collection = collection_value schedule_sha256 = collection_value.get("scheduleSha256") context_window_tokens = collection_value.get("contextWindowTokens") answer_tokens_per_window = collection_value.get("answerTokensPerWindow") total_rows = collection_value.get("rows") if ( not isinstance(schedule_sha256, str) or len(schedule_sha256) != 64 or not isinstance(context_window_tokens, int) or isinstance(context_window_tokens, bool) or context_window_tokens < 3 or not isinstance(answer_tokens_per_window, int) or isinstance(answer_tokens_per_window, bool) or answer_tokens_per_window < 1 or answer_tokens_per_window >= context_window_tokens or not isinstance(total_rows, int) or isinstance(total_rows, bool) or total_rows < 1 ): raise ValueError("full payload packed collection geometry differs") self.schedule_sha256 = schedule_sha256 self.maximum_proposal_rows = maximum_proposal_rows self._context_window_tokens = context_window_tokens self._answer_tokens_per_window = answer_tokens_per_window terminal = total_rows if assigned_cursor_end is None else assigned_cursor_end if ( isinstance(assigned_cursor_start, bool) or isinstance(terminal, bool) or assigned_cursor_start < 0 or terminal <= assigned_cursor_start or terminal > total_rows ): raise ValueError("full payload packed assigned cursor window is invalid") self.assigned_cursor_start = assigned_cursor_start self.assigned_cursor_end = terminal shard_values = collection_value.get("shards") if not isinstance(shard_values, list) or not shard_values: raise ValueError("full payload packed collection shards are absent") self._shard_records: list[dict[str, Any]] = [] prior_end = 0 observed_work_ids: list[str] = [] prior_source_ordinal = -1 for local_ordinal, raw_record in enumerate(shard_values): if not isinstance(raw_record, dict): raise ValueError("full payload packed shard record is malformed") schedule_ordinal = raw_record.get("scheduleOrdinal") start = raw_record.get("cursorStart") end = raw_record.get("cursorEnd") manifest_record = raw_record.get("manifest") shard_authority = raw_record.get("shardAuthoritySha256") payload_work_id = raw_record.get("payloadWorkId") source_ordinal = raw_record.get("sourceScheduleOrdinal") if ( not isinstance(payload_work_id, str) or len(payload_work_id) != 64 or payload_work_id in observed_work_ids or not isinstance(start, int) or isinstance(start, bool) or not isinstance(end, int) or isinstance(end, bool) or start != prior_end or end <= start or raw_record.get("rows") != end - start or not isinstance(manifest_record, dict) or not isinstance(shard_authority, str) or len(shard_authority) != 64 or ( incremental_cohort and ( type(schedule_ordinal) is not int or schedule_ordinal != local_ordinal or not isinstance(source_ordinal, int) or isinstance(source_ordinal, bool) or source_ordinal <= prior_source_ordinal ) ) or ( not incremental_cohort and ( type(schedule_ordinal) is not int or schedule_ordinal <= prior_source_ordinal or source_ordinal is not None ) ) ): raise ValueError("full payload packed shard cursor differs") self._shard_records.append(dict(raw_record)) observed_work_ids.append(payload_work_id) canonical_schedule_ordinal = ( source_ordinal if incremental_cohort else schedule_ordinal ) assert isinstance(canonical_schedule_ordinal, int) prior_source_ordinal = canonical_schedule_ordinal prior_end = end if ( prior_end != total_rows or len(self._shard_records) != collection_value.get("shardCount") or ( incremental_cohort and ( collection_value.get("payloadWorkIds") != observed_work_ids or collection_value.get("selectedPayloadWorkCount") != len(observed_work_ids) or collection_value.get("payloadWorkIdsSha256") != _sha256_bytes( _json_bytes( {"payloadWorkIds": observed_work_ids} ) ) ) ) ): raise ValueError("full payload packed collection cursor coverage differs") self._cursor_ends = [ int(record["cursorEnd"]) for record in self._shard_records ] self._open_shards: dict[int, _FullPayloadPackedShard] = {} self._last_used: dict[int, int] = {} self._use_counter = 0 def _validate_component_binding(self, collection: Mapping[str, Any]) -> None: """Validate packed schedule/ledger receipts without opening raw payloads.""" schedule_record = collection.get("scheduleReceipt") ledger_record = collection.get("hashLedger") if ( not isinstance(schedule_record, dict) or not isinstance(ledger_record, dict) or ( self.schedule_receipt_path is not None and Path(str(schedule_record.get("path", ""))) .expanduser() .resolve() != self.schedule_receipt_path ) or ( self.hash_ledger_path is not None and Path(str(ledger_record.get("path", ""))) .expanduser() .resolve() != self.hash_ledger_path ) or ( self.corpus_root is not None and collection.get("corpusRoot") != str(self.corpus_root) ) or not _full_payload_packed_artifact_matches(schedule_record) or ( not self.incremental_cohort and not _full_payload_packed_artifact_matches(ledger_record) ) or ( self.incremental_cohort and ( not isinstance(ledger_record.get("selectedRows"), int) or ledger_record.get("selectedRows") != collection.get("selectedPayloadWorkCount") or not isinstance( ledger_record.get("selectedRowSha256sSha256"), str, ) or len( str(ledger_record["selectedRowSha256sSha256"]) ) != 64 or ledger_record.get("appendMayContinueAfterCohortSeal") is not True ) ) ): raise RuntimeError("full payload packed component binding differs") schedule_receipt_value = json.loads( Path(str(schedule_record["path"])).expanduser().resolve().read_text( encoding="utf-8" ) ) if ( not isinstance(schedule_receipt_value, dict) or schedule_receipt_value.get("schema") != FULL_PAYLOAD_TRAINING_SCHEDULE_RECEIPT_SCHEMA or schedule_receipt_value.get("passed") is not True ): raise RuntimeError("full payload packed schedule receipt differs") current_schedule = schedule_receipt_value.get("schedule") collection_schedule = collection.get("schedule") if ( not isinstance(current_schedule, dict) or not isinstance(collection_schedule, dict) or current_schedule.get("sha256") != collection.get("scheduleSha256") or collection_schedule.get("sha256") != collection.get("scheduleSha256") or not _full_payload_packed_artifact_matches(collection_schedule) ): raise RuntimeError("full payload packed schedule binding differs") def __len__(self) -> int: return self.assigned_cursor_end - self.assigned_cursor_start def __getitem__(self, index: int) -> list[dict[str, Any]]: return self.batch_for_cursor(index) def _shard_index_for_global_cursor(self, global_cursor: int) -> int: index = bisect.bisect_right(self._cursor_ends, global_cursor) if index >= len(self._shard_records): raise IndexError("full payload packed cursor is unavailable") return index def _touch_shard(self, index: int) -> None: self._use_counter += 1 self._last_used[index] = self._use_counter @staticmethod def _relative_source_path(manifest: Mapping[str, Any]) -> Path: value = manifest.get("payloadRelativePath") if not isinstance(value, str) or not value: raise RuntimeError("full payload packed source path is absent") relative_path = Path(value) if relative_path.is_absolute() or ".." in relative_path.parts: raise RuntimeError("full payload packed source path is malformed") return relative_path def _open_shard(self, index: int) -> _FullPayloadPackedShard: existing = self._open_shards.get(index) if existing is not None: self._touch_shard(index) return existing if len(self._open_shards) >= self._MAXIMUM_OPEN_SHARDS: retired_index = min( self._last_used, key=lambda candidate: self._last_used[candidate], ) self._open_shards.pop(retired_index).close() self._last_used.pop(retired_index, None) record = self._shard_records[index] manifest_record = record["manifest"] if not _full_payload_packed_artifact_matches(manifest_record): raise RuntimeError("full payload packed shard manifest bytes differ") manifest_value = json.loads( Path(str(manifest_record["path"])).expanduser().resolve().read_text( encoding="utf-8" ) ) manifest_authority = ( manifest_value.pop("shardAuthoritySha256", None) if isinstance(manifest_value, dict) else None ) expected_manifest_schedule_ordinal = ( record.get("sourceScheduleOrdinal") if self.incremental_cohort else record.get("scheduleOrdinal") ) if ( not isinstance(manifest_value, dict) or manifest_value.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_SHARD_SCHEMA or manifest_value.get("passed") is not True or manifest_value.get("scheduleOrdinal") != expected_manifest_schedule_ordinal or manifest_value.get("scheduleSha256") != self.schedule_sha256 or manifest_value.get("payloadWorkId") != record.get("payloadWorkId") or manifest_value.get("rows") != int(record["cursorEnd"]) - int(record["cursorStart"]) or manifest_authority != _sha256_bytes(_json_bytes(manifest_value)) or manifest_authority != record.get("shardAuthoritySha256") or manifest_value.get("targetEnteredForward") is not False or manifest_value.get("windowLocatorHashesPersisted") is not False or manifest_value.get("windowIdentityDerivedAtReceipt") is not True ): raise RuntimeError("full payload packed shard authority differs") manifest_boundary_contract = manifest_value.get( "tokenBoundaryContract" ) if manifest_boundary_contract is None: manifest_value["tokenBoundaryContract"] = ( FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT ) manifest_boundary_contract = ( FULL_PAYLOAD_PACKED_TOKEN_BOUNDARY_CONTRACT ) elif ( manifest_boundary_contract not in FULL_PAYLOAD_PACKED_SUPPORTED_TOKEN_BOUNDARY_CONTRACTS ): raise RuntimeError( "full payload packed shard token boundary contract differs" ) if ( manifest_boundary_contract != self.collection["tokenBoundaryContract"] ): raise RuntimeError( "full payload packed shard and collection token boundaries differ" ) manifest_value["shardAuthoritySha256"] = manifest_authority artifacts = manifest_value.get("artifacts") if ( manifest_value.get("packedStorageLayout") == FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT ): token_compression = manifest_value.get("tokenCompression") record_storage = manifest_value.get("recordStorage") required_compact_artifacts = { "tokensZstd", "tokenChunkIndex", "recordsZstd", "recordChunkIndex", } record_count = manifest_value.get("recordCount") token_elements = manifest_value.get("tokenElements") rows = manifest_value.get("rows") token_chunk_count = ( token_compression.get("chunkCount") if isinstance(token_compression, dict) else None ) compact_chunk_count = ( record_storage.get("chunkCount") if isinstance(record_storage, dict) else None ) if ( not isinstance(artifacts, dict) or set(artifacts) != required_compact_artifacts or any( not _full_payload_packed_artifact_matches( artifacts[name] ) for name in required_compact_artifacts ) or not isinstance(token_compression, dict) or token_compression.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_ZSTD_SCHEMA or token_compression.get("compressed") != artifacts["tokensZstd"] or token_compression.get("chunkIndex") != artifacts["tokenChunkIndex"] or token_compression.get("boundedRandomAccess") is not True or not isinstance(record_storage, dict) or record_storage.get("schema") != FULL_PAYLOAD_PACKED_COMPACT_RECORD_SCHEMA or record_storage.get("compressed") != artifacts["recordsZstd"] or record_storage.get("chunkIndex") != artifacts["recordChunkIndex"] or record_storage.get("boundedRandomAccess") is not True or record_storage.get("targetEnteredForward") is not False or record_storage.get("targetValuesRecorded") is not False or not isinstance(record_count, int) or isinstance(record_count, bool) or record_count < 1 or not isinstance(token_elements, int) or isinstance(token_elements, bool) or token_elements < 1 or not isinstance(rows, int) or isinstance(rows, bool) or rows < 1 or not isinstance(token_chunk_count, int) or isinstance(token_chunk_count, bool) or token_chunk_count < 1 or not isinstance(compact_chunk_count, int) or isinstance(compact_chunk_count, bool) or compact_chunk_count < 1 or token_compression.get("uncompressedBytes") != token_elements * 4 or record_storage.get("recordCount") != record_count or record_storage.get("tokenElements") != token_elements or record_storage.get("rows") != rows or record_storage.get("chunkIndexRecordBytes") != FULL_PAYLOAD_PACKED_COMPACT_RECORD_INDEX_STRUCT.size ): raise RuntimeError( "full payload packed compact shard geometry differs" ) compressed_path = Path( str(artifacts["tokensZstd"]["path"]) ).resolve() token_index_path = Path( str(artifacts["tokenChunkIndex"]["path"]) ).resolve() record_path = Path( str(artifacts["recordsZstd"]["path"]) ).resolve() record_index_path = Path( str(artifacts["recordChunkIndex"]["path"]) ).resolve() if ( token_index_path.stat().st_size != token_chunk_count * FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.size or record_index_path.stat().st_size != compact_chunk_count * FULL_PAYLOAD_PACKED_COMPACT_RECORD_INDEX_STRUCT.size ): raise RuntimeError( "full payload packed compact shard artifact geometry differs" ) compressed_file = compressed_path.open("rb") token_index_file = token_index_path.open("rb") record_file = record_path.open("rb") record_index_file = record_index_path.open("rb") try: compressed_mmap = mmap.mmap( compressed_file.fileno(), 0, access=mmap.ACCESS_READ, ) token_index_mmap = mmap.mmap( token_index_file.fileno(), 0, access=mmap.ACCESS_READ, ) record_mmap = mmap.mmap( record_file.fileno(), 0, access=mmap.ACCESS_READ, ) record_index_mmap = mmap.mmap( record_index_file.fileno(), 0, access=mmap.ACCESS_READ, ) first_token = ( FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.unpack_from( token_index_mmap, 0, ) ) last_token = ( FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.unpack_from( token_index_mmap, (token_chunk_count - 1) * FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.size, ) ) if ( first_token[0] != 0 or first_token[2] != 0 or last_token[0] + last_token[1] != compressed_path.stat().st_size or last_token[2] + last_token[3] != token_elements * 4 ): raise RuntimeError( "full payload packed token chunk index geometry differs" ) expected_compressed = 0 expected_record = 0 expected_token = 0 expected_window = 0 for chunk_index in range(compact_chunk_count): ( compressed_offset, compressed_bytes, frame_record_start, frame_record_count, frame_token_start, frame_token_count, frame_window_start, frame_window_count, ) = ( FULL_PAYLOAD_PACKED_COMPACT_RECORD_INDEX_STRUCT .unpack_from( record_index_mmap, chunk_index * FULL_PAYLOAD_PACKED_COMPACT_RECORD_INDEX_STRUCT.size, ) ) if ( compressed_offset != expected_compressed or compressed_bytes < 1 or frame_record_start != expected_record or frame_record_count < 1 or frame_record_count > FULL_PAYLOAD_PACKED_COMPACT_RECORD_CHUNK_RECORDS or frame_token_start != expected_token or frame_token_count < 1 or frame_window_start != expected_window or frame_window_count < 1 ): raise RuntimeError( "full payload packed compact record index differs" ) expected_compressed += compressed_bytes expected_record += frame_record_count expected_token += frame_token_count expected_window += frame_window_count if ( expected_compressed != record_path.stat().st_size or expected_record != record_count or expected_token != token_elements or expected_window != rows ): raise RuntimeError( "full payload packed compact record frontier differs" ) shard = _FullPayloadPackedShard( cursor_start=int(record["cursorStart"]), cursor_end=int(record["cursorEnd"]), manifest=manifest_value, compressed_file=compressed_file, compressed_mmap=compressed_mmap, token_chunk_index_file=token_index_file, token_chunk_index_mmap=token_index_mmap, token_chunk_count=token_chunk_count, locator_file=None, locator_mmap=None, metadata_compressed_file=None, metadata_compressed_mmap=None, metadata_chunk_index_file=None, metadata_chunk_index_mmap=None, metadata_chunk_count=0, metadata_uncompressed_bytes=0, record_hash_file=None, record_hash_mmap=None, window_ends_file=None, window_ends_mmap=None, token_chunk_cache={}, token_chunk_last_used={}, metadata_chunk_cache={}, metadata_chunk_last_used={}, metadata_cache={}, metadata_last_used={}, compact_record_file=record_file, compact_record_mmap=record_mmap, compact_record_index_file=record_index_file, compact_record_index_mmap=record_index_mmap, compact_record_chunk_count=compact_chunk_count, compact_record_chunk_cache={}, compact_record_chunk_last_used={}, compact_record_by_index={}, ) except BaseException: for handle in ( compressed_file, token_index_file, record_file, record_index_file, ): handle.close() raise self._open_shards[index] = shard self._touch_shard(index) return shard required_artifacts = ( "tokensZstd", "tokenChunkIndex", "locators", "metadataZstd", "metadataChunkIndex", "recordLocatorHashes", "windowEnds", ) token_compression = manifest_value.get("tokenCompression") metadata_compression = manifest_value.get("metadataCompression") if ( not isinstance(artifacts, dict) or set(artifacts) != set(required_artifacts) or any( not _full_payload_packed_artifact_matches(artifacts[name]) for name in required_artifacts ) or not isinstance(token_compression, dict) or token_compression.get("schema") != FULL_PAYLOAD_PACKED_TOKEN_ZSTD_SCHEMA or token_compression.get("codec") != "zstd" or token_compression.get("independentFrames") is not True or token_compression.get("checksumPerFrame") is not True or token_compression.get("boundedRandomAccess") is not True or token_compression.get("compressed") != artifacts["tokensZstd"] or token_compression.get("chunkIndex") != artifacts["tokenChunkIndex"] or not isinstance(metadata_compression, dict) or metadata_compression.get("schema") != FULL_PAYLOAD_PACKED_METADATA_ZSTD_SCHEMA or metadata_compression.get("codec") != "zstd" or metadata_compression.get("independentFrames") is not True or metadata_compression.get("checksumPerFrame") is not True or metadata_compression.get("boundedRandomAccess") is not True or metadata_compression.get("compressed") != artifacts["metadataZstd"] or metadata_compression.get("chunkIndex") != artifacts["metadataChunkIndex"] or manifest_value.get("rawMetadataBytesRetained") is not False or manifest_value.get("compressedMetadataRandomAccess") is not True ): raise RuntimeError("full payload packed shard artifact differs") record_count = manifest_value.get("recordCount") token_elements = manifest_value.get("tokenElements") chunk_count = token_compression.get("chunkCount") metadata_bytes = manifest_value.get("metadataUncompressedBytes") metadata_compressed_bytes = manifest_value.get("metadataCompressedBytes") metadata_chunk_count = metadata_compression.get("chunkCount") if ( not isinstance(record_count, int) or isinstance(record_count, bool) or record_count < 1 or not isinstance(token_elements, int) or isinstance(token_elements, bool) or token_elements < 1 or not isinstance(chunk_count, int) or isinstance(chunk_count, bool) or chunk_count < 1 or token_compression.get("uncompressedBytes") != token_elements * 4 or token_compression.get("chunkIndexRecordBytes") != FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.size or not isinstance(metadata_bytes, int) or isinstance(metadata_bytes, bool) or metadata_bytes < 1 or manifest_value.get("metadataBytes") != metadata_bytes or metadata_compression.get("uncompressedBytes") != metadata_bytes or not isinstance(metadata_compressed_bytes, int) or isinstance(metadata_compressed_bytes, bool) or metadata_compressed_bytes < 1 or metadata_compression.get("compressedBytes") != metadata_compressed_bytes or artifacts["metadataZstd"].get("bytes") != metadata_compressed_bytes or not isinstance(metadata_chunk_count, int) or isinstance(metadata_chunk_count, bool) or metadata_chunk_count < 1 or metadata_compression.get("chunkIndexRecordBytes") != FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.size ): raise RuntimeError("full payload packed shard geometry differs") compressed_path = Path(str(artifacts["tokensZstd"]["path"])).resolve() chunk_index_path = Path( str(artifacts["tokenChunkIndex"]["path"]) ).resolve() locators_path = Path(str(artifacts["locators"]["path"])).resolve() metadata_compressed_path = Path( str(artifacts["metadataZstd"]["path"]) ).resolve() metadata_chunk_index_path = Path( str(artifacts["metadataChunkIndex"]["path"]) ).resolve() record_hashes_path = Path( str(artifacts["recordLocatorHashes"]["path"]) ).resolve() window_ends_path = Path(str(artifacts["windowEnds"]["path"])).resolve() if ( chunk_index_path.stat().st_size != chunk_count * FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.size or locators_path.stat().st_size != record_count * FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.size or metadata_chunk_index_path.stat().st_size != metadata_chunk_count * FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.size or record_hashes_path.stat().st_size != record_count * 32 or window_ends_path.stat().st_size != record_count * FULL_PAYLOAD_PACKED_TOKEN_WINDOW_INDEX_RECORD_BYTES ): raise RuntimeError("full payload packed shard artifact geometry differs") compressed_file = compressed_path.open("rb") token_chunk_index_file = chunk_index_path.open("rb") locator_file = locators_path.open("rb") metadata_compressed_file = metadata_compressed_path.open("rb") metadata_chunk_index_file = metadata_chunk_index_path.open("rb") record_hash_file = record_hashes_path.open("rb") window_ends_file = window_ends_path.open("rb") try: compressed_mmap = mmap.mmap( compressed_file.fileno(), 0, access=mmap.ACCESS_READ ) token_chunk_index_mmap = mmap.mmap( token_chunk_index_file.fileno(), 0, access=mmap.ACCESS_READ ) metadata_compressed_mmap = mmap.mmap( metadata_compressed_file.fileno(), 0, access=mmap.ACCESS_READ ) metadata_chunk_index_mmap = mmap.mmap( metadata_chunk_index_file.fileno(), 0, access=mmap.ACCESS_READ ) first_chunk = FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.unpack_from( token_chunk_index_mmap, 0, ) last_chunk = FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.unpack_from( token_chunk_index_mmap, (chunk_count - 1) * FULL_PAYLOAD_PACKED_TOKEN_ZSTD_INDEX_STRUCT.size, ) if ( first_chunk[0] != 0 or first_chunk[2] != 0 or last_chunk[0] + last_chunk[1] != compressed_path.stat().st_size or last_chunk[2] + last_chunk[3] != token_elements * 4 ): raise RuntimeError( "full payload packed token chunk index geometry differs" ) prior_compressed_end = 0 prior_uncompressed_end = 0 for metadata_chunk_index in range(metadata_chunk_count): ( metadata_compressed_offset, metadata_compressed_width, metadata_uncompressed_offset, metadata_uncompressed_width, ) = FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.unpack_from( metadata_chunk_index_mmap, metadata_chunk_index * FULL_PAYLOAD_PACKED_METADATA_ZSTD_INDEX_STRUCT.size, ) if ( metadata_compressed_offset != prior_compressed_end or metadata_uncompressed_offset != prior_uncompressed_end or metadata_compressed_width < 1 or metadata_uncompressed_width < 1 or metadata_uncompressed_width > FULL_PAYLOAD_PACKED_METADATA_ZSTD_CHUNK_BYTES ): raise RuntimeError( "full payload packed metadata chunk index geometry differs" ) prior_compressed_end += metadata_compressed_width prior_uncompressed_end += metadata_uncompressed_width if ( prior_compressed_end != metadata_compressed_bytes or prior_compressed_end != metadata_compressed_path.stat().st_size or prior_uncompressed_end != metadata_bytes ): raise RuntimeError( "full payload packed metadata chunk index geometry differs" ) shard = _FullPayloadPackedShard( cursor_start=int(record["cursorStart"]), cursor_end=int(record["cursorEnd"]), manifest=manifest_value, compressed_file=compressed_file, compressed_mmap=compressed_mmap, token_chunk_index_file=token_chunk_index_file, token_chunk_index_mmap=token_chunk_index_mmap, token_chunk_count=chunk_count, locator_file=locator_file, locator_mmap=mmap.mmap( locator_file.fileno(), 0, access=mmap.ACCESS_READ ), metadata_compressed_file=metadata_compressed_file, metadata_compressed_mmap=metadata_compressed_mmap, metadata_chunk_index_file=metadata_chunk_index_file, metadata_chunk_index_mmap=metadata_chunk_index_mmap, metadata_chunk_count=metadata_chunk_count, metadata_uncompressed_bytes=metadata_bytes, record_hash_file=record_hash_file, record_hash_mmap=mmap.mmap( record_hash_file.fileno(), 0, access=mmap.ACCESS_READ ), window_ends_file=window_ends_file, window_ends_mmap=mmap.mmap( window_ends_file.fileno(), 0, access=mmap.ACCESS_READ ), token_chunk_cache={}, token_chunk_last_used={}, metadata_chunk_cache={}, metadata_chunk_last_used={}, metadata_cache={}, metadata_last_used={}, ) except BaseException: for handle in ( compressed_file, token_chunk_index_file, locator_file, metadata_compressed_file, metadata_chunk_index_file, record_hash_file, window_ends_file, ): handle.close() raise self._open_shards[index] = shard self._touch_shard(index) return shard @staticmethod def _record_index_for_window( shard: _FullPayloadPackedShard, row_index: int, ) -> tuple[int, int]: if ( shard.manifest.get("packedStorageLayout") == FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT ): record, record_window_ordinal = ( shard.compact_record_for_window(row_index) ) return record.record_index, record_window_ordinal lower = 0 upper = int(shard.manifest["recordCount"]) while lower < upper: middle = (lower + upper) // 2 window_end = struct.unpack_from( "= int(shard.manifest["recordCount"]): raise RuntimeError("full payload packed window index differs") prior_end = ( 0 if lower == 0 else struct.unpack_from( " tuple[int, int, int, int, int, int, int]: record_index, record_window_ordinal = self._record_index_for_window( shard, row_index, ) if ( shard.manifest.get("packedStorageLayout") == FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT ): compact = shard.compact_record(record_index) token_start = record_window_ordinal * min( self._answer_tokens_per_window, self._context_window_tokens - compact.prefix_length, ) return ( record_index, compact.token_offset, compact.prefix_length, compact.content_length, 0, 0, token_start, ) ( token_offset, prefix_len, content_len, metadata_offset, metadata_len, ) = FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.unpack_from( shard.locator_mmap, record_index * FULL_PAYLOAD_PACKED_TOKEN_LOCATOR_STRUCT.size, ) answer_width = min( self._answer_tokens_per_window, self._context_window_tokens - prefix_len, ) token_start = record_window_ordinal * answer_width token_end = min(content_len, token_start + answer_width) if token_end <= token_start: raise RuntimeError("full payload packed target window differs") return ( record_index, token_offset, prefix_len, content_len, metadata_offset, metadata_len, token_start, ) def _row_from_open_shard( self, *, global_cursor: int, shard: _FullPayloadPackedShard, ) -> dict[str, Any]: """Materialize one row from an already-resolved immutable shard.""" import torch if ( global_cursor < self.assigned_cursor_start or global_cursor >= self.assigned_cursor_end or global_cursor < shard.cursor_start or global_cursor >= shard.cursor_end ): raise IndexError("full payload packed cursor is unavailable") row_index = global_cursor - shard.cursor_start ( record_index, token_offset, prefix_len, content_len, metadata_offset, metadata_len, token_start, ) = self._window_geometry(shard, row_index) answer_width = min( self._answer_tokens_per_window, self._context_window_tokens - prefix_len, ) token_end = min(content_len, token_start + answer_width) context_capacity = ( self._context_window_tokens - prefix_len - (token_end - token_start) ) context_start = max(0, token_start - context_capacity) if token_offset + prefix_len + content_len > int( shard.manifest["tokenElements"] ) or ( shard.manifest.get("packedStorageLayout") != FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT and metadata_offset + metadata_len > int(shard.manifest["metadataUncompressedBytes"]) ): raise RuntimeError("full payload packed row exceeds sealed storage") metadata_value = shard.metadata_record( record_index, metadata_offset, metadata_len, ) content_offset = token_offset + prefix_len prefix_ids = shard.token_slice(token_offset, content_offset) context_ids = ( shard.token_slice( content_offset + context_start, content_offset + token_start, ) if context_start < token_start else prefix_ids.new_empty((0,)) ) answer_ids = shard.token_slice( content_offset + token_start, content_offset + token_end, ) metadata = dict(metadata_value) # A standalone packed collection is its own stable component. The # federated reader replaces this with the federation's sealed component # ID, so both paths expose one identical sidecar authority field. metadata["full_payload_component_id"] = self.collection[ "collectionAuthoritySha256" ] metadata["prompt_ids"] = torch.cat((prefix_ids, context_ids)) metadata["answer_ids"] = answer_ids metadata["question_id"] = _stable_rank( "nnf.resynthesis.full_payload_token_window.v1", ( f"{metadata['payload_work_id']}\x00" f"{metadata['payload_record_locator']}\x00" f"{token_start}\x00{token_end}" ), ) record_hash = shard.record_identity_sha256(record_index) metadata["prompt_sha256"] = hashlib.sha256( b"nnf.resynthesis.full_payload_packed_prompt.v1\x00" + record_hash + struct.pack(" dict[str, Any]: if local_cursor < 0 or local_cursor >= len(self): raise IndexError("full payload packed cursor is unavailable") global_cursor = self.assigned_cursor_start + local_cursor shard_index = self._shard_index_for_global_cursor(global_cursor) shard = self._open_shard(shard_index) return self._row_from_open_shard( global_cursor=global_cursor, shard=shard, ) def batch_for_cursor(self, cursor: int) -> list[dict[str, Any]]: return [self._row(cursor)] @staticmethod def _bind_packed_batch_packet_boundary( rows: list[dict[str, Any]], *, packet: PackedTokenBatchPacket, global_cursor_start: int, ) -> None: """Bind metadata rows to one already-coalesced typed token arena. The direct packet builder owns the only token copy. This boundary replaces the compatibility row views with packet-arena views without parsing metadata again or concatenating per-row tensors. """ import torch if ( not rows or len(rows) != len(packet) or not torch.equal( packet.global_cursor_start_t, torch.tensor((global_cursor_start,), dtype=torch.int64), ) ): raise ValueError("packed token batch packet binding differs") for row_index, row in enumerate(rows): if not isinstance(row, PackedTokenTrainingRow): raise TypeError("packed token batch row type differs") prompt_sha256 = row.get("prompt_sha256") component_sha256 = row.get("full_payload_component_id") work_id = row.get("payload_work_id") cursor = row.get("full_payload_global_cursor") if ( not isinstance(prompt_sha256, str) or len(prompt_sha256) != 64 or not isinstance(component_sha256, str) or len(component_sha256) != 64 or not isinstance(work_id, str) or len(work_id) != 64 or cursor != global_cursor_start + row_index ): raise RuntimeError("packed token batch row authority differs") try: component_digest = bytes.fromhex(component_sha256) work_digest = bytes.fromhex(work_id) except ValueError as error: raise RuntimeError( "packed token batch digest authority differs" ) from error authority_index = int(packet.row_authority_index_t[row_index]) if ( bytes(packet.prompt_sha256_t[row_index]) != bytes.fromhex( prompt_sha256 ) or bytes( packet.authority_sha256_t[authority_index, 0] ) != component_digest or bytes( packet.authority_sha256_t[authority_index, 1] ) != work_digest ): raise RuntimeError( "packed token batch row authority differs" ) prompt_ids = packet.prompt_ids_boundary(row_index) answer_ids = packet.answer_ids_boundary(row_index) task_intent_targets = ( packet.task_intent_targets_boundary(row_index) ) row["prompt_ids"] = prompt_ids row["answer_ids"] = answer_ids if task_intent_targets is None: row.pop("task_intent_targets", None) else: row["task_intent_targets"] = task_intent_targets row.corrected_prompt_ids_t = prompt_ids row.batch_packet = packet row.batch_packet_row_index = row_index def batch_for_cursor_range( self, cursor_start: int, rows: int, ) -> list[dict[str, Any]]: if rows < 1 or cursor_start < 0 or cursor_start + rows > len(self): raise IndexError("full payload packed cursor range is unavailable") global_cursor = self.assigned_cursor_start + cursor_start global_end = global_cursor + rows materialized: list[dict[str, Any]] = [] while global_cursor < global_end: shard_index = self._shard_index_for_global_cursor(global_cursor) shard = self._open_shard(shard_index) shard_end = min(global_end, shard.cursor_end) materialized.extend( self._row_from_open_shard( global_cursor=row_cursor, shard=shard, ) for row_cursor in range(global_cursor, shard_end) ) global_cursor = shard_end if len(materialized) != rows: raise RuntimeError("full payload packed cursor range differs") if ( self.collection.get("tokenBoundaryContract") == FULL_PAYLOAD_PACKED_EXTENT_TOKEN_BOUNDARY_CONTRACT ): packet = self.packed_packet_for_cursor_range( cursor_start, rows, ) self._bind_packed_batch_packet_boundary( materialized, packet=packet, global_cursor_start=( self.assigned_cursor_start + cursor_start ), ) return materialized def packed_identity_rows_for_cursor_range( self, cursor_start: int, rows: int, ) -> tuple[PackedTokenTrainingRow, ...]: """Build compute identity views without legacy row materialization.""" return _packed_identity_rows_from_packet_boundary( self.packed_packet_for_cursor_range(cursor_start, rows), global_cursor_field="full_payload_global_cursor", ) def packed_packet_for_cursor_range( self, cursor_start: int, rows: int, ) -> PackedTokenBatchPacket: """Read one cursor range into a single tensor-native token arena.""" import torch if rows < 1 or cursor_start < 0 or cursor_start + rows > len(self): raise IndexError("full payload packed cursor range is unavailable") fast_global_start = self.assigned_cursor_start + cursor_start fast_global_end = fast_global_start + rows fast_shard_index = self._shard_index_for_global_cursor( fast_global_start ) fast_shard = self._open_shard(fast_shard_index) if fast_global_end <= fast_shard.cursor_end: fast_geometry = ( fast_shard.single_window_geometry_boundary() ) if fast_geometry is not None: fast_row_start = ( fast_global_start - fast_shard.cursor_start ) fast_row_end = fast_row_start + rows source_token_start = int( fast_geometry.token_offsets_t[fast_row_start] ) source_token_end = int( fast_geometry.token_offsets_t[fast_row_end] ) token_ids_t = fast_shard.token_slice( source_token_start, source_token_end, ).contiguous() row_offsets_t = ( fast_geometry.token_offsets_t[ fast_row_start : fast_row_end + 1 ] - source_token_start ).contiguous() component_digest = bytes.fromhex( str(self.collection["collectionAuthoritySha256"]) ) work_digest = bytes.fromhex( str(fast_shard.manifest["payloadWorkId"]) ) return PackedTokenBatchPacket( token_ids_t=token_ids_t, row_offsets_t=row_offsets_t, prompt_lengths_t=( fast_geometry.prompt_lengths_t[ fast_row_start:fast_row_end ].contiguous() ), prompt_sha256_t=( fast_geometry.prompt_sha256_t[ fast_row_start:fast_row_end ].contiguous() ), window_sha256_t=( fast_geometry.window_sha256_t[ fast_row_start:fast_row_end ].contiguous() ), authority_sha256_t=torch.frombuffer( bytearray(component_digest + work_digest), dtype=torch.uint8, ).reshape( 1, 2, hashlib.sha256().digest_size, ), row_authority_index_t=torch.zeros( (rows,), dtype=torch.int32, ), global_cursor_start_t=torch.tensor( (fast_global_start,), dtype=torch.int64, ), ) geometry_cursor = self.assigned_cursor_start + cursor_start geometry_end = geometry_cursor + rows token_arena_elements = 0 geometry_rows = 0 while geometry_cursor < geometry_end: shard_index = self._shard_index_for_global_cursor( geometry_cursor ) shard = self._open_shard(shard_index) shard_end = min(geometry_end, shard.cursor_end) while geometry_cursor < shard_end: ( _record_index, _token_offset, prefix_len, content_len, _metadata_offset, _metadata_len, token_start, ) = self._window_geometry( shard, geometry_cursor - shard.cursor_start, ) answer_width = min( self._answer_tokens_per_window, self._context_window_tokens - prefix_len, ) token_end = min( content_len, token_start + answer_width, ) context_capacity = ( self._context_window_tokens - prefix_len - (token_end - token_start) ) context_start = max( 0, token_start - context_capacity, ) row_token_elements = ( prefix_len + token_start - context_start + token_end - token_start ) if row_token_elements < 2: raise RuntimeError( "full payload packed packet row geometry differs" ) token_arena_elements += row_token_elements geometry_cursor += 1 geometry_rows += 1 if geometry_rows != rows or token_arena_elements < rows * 2: raise RuntimeError( "full payload packed packet geometry frontier differs" ) token_arena_t = torch.empty( (token_arena_elements,), dtype=torch.int32, ) row_offsets_t = torch.empty((rows + 1,), dtype=torch.int64) row_offsets_t[0] = 0 prompt_lengths_t = torch.empty((rows,), dtype=torch.int32) row_authority_index_t = torch.empty((rows,), dtype=torch.int32) prompt_sha256_bytes = bytearray() window_sha256_bytes = bytearray() authority_sha256_bytes = bytearray() authority_indices: dict[bytes, int] = {} token_copy_spans: list[tuple[int, int, int, int]] = [] component_sha256 = bytes.fromhex( str(self.collection["collectionAuthoritySha256"]) ) global_cursor = self.assigned_cursor_start + cursor_start global_end = global_cursor + rows row_offset = 0 token_cursor = 0 while global_cursor < global_end: shard_index = self._shard_index_for_global_cursor(global_cursor) shard = self._open_shard(shard_index) shard_end = min(global_end, shard.cursor_end) while global_cursor < shard_end: ( record_index, token_offset, prefix_len, content_len, metadata_offset, metadata_len, token_start, ) = self._window_geometry( shard, global_cursor - shard.cursor_start, ) answer_width = min( self._answer_tokens_per_window, self._context_window_tokens - prefix_len, ) token_end = min(content_len, token_start + answer_width) context_capacity = ( self._context_window_tokens - prefix_len - (token_end - token_start) ) context_start = max(0, token_start - context_capacity) if token_offset + prefix_len + content_len > int( shard.manifest["tokenElements"] ) or ( shard.manifest.get("packedStorageLayout") != FULL_PAYLOAD_PACKED_COMPACT_STORAGE_LAYOUT and metadata_offset + metadata_len > int(shard.manifest["metadataUncompressedBytes"]) ): raise RuntimeError( "full payload packed row exceeds sealed storage" ) content_offset = token_offset + prefix_len row_ranges = ( ( token_offset, content_offset + token_end, ), ) if context_start == 0 and token_start == 0 else ( (token_offset, content_offset), ( content_offset + context_start, content_offset + token_end, ), ) for source_start, source_end in row_ranges: if source_end <= source_start: continue range_width = source_end - source_start if ( token_copy_spans and token_copy_spans[-1][0] == shard_index and token_copy_spans[-1][3] == source_start and ( token_copy_spans[-1][1] + token_copy_spans[-1][3] - token_copy_spans[-1][2] ) == token_cursor ): ( prior_shard_index, prior_destination_start, prior_source_start, _prior_source_end, ) = token_copy_spans[-1] token_copy_spans[-1] = ( prior_shard_index, prior_destination_start, prior_source_start, source_end, ) else: token_copy_spans.append( ( shard_index, token_cursor, source_start, source_end, ) ) token_cursor += range_width prompt_length = ( prefix_len + token_start - context_start ) prompt_lengths_t[row_offset] = prompt_length row_offsets_t[row_offset + 1] = token_cursor record_hash = shard.record_identity_sha256(record_index) prompt_sha256_bytes.extend( hashlib.sha256( b"nnf.resynthesis.full_payload_packed_prompt.v1\x00" + record_hash + struct.pack( " tuple[PackedTokenPhysicalSourceRange, ...]: """Describe exact physical object ranges behind one packet arena.""" if rows < 1 or cursor_start < 0 or cursor_start + rows > len(self): raise IndexError("full payload packed cursor range is unavailable") component_sha256 = str( self.collection["collectionAuthoritySha256"] ) fast_global_start = self.assigned_cursor_start + cursor_start fast_global_end = fast_global_start + rows fast_shard_index = self._shard_index_for_global_cursor( fast_global_start ) fast_shard = self._open_shard(fast_shard_index) if fast_global_end <= fast_shard.cursor_end: fast_geometry = ( fast_shard.single_window_geometry_boundary() ) if fast_geometry is not None: fast_row_start = ( fast_global_start - fast_shard.cursor_start ) return ( fast_shard .single_window_physical_source_ranges_boundary( geometry=fast_geometry, row_start=fast_row_start, row_end=fast_row_start + rows, component_sha256=component_sha256, global_cursor_start=fast_global_start, ) ) global_cursor = self.assigned_cursor_start + cursor_start global_end = global_cursor + rows packet_byte_cursor = 0 physical_ranges: list[PackedTokenPhysicalSourceRange] = [] while global_cursor < global_end: shard_index = self._shard_index_for_global_cursor(global_cursor) shard = self._open_shard(shard_index) shard_end = min(global_end, shard.cursor_end) while global_cursor < shard_end: ( _record_index, token_offset, prefix_len, content_len, _metadata_offset, _metadata_len, token_start, ) = self._window_geometry( shard, global_cursor - shard.cursor_start, ) answer_width = min( self._answer_tokens_per_window, self._context_window_tokens - prefix_len, ) token_end = min(content_len, token_start + answer_width) context_capacity = ( self._context_window_tokens - prefix_len - (token_end - token_start) ) context_start = max(0, token_start - context_capacity) content_offset = token_offset + prefix_len row_ranges = ( ((token_offset, content_offset + token_end),) if context_start == 0 and token_start == 0 else ( (token_offset, content_offset), ( content_offset + context_start, content_offset + token_end, ), ) ) for source_start, source_end in row_ranges: if source_end <= source_start: continue source_ranges = ( shard.physical_source_ranges_boundary( token_start=source_start, token_end=source_end, packet_byte_start=packet_byte_cursor, component_sha256=component_sha256, global_cursor=global_cursor, ) ) if not source_ranges: raise RuntimeError( "full payload packed physical source range is empty" ) physical_ranges.extend(source_ranges) packet_byte_cursor = source_ranges[-1].packet_byte_end global_cursor += 1 if ( global_cursor != global_end or not physical_ranges or physical_ranges[0].packet_byte_start != 0 ): raise RuntimeError( "full payload packed physical source frontier differs" ) return tuple(physical_ranges) def proposal_window_limit(self, cursor: int) -> int: if cursor < 0 or cursor >= len(self): if self.training_complete(cursor): raise RuntimeError("full payload packed training schedule is complete") raise IndexError("full payload packed cursor is unavailable") return min(self.maximum_proposal_rows, len(self) - cursor) def await_proposal_window(self, cursor: int) -> int: return 0 if self.training_complete(cursor) else self.proposal_window_limit(cursor) def training_window_receipt( self, cursor_start: int, rows: int, ) -> dict[str, Any]: if rows < 1 or cursor_start < 0 or cursor_start + rows > len(self): raise RuntimeError("full payload packed window is unavailable") digest = hashlib.sha256() work_ids: list[str] = [] for local_cursor in range(cursor_start, cursor_start + rows): global_cursor = self.assigned_cursor_start + local_cursor shard_index = self._shard_index_for_global_cursor(global_cursor) shard = self._open_shard(shard_index) row_index = global_cursor - shard.cursor_start ( record_index, _token_offset, prefix_len, content_len, _metadata_offset, _metadata_len, token_start, ) = self._window_geometry(shard, row_index) answer_width = min( self._answer_tokens_per_window, self._context_window_tokens - prefix_len, ) token_end = min(content_len, token_start + answer_width) record_hash = shard.record_identity_sha256(record_index) digest.update( _full_payload_packed_window_identity( record_hash, token_start=token_start, token_end=token_end, ) ) work_id = str(shard.manifest["payloadWorkId"]) if not work_ids or work_ids[-1] != work_id: work_ids.append(work_id) return { "schema": FULL_PAYLOAD_TRAINING_WINDOW_RECEIPT_SCHEMA, "scheduleSha256": self.schedule_sha256, "collectionAuthoritySha256": self.collection[ "collectionAuthoritySha256" ], "cursorStart": cursor_start, "cursorEnd": cursor_start + rows, "globalCursorStart": self.assigned_cursor_start + cursor_start, "globalCursorEnd": self.assigned_cursor_start + cursor_start + rows, "assignedGlobalCursorStart": self.assigned_cursor_start, "assignedGlobalCursorEnd": self.assigned_cursor_end, "rows": rows, "windowIdentitySha256": digest.hexdigest(), "payloadWorkIds": work_ids, "assignedWindowConsumedAtCursorEnd": self.training_complete( cursor_start + rows ), "completeDatasetConsumedAtCursorEnd": bool( self.assigned_cursor_start == 0 and self.assigned_cursor_end == int(self.collection["rows"]) and self.training_complete(cursor_start + rows) ), "exactDisjointWindowAssignment": True, "targetValuesRecorded": False, "validationRowsObserved": 0, "targetEnteredForward": False, "rawDataDeletionAllowed": False, } def training_complete(self, cursor: int) -> bool: if cursor < 0: raise ValueError("full payload packed cursor cannot be negative") return cursor >= len(self) def cursor_receipt(self, cursor: int) -> dict[str, Any]: if cursor < 0: raise ValueError("full payload packed cursor cannot be negative") return { "schema": "nnf.resynthesis.full_payload_packed_training_cursor.v1", "scheduleSha256": self.schedule_sha256, "collectionAuthoritySha256": self.collection[ "collectionAuthoritySha256" ], "retainedTokenWindows": cursor, "globalRetainedCursor": self.assigned_cursor_start + cursor, "assignedGlobalCursorStart": self.assigned_cursor_start, "assignedGlobalCursorEnd": self.assigned_cursor_end, "assignedWindowConsumed": self.training_complete(cursor), "completeSelectedCohortConsumed": bool( self.incremental_cohort and self.assigned_cursor_start == 0 and self.assigned_cursor_end == int(self.collection["rows"]) and self.training_complete(cursor) ), "completeDatasetConsumed": bool( not self.incremental_cohort and self.assigned_cursor_start == 0 and self.assigned_cursor_end == int(self.collection["rows"]) and self.training_complete(cursor) ), "totalTokenWindows": len(self), "targetValuesRecorded": False, "rawDataDeletionAllowed": False, } def authority_receipt(self) -> dict[str, Any]: return { "schema": "nnf.resynthesis.full_payload_packed_training.v1", "collectionReceiptPath": str(self.collection_receipt_path), "collectionAuthoritySha256": self.collection[ "collectionAuthoritySha256" ], "scheduleSha256": self.schedule_sha256, "tokenizerAuthoritySha256": self.collection[ "tokenizerAuthoritySha256" ], "scheduledPayloadFiles": self.collection["shardCount"], "totalTokenWindows": self.collection["rows"], "assignedGlobalCursorStart": self.assigned_cursor_start, "assignedGlobalCursorEnd": self.assigned_cursor_end, "maximumProposalRows": self.maximum_proposal_rows, "contextWindowTokens": self.collection["contextWindowTokens"], "answerTokensPerWindow": self.collection[ "answerTokensPerWindow" ], "mmapReadable": True, "rawPayloadRequiredAtTraining": False, "rangeBatchTensorRows": True, "cudaWaveCompatible": True, "exactDisjointWindowAssignment": True, "lazyOpenShardLimit": self._MAXIMUM_OPEN_SHARDS, "incrementalCohort": self.incremental_cohort, "completeSelectedCohortSealed": ( self.collection.get("completeSelectedCohortSealed") if self.incremental_cohort else False ), "completeSourceScheduleClaimed": False, "globalDatasetTrainingClaimed": False, "completeDatasetClaimedTrained": False, "targetEnteredForward": False, "rawDataDeletionAllowed": False, } def close(self) -> None: for shard in self._open_shards.values(): shard.close() self._open_shards = {} self._last_used = {} class FullPayloadFederatedPackedTrainingBatches: """Lazy global cursor over independently packed native corpus roots. The class deliberately composes sealed token collections, not source trees. Consequently a learner reads only zstd/mmap token artifacts and schedule receipts after build; a missing raw science disk cannot silently cause retraining from an alternate source or a fallback tokenizer path. """ bulk_sequence_rows_per_update = 2_048 _MAXIMUM_OPEN_COMPONENTS = 4 def __init__( self, collection_receipt_path: Path, *, tokenizer_authority: Mapping[str, Any] | None = None, maximum_proposal_rows: int, assigned_cursor_start: int = 0, assigned_cursor_end: int | None = None, ) -> None: if ( isinstance(maximum_proposal_rows, bool) or maximum_proposal_rows < 1 ): raise ValueError("full payload federated proposal width must be positive") self.collection_receipt_path = collection_receipt_path.expanduser().resolve() validated_collection, federation, _validated_components = ( _validated_federated_full_payload_packed_collection( self.collection_receipt_path, tokenizer_authority=tokenizer_authority, validate_native_federation=False, ) ) loaded = dict(validated_collection) recorded_authority = ( loaded.pop("collectionAuthoritySha256", None) if isinstance(loaded, dict) else None ) if ( not isinstance(loaded, dict) or loaded.get("schema") != FULL_PAYLOAD_FEDERATED_PACKED_TOKEN_COLLECTION_SCHEMA or loaded.get("passed") is not True or not isinstance(recorded_authority, str) or len(recorded_authority) != 64 or recorded_authority != _sha256_bytes(_json_bytes(loaded)) or loaded.get("exactDisjointComponentWindows") is not True or loaded.get("rawPayloadCopied") is not False or loaded.get("rawPayloadRequiredAtTraining") is not False or loaded.get("targetEnteredForward") is not False ): raise ValueError("full payload federated packed authority differs") self._packed_ready_federation = ( loaded.get("federationMode") == FULL_PAYLOAD_PACKED_READY_FEDERATION_MODE ) if not self._packed_ready_federation: federation_record = loaded.get("federationReceipt") if not isinstance( federation_record, dict, ) or not _full_payload_packed_artifact_matches(federation_record): raise RuntimeError( "full payload federated authority receipt differs" ) federation, _membership_rows, _components = ( _validated_full_payload_multiroot_authority( Path(str(federation_record["path"])), validate_native_components=False, ) ) federation_authority = federation.get("federationAuthoritySha256") if ( not isinstance(federation_authority, str) or federation_authority != loaded.get("federationAuthoritySha256") or federation_authority != loaded.get("federatedScheduleSha256") ): raise RuntimeError("full payload federated collection binding differs") stored_tokenizer_authority = loaded.get("tokenizerAuthority") if not isinstance(stored_tokenizer_authority, dict): raise ValueError("full payload federated Fastokens authority is absent") ( _stored_tokenizer_record, stored_tokenizer_sha256, _stored_vocabulary_size, ) = _validated_full_payload_fastokens_authority(stored_tokenizer_authority) if loaded.get("tokenizerAuthoritySha256") != stored_tokenizer_sha256: raise RuntimeError("full payload federated Fastokens authority differs") if tokenizer_authority is not None: ( _runtime_tokenizer_record, runtime_tokenizer_sha256, _runtime_vocabulary_size, ) = _validated_full_payload_fastokens_authority(tokenizer_authority) if runtime_tokenizer_sha256 != stored_tokenizer_sha256: raise RuntimeError( "full payload federated runtime Fastokens authority differs" ) rows = loaded.get("rows") context_window_tokens = loaded.get("contextWindowTokens") answer_tokens_per_window = loaded.get("answerTokensPerWindow") if ( not isinstance(rows, int) or isinstance(rows, bool) or rows < 1 or not isinstance(context_window_tokens, int) or isinstance(context_window_tokens, bool) or context_window_tokens < 3 or not isinstance(answer_tokens_per_window, int) or isinstance(answer_tokens_per_window, bool) or answer_tokens_per_window < 1 or answer_tokens_per_window >= context_window_tokens ): raise ValueError("full payload federated packed geometry differs") terminal = rows if assigned_cursor_end is None else assigned_cursor_end if ( isinstance(assigned_cursor_start, bool) or isinstance(terminal, bool) or assigned_cursor_start < 0 or terminal <= assigned_cursor_start or terminal > rows ): raise ValueError("full payload federated assigned cursor window is invalid") component_values = loaded.get("components") if ( not isinstance(component_values, list) or not component_values or loaded.get("componentCount") != len(component_values) ): raise ValueError("full payload federated components are absent") self._component_records: list[dict[str, Any]] = [] prior_end = 0 for raw_component in component_values: if not isinstance(raw_component, dict): raise ValueError("full payload federated component is malformed") component = dict(raw_component) start = component.get("globalCursorStart") end = component.get("globalCursorEnd") collection_record = component.get("collectionReceipt") if ( not isinstance(start, int) or isinstance(start, bool) or not isinstance(end, int) or isinstance(end, bool) or start != prior_end or end <= start or component.get("componentCursorStart") != 0 or component.get("componentCursorEnd") != end - start or not isinstance(component.get("componentId"), str) or len(str(component["componentId"])) != 64 or ( not self._packed_ready_federation and not isinstance(component.get("corpusRoot"), str) ) or not isinstance(component.get("scheduleReceipt"), dict) or not isinstance(component.get("hashLedger"), dict) or not isinstance(collection_record, dict) or not _full_payload_packed_artifact_matches(collection_record) ): raise ValueError("full payload federated component cursor differs") component_collection = json.loads( Path(str(collection_record["path"])).expanduser().resolve().read_text( encoding="utf-8" ) ) component_schema = ( component_collection.get("schema") if isinstance(component_collection, dict) else None ) if ( not isinstance(component_collection, dict) or component_schema not in ( { FULL_PAYLOAD_PACKED_TOKEN_COLLECTION_SCHEMA, FULL_PAYLOAD_PACKED_TOKEN_COHORT_SCHEMA, } if self._packed_ready_federation else {FULL_PAYLOAD_PACKED_TOKEN_COLLECTION_SCHEMA} ) or ( self._packed_ready_federation and component.get("collectionSchema") != component_schema ) or component_collection.get("passed") is not True or component_collection.get("collectionAuthoritySha256") != component.get("collectionAuthoritySha256") or component_collection.get("rows") != end - start or component_collection.get("tokenizerAuthoritySha256") != stored_tokenizer_sha256 or component_collection.get("contextWindowTokens") != context_window_tokens or component_collection.get("answerTokensPerWindow") != answer_tokens_per_window ): raise RuntimeError("full payload federated child collection differs") self._component_records.append(component) prior_end = end if prior_end != rows: raise ValueError("full payload federated cursor coverage differs") self.collection = loaded self.collection["collectionAuthoritySha256"] = recorded_authority self.maximum_proposal_rows = maximum_proposal_rows self.assigned_cursor_start = assigned_cursor_start self.assigned_cursor_end = terminal self._cursor_ends = [ int(component["globalCursorEnd"]) for component in self._component_records ] self._open_components: dict[int, FullPayloadPackedTrainingBatches] = {} self._last_used: dict[int, int] = {} self._use_counter = 0 def __len__(self) -> int: return self.assigned_cursor_end - self.assigned_cursor_start def __getitem__(self, index: int) -> list[dict[str, Any]]: return self.batch_for_cursor(index) def _component_index_for_global_cursor(self, global_cursor: int) -> int: index = bisect.bisect_right(self._cursor_ends, global_cursor) if index >= len(self._component_records): raise IndexError("full payload federated cursor is unavailable") return index def _touch_component(self, index: int) -> None: self._use_counter += 1 self._last_used[index] = self._use_counter def _open_component(self, index: int) -> FullPayloadPackedTrainingBatches: existing = self._open_components.get(index) if existing is not None: self._touch_component(index) return existing if len(self._open_components) >= self._MAXIMUM_OPEN_COMPONENTS: retired_index = min( self._last_used, key=lambda candidate: self._last_used[candidate], ) self._open_components.pop(retired_index).close() self._last_used.pop(retired_index, None) component = self._component_records[index] collection_record = component["collectionReceipt"] component_path = Path(str(collection_record["path"])) child = ( FullPayloadPackedTrainingBatches( component_path, tokenizer_authority=self.collection["tokenizerAuthority"], maximum_proposal_rows=self.maximum_proposal_rows, ) if self._packed_ready_federation else FullPayloadPackedTrainingBatches( component_path, corpus_root=Path(str(component["corpusRoot"])), schedule_receipt_path=Path( str(component["scheduleReceipt"]["path"]) ), hash_ledger_path=Path(str(component["hashLedger"]["path"])), tokenizer_authority=self.collection["tokenizerAuthority"], maximum_proposal_rows=self.maximum_proposal_rows, ) ) self._open_components[index] = child self._touch_component(index) return child def _row(self, local_cursor: int) -> dict[str, Any]: if local_cursor < 0 or local_cursor >= len(self): raise IndexError("full payload federated cursor is unavailable") global_cursor = self.assigned_cursor_start + local_cursor component_index = self._component_index_for_global_cursor(global_cursor) component = self._component_records[component_index] child = self._open_component(component_index) child_cursor = global_cursor - int(component["globalCursorStart"]) # The child materializes a fresh row per call; annotate it directly so # the packed row class (and its tensor-backed train-fn boundary) # survives the federation hop instead of being stripped by dict(). row = child.batch_for_cursor(child_cursor)[0] row["full_payload_federated_global_cursor"] = global_cursor row["full_payload_component_id"] = component["componentId"] return row def batch_for_cursor(self, cursor: int) -> list[dict[str, Any]]: return [self._row(cursor)] def batch_for_cursor_range( self, cursor_start: int, rows: int, ) -> list[dict[str, Any]]: if rows < 1 or cursor_start < 0 or cursor_start + rows > len(self): raise IndexError("full payload federated cursor range is unavailable") global_cursor = self.assigned_cursor_start + cursor_start global_end = global_cursor + rows materialized: list[dict[str, Any]] = [] while global_cursor < global_end: component_index = self._component_index_for_global_cursor( global_cursor ) component = self._component_records[component_index] component_end = min( global_end, int(component["globalCursorEnd"]), ) child = self._open_component(component_index) child_cursor = global_cursor - int(component["globalCursorStart"]) component_rows = child.batch_for_cursor_range( child_cursor, component_end - global_cursor, ) for row_offset, row in enumerate(component_rows): row["full_payload_federated_global_cursor"] = ( global_cursor + row_offset ) row["full_payload_component_id"] = component["componentId"] materialized.append(row) global_cursor = component_end if len(materialized) != rows: raise RuntimeError("full payload federated cursor range differs") return materialized def packed_identity_rows_for_cursor_range( self, cursor_start: int, rows: int, ) -> tuple[PackedTokenTrainingRow, ...]: """Build federated identity views over one exact packet arena.""" return _packed_identity_rows_from_packet_boundary( self.packed_packet_for_cursor_range(cursor_start, rows), global_cursor_field="full_payload_federated_global_cursor", ) def packed_packet_for_cursor_range( self, cursor_start: int, rows: int, ) -> PackedTokenBatchPacket: """Compose child packet arenas without materializing row dictionaries.""" import torch if rows < 1 or cursor_start < 0 or cursor_start + rows > len(self): raise IndexError("full payload federated cursor range is unavailable") global_cursor = self.assigned_cursor_start + cursor_start global_end = global_cursor + rows child_packets: list[ tuple[dict[str, Any], PackedTokenBatchPacket, int] ] = [] exact_token_elements = 0 while global_cursor < global_end: component_index = self._component_index_for_global_cursor( global_cursor ) component = self._component_records[component_index] component_end = min( global_end, int(component["globalCursorEnd"]), ) child = self._open_component(component_index) child_cursor = global_cursor - int( component["globalCursorStart"] ) component_rows = component_end - global_cursor child_packet = child.packed_packet_for_cursor_range( child_cursor, component_rows, ) if len(child_packet) != component_rows: raise RuntimeError( "full payload federated child packet rows differ" ) child_packets.append( (component, child_packet, component_rows) ) exact_token_elements += child_packet.token_ids_t.numel() global_cursor = component_end if global_cursor != global_end or not child_packets: raise RuntimeError("full payload federated child frontier differs") child_intent_arenas_t = tuple( child_packet.task_intent_targets_t for _component, child_packet, _component_rows in child_packets ) if not ( all(value is None for value in child_intent_arenas_t) or all( isinstance(value, torch.Tensor) for value in child_intent_arenas_t ) ): raise RuntimeError( "full payload federated task-intent coverage differs" ) task_intent_width = 0 if all( isinstance(value, torch.Tensor) for value in child_intent_arenas_t ): task_intent_widths = { cast(torch.Tensor, value).shape[1] for value in child_intent_arenas_t } if len(task_intent_widths) != 1: raise RuntimeError( "full payload federated task-intent geometry differs" ) task_intent_width = next(iter(task_intent_widths)) single_child = len(child_packets) == 1 if single_child: only_packet = child_packets[0][1] token_arena_t = only_packet.token_ids_t row_offsets_t = only_packet.row_offsets_t prompt_lengths_t = only_packet.prompt_lengths_t prompt_sha256_t = only_packet.prompt_sha256_t window_sha256_t = only_packet.window_sha256_t task_intent_targets_t = ( only_packet.task_intent_targets_t ) else: token_arena_t = torch.empty( (exact_token_elements,), dtype=torch.int32, ) row_offsets_t = torch.empty((rows + 1,), dtype=torch.int64) row_offsets_t[0] = 0 prompt_lengths_t = torch.empty((rows,), dtype=torch.int32) prompt_sha256_t = torch.empty((rows, 32), dtype=torch.uint8) window_sha256_t = torch.empty((rows, 32), dtype=torch.uint8) task_intent_targets_t = ( torch.empty( (rows, task_intent_width), dtype=torch.float32, ) if task_intent_width > 0 else None ) row_authority_index_t = torch.empty((rows,), dtype=torch.int32) authority_sha256_bytes = bytearray() authority_indices: dict[bytes, int] = {} row_cursor = 0 token_cursor = 0 for component, child_packet, component_rows in child_packets: child_token_count = child_packet.token_ids_t.numel() if not single_child: token_arena_t[ token_cursor : token_cursor + child_token_count ].copy_(child_packet.token_ids_t) row_offsets_t[ row_cursor + 1 : row_cursor + component_rows + 1 ].copy_(child_packet.row_offsets_t[1:] + token_cursor) prompt_lengths_t[ row_cursor : row_cursor + component_rows ].copy_(child_packet.prompt_lengths_t) prompt_sha256_t[ row_cursor : row_cursor + component_rows ].copy_(child_packet.prompt_sha256_t) window_sha256_t[ row_cursor : row_cursor + component_rows ].copy_(child_packet.window_sha256_t) if task_intent_targets_t is not None: child_task_intent_targets_t = ( child_packet.task_intent_targets_t ) if child_task_intent_targets_t is None: raise RuntimeError( "full payload federated task-intent coverage differs" ) task_intent_targets_t[ row_cursor : row_cursor + component_rows ].copy_(child_task_intent_targets_t) component_sha256 = bytes.fromhex(str(component["componentId"])) authority_remap_t = torch.empty( (child_packet.authority_sha256_t.shape[0],), dtype=torch.int32, ) for child_authority_index in range( child_packet.authority_sha256_t.shape[0] ): payload_work_sha256 = ( child_packet.authority_sha256_t[ child_authority_index, 1, ] .numpy() .tobytes() ) authority = component_sha256 + payload_work_sha256 authority_index = authority_indices.get(authority) if authority_index is None: authority_index = len(authority_indices) authority_indices[authority] = authority_index authority_sha256_bytes.extend(authority) authority_remap_t[child_authority_index] = authority_index row_authority_index_t[ row_cursor : row_cursor + component_rows ].copy_( authority_remap_t[ child_packet.row_authority_index_t.to(dtype=torch.int64) ] ) token_cursor += child_token_count row_cursor += component_rows if ( row_cursor != rows or token_cursor != exact_token_elements or token_arena_t.numel() != exact_token_elements ): raise RuntimeError("full payload federated packet frontier differs") return PackedTokenBatchPacket( token_ids_t=token_arena_t, row_offsets_t=row_offsets_t, prompt_lengths_t=prompt_lengths_t, prompt_sha256_t=prompt_sha256_t, window_sha256_t=window_sha256_t, authority_sha256_t=torch.frombuffer( authority_sha256_bytes, dtype=torch.uint8, ) .clone() .reshape(len(authority_indices), 2, 32), row_authority_index_t=row_authority_index_t, global_cursor_start_t=torch.tensor( [self.assigned_cursor_start + cursor_start], dtype=torch.int64, ), task_intent_targets_t=task_intent_targets_t, ) def packed_physical_source_ranges_for_cursor_range( self, cursor_start: int, rows: int, ) -> tuple[PackedTokenPhysicalSourceRange, ...]: """Compose child physical byte provenance without reading token data.""" if rows < 1 or cursor_start < 0 or cursor_start + rows > len(self): raise IndexError("full payload federated cursor range is unavailable") global_cursor = self.assigned_cursor_start + cursor_start global_end = global_cursor + rows packet_byte_cursor = 0 physical_ranges: list[PackedTokenPhysicalSourceRange] = [] while global_cursor < global_end: component_index = self._component_index_for_global_cursor( global_cursor ) component = self._component_records[component_index] component_end = min( global_end, int(component["globalCursorEnd"]), ) child = self._open_component(component_index) child_cursor = global_cursor - int( component["globalCursorStart"] ) component_rows = component_end - global_cursor child_ranges = ( child.packed_physical_source_ranges_for_cursor_range( child_cursor, component_rows, ) ) if not child_ranges or child_ranges[0].packet_byte_start != 0: raise RuntimeError( "full payload federated physical child range differs" ) for child_range in child_ranges: physical_ranges.append( PackedTokenPhysicalSourceRange( component_sha256=str(component["componentId"]), payload_work_id=child_range.payload_work_id, object_path=child_range.object_path, object_sha256=child_range.object_sha256, object_byte_count=child_range.object_byte_count, object_compressed_byte_start=( child_range.object_compressed_byte_start ), object_compressed_byte_end=( child_range.object_compressed_byte_end ), source_uncompressed_byte_start=( child_range.source_uncompressed_byte_start ), source_uncompressed_byte_end=( child_range.source_uncompressed_byte_end ), packet_byte_start=( packet_byte_cursor + child_range.packet_byte_start ), packet_byte_end=( packet_byte_cursor + child_range.packet_byte_end ), global_cursor=( global_cursor + child_range.global_cursor - child_cursor ), ) ) packet_byte_cursor += child_ranges[-1].packet_byte_end global_cursor = component_end if ( global_cursor != global_end or not physical_ranges or physical_ranges[0].packet_byte_start != 0 ): raise RuntimeError( "full payload federated physical source frontier differs" ) return tuple(physical_ranges) def proposal_window_limit(self, cursor: int) -> int: if cursor < 0 or cursor >= len(self): if self.training_complete(cursor): raise RuntimeError("full payload federated training schedule is complete") raise IndexError("full payload federated cursor is unavailable") return min(self.maximum_proposal_rows, len(self) - cursor) def await_proposal_window(self, cursor: int) -> int: return 0 if self.training_complete(cursor) else self.proposal_window_limit(cursor) def training_window_receipt( self, cursor_start: int, rows: int, ) -> dict[str, Any]: if rows < 1 or cursor_start < 0 or cursor_start + rows > len(self): raise RuntimeError("full payload federated window is unavailable") global_cursor = self.assigned_cursor_start + cursor_start remaining = rows digest = hashlib.sha256() component_windows: list[dict[str, Any]] = [] payload_work_ids: list[str] = [] while remaining > 0: component_index = self._component_index_for_global_cursor(global_cursor) component = self._component_records[component_index] component_end = int(component["globalCursorEnd"]) chunk_rows = min(remaining, component_end - global_cursor) child = self._open_component(component_index) child_start = global_cursor - int(component["globalCursorStart"]) child_receipt = child.training_window_receipt(child_start, chunk_rows) child_identity = child_receipt.get("windowIdentitySha256") if not isinstance(child_identity, str) or len(child_identity) != 64: raise RuntimeError("full payload federated child window differs") digest.update(str(component["componentId"]).encode("ascii")) digest.update(bytes.fromhex(child_identity)) component_windows.append( { "componentId": component["componentId"], "globalCursorStart": global_cursor, "globalCursorEnd": global_cursor + chunk_rows, "componentCursorStart": child_start, "componentCursorEnd": child_start + chunk_rows, "windowIdentitySha256": child_identity, } ) for work_id in child_receipt.get("payloadWorkIds", []): qualified_work_id = f"{component['componentId']}:{work_id}" if not payload_work_ids or payload_work_ids[-1] != qualified_work_id: payload_work_ids.append(qualified_work_id) global_cursor += chunk_rows remaining -= chunk_rows return { "schema": FULL_PAYLOAD_FEDERATED_TRAINING_WINDOW_RECEIPT_SCHEMA, "scheduleSha256": self.collection["federatedScheduleSha256"], "sourceSha256": self.collection["federationAuthoritySha256"], "collectionAuthoritySha256": self.collection[ "collectionAuthoritySha256" ], "cursorStart": cursor_start, "cursorEnd": cursor_start + rows, "globalCursorStart": self.assigned_cursor_start + cursor_start, "globalCursorEnd": self.assigned_cursor_start + cursor_start + rows, "assignedGlobalCursorStart": self.assigned_cursor_start, "assignedGlobalCursorEnd": self.assigned_cursor_end, "rows": rows, "windowIdentitySha256": digest.hexdigest(), "componentWindows": component_windows, "payloadWorkIds": payload_work_ids, "assignedWindowConsumedAtCursorEnd": self.training_complete( cursor_start + rows ), "completeDatasetConsumedAtCursorEnd": bool( self.assigned_cursor_start == 0 and self.assigned_cursor_end == int(self.collection["rows"]) and self.training_complete(cursor_start + rows) ), "exactDisjointWindowAssignment": True, "targetValuesRecorded": False, "validationRowsObserved": 0, "targetEnteredForward": False, "rawDataDeletionAllowed": False, } def training_complete(self, cursor: int) -> bool: if cursor < 0: raise ValueError("full payload federated cursor cannot be negative") return cursor >= len(self) def cursor_receipt(self, cursor: int) -> dict[str, Any]: if cursor < 0: raise ValueError("full payload federated cursor cannot be negative") return { "schema": "nnf.resynthesis.full_payload_federated_packed_training_cursor.v1", "scheduleSha256": self.collection["federatedScheduleSha256"], "federationAuthoritySha256": self.collection[ "federationAuthoritySha256" ], "collectionAuthoritySha256": self.collection[ "collectionAuthoritySha256" ], "retainedTokenWindows": cursor, "globalRetainedCursor": self.assigned_cursor_start + cursor, "assignedGlobalCursorStart": self.assigned_cursor_start, "assignedGlobalCursorEnd": self.assigned_cursor_end, "assignedWindowConsumed": self.training_complete(cursor), "completeDatasetConsumed": bool( self.assigned_cursor_start == 0 and self.assigned_cursor_end == int(self.collection["rows"]) and self.training_complete(cursor) ), "totalTokenWindows": len(self), "targetValuesRecorded": False, "rawDataDeletionAllowed": False, } def authority_receipt(self) -> dict[str, Any]: return { "schema": "nnf.resynthesis.full_payload_federated_packed_training.v1", "collectionReceiptPath": str(self.collection_receipt_path), "collectionAuthoritySha256": self.collection[ "collectionAuthoritySha256" ], "federationAuthoritySha256": self.collection[ "federationAuthoritySha256" ], "federationMode": self.collection.get("federationMode"), "scheduleSha256": self.collection["federatedScheduleSha256"], "tokenizerAuthoritySha256": self.collection[ "tokenizerAuthoritySha256" ], "scheduledPayloadFiles": self.collection[ "canonicalPayloadFileCount" ], "totalTokenWindows": self.collection["rows"], "recordCount": self.collection.get("recordCount"), "tokenElements": self.collection.get("tokenElements"), "sourceBytes": self.collection.get("sourceBytes"), "assignedGlobalCursorStart": self.assigned_cursor_start, "assignedGlobalCursorEnd": self.assigned_cursor_end, "maximumProposalRows": self.maximum_proposal_rows, "contextWindowTokens": self.collection["contextWindowTokens"], "answerTokensPerWindow": self.collection[ "answerTokensPerWindow" ], "mmapReadable": True, "rawPayloadRequiredAtTraining": False, "rangeBatchTensorRows": True, "cudaWaveCompatible": True, "exactDisjointWindowAssignment": True, "lazyOpenComponentLimit": self._MAXIMUM_OPEN_COMPONENTS, "completeDatasetClaimedTrained": False, "targetEnteredForward": False, "rawDataDeletionAllowed": False, } def close(self) -> None: for component in self._open_components.values(): component.close() self._open_components = {} self._last_used = {} def _looks_textual_payload(path: str) -> bool: suffixes = Path(path).suffixes if suffixes and suffixes[-1] == ".gz" and len(suffixes) >= 2: return suffixes[-2] in TEXT_SUFFIXES return bool(suffixes and suffixes[-1] in TEXT_SUFFIXES) def _text_probe(path: Path, *, byte_budget: int) -> str: if byte_budget <= 0 or not path.is_file(): return "" try: if path.suffix == ".gz" and len(path.suffixes) >= 2: with gzip.open(path, "rt", encoding="utf-8", errors="replace") as handle: return _clip_text(handle.read(byte_budget), limit=byte_budget) with path.open("r", encoding="utf-8", errors="replace") as handle: return _clip_text(handle.read(byte_budget), limit=byte_budget) except OSError: return "" def _candidate( *, record: dict[str, Any], source_sha256: str, family: str, surface: str, prompt: str, answer: str, evidence_document_id: str, evidence_text: str, source_record_suffix: str, ) -> CorpusTrainingCandidate: claims = _as_list(record.get("content_claims")) source_id = str(record["source_id"]) identity = _stable_rank( "nnf.resynthesis.corpus_training.question.v1", f"{source_id}\x00{family}\x00{surface}\x00{prompt}\x00{answer}", ) return CorpusTrainingCandidate( surface=surface, family=family, answer=answer, domain=_domain_for_claims(claims), evidence_document_id=evidence_document_id, evidence_text=evidence_text, prompt=prompt, question_id=identity[:24], source_id=source_id, source_record_id=f"{source_id}:{source_record_suffix}:{identity[:16]}", source_sha256=source_sha256, ) def _source_level_candidates( record: dict[str, Any], *, source_sha256: str, evidence_document_id: str, evidence_text: str, ) -> list[CorpusTrainingCandidate]: source_id = str(record["source_id"]) title = str(record.get("title") or "NO_TITLE_DECLARED").strip() publisher = str(record.get("publisher") or "NO_PUBLISHER_DECLARED").strip() rights = str(record.get("rights_class") or "unknown_hold").strip() license_name = str(record.get("license_name") or "NO_LICENSE_DECLARED").strip() cas_usage = str(record.get("cas_usage_class", "none")).strip() release_id = str(record.get("release_id") or "NO_RELEASE_ID").strip() claims = _as_list(record.get("content_claims")) primary_claim = claims[0] if claims else "NO_CONTENT_CLAIM" rubric_factor = _primary_rubric_factor(claims) base = ( "Use the source evidence below. Reply with exactly the requested value.\n" f"Evidence:\n{evidence_text}\n" ) candidates = [ _candidate( record=record, source_sha256=source_sha256, family="source_identity", surface="train", prompt=base + "Question: What is the source_id?\nAnswer:\n", answer=source_id, evidence_document_id=evidence_document_id, evidence_text=evidence_text, source_record_suffix="source_identity", ), _candidate( record=record, source_sha256=source_sha256, family="title_reconstruction", surface="train", prompt=base + "Question: What title is declared for this corpus source?\nAnswer:\n", answer=title, evidence_document_id=evidence_document_id, evidence_text=evidence_text, source_record_suffix="title", ), _candidate( record=record, source_sha256=source_sha256, family="primary_content_claim", surface="train", prompt=base + "Question: What is the first declared content claim?\nAnswer:\n", answer=primary_claim, evidence_document_id=evidence_document_id, evidence_text=evidence_text, source_record_suffix="primary_claim", ), _candidate( record=record, source_sha256=source_sha256, family="route_rubric_alignment", surface="train", prompt=base + "Question: Which route-evaluation rubric factor is most directly supported?\nAnswer:\n", answer=rubric_factor, evidence_document_id=evidence_document_id, evidence_text=evidence_text, source_record_suffix="rubric", ), _candidate( record=record, source_sha256=source_sha256, family="training_admissibility", surface="validation", prompt=base + "Question: What rights class governs model-training use?\nAnswer:\n", answer=rights, evidence_document_id=evidence_document_id, evidence_text=evidence_text, source_record_suffix="rights", ), _candidate( record=record, source_sha256=source_sha256, family="publisher_generalization", surface="validation", prompt=base + "Question: Which publisher is responsible for this source?\nAnswer:\n", answer=publisher, evidence_document_id=evidence_document_id, evidence_text=evidence_text, source_record_suffix="publisher", ), _candidate( record=record, source_sha256=source_sha256, family="cas_usage_holdout", surface="heldout", prompt=base + "Question: What CAS usage class is declared?\nAnswer:\n", answer=cas_usage, evidence_document_id=evidence_document_id, evidence_text=evidence_text, source_record_suffix="cas_usage", ), _candidate( record=record, source_sha256=source_sha256, family="release_holdout", surface="heldout", prompt=base + "Question: What release identifier is declared?\nAnswer:\n", answer=release_id, evidence_document_id=evidence_document_id, evidence_text=evidence_text, source_record_suffix="release", ), _candidate( record=record, source_sha256=source_sha256, family="license_correction", surface="correction_stress", prompt=base + "Question: What license statement should be preserved with this source?\nAnswer:\n", answer=license_name, evidence_document_id=evidence_document_id, evidence_text=evidence_text, source_record_suffix="license", ), ] for ( family, rubric_label, required_claims, rubric_factor, surface, ) in MHC_ROUTE_RUBRIC_FACTORS: support_label = _source_scoped_support_label(claims, required_claims) mhc_evidence = _clip_text( "\n".join( [ evidence_text, f"MHC rubric factor: {rubric_factor}", f"Rubric description: {rubric_label}", "Disposition labels are source-scoped. They are not a " "laboratory recommendation, synthetic-route verdict, or " "claim that another source agrees.", "Allowed labels: SUPPORTED_WITH_PROVENANCE_AND_EXPLICIT_UNCERTAINTY | " "PARTIAL_REQUIRES_CROSS_SOURCE_EVIDENCE | " "UNSUPPORTED_BY_THIS_SOURCE_ALONE", ] ), limit=3600, ) candidates.append( _candidate( record=record, source_sha256=source_sha256, family=family, surface=surface, prompt=( "Use the source-scoped MHC evidence below. Reply with exactly " "one allowed disposition label.\n" f"Evidence:\n{mhc_evidence}\n" f"Question: What is the support disposition for {rubric_label}?\n" "Answer:\n" ), answer=support_label, evidence_document_id=evidence_document_id, evidence_text=mhc_evidence, source_record_suffix=family, ) ) return candidates def _none_growth_candidates( record: dict[str, Any], *, source_sha256: str, evidence_document_id: str, evidence_text: str, payload_file_count: int, payload_bytes: int, ) -> list[CorpusTrainingCandidate]: """Create active NoNE/RBO growth objectives from source-sealed evidence.""" claims = _as_list(record.get("content_claims")) labels = _none_growth_labels( claims, payload_file_count=payload_file_count, payload_bytes=payload_bytes, ) axis_summary = _none_axis_summary(claims) growth_evidence = _clip_text( "\n".join( [ evidence_text, f"NoNE capability axes: {axis_summary}", f"Payload file count: {payload_file_count}", f"Payload bytes: {payload_bytes}", ( "NoNE action policy: train current expert/layer routing, " "knowledge transfer, recursive traversal, and MHC stability " "inside the RBO forward path. Additional expert/layer tensors " "require a separately checkpointed geometry migration before " "promotion." ), ( "Allowed expansion labels: " + " | ".join(sorted(set(labels.values()))) ), ] ), limit=4200, ) questions = { "none_expert_specialization_pressure": ( "Which NoNE expert-specialization pressure should this source add?" ), "none_knowledge_transfer_requirement": ( "What learned knowledge-transfer requirement should route this source?" ), "none_recursive_traversal_requirement": ( "What recursive traversal requirement should the RBO learn for this source?" ), "none_rbo_rotation_requirement": ( "What expert/layer rotation requirement must remain model-owned?" ), "none_expert_geometry_migration": ( "What expert-geometry migration disposition is source-scoped here?" ), "none_layer_geometry_migration": ( "What layer-geometry migration disposition is source-scoped here?" ), "none_mhc_stability_requirement": ( "What MHC stability requirement must be retained during this update?" ), "none_question_expansion_policy": ( "How should the model expand follow-up questions for this source?" ), } candidates: list[CorpusTrainingCandidate] = [] for family in sorted(NONE_OBJECTIVE_FAMILIES): candidates.append( _candidate( record=record, source_sha256=source_sha256, family=family, surface="train", prompt=( "Use the NoNE/RBO source evidence below. Reply with exactly " "one allowed expansion label.\n" f"Evidence:\n{growth_evidence}\n" f"Question: {questions[family]}\n" "Answer:\n" ), answer=labels[family], evidence_document_id=evidence_document_id, evidence_text=growth_evidence, source_record_suffix=family, ) ) return candidates def _payload_candidates( corpus_root: Path, record: dict[str, Any], manifest: dict[str, Any] | None, *, source_sha256: str, payload_text_probe_bytes: int, ) -> tuple[list[CorpusTrainingCandidate], list[dict[str, Any]]]: if manifest is None: return [], [] source_id = str(record["source_id"]) files = manifest.get("files") if not isinstance(files, list): return [], [] candidates: list[CorpusTrainingCandidate] = [] evidence_rows: list[dict[str, Any]] = [] for index, entry in enumerate(files): if not isinstance(entry, dict): continue rel_path = str(entry.get("path", "")).strip() if not rel_path: continue integrity_status = str(entry.get("integrity_status", "not_applicable")) source_checksum = str(entry.get("source_checksum") or "NO_SOURCE_CHECKSUM") bytes_text = str(int(entry.get("bytes", 0))) local_path = corpus_root / rel_path text_probe = ( _text_probe(local_path, byte_budget=payload_text_probe_bytes) if _looks_textual_payload(rel_path) else "" ) evidence_text = _clip_text( "\n".join( [ f"Payload owner source_id: {source_id}", f"Payload path: {rel_path}", f"Payload bytes: {bytes_text}", f"Integrity status: {integrity_status}", f"Source checksum: {source_checksum}", f"Text probe: {text_probe or 'NO_TEXT_PROBE'}", ] ), limit=3600, ) evidence_document_id = ( f"corpus-payload:{source_id}:{_stable_rank('payload', rel_path)[:24]}" ) evidence_rows.append( { "document_id": evidence_document_id, "source_id": source_id, "source_sha256": source_sha256, "text": evidence_text, "text_sha256": _sha256_bytes(evidence_text.encode("utf-8")), "rights_disposition": "training_admissible", "license": str(record.get("license_name") or "NO_LICENSE_DECLARED"), "attribution": str(record.get("publisher") or source_id), } ) surface = "train" family = "payload_owner" question = "Which corpus source_id owns this payload file?" answer = source_id if index % 17 == 0: surface = "validation" family = "payload_integrity_validation" question = "What integrity status is recorded for this payload?" answer = integrity_status elif index % 29 == 0: surface = "heldout" family = "payload_checksum_holdout" question = "What provider checksum is recorded for this payload?" answer = source_checksum elif text_probe and index % 13 == 0: surface = "correction_stress" family = "payload_text_probe_correction" question = "Which source_id should be attributed for this text probe?" answer = source_id prompt = ( "Use the payload evidence below. Reply with exactly the requested value.\n" f"Evidence:\n{evidence_text}\n" f"Question: {question}\nAnswer:\n" ) candidates.append( _candidate( record=record, source_sha256=source_sha256, family=family, surface=surface, prompt=prompt, answer=answer, evidence_document_id=evidence_document_id, evidence_text=evidence_text, source_record_suffix=family, ) ) return candidates, evidence_rows def _rows_from_candidates( candidates: list[CorpusTrainingCandidate], *, tokenizer: Any, train: bool, ) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] from resynthesis.execution_grounding import ( compose_runtime_evidence_prompt_ids, ) for candidate in candidates: objective_metadata: dict[str, Any] = {} if candidate.family in NONE_OBJECTIVE_FAMILIES: objective_metadata = { "none_capability_objective": candidate.family, "rbo_training_intent": ( "train_route_transfer_rotation_recursion_and_mhc_stability" ), "geometry_growth_claimed_live": False, "checkpointed_geometry_migration_required": ( "GEOMETRY_MIGRATION" in candidate.answer ), } prompt_ids, answer_ids = _tokenize_prefix_and_answer( tokenizer, candidate.prompt, candidate.answer, ) if train: corrected_prompt_ids = compose_runtime_evidence_prompt_ids( tokenizer, prompt_ids, candidate.evidence_text, ) rows.append( { "schema": CORPUS_TRAINING_ROW_SCHEMA, "domain": candidate.domain, "generalization_axis": "task_family", "generalization_group": f"corpus:{candidate.family}:train", "question_id": candidate.question_id, "source_id": candidate.source_id, "source_record_id": candidate.source_record_id, "source_sha256": candidate.source_sha256, "prompt_sha256": _sha256_bytes(candidate.prompt.encode("utf-8")), "prompt_ids": prompt_ids, "answer_ids": answer_ids, "input_ids": prompt_ids + answer_ids, "target_ids": [-100] * len(prompt_ids) + answer_ids, "corrected_input_ids": corrected_prompt_ids + answer_ids, "corrected_prompt_ids": corrected_prompt_ids, "corrected_target_ids": ( [-100] * len(corrected_prompt_ids) + answer_ids ), "training_evidence_document_id": candidate.evidence_document_id, "rights_disposition": "training_admissible", "corpus_surface_family": candidate.family, **objective_metadata, } ) else: rows.append( { "schema": f"{CORPUS_EVAL_ROW_SCHEMA_PREFIX}_{candidate.surface}.v1", "evaluation_surface": candidate.surface, "domain": candidate.domain, "generalization_axis": "task_family", "generalization_group": ( f"corpus:{candidate.family}:{candidate.surface}" ), "question_id": candidate.question_id, "source_id": candidate.source_id, "source_record_id": candidate.source_record_id, "source_sha256": candidate.source_sha256, "prompt_sha256": _sha256_bytes(candidate.prompt.encode("utf-8")), "prompt_ids": prompt_ids, "answer_ids": answer_ids, "rights_disposition": "evaluation_evidence_only", "corpus_surface_family": candidate.family, **objective_metadata, } ) return rows def _select_surface( candidates: list[CorpusTrainingCandidate], surface: str, *, target_rows: int | None, ) -> list[CorpusTrainingCandidate]: selected = [candidate for candidate in candidates if candidate.surface == surface] selected.sort( key=lambda candidate: _stable_rank( "nnf.resynthesis.corpus_training.surface_order.v1", f"{surface}\x00{candidate.question_id}\x00{candidate.source_record_id}", ) ) if surface == "train": selected.sort( key=lambda candidate: ( 0 if candidate.family in NONE_OBJECTIVE_FAMILIES or candidate.family.startswith("mhc_") else 1, _stable_rank( "nnf.resynthesis.corpus_training.priority_train_order.v1", f"{candidate.family}\x00{candidate.question_id}\x00{candidate.source_record_id}", ), ) ) if target_rows is not None and target_rows > 0: selected = selected[:target_rows] if not selected: raise ValueError(f"no corpus candidates available for {surface}") return selected def _artifact_receipt(path: Path) -> dict[str, Any]: with path.open(encoding="utf-8") as handle: rows = sum(1 for line in handle if line.strip()) return { "path": str(path), "rows": rows, "sha256": file_sha256(path), } def _json_artifact_receipt(path: Path) -> dict[str, Any]: return { "path": str(path), "bytes": path.stat().st_size, "sha256": file_sha256(path), } def _none_growth_entry( record: dict[str, Any], *, payload_file_count: int, payload_bytes: int, ) -> dict[str, Any]: claims = _as_list(record.get("content_claims")) labels = _none_growth_labels( claims, payload_file_count=payload_file_count, payload_bytes=payload_bytes, ) return { "source_id": str(record["source_id"]), "title": str(record.get("title") or "NO_TITLE_DECLARED"), "publisher": str(record.get("publisher") or "NO_PUBLISHER_DECLARED"), "content_claims": claims, "capability_axes": [ {"axis": axis_id, "description": description} for axis_id, description in _none_capability_axes(claims) ], "functional_expert_families": [ {"family": family_id, "description": description} for family_id, description in _none_functional_expert_families( claims ) ], "payload_file_count": payload_file_count, "payload_bytes": payload_bytes, "growth_labels": labels, } def _none_growth_plan(entries: list[dict[str, Any]]) -> dict[str, Any]: if not entries: raise ValueError("NoNE growth plan requires at least one admitted source") from resynthesis.config import ResynthesisConfig config = ResynthesisConfig() axis_counts: dict[str, int] = {} family_counts: dict[str, int] = {} planned_entries: list[dict[str, Any]] = [] source_claim_sets: list[frozenset[str]] = [] logical_expert_page_objectives = 0 for entry in entries: source_claim_sets.append( frozenset(_as_list(entry.get("content_claims"))) ) axes = entry.get("capability_axes") axis_ids: set[str] = set() if isinstance(axes, list): for axis in axes: if not isinstance(axis, dict): continue axis_id = str(axis.get("axis") or "").strip() if axis_id: axis_ids.add(axis_id) axis_counts[axis_id] = axis_counts.get(axis_id, 0) + 1 families = entry.get("functional_expert_families") family_ids: set[str] = set() if isinstance(families, list): for family in families: if not isinstance(family, dict): continue family_id = str(family.get("family") or "").strip() if family_id: family_ids.add(family_id) family_counts[family_id] = ( family_counts.get(family_id, 0) + 1 ) payload_bytes = entry.get("payload_bytes") source_payload_bytes = ( payload_bytes if isinstance(payload_bytes, int) and not isinstance(payload_bytes, bool) and payload_bytes >= 0 else 0 ) payload_page_objectives = max( 1, ( source_payload_bytes + NONE_EXPERT_PAGE_PLANNING_SHARD_BYTES - 1 ) // NONE_EXPERT_PAGE_PLANNING_SHARD_BYTES, ) source_page_objectives = max( payload_page_objectives, len(axis_ids), len(family_ids), 1, ) logical_expert_page_objectives += source_page_objectives planned_entry = dict(entry) planned_entry["planned_expert_page_objectives"] = ( source_page_objectives ) planned_entry["payload_page_objectives"] = payload_page_objectives planned_entries.append(planned_entry) active_axis_count = len(axis_counts) expert_migration_sources = sum( 1 for entry in entries if str( dict(entry.get("growth_labels", {})).get( "none_expert_geometry_migration", "", ) ) == "PLAN_CHECKPOINTED_EXPERT_GEOMETRY_MIGRATION" ) layer_migration_sources = sum( 1 for entry in entries if str( dict(entry.get("growth_labels", {})).get( "none_layer_geometry_migration", "", ) ) == "PLAN_CHECKPOINTED_LAYER_GEOMETRY_MIGRATION" ) additional_expert_slots = max( active_axis_count, len(NONE_FUNCTIONAL_EXPERT_FAMILIES), ) additional_layer_slots = len( { axis for axis in axis_counts if axis in { "patent_literature_route_context", "clinical_regulatory_safety", "production_process_scale", "evidence_quality_uncertainty", "all_atom_structure_geometry", "affinity_developability_antibody", "reaction_ord_process", } } ) if layer_migration_sources and additional_layer_slots < 1: additional_layer_slots = 1 from resynthesis.science_layers import ResynthesisScienceLayer structural = int(ResynthesisScienceLayer.structural_expert_count) functional_catalog = len(NONE_FUNCTIONAL_EXPERT_FAMILIES) legacy_scientific_catalog = len( NONE_SCIENTIFIC_FUNCTIONAL_EXPERT_FAMILIES ) language_catalog = len(NONE_LANGUAGE_EXPERT_FAMILIES) specialist_catalog = len( NONE_V2_PLUS_SCIENCE_SPECIALIST_DEFINITIONS ) scientific_catalog = legacy_scientific_catalog + specialist_catalog v18_catalog = len(NONE_V18_FUNCTIONAL_EXPERT_FAMILIES) legacy_scientific_family_ids = frozenset( family_id for family_id, _description, _required_claims in ( NONE_SCIENTIFIC_FUNCTIONAL_EXPERT_FAMILIES ) ) language_family_ids = frozenset( family_id for family_id, _description, _required_claims in ( NONE_LANGUAGE_EXPERT_FAMILIES ) ) language_ability_ids_by_family = dict( NONE_LANGUAGE_EXPERT_ABILITY_ASSIGNMENTS ) specialist_definition_by_id = { definition.family_id: definition for definition in NONE_V2_PLUS_SCIENCE_SPECIALIST_DEFINITIONS } specialist_family_ids = frozenset(specialist_definition_by_id) specialist_readiness_counts = { definition.family_id: sum( 1 for claims in source_claim_sets if definition.readiness_claims.issubset(claims) ) for definition in NONE_V2_PLUS_SCIENCE_SPECIALIST_DEFINITIONS } inherited_seed_page_count = int(config.num_layers) * structural current_physical_graph_layer_count = ( inherited_seed_page_count + functional_catalog ) proposed_physical_graph_layer_count = ( current_physical_graph_layer_count + logical_expert_page_objectives ) current_geometry = { "scienceLayers": int(config.num_layers), "residentScienceLayers": int(config.num_layers), "inheritedSeedPageCount": inherited_seed_page_count, "functionalFamilyRootPageCount": functional_catalog, "physicalNoNELayers": current_physical_graph_layer_count, "physicalGraphLayerCount": current_physical_graph_layer_count, "physicalGraphNodeCount": current_physical_graph_layer_count, "physicalPageExperts": current_physical_graph_layer_count, "scienceExperts": int(config.num_experts), "mhcHeads": int(config.mhc_heads), "structuralExperts": structural, "functionalExpertFamiliesCatalog": functional_catalog, "scientificFunctionalExpertFamiliesCatalog": scientific_catalog, "legacyScientificFunctionalExpertFamiliesCatalog": ( legacy_scientific_catalog ), "languageExpertPacksCatalog": language_catalog, "v2PlusScienceSpecialistsCatalog": specialist_catalog, } proposed_experts = max( current_geometry["scienceExperts"] + additional_expert_slots, structural + functional_catalog, ) resident_target_layers = ( current_geometry["scienceLayers"] + additional_layer_slots ) target_geometry = { "scienceLayers": resident_target_layers, "residentScienceLayers": resident_target_layers, "scienceLayersCountScope": "resident_dense_traversal_only", "scienceLayersAreTotalNoNELayers": False, "physicalNoNELayers": proposed_physical_graph_layer_count, "totalPhysicalNoNELayers": proposed_physical_graph_layer_count, "sparsePhysicalNoNELayers": proposed_physical_graph_layer_count, "physicalGraphLayerCount": proposed_physical_graph_layer_count, "physicalGraphNodeCount": proposed_physical_graph_layer_count, "physicalPageExperts": proposed_physical_graph_layer_count, "scienceExperts": proposed_experts, "mhcHeads": current_geometry["mhcHeads"], "structuralExperts": structural, "functionalExpertFamilies": functional_catalog, "scientificFunctionalExpertFamilies": scientific_catalog, "legacyScientificFunctionalExpertFamilies": ( legacy_scientific_catalog ), "languageExpertPacks": language_catalog, "v2PlusScienceSpecialists": specialist_catalog, } functional_family_roadmap: list[dict[str, Any]] = [] for catalog_ordinal, ( family_id, description, _required_claims, ) in enumerate(NONE_FUNCTIONAL_EXPERT_FAMILIES): row: dict[str, Any] = { "family": family_id, "description": description, "admittedSourceCount": family_counts.get(family_id, 0), "sourceEvidenceGap": family_counts.get(family_id, 0) == 0, "catalogOrdinal": catalog_ordinal, "catalogSegment": ( "v18_science" if family_id in legacy_scientific_family_ids else ( "v18_language" if family_id in language_family_ids else "v2_plus_science_specialist" ) ), } if family_id in language_family_ids: row.update( { "sharedLanguageAbilityIds": list( language_ability_ids_by_family[family_id] ), "registeredCapabilityClaimed": False, "trainedCapabilityClaimed": False, } ) specialist_definition = specialist_definition_by_id.get(family_id) if specialist_definition is not None: readiness_count = specialist_readiness_counts[family_id] row.update( { "readinessClaims": sorted( specialist_definition.readiness_claims ), "readinessEvidenceSourceCount": readiness_count, "readinessEvidenceGap": readiness_count == 0, "rubricFactors": list( specialist_definition.rubric_factors ), "registeredCapabilityClaimed": False, "trainedCapabilityClaimed": False, } ) functional_family_roadmap.append(row) language_pack_roadmap = [ row for row in functional_family_roadmap if row["family"] in language_family_ids ] specialist_roadmap = [ row for row in functional_family_roadmap if row["family"] in specialist_family_ids ] return { "schema": NONE_GROWTH_PLAN_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "sourceCount": len(entries), "axisSourceCounts": dict(sorted(axis_counts.items())), "functionalExpertFamilySourceCounts": dict( sorted(family_counts.items()) ), "functionalExpertFamilyRoadmap": functional_family_roadmap, "functionalExpertFamilyCount": functional_catalog, "languageExpertCatalogSchema": ( NONE_LANGUAGE_EXPERT_CATALOG_SCHEMA ), "languageExpertPackCount": language_catalog, "languageExpertPackIdsSha256": NONE_LANGUAGE_EXPERT_IDS_SHA256, "nativeLanguagePackPrefixSchema": ( NATIVE_LANGUAGE_PACK_PREFIX_SCHEMA ), "nativeLanguagePackPrefixCount": len( NATIVE_LANGUAGE_PACK_PREFIX_IDS ), "nativeLanguagePackPrefixIdsSha256": ( NATIVE_LANGUAGE_PACK_PREFIX_IDS_SHA256 ), "broadLanguagePackCount": len(BROAD_LANGUAGE_PACK_IDS), "broadLanguagePackIdsSha256": BROAD_LANGUAGE_PACK_IDS_SHA256, "broadLanguageSupplementalPackCount": ( len(BROAD_LANGUAGE_PACK_IDS) - len(NATIVE_LANGUAGE_PACK_PREFIX_IDS) ), "broadLanguageSourceSchema": LINGUIST_LANGUAGE_SOURCE_SCHEMA, "broadLanguageSourceRepository": ( LINGUIST_LANGUAGE_SOURCE_REPOSITORY ), "broadLanguageSourceCommit": LINGUIST_LANGUAGE_SOURCE_COMMIT, "broadLanguageSourcePath": LINGUIST_LANGUAGE_SOURCE_PATH, "broadLanguageSourceFileSha256": ( LINGUIST_LANGUAGE_SOURCE_FILE_SHA256 ), "broadLanguageSourceNamesSha256": ( LINGUIST_LANGUAGE_SOURCE_NAMES_SHA256 ), "broadLanguageSourcePackIdsSha256": ( LINGUIST_LANGUAGE_SOURCE_PACK_IDS_SHA256 ), "broadLanguageSourceRecordCount": ( LINGUIST_LANGUAGE_SOURCE_RECORD_COUNT ), "broadLanguageUniquePackCount": len( LINGUIST_LANGUAGE_UNIQUE_SOURCE_PACK_IDS ), "broadLanguageSourceTypeCounts": dict( LINGUIST_LANGUAGE_SOURCE_TYPE_COUNTS ), "languageAbilitySetSchema": LANGUAGE_ABILITY_SET_SCHEMA, "languageAbilityAxisCount": len(LANGUAGE_ABILITY_AXIS_IDS), "languageAbilityAxisIdsSha256": ( LANGUAGE_ABILITY_AXIS_IDS_SHA256 ), "languageAbilityAxisRoadmap": [ {"ability": ability_id, "description": description} for ability_id, description in LANGUAGE_ABILITY_AXES ], "languageAbilityPackAssignmentsSha256": ( NONE_LANGUAGE_EXPERT_ABILITY_ASSIGNMENTS_SHA256 ), "languageExpertPackSourceCounts": { family_id: family_counts.get(family_id, 0) for family_id in sorted(language_family_ids) }, "languageExpertPackRoadmap": language_pack_roadmap, "scienceSpecialistCatalogSchema": ( NONE_SCIENCE_SPECIALIST_CATALOG_SCHEMA ), "scienceSpecialistCount": specialist_catalog, "scienceSpecialistIdsSha256": ( NONE_V2_PLUS_SCIENCE_SPECIALIST_IDS_SHA256 ), "scienceSpecialistSourceCounts": { family_id: family_counts.get(family_id, 0) for family_id in sorted(specialist_family_ids) }, "scienceSpecialistReadinessSourceCounts": dict( sorted(specialist_readiness_counts.items()) ), "scienceSpecialistRoadmap": specialist_roadmap, "acceptedV18FunctionalCatalogCount": v18_catalog, "currentSeedGeometry": current_geometry, "proposedMinimumAdditionalExpertSlots": additional_expert_slots, "proposedMinimumAdditionalLayerSlots": additional_layer_slots, "proposedMinimumAdditionalResidentLayerSlots": additional_layer_slots, "proposedMinimumAdditionalPhysicalNoNELayers": ( logical_expert_page_objectives ), "proposedMinimumTargetGeometry": target_geometry, "proposedMinimumDenseSeedGeometry": { **target_geometry, "scienceLayers": resident_target_layers, }, "initialLogicalExpertPageObjectives": ( logical_expert_page_objectives ), "physicalGraphPlan": { "schema": "nnf.resynthesis.physical_graph_plan.v1", "countScope": "source_derived_additive_page_objectives", "physicalGraphLayerKind": "sparse_page_backed_none_expert", "inheritedSeedPageCount": inherited_seed_page_count, "inheritedFunctionalFamilyRootPageCount": functional_catalog, "inheritedPhysicalGraphLayerCount": ( current_physical_graph_layer_count ), "additivePageObjectiveCount": logical_expert_page_objectives, "physicalGraphLayerCount": proposed_physical_graph_layer_count, "physicalGraphNodeCount": proposed_physical_graph_layer_count, "modelRoutedPageNodeCount": proposed_physical_graph_layer_count, "pageToPhysicalGraphLayerCardinality": "one_to_one", "pageToPhysicalGraphNodeCardinality": "one_to_one", "denseResidentScienceLayerCount": resident_target_layers, "pageBackedLayersReportedAsDenseLayers": False, "physicalGraphLayerCountMaximum": None, "pageParameterWidthMaximum": None, "hiddenDimensionMaximum": None, "expertRankMaximum": None, "transferDimensionMaximum": None, "successorCompactBanksRemainAdditive": True, "successorTensorGeometryMayExpand": True, "perLayerUniqueIdentityRequired": True, "perLayerPhysicalWeightsRequired": True, "perLayerModelOwnedRoutingRequired": True, "perLayerGradientAndDeltaProofRequired": True, "trainingClaimed": False, "promotionEligible": False, }, "expertPagePlanningShardBytes": ( NONE_EXPERT_PAGE_PLANNING_SHARD_BYTES ), "expertMigrationSourceCount": expert_migration_sources, "layerMigrationSourceCount": layer_migration_sources, "migrationPolicy": { "currentTraining": ( "train existing RBO/NoNE routing, transfer, recursive traversal, " "rotation pressure, MHC stability, and correction policy" ), "geometryExpansion": ( "the monolithic target is a compatibility seed only; scalable " "growth uses immutable content-addressed expert-page generations " "that pass cold-reload, heldout, transfer, and ablation proof" ), "minimumSlotsAreNotCaps": True, "denseSeedIsNotTotalExpertCount": True, "residentScienceLayersAreNotTotalNoNELayers": True, "physicalNoNELayerIdentity": ( "one_content_addressed_page_objective_per_logical_layer_v1" ), "modelOwnedRoutingRequired": True, }, "pagingPolicy": { "catalogMode": ( "content_addressed_immutable_generation_bundles" ), "acceptedGenerationPointer": True, "sessionOwnedGeneration": True, "modelOwnedGraphEndstateRouting": ( "family_to_cluster_to_page_to_expert — graph endstate-backward, " "never hierarchical intent ladders" ), "hierarchicalReasoning": False, "storageBoundaryMayReroute": False, "hostFixedResidentExpertLimit": None, "trainedResidencyAndPrefetchRequired": True, "unchangedPagesSharedAcrossGenerations": True, "perPageOptimizerState": True, "perPageDataAndEvaluationLineage": True, "planningShardBytesIsNotAResidencyCap": True, }, "parameterScalePolicy": { "registeredParameters": ( "count only serialized core plus immutable expert pages" ), "trainedValidatedParameters": ( "count only pages with committed gradients, heldout gain, " "source retention, transfer or ablation value, and cold reload" ), "residentParameters": ( "report the model-selected working set separately" ), "activatedParameters": ( "report the exact request route separately" ), "untrainedRegisteredPagesAreCapabilities": False, "noPermanentExpertCountCap": True, }, "capacityExpansionPolicy": { "triggerAuthority": ( "model_owned_gap_route_gradient_and_accepted_saturation_tensors" ), "hostSelectedFamilyOrLayerAllowed": False, "additiveSuccessorBanksAllowed": True, "staticPhysicalPageCeiling": None, "staticParameterCeiling": None, "staticPageParameterWidthCeiling": None, "staticHiddenDimensionCeiling": None, "staticExpertRankCeiling": None, "staticTransferDimensionCeiling": None, "existingPageOrParameterIdentityMayBeReused": False, "newPageAndParameterIdentityRequired": True, "successorBanksMayUseExpandedTensorGeometry": True, "crossGeometryTransferRequiresLearnedProjection": True, "acceptedGenerationMutationRequiresImmutableTransaction": True, }, "proofAccountingPolicy": { "schema": NONE_GROWTH_PROOF_ACCOUNTING_SCHEMA, "scope": "planning_only_no_materialization_or_training_claim", "plannedPhysicalCapacityPageCount": ( proposed_physical_graph_layer_count ), "materializedPhysicalCapacityPageCount": None, "residentPageCount": None, "activeRoutePageCount": None, "uniqueUpdatedPageCount": None, "validatedTrainedPageCount": None, "trainedKnowledgePageCount": None, "acceptedUnionPageCount": None, "coldReloadValidatedPageCount": None, "proofStageCountsAreNotInterchangeable": True, "metadataOrAllocationCannotProveTraining": True, "transferInitializedOrPreallocatedPagesCountAsTrained": False, "physicalOrResidentCapacityCountsAsTrainedKnowledge": False, "acceptedUnionMembershipAloneCountsAsTrainedKnowledge": False, }, "sourceObjectives": planned_entries, "checks": { "entriesNonEmpty": bool(entries), "axisCountsNonEmpty": bool(axis_counts), "functionalExpertRoadmapPresent": bool( functional_family_roadmap ), "functionalExpertCatalogAtLeast24": functional_catalog >= 24, "scientificExpertPrefixPreserved": ( NONE_FUNCTIONAL_EXPERT_FAMILIES[ :legacy_scientific_catalog ] == NONE_SCIENTIFIC_FUNCTIONAL_EXPERT_FAMILIES ), "acceptedV18CatalogPrefixPreserved": ( NONE_FUNCTIONAL_EXPERT_FAMILIES[:v18_catalog] == NONE_V18_FUNCTIONAL_EXPERT_FAMILIES ), "languageExpertCatalogAtLeast60": language_catalog >= 60, "languageExpertCatalogExactBroadIdentity": ( language_catalog == 936 and NONE_LANGUAGE_EXPERT_IDS_SHA256 == BROAD_LANGUAGE_PACK_IDS_SHA256 ), "nativeLanguagePackPrefixPreserved": ( tuple( family_id for family_id, _description, _claims in ( NONE_LANGUAGE_EXPERT_FAMILIES[ : len(NATIVE_LANGUAGE_PACK_PREFIX_IDS) ] ) ) == NATIVE_LANGUAGE_PACK_PREFIX_IDS ), "sharedLanguageAbilitySetComplete": ( len(LANGUAGE_ABILITY_AXIS_IDS) == 53 and len(NONE_LANGUAGE_EXPERT_ABILITY_ASSIGNMENTS) == language_catalog and all( bool(ability_ids) for _family_id, ability_ids in ( NONE_LANGUAGE_EXPERT_ABILITY_ASSIGNMENTS ) ) ), "languagePacksIncludedInFunctionalRoadmap": ( len(language_pack_roadmap) == language_catalog and { row["family"] for row in language_pack_roadmap } == language_family_ids ), "languagePacksRemainUntrainedUntilPromotionProof": all( row.get("registeredCapabilityClaimed") is False and row.get("trainedCapabilityClaimed") is False for row in language_pack_roadmap ), "scienceSpecialistCatalogAtLeast33": ( specialist_catalog >= 33 ), "scienceSpecialistsAppendedAfterAcceptedV18Prefix": ( NONE_FUNCTIONAL_EXPERT_FAMILIES[v18_catalog:] == NONE_V2_PLUS_SCIENCE_SPECIALIST_FAMILIES ), "functionalCatalogRetainsScienceLanguageAndSpecialists": ( functional_catalog == legacy_scientific_catalog + language_catalog + specialist_catalog ), "scienceSpecialistRoadmapExact": ( len(specialist_roadmap) == specialist_catalog and tuple( str(row["family"]) for row in specialist_roadmap ) == tuple( definition.family_id for definition in ( NONE_V2_PLUS_SCIENCE_SPECIALIST_DEFINITIONS ) ) ), "scienceSpecialistReadinessGapsExplicit": all( isinstance(row.get("readinessEvidenceGap"), bool) and row.get("registeredCapabilityClaimed") is False and row.get("trainedCapabilityClaimed") is False for row in specialist_roadmap ), "targetExpertsCoverFunctionalCatalog": ( proposed_experts >= structural + functional_catalog ), "logicalExpertPagesDerivedFromSources": ( logical_expert_page_objectives >= len(entries) ), "currentPhysicalGraphDerivedFromSeedAndFamilyRoots": ( inherited_seed_page_count == int(config.num_layers) * structural and current_physical_graph_layer_count == inherited_seed_page_count + functional_catalog ), "plannedPhysicalGraphIsInheritedPlusAdditive": ( proposed_physical_graph_layer_count == current_physical_graph_layer_count + logical_expert_page_objectives ), "expertSlotsCoverFunctionalCatalog": ( additional_expert_slots >= functional_catalog ), "layerSlotsDerivedFromComplexAxes": additional_layer_slots >= 0, "currentTrainingDoesNotClaimLiveGeometryExpansion": True, "checkpointedMigrationRequiredBeforePromotion": True, "denseSeedNotReportedAsTotalNoNE": True, "fixedResidentExpertCapAbsent": True, "proofAccountingStagesSeparated": True, "capacityExpansionHasNoStaticCeiling": True, }, } def _validated_federated_growth_demand( federation: Mapping[str, Any], membership_rows: Sequence[Mapping[str, Any]], collection: Mapping[str, Any], ) -> tuple[list[dict[str, Any]], dict[str, Any]]: """Collapse canonical federation content into immutable NoNE objectives. The input membership has already performed byte-plus-format deduplication. This boundary groups only canonical rows by their sealed source record, namespacing identities by component, so same-named source records on independent science disks cannot collide. The result is demand evidence only: it cannot allocate pages, train a model, or advance an accepted generation. """ federation_authority = federation.get("federationAuthoritySha256") membership_record = federation.get("membership") collection_authority = collection.get("collectionAuthoritySha256") tokenizer_authority = collection.get("tokenizerAuthoritySha256") if ( not isinstance(federation_authority, str) or len(federation_authority) != 64 or not isinstance(membership_record, dict) or not _full_payload_packed_artifact_matches(membership_record) or not isinstance(collection_authority, str) or len(collection_authority) != 64 or not isinstance(tokenizer_authority, str) or len(tokenizer_authority) != 64 ): raise ValueError("full payload federated growth binding is malformed") aggregated: dict[str, dict[str, Any]] = {} canonical_bytes = 0 duplicate_provenance = 0 for membership in membership_rows: canonical = membership.get("canonical") provenance = membership.get("duplicateProvenance") if ( membership.get("schema") != FULL_PAYLOAD_MULTIROOT_MEMBERSHIP_SCHEMA or not isinstance(canonical, Mapping) or not isinstance(provenance, list) or not provenance ): raise ValueError("full payload federated growth membership differs") component_id = canonical.get("componentId") source_record_sha256 = canonical.get("sourceRecordSha256") source_id = canonical.get("sourceId") payload_bytes = canonical.get("payloadBytes") domain_authority = canonical.get("domainAuthority") content_key = membership.get("contentKeySha256") if ( not isinstance(component_id, str) or len(component_id) != 64 or not isinstance(source_record_sha256, str) or len(source_record_sha256) != 64 or not isinstance(source_id, str) or not source_id or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or payload_bytes < 0 or not isinstance(domain_authority, Mapping) or not isinstance(content_key, str) or len(content_key) != 64 ): raise ValueError("full payload federated growth canonical row differs") content_claims = domain_authority.get("contentClaims") if ( not isinstance(content_claims, list) or not content_claims or any(not isinstance(claim, str) or not claim for claim in content_claims) ): raise ValueError("full payload federated content claims are absent") source_identity = _sha256_bytes( _json_bytes( { "schema": "nnf.resynthesis.full_payload_federated_source_identity.v1", "componentId": component_id, "sourceRecordSha256": source_record_sha256, "sourceId": source_id, } ) ) identity = { "componentId": component_id, "sourceRecordSha256": source_record_sha256, "sourceId": source_id, "contentClaims": sorted(set(content_claims)), "corpusRoot": canonical.get("corpusRoot"), } existing = aggregated.get(source_identity) if existing is None: aggregated[source_identity] = { "identity": identity, "payloadFileCount": 1, "payloadBytes": payload_bytes, "canonicalContentKeys": [content_key], "duplicateProvenanceCount": len(provenance) - 1, } else: if existing["identity"] != identity: raise RuntimeError("full payload federated source identity conflicts") existing["payloadFileCount"] += 1 existing["payloadBytes"] += payload_bytes existing["canonicalContentKeys"].append(content_key) existing["duplicateProvenanceCount"] += len(provenance) - 1 canonical_bytes += payload_bytes duplicate_provenance += len(provenance) - 1 if canonical_bytes != federation.get("canonicalPayloadBytes"): raise RuntimeError("full payload federated growth byte denominator differs") if duplicate_provenance != federation.get("duplicatePayloadProvenanceCount"): raise RuntimeError("full payload federated growth duplicate denominator differs") entries: list[dict[str, Any]] = [] source_evidence: list[dict[str, Any]] = [] for source_identity in sorted(aggregated): value = aggregated[source_identity] identity = value["identity"] entry = _none_growth_entry( { "source_id": source_identity, "title": identity["sourceId"], "publisher": identity["corpusRoot"] or identity["componentId"], "content_claims": identity["contentClaims"], }, payload_file_count=int(value["payloadFileCount"]), payload_bytes=int(value["payloadBytes"]), ) entries.append(entry) source_evidence.append( { "sourceId": source_identity, "componentId": identity["componentId"], "sourceRecordSha256": identity["sourceRecordSha256"], "originalSourceId": identity["sourceId"], "contentClaims": identity["contentClaims"], "canonicalPayloadFileCount": value["payloadFileCount"], "canonicalPayloadBytes": value["payloadBytes"], "canonicalContentKeysSha256": _sha256_bytes( _json_bytes(sorted(value["canonicalContentKeys"])) ), "duplicateProvenanceCount": value["duplicateProvenanceCount"], } ) if not entries: raise RuntimeError("full payload federated growth has no canonical sources") demand = { "schema": FULL_PAYLOAD_FEDERATED_NONE_GROWTH_PLAN_SCHEMA, "federationAuthoritySha256": federation_authority, "membership": dict(membership_record), "membershipSha256": membership_record.get("sha256"), "packedCollectionAuthoritySha256": collection_authority, "tokenizerAuthoritySha256": tokenizer_authority, "contextWindowTokens": collection.get("contextWindowTokens"), "answerTokensPerWindow": collection.get("answerTokensPerWindow"), "canonicalPayloadFileCount": len(membership_rows), "canonicalPayloadBytes": canonical_bytes, "duplicatePayloadProvenanceCount": duplicate_provenance, "sourceIdentityNamespace": ( "component_id_plus_source_record_sha256_plus_original_source_id" ), "sourceEvidence": source_evidence, "rawPayloadCopied": False, "rawPayloadRequiredAtTraining": False, "modelTrainingClaimed": False, "pageAllocationClaimed": False, "parameterExpansionClaimed": False, "promotionEligible": False, "acceptedGenerationMutationAllowed": False, } return entries, demand def _validated_packed_ready_growth_demand( ready_federation: Mapping[str, Any], collection: Mapping[str, Any], ) -> tuple[list[dict[str, Any]], dict[str, Any]]: """Collapse packed-ready federation components into NoNE objectives. A packed-ready federation is metadata-only: it seals disjoint packed component authorities instead of a multiroot membership, so it carries no canonical membership rows or duplicate provenance. Each sealed component is turned directly into one NoNE source objective, namespaced by its component authority so independent science disks cannot collide. The result is demand evidence only: it cannot allocate pages, train a model, or advance an accepted generation. """ federation_authority = ready_federation.get("federationAuthoritySha256") collection_authority = collection.get("collectionAuthoritySha256") tokenizer_authority = collection.get("tokenizerAuthoritySha256") component_values = collection.get("components") if ( not isinstance(federation_authority, str) or len(federation_authority) != 64 or not isinstance(collection_authority, str) or len(collection_authority) != 64 or not isinstance(tokenizer_authority, str) or len(tokenizer_authority) != 64 or not isinstance(component_values, list) or not component_values ): raise ValueError("full payload packed-ready growth binding is malformed") entries: list[dict[str, Any]] = [] source_evidence: list[dict[str, Any]] = [] canonical_files = 0 canonical_bytes = 0 observed_component_ids: set[str] = set() for component in component_values: component_id = component.get("componentId") component_authority = component.get("componentAuthoritySha256") corpus_root = component.get("corpusRoot") collection_receipt = component.get("collectionReceipt") payload_file_count = component.get("selectedPayloadWorkCount") payload_bytes = component.get("sourceBytes") if ( not isinstance(component, Mapping) or not isinstance(component_id, str) or len(component_id) != 64 or component_id in observed_component_ids or not isinstance(component_authority, str) or component_authority != component_id or not isinstance(collection_receipt, Mapping) or not _full_payload_packed_artifact_matches(collection_receipt) or not isinstance(payload_file_count, int) or isinstance(payload_file_count, bool) or payload_file_count < 1 or not isinstance(payload_bytes, int) or isinstance(payload_bytes, bool) or payload_bytes < 0 or (corpus_root is not None and not isinstance(corpus_root, str)) ): raise ValueError("full payload packed-ready growth component differs") observed_component_ids.add(component_id) source_identity = _sha256_bytes( _json_bytes( { "schema": ( "nnf.resynthesis.full_payload_packed_ready_source_identity.v1" ), "componentId": component_id, "componentAuthoritySha256": component_authority, } ) ) publisher = corpus_root or component_id entry = _none_growth_entry( { "source_id": source_identity, "title": publisher, "publisher": publisher, "content_claims": [], }, payload_file_count=int(payload_file_count), payload_bytes=int(payload_bytes), ) entries.append(entry) source_evidence.append( { "sourceId": source_identity, "componentId": component_id, "componentAuthoritySha256": component_authority, "collectionAuthoritySha256": component.get( "collectionAuthoritySha256" ), "corpusRoot": corpus_root, "canonicalPayloadFileCount": payload_file_count, "canonicalPayloadBytes": payload_bytes, "duplicateProvenanceCount": 0, } ) canonical_files += int(payload_file_count) canonical_bytes += int(payload_bytes) if not entries: raise RuntimeError("full payload packed-ready growth has no components") if canonical_files != collection.get("canonicalPayloadFileCount"): raise RuntimeError( "full payload packed-ready growth file denominator differs" ) if canonical_bytes != collection.get("canonicalPayloadBytes"): raise RuntimeError( "full payload packed-ready growth byte denominator differs" ) demand = { "schema": FULL_PAYLOAD_FEDERATED_NONE_GROWTH_PLAN_SCHEMA, "federationMode": FULL_PAYLOAD_PACKED_READY_FEDERATION_MODE, "federationAuthoritySha256": federation_authority, "packedCollectionAuthoritySha256": collection_authority, "tokenizerAuthoritySha256": tokenizer_authority, "contextWindowTokens": collection.get("contextWindowTokens"), "answerTokensPerWindow": collection.get("answerTokensPerWindow"), "canonicalPayloadFileCount": canonical_files, "canonicalPayloadBytes": canonical_bytes, "duplicatePayloadProvenanceCount": 0, "sourceIdentityNamespace": "packed_ready_component_authority_sha256", "sourceEvidence": source_evidence, "rawPayloadCopied": False, "rawPayloadRequiredAtTraining": False, "modelTrainingClaimed": False, "pageAllocationClaimed": False, "parameterExpansionClaimed": False, "promotionEligible": False, "acceptedGenerationMutationAllowed": False, } return entries, demand def build_federated_full_payload_none_growth_plan( federation_authority_receipt_path: Path, packed_collection_receipt_path: Path, output_plan_path: Path, output_receipt_path: Path, ) -> dict[str, Any]: """Create sealed content-demand evidence for future global NoNE growth. This planning action deliberately does not create a page, parameter, model generation, or training branch. It only turns exactly packed, reader-ready federation content into the existing NoNE source-objective language. Later global union/retained-evidence admission decides if and how pages/parameters are physically allocated. """ federation_path = federation_authority_receipt_path.expanduser().resolve() collection_path = packed_collection_receipt_path.expanduser().resolve() plan_path = output_plan_path.expanduser().resolve() receipt_path = output_receipt_path.expanduser().resolve() collection_probe = json.loads(collection_path.read_text(encoding="utf-8")) packed_ready_mode = ( isinstance(collection_probe, dict) and collection_probe.get("federationMode") == FULL_PAYLOAD_PACKED_READY_FEDERATION_MODE ) if packed_ready_mode: # A packed-ready federation is self-describing: the sealed collection # receipt is its own federation authority, so it is passed as both the # federation authority and the packed collection. Validation synthesizes # the packed-ready federation authority (no native multiroot authority # exists), and demand is built directly from the sealed components. collection, ready_federation, _packed_components = ( _validated_federated_full_payload_packed_collection( collection_path, validate_native_federation=False, ) ) federation = ready_federation if federation.get("federationAuthoritySha256") != collection.get( "federationAuthoritySha256" ): raise RuntimeError("full payload federated growth federation differs") entries, demand = _validated_packed_ready_growth_demand( ready_federation, collection, ) else: federation, membership_rows, _components = ( _validated_full_payload_multiroot_authority( federation_path, validate_native_components=True, ) ) collection, collection_federation, _packed_components = ( _validated_federated_full_payload_packed_collection( collection_path, validate_native_federation=True, ) ) if ( collection_federation.get("federationAuthoritySha256") != federation.get("federationAuthoritySha256") ): raise RuntimeError("full payload federated growth federation differs") entries, demand = _validated_federated_growth_demand( federation, membership_rows, collection, ) if plan_path.is_file(): plan = json.loads(plan_path.read_text(encoding="utf-8")) if ( not isinstance(plan, dict) or plan.get("schema") != NONE_GROWTH_PLAN_SCHEMA or plan.get("federatedContentDemand") != demand or not isinstance(plan.get("planAuthoritySha256"), str) ): raise RuntimeError("full payload federated growth plan differs") recorded_plan_authority = plan.pop("planAuthoritySha256") if recorded_plan_authority != _sha256_bytes(_json_bytes(plan)): raise RuntimeError("full payload federated growth plan authority differs") plan["planAuthoritySha256"] = recorded_plan_authority else: plan = _none_growth_plan(entries) plan["federatedContentDemand"] = demand plan["checks"] = { **dict(plan["checks"]), "federatedMembershipBound": True, "packedCollectionBound": True, "deduplicatedCanonicalPayloadsOnly": True, "sourceIdentityNamespacedAcrossRoots": True, "rawPayloadsRemainAtNativeRoots": True, "growthPlanDoesNotAllocatePages": True, "growthPlanDoesNotExpandParameters": True, "growthPlanDoesNotTrainModel": True, "growthPlanDoesNotMutateAcceptedGeneration": True, } plan["planAuthoritySha256"] = _sha256_bytes(_json_bytes(plan)) _atomic_json(plan_path, plan) checks = { "planAuthorityBound": file_sha256(plan_path) == _packed_artifact(plan_path)["sha256"], "federationAuthorityBound": demand["federationAuthoritySha256"] == federation["federationAuthoritySha256"], "packedCollectionAuthorityBound": demand[ "packedCollectionAuthoritySha256" ] == collection["collectionAuthoritySha256"], "noAllocationClaim": demand["pageAllocationClaimed"] is False, "noParameterExpansionClaim": demand["parameterExpansionClaimed"] is False, "noTrainingClaim": demand["modelTrainingClaimed"] is False, "noPointerMutation": demand["acceptedGenerationMutationAllowed"] is False, } receipt = { "schema": FULL_PAYLOAD_FEDERATED_NONE_GROWTH_PLAN_SCHEMA, "passed": all(checks.values()), "federationReceipt": _packed_artifact(federation_path), "federationAuthoritySha256": federation["federationAuthoritySha256"], "packedCollectionReceipt": _packed_artifact(collection_path), "packedCollectionAuthoritySha256": collection[ "collectionAuthoritySha256" ], "outputPlan": _packed_artifact(plan_path), "planAuthoritySha256": plan["planAuthoritySha256"], "canonicalPayloadFileCount": demand["canonicalPayloadFileCount"], "canonicalPayloadBytes": demand["canonicalPayloadBytes"], "sourceCount": len(entries), "checks": checks, "modelTrainingClaimed": False, "pageAllocationClaimed": False, "parameterExpansionClaimed": False, "promotionEligible": False, "acceptedGenerationMutationAllowed": False, } receipt["growthPlanReceiptSha256"] = _sha256_bytes(_json_bytes(receipt)) if receipt_path.is_file(): existing = json.loads(receipt_path.read_text(encoding="utf-8")) if existing != receipt: raise RuntimeError("full payload federated growth receipt differs") else: _atomic_json(receipt_path, receipt) return receipt def _verified_compact_capacity_alignment( compact_summary_path: Path, ) -> dict[str, Any]: """Resolve one complete compact bank into planning-only authority.""" from resynthesis.training_throughput import ( discover_compact_transfer_page_bank, ) summary_path = compact_summary_path.expanduser().resolve() bank = discover_compact_transfer_page_bank(summary_path) summary = json.loads(summary_path.read_text(encoding="utf-8")) if not isinstance(summary, dict): raise RuntimeError("compact capacity summary is malformed") target = int(bank.page_ids_t.numel()) page_parameter_elements = int(bank.page_parameter_elements_t.reshape(())) physical_parameter_elements = int( bank.physical_parameter_elements_t.reshape(()) ) object_bytes = int(bank.total_object_bytes_t.reshape(())) if ( target < 1 or summary.get("requestedPageCount") != target or summary.get("trainedPageCount") != 0 or summary.get("acceptedGenerationCommitted") is not False or summary.get("pageParameterElements") != page_parameter_elements or summary.get("physicalParameterElementsInitialized") != physical_parameter_elements or physical_parameter_elements != target * page_parameter_elements or summary.get("objectBytes") != object_bytes ): raise RuntimeError("compact capacity planning authority differs") from resynthesis.none_catalog_expand import ( _untrained_capacity_proof_accounting, ) proof_accounting = _untrained_capacity_proof_accounting( physical_page_count=target, physical_parameter_elements=physical_parameter_elements, unique_object_identity_count=len( { bytes(row) for row in bank.object_sha256s_t.detach().cpu().tolist() } ), ) return { "schema": NONE_GROWTH_PLAN_CAPACITY_ALIGNMENT_SCHEMA, "bankSummaryPath": str(bank.summary_path), "bankSummarySha256": file_sha256(bank.summary_path), "bankJournalPath": str(bank.journal_path), "bankJournalSha256": file_sha256(bank.journal_path), "bankStoreRoot": str(bank.store_root), "bankSessionId": [int(value) for value in bank.session_id_t.tolist()], "bankSourceCheckpointSha256": bytes( bank.source_checkpoint_sha256_t.tolist() ).hex(), "bankStartPageId": int(bank.page_ids_t[0]), "bankEndPageId": int(bank.page_ids_t[-1]), "bankPageCount": target, "bankPhysicalGraphLayerCount": target, "bankPhysicalGraphNodeCount": target, "bankModelRoutedPageNodeCount": target, "bankPageToPhysicalGraphLayerCardinality": "one_to_one", "bankPageToPhysicalGraphNodeCardinality": "one_to_one", "bankPageParameterElements": page_parameter_elements, "bankPhysicalParameterElements": physical_parameter_elements, "bankObjectBytes": object_bytes, "allocationPolicy": NONE_GROWTH_PLAN_CAPACITY_ALLOCATION_POLICY, "modelTrainingClaimed": False, "promotionEligibilityClaimed": False, "bankProofAccounting": proof_accounting, } def _align_none_growth_plan_to_compact_capacity( plan: dict[str, Any], alignment: Mapping[str, Any], ) -> dict[str, Any]: """Expand source objectives and traversal depth from physical capacity. Compact capacity adds page breadth within each functional family. The graph therefore needs more recurrent layer opportunities as pages-per-family grows; otherwise roots scale while traversal depth stays frozen. The additive depth is derived from bank geometry and has no host-defined maximum. """ additive_bank_page_count = alignment.get("bankPageCount") bank_proof_value = alignment.get("bankProofAccounting") source_rows_value = plan.get("sourceObjectives") if ( not isinstance(additive_bank_page_count, int) or isinstance(additive_bank_page_count, bool) or additive_bank_page_count < 1 or not isinstance(bank_proof_value, dict) or bank_proof_value.get("schema") != NONE_GROWTH_PROOF_ACCOUNTING_SCHEMA or bank_proof_value.get("physicalCapacityPageCount") != additive_bank_page_count or bank_proof_value.get("uniqueMaterializedPageCount") != additive_bank_page_count or bank_proof_value.get("uniqueMaterializedObjectIdentityCount") != additive_bank_page_count or bank_proof_value.get("transferInitializedUntrainedPageCount") != additive_bank_page_count or bank_proof_value.get("uniqueUpdatedPageCount") != 0 or bank_proof_value.get("validatedTrainedPageCount") != 0 or bank_proof_value.get("trainedKnowledgePageCount") != 0 or bank_proof_value.get("acceptedUnionPageCount") != 0 or bank_proof_value.get("coldReloadValidatedPageCount") != 0 or bank_proof_value.get("trainedCapabilityClaimed") is not False or bank_proof_value.get( "physicalOrResidentCapacityCountsAsTrainedKnowledge" ) is not False or bank_proof_value.get( "acceptedUnionMembershipAloneCountsAsTrainedKnowledge" ) is not False or not isinstance(bank_proof_value.get("hostFixedMaximums"), dict) or any( value is not None for value in bank_proof_value["hostFixedMaximums"].values() ) or bank_proof_value["hostFixedMaximums"].get("hiddenDimension", False) is not None or not isinstance(source_rows_value, list) or not source_rows_value ): raise RuntimeError("NoNE compact capacity target is malformed") bank_proof = dict(bank_proof_value) source_rows: list[dict[str, Any]] = [] source_ids: set[str] = set() baseline = 0 payload_bytes = 0 for value in source_rows_value: if not isinstance(value, dict): raise RuntimeError("NoNE capacity source objective is malformed") row = dict(value) source_id = row.get("source_id") planned = row.get("planned_expert_page_objectives") payload_pages = row.get("payload_page_objectives") source_bytes = row.get("payload_bytes") if ( not isinstance(source_id, str) or not source_id.strip() or source_id in source_ids or not isinstance(planned, int) or isinstance(planned, bool) or planned < 1 or not isinstance(payload_pages, int) or isinstance(payload_pages, bool) or payload_pages < 1 or payload_pages > planned or not isinstance(source_bytes, int) or isinstance(source_bytes, bool) or source_bytes < 0 ): raise RuntimeError("NoNE capacity source objective differs") source_ids.add(source_id) source_rows.append(row) baseline += planned payload_bytes += source_bytes prior_alignment_value = plan.get("capacityAlignment") if prior_alignment_value is not None and ( not isinstance(prior_alignment_value, dict) or prior_alignment_value.get("schema") != NONE_GROWTH_PLAN_CAPACITY_ALIGNMENT_SCHEMA ): raise RuntimeError("NoNE inherited compact capacity is malformed") prior_alignment = ( dict(prior_alignment_value) if isinstance(prior_alignment_value, dict) else None ) inherited_compact_page_count = 0 if prior_alignment is not None: inherited_compact_page_count_value = prior_alignment.get( "alignedLogicalExpertPageObjectives" ) if ( not isinstance(inherited_compact_page_count_value, int) or isinstance(inherited_compact_page_count_value, bool) or inherited_compact_page_count_value < 1 or inherited_compact_page_count_value != baseline ): raise RuntimeError( "NoNE inherited compact objective denominator differs" ) inherited_compact_page_count = inherited_compact_page_count_value aligned_objective_count = ( additive_bank_page_count if prior_alignment is None else inherited_compact_page_count + additive_bank_page_count ) if aligned_objective_count < baseline: raise RuntimeError( "compact capacity cannot reduce existing NoNE page objectives" ) additional = aligned_objective_count - baseline positive_rows = [row for row in source_rows if int(row["payload_bytes"]) > 0] positive_payload_bytes = sum(int(row["payload_bytes"]) for row in positive_rows) if additional > 0 and positive_payload_bytes < 1: raise RuntimeError( "NoNE capacity growth has no payload bytes for objective allocation" ) allocations = {str(row["source_id"]): 0 for row in source_rows} remainders: list[tuple[int, str]] = [] allocated = 0 if additional > 0: for row in positive_rows: source_id = str(row["source_id"]) numerator = additional * int(row["payload_bytes"]) quotient, remainder = divmod(numerator, positive_payload_bytes) allocations[source_id] = quotient allocated += quotient remainders.append((remainder, source_id)) remaining = additional - allocated for _remainder, source_id in sorted( remainders, key=lambda value: (-value[0], value[1]), )[:remaining]: allocations[source_id] += 1 aligned_rows: list[dict[str, Any]] = [] for row in source_rows: source_id = str(row["source_id"]) extra = allocations[source_id] aligned_row = dict(row) aligned_row["planned_expert_page_objectives"] = ( int(row["planned_expert_page_objectives"]) + extra ) if int(row["payload_bytes"]) > 0: aligned_row["payload_page_objectives"] = ( int(row["payload_page_objectives"]) + extra ) aligned_rows.append(aligned_row) aligned_total = sum( int(row["planned_expert_page_objectives"]) for row in aligned_rows ) if ( aligned_total != aligned_objective_count or sum(allocations.values()) != additional ): raise RuntimeError("NoNE compact capacity allocation is not exact") aligned = dict(plan) target_geometry_value = aligned.get("proposedMinimumTargetGeometry") dense_geometry_value = aligned.get("proposedMinimumDenseSeedGeometry") current_geometry_value = aligned.get("currentSeedGeometry") baseline_additional_layers = aligned.get( "proposedMinimumAdditionalResidentLayerSlots", aligned.get("proposedMinimumAdditionalLayerSlots"), ) if ( not isinstance(target_geometry_value, dict) or not isinstance(dense_geometry_value, dict) or not isinstance(current_geometry_value, dict) or not isinstance(baseline_additional_layers, int) or isinstance(baseline_additional_layers, bool) or baseline_additional_layers < 0 ): raise RuntimeError("NoNE capacity layer geometry is malformed") target_geometry = dict(target_geometry_value) dense_geometry = dict(dense_geometry_value) functional_family_count = target_geometry.get("functionalExpertFamilies") baseline_target_layers = target_geometry.get( "residentScienceLayers", target_geometry.get("scienceLayers"), ) baseline_dense_layers = dense_geometry.get("scienceLayers") current_layers = current_geometry_value.get("scienceLayers") seed_physical_graph_layer_count = current_geometry_value.get( "physicalGraphLayerCount", current_geometry_value.get("physicalNoNELayers", 0), ) if ( not isinstance(functional_family_count, int) or isinstance(functional_family_count, bool) or functional_family_count < 1 or not isinstance(baseline_target_layers, int) or isinstance(baseline_target_layers, bool) or baseline_target_layers < 1 or not isinstance(baseline_dense_layers, int) or isinstance(baseline_dense_layers, bool) or baseline_dense_layers < 1 or not isinstance(current_layers, int) or isinstance(current_layers, bool) or current_layers < 1 or not isinstance(seed_physical_graph_layer_count, int) or isinstance(seed_physical_graph_layer_count, bool) or seed_physical_graph_layer_count < 0 or baseline_target_layers < current_layers ): raise RuntimeError("NoNE capacity layer geometry differs") current_physical_graph_layer_count = seed_physical_graph_layer_count inherited_capacity_routing_layer_slots = 0 if prior_alignment is not None: prior_physical_plan = aligned.get("physicalGraphPlan") prior_physical_count = ( prior_physical_plan.get("physicalGraphLayerCount") if isinstance(prior_physical_plan, dict) else None ) prior_aligned_physical_count = prior_alignment.get( "alignedPhysicalNoNELayerCount" ) prior_aligned_target_layers = prior_alignment.get( "residentTraversalLayerCount", prior_alignment.get("alignedTargetScienceLayers"), ) prior_family_count = prior_alignment.get( "functionalExpertFamilyCount" ) prior_pages_per_family = ( inherited_compact_page_count + functional_family_count - 1 ) // functional_family_count derived_inherited_routing_slots = ( prior_pages_per_family - 1 ).bit_length() recorded_inherited_routing_slots = prior_alignment.get( "totalCapacityRoutingLayerSlots", derived_inherited_routing_slots, ) if ( not isinstance(prior_physical_count, int) or isinstance(prior_physical_count, bool) or prior_physical_count < 1 or prior_aligned_physical_count != prior_physical_count or prior_aligned_target_layers != baseline_target_layers or prior_family_count != functional_family_count or not isinstance(recorded_inherited_routing_slots, int) or isinstance(recorded_inherited_routing_slots, bool) or recorded_inherited_routing_slots < 0 or recorded_inherited_routing_slots != derived_inherited_routing_slots ): raise RuntimeError("NoNE inherited capacity geometry differs") current_physical_graph_layer_count = prior_physical_count inherited_capacity_routing_layer_slots = ( recorded_inherited_routing_slots ) page_objectives_per_family = ( aligned_objective_count + functional_family_count - 1 ) // functional_family_count total_capacity_routing_layer_slots = ( page_objectives_per_family - 1 ).bit_length() if ( total_capacity_routing_layer_slots < inherited_capacity_routing_layer_slots ): raise RuntimeError("NoNE cumulative capacity routing depth regressed") capacity_routing_layer_slots = ( total_capacity_routing_layer_slots - inherited_capacity_routing_layer_slots ) aligned_target_layers = ( baseline_target_layers + capacity_routing_layer_slots ) aligned_dense_layers = baseline_dense_layers + capacity_routing_layer_slots total_physical_graph_layer_count = ( current_physical_graph_layer_count + additive_bank_page_count ) page_parameter_elements_value = alignment.get( "bankPageParameterElements" ) page_parameter_elements = ( page_parameter_elements_value if isinstance(page_parameter_elements_value, int) and not isinstance(page_parameter_elements_value, bool) and page_parameter_elements_value > 0 else None ) bank_physical_parameter_elements = alignment.get( "bankPhysicalParameterElements" ) if ( page_parameter_elements is None or not isinstance(bank_physical_parameter_elements, int) or isinstance(bank_physical_parameter_elements, bool) or bank_physical_parameter_elements != additive_bank_page_count * page_parameter_elements ): raise RuntimeError("NoNE additive bank parameter geometry differs") if prior_alignment is None: inherited_physical_parameter_elements = ( current_physical_graph_layer_count * page_parameter_elements ) page_parameter_element_widths = (page_parameter_elements,) else: inherited_physical_parameter_elements_value = prior_alignment.get( "alignedPhysicalParameterElements" ) inherited_widths_value = prior_alignment.get( "pageModelParameterElementWidths" ) legacy_inherited_width = prior_alignment.get( "pageModelParameterElements" ) inherited_widths = ( inherited_widths_value if isinstance(inherited_widths_value, list) else [legacy_inherited_width] ) if ( not isinstance(inherited_physical_parameter_elements_value, int) or isinstance(inherited_physical_parameter_elements_value, bool) or inherited_physical_parameter_elements_value < 1 or not inherited_widths or any( not isinstance(width, int) or isinstance(width, bool) or width < 1 for width in inherited_widths ) ): raise RuntimeError( "NoNE inherited capacity parameter geometry differs" ) inherited_physical_parameter_elements = ( inherited_physical_parameter_elements_value ) page_parameter_element_widths = tuple( sorted({*inherited_widths, page_parameter_elements}) ) total_physical_parameter_elements = ( inherited_physical_parameter_elements + bank_physical_parameter_elements ) uniform_page_parameter_width = ( page_parameter_element_widths[0] if len(page_parameter_element_widths) == 1 else None ) # ``scienceLayers`` remains the executable resident traversal depth for # compatibility with checkpoint/RBO lineage. The much larger sparse # graph is counted separately and exactly one-to-one with page objects. target_geometry["scienceLayers"] = aligned_target_layers target_geometry["residentScienceLayers"] = aligned_target_layers target_geometry["physicalNoNELayers"] = total_physical_graph_layer_count target_geometry["totalPhysicalNoNELayers"] = ( total_physical_graph_layer_count ) target_geometry["sparsePhysicalNoNELayers"] = ( total_physical_graph_layer_count ) target_geometry["scienceLayersCountScope"] = ( "resident_dense_traversal_only" ) target_geometry["scienceLayersAreTotalNoNELayers"] = False target_geometry["physicalGraphLayerCount"] = ( total_physical_graph_layer_count ) target_geometry["physicalGraphNodeCount"] = ( total_physical_graph_layer_count ) target_geometry["physicalPageExperts"] = total_physical_graph_layer_count dense_geometry["scienceLayers"] = aligned_dense_layers dense_geometry["residentScienceLayers"] = aligned_dense_layers dense_geometry["physicalNoNELayers"] = total_physical_graph_layer_count dense_geometry["totalPhysicalNoNELayers"] = ( total_physical_graph_layer_count ) dense_geometry["sparsePhysicalNoNELayers"] = ( total_physical_graph_layer_count ) dense_geometry["scienceLayersCountScope"] = ( "resident_dense_traversal_only" ) dense_geometry["scienceLayersAreTotalNoNELayers"] = False dense_geometry["physicalGraphLayerCount"] = ( total_physical_graph_layer_count ) dense_geometry["physicalGraphNodeCount"] = ( total_physical_graph_layer_count ) dense_geometry["physicalPageExperts"] = total_physical_graph_layer_count aligned["proposedMinimumTargetGeometry"] = target_geometry aligned["proposedMinimumDenseSeedGeometry"] = dense_geometry aligned["proposedMinimumAdditionalLayerSlots"] = ( baseline_additional_layers + capacity_routing_layer_slots ) aligned["proposedMinimumAdditionalResidentLayerSlots"] = ( baseline_additional_layers + capacity_routing_layer_slots ) aligned["proposedMinimumAdditionalPhysicalNoNELayers"] = ( additive_bank_page_count ) aligned["proposedSourceObjectiveExpansionPageCount"] = additional aligned["sourceObjectives"] = aligned_rows aligned["initialLogicalExpertPageObjectives"] = aligned_objective_count aligned["physicalGraphPlan"] = { "schema": "nnf.resynthesis.physical_graph_plan.v1", "countScope": "verified_compact_bank_additive_capacity", "physicalGraphLayerKind": "sparse_page_backed_none_expert", "inheritedPhysicalGraphLayerCount": ( current_physical_graph_layer_count ), "inheritedCompactPageCount": inherited_compact_page_count, "additiveCompactPageCount": additive_bank_page_count, "totalCompactPageCount": aligned_objective_count, "additiveBankPageCount": additive_bank_page_count, "additiveBankPhysicalGraphLayerCount": additive_bank_page_count, "additiveBankPhysicalGraphNodeCount": additive_bank_page_count, "physicalGraphLayerCount": total_physical_graph_layer_count, "physicalGraphNodeCount": total_physical_graph_layer_count, "modelRoutedPageNodeCount": total_physical_graph_layer_count, "pageToPhysicalGraphLayerCardinality": "one_to_one", "pageToPhysicalGraphNodeCardinality": "one_to_one", "denseResidentScienceLayerCount": aligned_dense_layers, "inheritedCapacityRoutingLayerSlots": ( inherited_capacity_routing_layer_slots ), "additionalCapacityRoutingLayerSlots": ( capacity_routing_layer_slots ), "totalCapacityRoutingLayerSlots": ( total_capacity_routing_layer_slots ), "pageBackedLayersReportedAsDenseLayers": False, "physicalGraphLayerCountMaximum": None, "pageParameterWidthMaximum": None, "hiddenDimensionMaximum": None, "expertRankMaximum": None, "transferDimensionMaximum": None, "successorCompactBanksRemainAdditive": True, "successorTensorGeometryMayExpand": True, "perLayerUniqueIdentityRequired": True, "perLayerPhysicalWeightsRequired": True, "perLayerModelOwnedRoutingRequired": True, "perLayerGradientAndDeltaProofRequired": True, "trainingClaimed": False, "promotionEligible": False, } aligned["physicalGraphPlan"].update( { "pageModelParameterElementWidths": list( page_parameter_element_widths ), "uniformPageModelParameterWidth": ( uniform_page_parameter_width is not None ), "additiveBankPageModelParameterElements": ( page_parameter_elements ), "inheritedPhysicalParameterElements": ( inherited_physical_parameter_elements ), "additiveBankPhysicalParameterElements": ( bank_physical_parameter_elements ), "physicalGraphParameterElements": ( total_physical_parameter_elements ), } ) if uniform_page_parameter_width is not None: aligned["physicalGraphPlan"]["pageModelParameterElements"] = ( uniform_page_parameter_width ) aligned["capacityAlignment"] = { **dict(alignment), "baselineLogicalExpertPageObjectives": baseline, "additionalLogicalExpertPageObjectives": additional, "alignedLogicalExpertPageObjectives": aligned_objective_count, "sourceObjectiveExpansionPageCount": additional, "payloadBytes": payload_bytes, "positivePayloadSourceCount": len(positive_rows), "metadataOnlySourceCount": len(source_rows) - len(positive_rows), "sourceCount": len(source_rows), "layerAllocationPolicy": ( "one_content_addressed_page_objective_per_logical_layer_v1" ), "residentRoutingDepthPolicy": ( "additive_ceil_log2_page_objectives_per_functional_family_v1" ), "functionalExpertFamilyCount": functional_family_count, # This is ceil-division arithmetic used to derive recurrent depth, not # a page-count ceiling. Name it as a rounded-up observation so no # consumer can accidentally reinterpret it as admission authority. "pageObjectivesPerFunctionalFamilyRoundedUp": ( page_objectives_per_family ), "baselineResidentTraversalLayerCount": baseline_target_layers, "additionalCapacityRoutingLayerSlots": ( capacity_routing_layer_slots ), "inheritedCapacityRoutingLayerSlots": ( inherited_capacity_routing_layer_slots ), "totalCapacityRoutingLayerSlots": ( total_capacity_routing_layer_slots ), "sparsePageLayerRoutingOwnership": ( "model_owned_tensor_routes_recurrently_bind_physical_pages" ), "residentTraversalLayerCount": aligned_target_layers, "inheritedPhysicalNoNELayerCount": ( current_physical_graph_layer_count ), "baselinePhysicalNoNELayerCount": ( current_physical_graph_layer_count ), "inheritedCompactPageCount": inherited_compact_page_count, "additiveCompactPageCount": additive_bank_page_count, "totalCompactPageCount": aligned_objective_count, "additiveBankPageCount": additive_bank_page_count, "additionalPhysicalNoNELayerCount": additive_bank_page_count, "alignedPhysicalNoNELayerCount": total_physical_graph_layer_count, "alignedPhysicalPageExpertCount": total_physical_graph_layer_count, "physicalGraphLayerCount": total_physical_graph_layer_count, "physicalGraphNodeCount": total_physical_graph_layer_count, "modelRoutedCompactPageNodeCount": aligned_objective_count, "modelRoutedPhysicalGraphNodeCount": total_physical_graph_layer_count, "pageToPhysicalGraphLayerCardinality": "one_to_one", "pageToPhysicalGraphNodeCardinality": "one_to_one", "pageBackedLayersReportedAsDenseLayers": False, "physicalGraphLayerCountMaximum": None, "pageParameterWidthMaximum": None, "hiddenDimensionMaximum": None, "expertRankMaximum": None, "transferDimensionMaximum": None, "successorCompactBanksRemainAdditive": True, "successorTensorGeometryMayExpand": True, "physicalNoNELayerIdentity": ( "one_content_addressed_page_objective_per_logical_layer_v1" ), "bankProofAccounting": bank_proof, } aligned["capacityAlignment"].update( { "pageModelParameterElementWidths": list( page_parameter_element_widths ), "uniformPageModelParameterWidth": ( uniform_page_parameter_width is not None ), "additiveBankPageModelParameterElements": ( page_parameter_elements ), "inheritedPhysicalParameterElements": ( inherited_physical_parameter_elements ), "additiveBankPhysicalParameterElements": ( bank_physical_parameter_elements ), "alignedPhysicalParameterElements": ( total_physical_parameter_elements ), } ) if uniform_page_parameter_width is not None: aligned["capacityAlignment"]["pageModelParameterElements"] = ( uniform_page_parameter_width ) else: aligned["capacityAlignment"].pop( "pageModelParameterElements", None, ) checks_value = aligned.get("checks") checks = dict(checks_value) if isinstance(checks_value, dict) else {} checks.update( { "compactCapacityTargetVerified": True, "capacityAlignmentPreservesBaseline": all( int(aligned_row["planned_expert_page_objectives"]) >= int(source_row["planned_expert_page_objectives"]) for source_row, aligned_row in zip( source_rows, aligned_rows, strict=True, ) ), "capacityAlignmentExact": ( aligned_total == aligned_objective_count ), "capacityAlignmentClaimsNoTraining": ( alignment.get("modelTrainingClaimed") is False and alignment.get("promotionEligibilityClaimed") is False ), "capacityLayerGeometryDerived": ( aligned_target_layers == baseline_target_layers + capacity_routing_layer_slots and aligned_dense_layers == baseline_dense_layers + capacity_routing_layer_slots and total_capacity_routing_layer_slots == inherited_capacity_routing_layer_slots + capacity_routing_layer_slots and target_geometry["scienceLayers"] == aligned_target_layers and target_geometry["residentScienceLayers"] == aligned_target_layers and dense_geometry["scienceLayers"] == aligned_dense_layers and target_geometry["physicalNoNELayers"] == total_physical_graph_layer_count and dense_geometry["physicalNoNELayers"] == total_physical_graph_layer_count ), "capacityLayerGeometryMonotonic": ( aligned_target_layers >= baseline_target_layers and aligned_dense_layers >= baseline_dense_layers ), "capacityLayerGeometryHasNoFixedMaximum": True, "everyPhysicalPageObjectiveHasLogicalLayerIdentity": ( target_geometry["physicalNoNELayers"] == total_physical_graph_layer_count and target_geometry["physicalGraphLayerCount"] == total_physical_graph_layer_count and target_geometry["physicalGraphNodeCount"] == total_physical_graph_layer_count and current_physical_graph_layer_count + additive_bank_page_count == total_physical_graph_layer_count and aligned_total == aligned_objective_count ), "compactBankPagesMapOneToOneToNewPhysicalGraphLayers": ( alignment.get( "bankPhysicalGraphLayerCount", additive_bank_page_count, ) == additive_bank_page_count and alignment.get( "bankPhysicalGraphNodeCount", additive_bank_page_count, ) == additive_bank_page_count and alignment.get( "bankPageToPhysicalGraphLayerCardinality", "one_to_one", ) == "one_to_one" and alignment.get( "bankPageToPhysicalGraphNodeCardinality", "one_to_one", ) == "one_to_one" ), "physicalGraphPagesRemainSeparateFromDenseLayers": ( aligned["physicalGraphPlan"][ "pageBackedLayersReportedAsDenseLayers" ] is False and aligned["physicalGraphPlan"][ "denseResidentScienceLayerCount" ] == aligned_dense_layers and aligned["physicalGraphPlan"][ "physicalGraphLayerCount" ] == total_physical_graph_layer_count and target_geometry["scienceLayers"] == aligned_target_layers ), "physicalGraphLayerCountHasNoFixedMaximum": ( aligned["physicalGraphPlan"][ "physicalGraphLayerCountMaximum" ] is None and aligned["physicalGraphPlan"][ "successorCompactBanksRemainAdditive" ] is True ), "physicalGraphTrainingProofRequiredPerLayer": ( aligned["physicalGraphPlan"][ "perLayerUniqueIdentityRequired" ] is True and aligned["physicalGraphPlan"][ "perLayerPhysicalWeightsRequired" ] is True and aligned["physicalGraphPlan"][ "perLayerModelOwnedRoutingRequired" ] is True and aligned["physicalGraphPlan"][ "perLayerGradientAndDeltaProofRequired" ] is True ), "physicalGraphParameterCountIsExact": ( bank_physical_parameter_elements == additive_bank_page_count * page_parameter_elements and total_physical_parameter_elements == inherited_physical_parameter_elements + bank_physical_parameter_elements ), "successorPageWidthMayExpandWithoutRewritingInheritedTensors": ( aligned["physicalGraphPlan"][ "pageParameterWidthMaximum" ] is None and aligned["physicalGraphPlan"][ "hiddenDimensionMaximum" ] is None and aligned["physicalGraphPlan"][ "expertRankMaximum" ] is None and aligned["physicalGraphPlan"][ "transferDimensionMaximum" ] is None and aligned["physicalGraphPlan"][ "successorTensorGeometryMayExpand" ] is True and total_physical_parameter_elements == inherited_physical_parameter_elements + bank_physical_parameter_elements ), "capacityRoutingDepthAppliedIncrementally": ( total_capacity_routing_layer_slots == inherited_capacity_routing_layer_slots + capacity_routing_layer_slots and aligned_target_layers == baseline_target_layers + capacity_routing_layer_slots and aligned_dense_layers == baseline_dense_layers + capacity_routing_layer_slots ), "compactCapacityCountsAreInheritedPlusAdditive": ( aligned_objective_count == inherited_compact_page_count + additive_bank_page_count and total_physical_graph_layer_count == current_physical_graph_layer_count + additive_bank_page_count ), "compactCapacityProofStagesSeparated": ( bank_proof["physicalCapacityPageCount"] == additive_bank_page_count and bank_proof["uniqueUpdatedPageCount"] == 0 and bank_proof["validatedTrainedPageCount"] == 0 and bank_proof["trainedKnowledgePageCount"] == 0 and bank_proof["acceptedUnionPageCount"] == 0 and bank_proof["coldReloadValidatedPageCount"] == 0 and bank_proof[ "physicalOrResidentCapacityCountsAsTrainedKnowledge" ] is False ), } ) aligned["checks"] = checks return aligned def _retained_capacity_alignment( loaded: Mapping[str, Any], ) -> dict[str, Any] | None: value = loaded.get("capacityAlignment") if value is None: return None if not isinstance(value, dict): raise ValueError("NoNE capacity alignment is malformed") summary_value = value.get("bankSummaryPath") expected_summary_sha256 = value.get("bankSummarySha256") if ( value.get("schema") != NONE_GROWTH_PLAN_CAPACITY_ALIGNMENT_SCHEMA or not isinstance(summary_value, str) or not summary_value or not isinstance(expected_summary_sha256, str) or len(expected_summary_sha256) != 64 ): raise ValueError("NoNE capacity alignment authority is malformed") alignment = _verified_compact_capacity_alignment(Path(summary_value)) retained_fields = ( "bankSummarySha256", "bankJournalSha256", "bankPageCount", "bankPageParameterElements", "bankPhysicalParameterElements", "bankObjectBytes", "allocationPolicy", "modelTrainingClaimed", "promotionEligibilityClaimed", "bankProofAccounting", ) if any(value.get(field) != alignment.get(field) for field in retained_fields): raise ValueError("NoNE capacity alignment bytes differ") return alignment def derive_paged_none_growth_plan_from_existing( source_plan: Path, ) -> dict[str, Any]: """Derive the current catalog graph from one immutable source plan. This pure planning boundary lets checkpoint adaptation bind the current functional graph without rewriting the source plan or claiming training. """ source = source_plan.resolve() loaded = json.loads(source.read_text(encoding="utf-8")) if ( not isinstance(loaded, dict) or loaded.get("schema") not in { LEGACY_NONE_GROWTH_PLAN_SCHEMA, NONE_GROWTH_PLAN_SCHEMA, } ): raise ValueError("source NoNE growth plan schema is unsupported") loaded_alignment = loaded.get("capacityAlignment") if isinstance(loaded_alignment, dict) and loaded_alignment.get( "parentCapacityPlanPath" ) is not None: parent_value = loaded_alignment.get("parentCapacityPlanPath") parent_sha256 = loaded_alignment.get("parentCapacityPlanSha256") summary_value = loaded_alignment.get("bankSummaryPath") if ( not isinstance(parent_value, str) or not parent_value or not isinstance(parent_sha256, str) or len(parent_sha256) != 64 or not isinstance(summary_value, str) or not summary_value ): raise ValueError("additive NoNE capacity lineage is malformed") parent_path = Path(parent_value).expanduser().resolve() if not parent_path.is_file() or file_sha256(parent_path) != parent_sha256: raise ValueError("additive NoNE capacity parent bytes differ") accepted_catalog_value = loaded_alignment.get( "acceptedPageCatalogPath" ) accepted_catalog_sha256 = loaded_alignment.get( "acceptedPageCatalogSha256" ) accepted_catalog_path: Path | None = None if accepted_catalog_value is not None or accepted_catalog_sha256 is not None: if ( not isinstance(accepted_catalog_value, str) or not accepted_catalog_value or not isinstance(accepted_catalog_sha256, str) or len(accepted_catalog_sha256) != 64 ): raise ValueError( "additive NoNE accepted catalog authority is malformed" ) accepted_catalog_path = Path( accepted_catalog_value ).expanduser().resolve() if ( not accepted_catalog_path.is_file() or file_sha256(accepted_catalog_path) != accepted_catalog_sha256 ): raise ValueError( "additive NoNE accepted catalog bytes differ" ) plan = derive_additive_capacity_aligned_none_growth_plan_from_existing( parent_path, Path(summary_value), accepted_page_catalog=accepted_catalog_path, ) if accepted_catalog_path is not None and ( not isinstance(plan.get("capacityAlignment"), dict) or plan["capacityAlignment"].get("acceptedPageCatalogSha256") != accepted_catalog_sha256 ): raise ValueError("additive NoNE accepted catalog bytes changed") plan["lineage"] = { "sourcePlan": str(source), "sourcePlanSha256": file_sha256(source), "sourcePlanSchema": loaded.get("schema"), "sourcePlanPreserved": True, "modelTrainingClaimed": False, "promotionEligibilityClaimed": False, } return plan raw_entries = loaded.get("sourceObjectives") if not isinstance(raw_entries, list) or not raw_entries: raise ValueError("source NoNE growth plan has no source objectives") entries: list[dict[str, Any]] = [] for raw_entry in raw_entries: if not isinstance(raw_entry, dict): raise ValueError("source NoNE growth objective is invalid") entry = dict(raw_entry) claims = _as_list(entry.get("content_claims")) entry["capability_axes"] = [ {"axis": axis_id, "description": description} for axis_id, description in _none_capability_axes(claims) ] entry["functional_expert_families"] = [ {"family": family_id, "description": description} for family_id, description in _none_functional_expert_families( claims ) ] entries.append(entry) plan = _none_growth_plan(entries) federated_demand = loaded.get("federatedContentDemand") if federated_demand is not None: if ( not isinstance(federated_demand, dict) or federated_demand.get("schema") != FULL_PAYLOAD_FEDERATED_NONE_GROWTH_PLAN_SCHEMA or federated_demand.get("modelTrainingClaimed") is not False or federated_demand.get("pageAllocationClaimed") is not False or federated_demand.get("parameterExpansionClaimed") is not False or federated_demand.get("acceptedGenerationMutationAllowed") is not False ): raise ValueError("federated NoNE growth demand is malformed") # Keep the sealed cross-root content demand through every derived # planning form. Capacity alignment may add a bank later, but cannot # substitute or silently drop the source evidence that requested it. plan["federatedContentDemand"] = dict(federated_demand) retained_alignment = _retained_capacity_alignment(loaded) if retained_alignment is not None: plan = _align_none_growth_plan_to_compact_capacity( plan, retained_alignment, ) plan["lineage"] = { "sourcePlan": str(source), "sourcePlanSha256": file_sha256(source), "sourcePlanSchema": loaded.get("schema"), "sourcePlanPreserved": True, "modelTrainingClaimed": False, "promotionEligibilityClaimed": False, } return plan def derive_capacity_aligned_none_growth_plan_from_existing( source_plan: Path, compact_summary_path: Path, ) -> dict[str, Any]: """Bind an immutable source plan to one verified physical page bank.""" plan = derive_paged_none_growth_plan_from_existing(source_plan) alignment = _verified_compact_capacity_alignment(compact_summary_path) return _align_none_growth_plan_to_compact_capacity(plan, alignment) def _rebase_additive_growth_plan_to_accepted_catalog_boundary( plan: dict[str, Any], *, parent_plan: Mapping[str, Any], accepted_page_catalog: Path, ) -> dict[str, Any]: """Preserve every admitted objective while allocating successor capacity. A retained scale cohort is model-routed and therefore need not match an independently recomputed proportional allocation for the same page count. The accepted catalog is the immutable identity authority. The parent plan remains a second lower bound so successor discovery also preserves every previously planned ordinal. Remaining capacity is allocated from payload bytes only after both exact prefixes have been retained. """ catalog_path = accepted_page_catalog.expanduser().resolve() with catalog_path.open("rb") as handle: opened_before = os.fstat(handle.fileno()) catalog_bytes = handle.read() opened_after = os.fstat(handle.fileno()) path_after = catalog_path.stat() opened_identity = ( opened_before.st_dev, opened_before.st_ino, opened_before.st_size, opened_before.st_mtime_ns, opened_before.st_ctime_ns, ) if opened_identity != ( opened_after.st_dev, opened_after.st_ino, opened_after.st_size, opened_after.st_mtime_ns, opened_after.st_ctime_ns, ) or opened_identity != ( path_after.st_dev, path_after.st_ino, path_after.st_size, path_after.st_mtime_ns, path_after.st_ctime_ns, ): raise RuntimeError("NoNE accepted page catalog changed during read") accepted_catalog_sha256 = hashlib.sha256(catalog_bytes).hexdigest() catalog = json.loads(catalog_bytes) pages = catalog.get("pages") if isinstance(catalog, dict) else None if ( not isinstance(catalog, dict) or catalog.get("schema") != NONE_V2_PLUS_PAGE_CATALOG_SCHEMA or not isinstance(pages, list) or catalog.get("pageCount") != len(pages) ): raise RuntimeError("NoNE accepted page catalog authority differs") source_rows_value = plan.get("sourceObjectives") parent_rows_value = parent_plan.get("sourceObjectives") target_total = plan.get("initialLogicalExpertPageObjectives") parent_total = parent_plan.get("initialLogicalExpertPageObjectives") if ( not isinstance(source_rows_value, list) or not source_rows_value or not isinstance(parent_rows_value, list) or len(parent_rows_value) != len(source_rows_value) or not isinstance(target_total, int) or isinstance(target_total, bool) or not isinstance(parent_total, int) or isinstance(parent_total, bool) or target_total < parent_total ): raise RuntimeError("NoNE accepted-prefix plan geometry differs") def identity_vector( row: Mapping[str, Any], *, field: str, identity: str, ) -> tuple[str, ...]: values = row.get(field) if not isinstance(values, list) or not values: raise RuntimeError("NoNE accepted-prefix identity vector is absent") identifiers: list[str] = [] for value in values: identifier = value.get(identity) if isinstance(value, dict) else None if not isinstance(identifier, str) or not identifier.strip(): raise RuntimeError( "NoNE accepted-prefix identity vector is malformed" ) identifiers.append(identifier.strip()) if len(set(identifiers)) != len(identifiers): raise RuntimeError("NoNE accepted-prefix identity vector repeats") return tuple(identifiers) source_rows: dict[str, dict[str, Any]] = {} parent_counts: dict[str, int] = {} source_payload_counts: dict[str, int] = {} for target_value, parent_value in zip( source_rows_value, parent_rows_value, strict=True, ): if not isinstance(target_value, dict) or not isinstance(parent_value, dict): raise RuntimeError("NoNE accepted-prefix source row is malformed") source_id = target_value.get("source_id") parent_source_id = parent_value.get("source_id") parent_count = parent_value.get("planned_expert_page_objectives") target_count = target_value.get("planned_expert_page_objectives") parent_payload_count = parent_value.get("payload_page_objectives") target_payload_count = target_value.get("payload_page_objectives") if ( not isinstance(source_id, str) or not source_id.strip() or source_id in source_rows or parent_source_id != source_id or not isinstance(parent_count, int) or isinstance(parent_count, bool) or parent_count < 1 or not isinstance(target_count, int) or isinstance(target_count, bool) or target_count < parent_count or not isinstance(parent_payload_count, int) or isinstance(parent_payload_count, bool) or not 1 <= parent_payload_count <= parent_count or not isinstance(target_payload_count, int) or isinstance(target_payload_count, bool) or not 1 <= target_payload_count <= target_count or identity_vector( target_value, field="functional_expert_families", identity="family", ) != identity_vector( parent_value, field="functional_expert_families", identity="family", ) or identity_vector( target_value, field="capability_axes", identity="axis", ) != identity_vector( parent_value, field="capability_axes", identity="axis", ) ): raise RuntimeError("NoNE accepted-prefix parent identity differs") source_rows[source_id] = dict(target_value) parent_counts[source_id] = parent_count source_payload_counts[source_id] = target_payload_count if sum(parent_counts.values()) != parent_total: raise RuntimeError("NoNE accepted-prefix parent count differs") accepted_ordinals: dict[str, set[int]] = {} accepted_payload_ordinals: dict[str, set[int]] = {} accepted_identity_rows: list[dict[str, Any]] = [] accepted_objective_ids: set[str] = set() for page in pages: if not isinstance(page, dict): raise RuntimeError("NoNE accepted page catalog row is malformed") objective_id = page.get("objectiveId") if objective_id is None: continue source_id = page.get("objectiveSourceId") ordinal = page.get("objectiveOrdinal") families = page.get("functionalFamilies") axes = page.get("capabilityAxes") payload_shard_backed = page.get("payloadShardBacked") source_row = source_rows.get(str(source_id)) expected_objective_id = ( "objective_" + hashlib.sha256( b"nnf-resynthesis-none-objective-page-v1\0" + str(source_id).encode("utf-8") + b"\0" + int(ordinal).to_bytes(8, "little", signed=False) ).hexdigest() if isinstance(source_id, str) and isinstance(ordinal, int) and not isinstance(ordinal, bool) and 0 <= ordinal < target_total else None ) if ( not isinstance(objective_id, str) or objective_id in accepted_objective_ids or source_row is None or expected_objective_id != objective_id or not isinstance(payload_shard_backed, bool) or families != list( identity_vector( source_row, field="functional_expert_families", identity="family", ) ) or axes != list( identity_vector( source_row, field="capability_axes", identity="axis", ) ) ): raise RuntimeError("NoNE accepted objective identity differs") assert isinstance(source_id, str) assert isinstance(ordinal, int) accepted_objective_ids.add(objective_id) accepted_ordinals.setdefault(source_id, set()).add(ordinal) if payload_shard_backed: accepted_payload_ordinals.setdefault(source_id, set()).add(ordinal) accepted_identity_rows.append( { "objectiveId": objective_id, "objectiveSourceId": source_id, "objectiveOrdinal": ordinal, "functionalFamilies": list(families), "capabilityAxes": list(axes), "payloadShardBacked": payload_shard_backed, } ) accepted_count_value = catalog.get("objectivePageCount") if ( not accepted_objective_ids or accepted_count_value != len(accepted_objective_ids) ): raise RuntimeError("NoNE accepted objective denominator differs") accepted_counts: dict[str, int] = {} accepted_payload_counts: dict[str, int] = {} for source_id, ordinals in accepted_ordinals.items(): accepted_count = max(ordinals) + 1 if ordinals != set(range(accepted_count)): raise RuntimeError("NoNE accepted objective prefix is not contiguous") accepted_counts[source_id] = accepted_count payload_ordinals = accepted_payload_ordinals.get(source_id, set()) payload_count = len(payload_ordinals) if payload_count < 1 or payload_ordinals != set(range(payload_count)): raise RuntimeError( "NoNE accepted payload-backed prefix is not contiguous" ) accepted_payload_counts[source_id] = payload_count floors = { source_id: max(parent_counts[source_id], accepted_counts.get(source_id, 0)) for source_id in source_rows } remaining = target_total - sum(floors.values()) if remaining < 0: raise RuntimeError("NoNE accepted objective prefixes exceed capacity") positive_rows = [ row for row in source_rows.values() if isinstance(row.get("payload_bytes"), int) and not isinstance(row.get("payload_bytes"), bool) and int(row["payload_bytes"]) > 0 ] positive_bytes = sum(int(row["payload_bytes"]) for row in positive_rows) if remaining and positive_bytes < 1: raise RuntimeError("NoNE accepted-prefix allocation has no payload bytes") allocations = {source_id: 0 for source_id in source_rows} remainders: list[tuple[int, str]] = [] allocated = 0 for row in positive_rows: source_id = str(row["source_id"]) quotient, remainder = divmod( remaining * int(row["payload_bytes"]), positive_bytes, ) allocations[source_id] = quotient allocated += quotient remainders.append((remainder, source_id)) for _remainder, source_id in sorted( remainders, key=lambda value: (-value[0], value[1]), )[: remaining - allocated]: allocations[source_id] += 1 rebased_rows: list[dict[str, Any]] = [] for source_id, source_row in source_rows.items(): planned_count = floors[source_id] + allocations[source_id] rebased = dict(source_row) rebased["planned_expert_page_objectives"] = planned_count accepted_floor_count = accepted_counts.get(source_id) accepted_payload_count = accepted_payload_counts.get(source_id) if ( accepted_floor_count is not None and accepted_payload_count is not None ): if accepted_payload_count < accepted_floor_count: payload_count = accepted_payload_count else: payload_count = min( planned_count, max( accepted_floor_count, source_payload_counts[source_id], ), ) else: payload_count = min( planned_count, source_payload_counts[source_id], ) rebased["payload_page_objectives"] = payload_count rebased_rows.append(rebased) if ( sum(int(row["planned_expert_page_objectives"]) for row in rebased_rows) != target_total or any( int(row["planned_expert_page_objectives"]) < parent_counts[str(row["source_id"])] for row in rebased_rows ) ): raise RuntimeError("NoNE accepted-prefix allocation is not exact") accepted_identity_sha256 = hashlib.sha256( _json_bytes( sorted( accepted_identity_rows, key=lambda row: ( str(row["objectiveSourceId"]), int(row["objectiveOrdinal"]), ), ) ) ).hexdigest() rebased_plan = dict(plan) rebased_plan["sourceObjectives"] = rebased_rows alignment_value = rebased_plan.get("capacityAlignment") physical_value = rebased_plan.get("physicalGraphPlan") checks_value = rebased_plan.get("checks") if ( not isinstance(alignment_value, dict) or not isinstance(physical_value, dict) or not isinstance(checks_value, dict) ): raise RuntimeError("NoNE accepted-prefix plan authority is incomplete") alignment = dict(alignment_value) alignment.update( { "acceptedPageCatalogPath": str(catalog_path), "acceptedPageCatalogSha256": accepted_catalog_sha256, "acceptedPageCatalogBytes": len(catalog_bytes), "acceptedObjectivePageCount": len(accepted_objective_ids), "acceptedObjectiveIdentitySha256": accepted_identity_sha256, "objectiveAllocationPolicy": ( "payload_byte_weighted_largest_remainder_" "parent_and_accepted_prefix_preserving_v1" ), } ) rebased_plan["capacityAlignment"] = alignment physical = dict(physical_value) physical["acceptedObjectivePageCount"] = len(accepted_objective_ids) physical["acceptedObjectiveIdentitySha256"] = accepted_identity_sha256 rebased_plan["physicalGraphPlan"] = physical checks = dict(checks_value) checks.update( { "acceptedObjectiveIdentityPrefixPreserved": True, "acceptedPayloadBackingPrefixPreserved": all( ( int(row["objectiveOrdinal"]) < int( next( source_row["payload_page_objectives"] for source_row in rebased_rows if source_row["source_id"] == row["objectiveSourceId"] ) ) ) is row["payloadShardBacked"] for row in accepted_identity_rows ), "parentObjectiveIdentityPrefixPreserved": True, "acceptedCatalogAdditiveDeltaExact": ( target_total - len(accepted_objective_ids) == alignment.get("additiveBankPageCount") ), } ) if not all(checks.values()): raise RuntimeError("NoNE accepted-prefix plan checks did not pass") rebased_plan["checks"] = checks return rebased_plan def derive_additive_capacity_aligned_none_growth_plan_from_existing( parent_capacity_plan: Path, compact_summary_path: Path, *, accepted_page_catalog: Path | None = None, ) -> dict[str, Any]: """Extend one capacity plan with a disjoint verified compact bank. The parent plan owns cumulative objective and sparse-graph geometry. The new bank remains an independent immutable authority, so its page count is the transaction delta while ``initialLogicalExpertPageObjectives`` remains the cumulative total. This is an external planning boundary and claims no training or promotion. """ parent_path = parent_capacity_plan.expanduser().resolve() parent = json.loads(parent_path.read_text(encoding="utf-8")) bank = _verified_compact_capacity_alignment(compact_summary_path) parent_alignment = parent.get("capacityAlignment") parent_physical = parent.get("physicalGraphPlan") parent_target = parent.get("proposedMinimumTargetGeometry") parent_dense = parent.get("proposedMinimumDenseSeedGeometry") parent_lineage = parent.get("lineage") parent_checks = parent.get("checks") parent_objectives = parent.get("initialLogicalExpertPageObjectives") additive_pages = bank.get("bankPageCount") page_elements = bank.get("bankPageParameterElements") if ( parent.get("schema") != NONE_GROWTH_PLAN_SCHEMA or not isinstance(parent_alignment, dict) or not isinstance(parent_physical, dict) or not isinstance(parent_target, dict) or not isinstance(parent_dense, dict) or not isinstance(parent_lineage, dict) or not isinstance(parent_checks, dict) or not parent_checks or not all(value is True for value in parent_checks.values()) or not isinstance(parent_objectives, int) or isinstance(parent_objectives, bool) or parent_objectives < 1 or not isinstance(additive_pages, int) or isinstance(additive_pages, bool) or additive_pages < 1 or not isinstance(page_elements, int) or isinstance(page_elements, bool) or page_elements < 1 or not isinstance( parent_alignment.get("bankPageParameterElements"), int, ) or isinstance( parent_alignment.get("bankPageParameterElements"), bool, ) or parent_alignment.get("bankPageParameterElements", 0) < 1 ): raise RuntimeError("NoNE additive capacity parent authority differs") parent_summary_value = parent_alignment.get("bankSummaryPath") inherited_layers = parent_physical.get("physicalGraphLayerCount") inherited_parameters = parent_physical.get("physicalGraphParameterElements") parent_resident_layers = parent_target.get( "residentScienceLayers", parent_target.get("scienceLayers"), ) family_count = parent_target.get("functionalExpertFamilies") if ( not isinstance(parent_summary_value, str) or not parent_summary_value or not isinstance(inherited_layers, int) or isinstance(inherited_layers, bool) or inherited_layers < 1 or not isinstance(inherited_parameters, int) or isinstance(inherited_parameters, bool) or inherited_parameters < 1 # The parent owns its own possibly heterogeneous tensor widths. Bind # its exact cumulative element count rather than multiplying every # inherited layer by the new bank's width, which would silently turn # successor geometry into a global fixed-width ceiling. or inherited_parameters != parent_alignment.get("alignedPhysicalParameterElements") or not isinstance(parent_resident_layers, int) or isinstance(parent_resident_layers, bool) or parent_resident_layers < 1 or not isinstance(family_count, int) or isinstance(family_count, bool) or family_count < 1 or parent_alignment.get("alignedLogicalExpertPageObjectives") != parent_objectives or parent_alignment.get("alignedPhysicalNoNELayerCount") != inherited_layers or parent_alignment.get( "residentTraversalLayerCount", parent_alignment.get("alignedTargetScienceLayers"), ) != parent_resident_layers ): raise RuntimeError("NoNE additive capacity parent geometry differs") parent_bank = _verified_compact_capacity_alignment( Path(parent_summary_value) ) retained_bank_fields = ( "bankSummarySha256", "bankJournalSha256", "bankSessionId", "bankPageCount", "bankPageParameterElements", "bankPhysicalParameterElements", "bankObjectBytes", ) if any( parent_alignment.get(field) != parent_bank.get(field) for field in retained_bank_fields ): raise RuntimeError("NoNE additive capacity parent bank bytes differ") parent_bank_end = parent_bank.get("bankEndPageId") additive_bank_start = bank.get("bankStartPageId") if ( parent_bank.get("bankSessionId") != bank.get("bankSessionId") or parent_bank.get("bankSourceCheckpointSha256") != bank.get("bankSourceCheckpointSha256") or not isinstance(parent_bank_end, int) or isinstance(parent_bank_end, bool) or not isinstance(additive_bank_start, int) or isinstance(additive_bank_start, bool) or additive_bank_start <= parent_bank_end ): raise RuntimeError("NoNE additive compact bank is not disjoint") plan = _align_none_growth_plan_to_compact_capacity(parent, bank) if accepted_page_catalog is not None: plan = _rebase_additive_growth_plan_to_accepted_catalog_boundary( plan, parent_plan=parent, accepted_page_catalog=accepted_page_catalog, ) alignment = dict(plan["capacityAlignment"]) alignment.update( { "parentCapacityPlanPath": str(parent_path), "parentCapacityPlanSha256": file_sha256(parent_path), "parentBankSummarySha256": parent_bank["bankSummarySha256"], "parentBankEndPageId": parent_bank_end, } ) plan["capacityAlignment"] = alignment physical = dict(plan["physicalGraphPlan"]) total_layers = physical.get("physicalGraphLayerCount") cumulative_objectives = plan.get("initialLogicalExpertPageObjectives") target_resident_layers = physical.get("denseResidentScienceLayerCount") total_capacity_routing_slots = alignment.get( "totalCapacityRoutingLayerSlots" ) inherited_capacity_routing_slots = alignment.get( "inheritedCapacityRoutingLayerSlots" ) additional_capacity_routing_slots = alignment.get( "additionalCapacityRoutingLayerSlots" ) if ( not isinstance(total_layers, int) or isinstance(total_layers, bool) or not isinstance(cumulative_objectives, int) or isinstance(cumulative_objectives, bool) or not isinstance(target_resident_layers, int) or isinstance(target_resident_layers, bool) or not isinstance(total_capacity_routing_slots, int) or isinstance(total_capacity_routing_slots, bool) or not isinstance(inherited_capacity_routing_slots, int) or isinstance(inherited_capacity_routing_slots, bool) or not isinstance(additional_capacity_routing_slots, int) or isinstance(additional_capacity_routing_slots, bool) ): raise RuntimeError("NoNE additive capacity result geometry differs") plan["successorCapacityParent"] = { "path": str(parent_path), "sha256": file_sha256(parent_path), } checks = dict(plan["checks"]) checks.update( { "additiveCapacityParentExact": True, "additiveBankDisjointAuthorityBound": True, "cumulativeSparseGraphIsInheritedPlusAdditive": ( total_layers == inherited_layers + additive_pages ), "cumulativeObjectiveCountIsParentPlusAdditive": ( cumulative_objectives == parent_objectives + additive_pages ), "residentTraversalDepthDerivedFromCumulativeBreadth": ( target_resident_layers == parent_resident_layers + additional_capacity_routing_slots and total_capacity_routing_slots == inherited_capacity_routing_slots + additional_capacity_routing_slots ), "successorClaimsNoTrainingOrPromotion": ( alignment.get("modelTrainingClaimed") is False and alignment.get("promotionEligibilityClaimed") is False ), } ) plan["checks"] = checks return plan def build_additive_capacity_aligned_none_growth_plan_from_existing( parent_capacity_plan: Path, compact_summary_path: Path | None, output_plan: Path, receipt_path: Path, *, accepted_page_catalog: Path | None = None, successor_capability_catalog: Path | None = None, family_root_bank_plan: Path | None = None, ) -> dict[str, Any]: """Write one disjoint-bank successor capacity/root plan and receipt.""" parent = parent_capacity_plan.expanduser().resolve() summary = ( compact_summary_path.expanduser().resolve() if compact_summary_path is not None else None ) output = output_plan.expanduser().resolve() receipt = receipt_path.expanduser().resolve() accepted = ( accepted_page_catalog.expanduser().resolve() if accepted_page_catalog is not None else None ) capability_catalog = ( successor_capability_catalog.expanduser().resolve() if successor_capability_catalog is not None else None ) root_bank = ( family_root_bank_plan.expanduser().resolve() if family_root_bank_plan is not None else None ) if (capability_catalog is None) != (root_bank is None): raise ValueError( "successor capability catalog and family-root bank plan " "must be provided together" ) if summary is None and capability_catalog is None: raise ValueError( "compact summary is required unless an exact-prefix " "successor capability catalog is provided" ) immutable_inputs = {parent} if summary is not None: immutable_inputs.add(summary) if accepted is not None: immutable_inputs.add(accepted) if capability_catalog is not None: immutable_inputs.add(capability_catalog) if ( output in immutable_inputs or receipt in immutable_inputs or output == receipt or root_bank in immutable_inputs or root_bank == output or root_bank == receipt ): raise ValueError( "additive capacity outputs must be separate from immutable inputs" ) if summary is not None: plan = derive_additive_capacity_aligned_none_growth_plan_from_existing( parent, summary, accepted_page_catalog=accepted, ) else: plan = json.loads(parent.read_text(encoding="utf-8")) root_expansion: dict[str, Any] | None = None if capability_catalog is not None and root_bank is not None: successor = json.loads(capability_catalog.read_text(encoding="utf-8")) if ( successor.get("schema") != "nnf.resynthesis.none_capability_root_successor.v1" ): raise ValueError("unsupported successor capability-root catalog") if successor.get("parentCapacityPlanSha256") != file_sha256(parent): raise ValueError( "successor capability-root catalog does not bind the exact parent" ) parent_roadmap = plan.get("functionalExpertFamilyRoadmap") successor_roadmap = successor.get("functionalExpertFamilyRoadmap") if not isinstance(parent_roadmap, list) or not isinstance( successor_roadmap, list, ): raise ValueError("functional expert family roadmaps must be lists") if len(successor_roadmap) <= len(parent_roadmap): raise ValueError( "successor capability-root catalog must append at least one root" ) if successor_roadmap[: len(parent_roadmap)] != parent_roadmap: raise ValueError( "successor capability-root catalog changed the accepted prefix" ) suffix = successor_roadmap[len(parent_roadmap) :] known_families = { row.get("family") for row in parent_roadmap if isinstance(row, dict) } new_families: set[str] = set() new_capabilities: set[str] = set() for offset, row in enumerate(suffix): if not isinstance(row, dict): raise ValueError("successor family-root rows must be objects") ordinal = len(parent_roadmap) + offset family = row.get("family") capability_ids = row.get("capabilityIds") source_evidence = row.get("sourceEvidence") if row.get("catalogOrdinal") != ordinal: raise ValueError( "successor family-root ordinals must be contiguous" ) if ( not isinstance(family, str) or not family or family in known_families or family in new_families ): raise ValueError( "successor family-root names must be non-empty and disjoint" ) if ( not isinstance(capability_ids, list) or not capability_ids or any( not isinstance(capability_id, str) or not capability_id for capability_id in capability_ids ) ): raise ValueError( "every successor family root must declare capabilities" ) if any( capability_id in new_capabilities for capability_id in capability_ids ): raise ValueError( "successor capability identifiers must be disjoint" ) if ( not isinstance(source_evidence, list) or not source_evidence or any( not isinstance(evidence, dict) or not isinstance(evidence.get("authorityPath"), str) or not evidence["authorityPath"] or not isinstance(evidence.get("authoritySha256"), str) or len(evidence["authoritySha256"]) != 64 or not isinstance(evidence.get("objectiveId"), str) or not evidence["objectiveId"] or not isinstance(evidence.get("evidenceCount"), int) or isinstance(evidence["evidenceCount"], bool) or evidence["evidenceCount"] <= 0 for evidence in source_evidence ) ): raise ValueError( "every successor family root must bind non-empty " "source authority evidence" ) if ( row.get("registeredCapabilityClaimed") is not False or row.get("trainedCapabilityClaimed") is not False or row.get("promotionEligibilityClaimed") is not False ): raise ValueError( "new roots must withhold registration, training, and " "promotion claims" ) new_families.add(family) new_capabilities.update(capability_ids) alignment = plan["capacityAlignment"] physical = plan["physicalGraphPlan"] prior_root_expansion = plan.get("familyRootExpansion") prior_planned_page_id = int(alignment["bankEndPageId"]) parent_planned_physical_count = int( physical["physicalGraphLayerCount"] ) if isinstance(prior_root_expansion, dict): prior_planned_page_id = max( prior_planned_page_id, int( prior_root_expansion.get( "lastPlannedPageId", prior_planned_page_id, ) ), ) parent_planned_physical_count = max( parent_planned_physical_count, int( prior_root_expansion.get( "plannedPhysicalGraphLayerCount", parent_planned_physical_count, ) ), ) first_page_id = prior_planned_page_id + 1 root_pages = [ { "catalogOrdinal": row["catalogOrdinal"], "family": row["family"], "capabilityIds": list(row["capabilityIds"]), "sourceEvidence": list(row["sourceEvidence"]), "pageId": first_page_id + offset, "physicalMaterializationClaimed": False, "registeredCapabilityClaimed": False, "trainedCapabilityClaimed": False, "promotionEligibilityClaimed": False, } for offset, row in enumerate(suffix) ] physical_count = int(physical["physicalGraphLayerCount"]) canonical_json = json.dumps( parent_roadmap, sort_keys=True, separators=(",", ":"), ).encode("utf-8") parent_roadmap_sha256 = __import__("hashlib").sha256( canonical_json ).hexdigest() root_bank_payload = { "schema": "nnf.resynthesis.none_family_root_bank_plan.v1", "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "status": "PLANNED_UNMATERIALIZED_UNTRAINED", "passed": True, "parentCapacityPlan": { "path": str(parent), "sha256": file_sha256(parent), "functionalExpertFamilyCount": len(parent_roadmap), "functionalExpertFamilyRoadmapSha256": ( parent_roadmap_sha256 ), "physicalGraphLayerCount": physical_count, "bankEndPageId": int(alignment["bankEndPageId"]), "lastPlannedFamilyRootPageId": prior_planned_page_id, "plannedPhysicalGraphLayerCount": ( parent_planned_physical_count ), }, "successorCapabilityCatalog": { "path": str(capability_catalog), "sha256": file_sha256(capability_catalog), }, "familyRootCountBefore": len(parent_roadmap), "familyRootCountAfter": len(successor_roadmap), "addedFamilyRootCount": len(suffix), "addedCapabilityCount": len(new_capabilities), "plannedPageCount": len(root_pages), "materializedPageCount": 0, "trainedPageCount": 0, "firstPlannedPageId": first_page_id, "lastPlannedPageId": first_page_id + len(root_pages) - 1, "materializedPhysicalGraphLayerCount": physical_count, "parentPlannedPhysicalGraphLayerCount": ( parent_planned_physical_count ), "plannedPhysicalGraphLayerCount": ( parent_planned_physical_count + len(root_pages) ), "pageToPhysicalGraphLayerCardinality": "one_to_one", "rootPages": root_pages, "acceptedPointerWritePlanned": False, "physicalAdmissionClaimed": False, "modelTrainingClaimed": False, "promotionEligibilityClaimed": False, "checks": { "parentCapacityPlanExact": True, "functionalRootPrefixExact": True, "familyNamesDisjoint": True, "capabilityIdsDisjoint": True, "sourceEvidenceBound": True, "pageIdsDisjointAndContiguous": True, "newRootsExplicitlyUnmaterialized": True, "newRootsExplicitlyUntrained": True, "acceptedPointerWriteAbsent": True, "promotionNotClaimed": True, }, } _atomic_json(root_bank, root_bank_payload) plan["builtAt"] = time.strftime( "%Y-%m-%dT%H:%M:%SZ", time.gmtime(), ) plan["functionalExpertFamilyRoadmap"] = successor_roadmap plan["functionalExpertFamilyCount"] = len(successor_roadmap) plan["proposedMinimumAdditionalExpertSlots"] = len( successor_roadmap ) plan["proposedMinimumAdditionalPhysicalNoNELayers"] = int( plan["proposedMinimumAdditionalPhysicalNoNELayers"] ) + len(root_pages) target_geometry = plan["proposedMinimumTargetGeometry"] target_geometry["functionalExpertFamilies"] = len(successor_roadmap) target_geometry["plannedInheritedGraphLayerCount"] = int( target_geometry["plannedInheritedGraphLayerCount"] ) + len(root_pages) target_geometry["plannedTotalGraphLayerCount"] = int( target_geometry["plannedTotalGraphLayerCount"] ) + len(root_pages) target_geometry["physicalGraphLayerCount"] = ( parent_planned_physical_count + len(root_pages) ) target_geometry["physicalGraphNodeCount"] = ( parent_planned_physical_count + len(root_pages) ) target_geometry["physicalNoNELayers"] = ( parent_planned_physical_count + len(root_pages) ) target_geometry["physicalPageExperts"] = ( parent_planned_physical_count + len(root_pages) ) target_geometry["sparsePhysicalNoNELayers"] = ( parent_planned_physical_count + len(root_pages) ) target_geometry["totalPhysicalNoNELayers"] = ( parent_planned_physical_count + len(root_pages) ) logical = plan["logicalGraphPlan"] logical["plannedInheritedGraphLayerCount"] = int( logical["plannedInheritedGraphLayerCount"] ) + len(root_pages) logical["plannedTotalGraphLayerCount"] = int( logical["plannedTotalGraphLayerCount"] ) + len(root_pages) plan["successorCapabilityCatalog"] = { "path": str(capability_catalog), "sha256": file_sha256(capability_catalog), "schema": successor["schema"], "functionalExpertFamilyCountBefore": len(parent_roadmap), "functionalExpertFamilyCountAfter": len(successor_roadmap), "addedCapabilityCount": len(new_capabilities), } root_expansion = { "schema": "nnf.resynthesis.none_family_root_expansion.v1", "familyRootBankPlanPath": str(root_bank), "familyRootBankPlanSha256": file_sha256(root_bank), "addedFamilyRootCount": len(root_pages), "addedCapabilityCount": len(new_capabilities), "firstPlannedPageId": first_page_id, "lastPlannedPageId": first_page_id + len(root_pages) - 1, "materializedPhysicalGraphLayerCount": physical_count, "parentPlannedPhysicalGraphLayerCount": ( parent_planned_physical_count ), "plannedPhysicalGraphLayerCount": ( parent_planned_physical_count + len(root_pages) ), "physicalAdmissionClaimed": False, "modelTrainingClaimed": False, "promotionEligibilityClaimed": False, } plan["familyRootExpansion"] = root_expansion plan["checks"]["successorCapabilityCatalogPrefixExact"] = True plan["checks"]["successorFamilyRootsDisjoint"] = True plan["checks"]["successorCapabilitiesDisjoint"] = True plan["checks"]["successorFamilyRootSourceEvidenceBound"] = True plan["checks"]["successorFamilyRootPagesUnmaterialized"] = True plan["checks"]["successorFamilyRootPagesUntrained"] = True plan["checks"]["successorFamilyRootPromotionNotClaimed"] = True _atomic_json(output, plan) alignment = plan["capacityAlignment"] physical = plan["physicalGraphPlan"] if root_expansion is None: if summary is None: raise RuntimeError("additive capacity summary authority disappeared") checks = { "parentPlanExact": alignment.get("parentCapacityPlanSha256") == file_sha256(parent), "additiveBankExact": alignment.get("bankSummarySha256") == file_sha256(summary), "cumulativeObjectivesGrow": plan[ "initialLogicalExpertPageObjectives" ] > json.loads(parent.read_text(encoding="utf-8"))[ "initialLogicalExpertPageObjectives" ], "physicalGraphIsInheritedPlusAdditive": physical[ "physicalGraphLayerCount" ] == physical["inheritedPhysicalGraphLayerCount"] + physical["additiveBankPageCount"], "allPlanChecksPass": all(plan["checks"].values()), "trainingNotClaimed": ( alignment.get("modelTrainingClaimed") is False ), "promotionNotClaimed": ( alignment.get("promotionEligibilityClaimed") is False ), } else: if capability_catalog is None or root_bank is None: raise RuntimeError("successor family-root authority disappeared") parent_payload = json.loads(parent.read_text(encoding="utf-8")) checks = { "parentPlanExact": ( root_expansion["materializedPhysicalGraphLayerCount"] == parent_payload["physicalGraphPlan"][ "physicalGraphLayerCount" ] ), "successorCapabilityCatalogExact": ( plan["successorCapabilityCatalog"]["sha256"] == file_sha256(capability_catalog) ), "familyRootBankPlanExact": ( root_expansion["familyRootBankPlanSha256"] == file_sha256(root_bank) ), "functionalRootPrefixExact": ( plan["functionalExpertFamilyRoadmap"][ : parent_payload["functionalExpertFamilyCount"] ] == parent_payload["functionalExpertFamilyRoadmap"] ), "functionalRootsGrow": ( plan["functionalExpertFamilyCount"] > parent_payload["functionalExpertFamilyCount"] ), "plannedPhysicalGraphGrows": ( root_expansion["plannedPhysicalGraphLayerCount"] > root_expansion["parentPlannedPhysicalGraphLayerCount"] ), "materializedPhysicalGraphUnchanged": ( physical["physicalGraphLayerCount"] == parent_payload["physicalGraphPlan"][ "physicalGraphLayerCount" ] ), "physicalAdmissionNotClaimed": ( root_expansion["physicalAdmissionClaimed"] is False ), "trainingNotClaimed": ( root_expansion["modelTrainingClaimed"] is False ), "promotionNotClaimed": ( root_expansion["promotionEligibilityClaimed"] is False ), "allPlanChecksPass": all(plan["checks"].values()), } payload = { "schema": NONE_GROWTH_PLAN_CAPACITY_ALIGNMENT_RECEIPT_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(checks.values()), "parentPlan": {"path": str(parent), "sha256": file_sha256(parent)}, "acceptedPageCatalog": ( { "path": str(accepted), "sha256": alignment["acceptedPageCatalogSha256"], } if accepted is not None else None ), "additiveBank": dict(alignment) if summary is not None else None, "familyRootBankPlan": ( { "path": str(root_bank), "sha256": root_expansion["familyRootBankPlanSha256"], "addedFamilyRootCount": root_expansion[ "addedFamilyRootCount" ], "addedCapabilityCount": root_expansion[ "addedCapabilityCount" ], "materializedPhysicalGraphLayerCount": root_expansion[ "materializedPhysicalGraphLayerCount" ], "parentPlannedPhysicalGraphLayerCount": root_expansion[ "parentPlannedPhysicalGraphLayerCount" ], "plannedPhysicalGraphLayerCount": root_expansion[ "plannedPhysicalGraphLayerCount" ], } if root_expansion is not None else None ), "outputPlan": { "path": str(output), "sha256": file_sha256(output), "logicalExpertPageObjectives": plan[ "initialLogicalExpertPageObjectives" ], "additiveBankPageCount": physical["additiveBankPageCount"], "physicalGraphLayerCount": physical["physicalGraphLayerCount"], "physicalGraphParameterElements": physical[ "physicalGraphParameterElements" ], "denseResidentScienceLayerCount": physical[ "denseResidentScienceLayerCount" ], "functionalExpertFamilyCount": plan[ "functionalExpertFamilyCount" ], "plannedPhysicalGraphLayerCount": ( root_expansion["plannedPhysicalGraphLayerCount"] if root_expansion is not None else physical["physicalGraphLayerCount"] ), }, "checks": checks, "physicalAdmissionClaimed": False, "trainingClaimed": False, "promotionEligible": False, } _atomic_json(receipt, payload) return payload def build_capacity_aligned_none_growth_plan_from_existing( source_plan: Path, compact_summary_path: Path, output_plan: Path, receipt_path: Path, ) -> dict[str, Any]: """Write a separate bank-aligned plan without training or promotion.""" source = source_plan.expanduser().resolve() summary = compact_summary_path.expanduser().resolve() output = output_plan.expanduser().resolve() receipt = receipt_path.expanduser().resolve() if output in {source, summary}: raise ValueError("capacity-aligned NoNE plan must be a separate artifact") source_sha256 = file_sha256(source) plan = derive_capacity_aligned_none_growth_plan_from_existing( source, summary, ) _atomic_json(output, plan) alignment = plan.get("capacityAlignment") physical_graph_plan = plan.get("physicalGraphPlan") checks = { "sourcePlanPreserved": file_sha256(source) == source_sha256, "outputUsesPagedSchema": plan.get("schema") == NONE_GROWTH_PLAN_SCHEMA, "compactBankVerified": isinstance(alignment, dict) and alignment.get("bankSummarySha256") == file_sha256(summary), "objectiveTargetMatchesBank": isinstance(alignment, dict) and plan.get("initialLogicalExpertPageObjectives") == alignment.get("alignedLogicalExpertPageObjectives") and alignment.get("totalCompactPageCount") == alignment.get("alignedLogicalExpertPageObjectives"), "baselineObjectivesPreserved": isinstance(alignment, dict) and int(alignment.get("baselineLogicalExpertPageObjectives", -1)) <= int(alignment.get("alignedLogicalExpertPageObjectives", -2)), "modelTrainingNotClaimed": isinstance(alignment, dict) and alignment.get("modelTrainingClaimed") is False, "promotionNotClaimed": isinstance(alignment, dict) and alignment.get("promotionEligibilityClaimed") is False, "layerTargetMatchesCapacity": ( isinstance(alignment, dict) and isinstance( plan.get("proposedMinimumTargetGeometry"), dict, ) and plan["proposedMinimumTargetGeometry"].get("physicalNoNELayers") == alignment.get("alignedPhysicalNoNELayerCount") and plan["proposedMinimumTargetGeometry"].get( "residentScienceLayers" ) == alignment.get("residentTraversalLayerCount") and plan["proposedMinimumTargetGeometry"].get("scienceLayers") == alignment.get("residentTraversalLayerCount") ), "physicalGraphTargetMatchesInheritedPlusAdditiveBank": ( isinstance(alignment, dict) and isinstance(physical_graph_plan, dict) and physical_graph_plan.get("additiveBankPageCount") == alignment.get("bankPageCount") and physical_graph_plan.get("inheritedPhysicalGraphLayerCount") == alignment.get("inheritedPhysicalNoNELayerCount") and physical_graph_plan.get("physicalGraphLayerCount") == alignment.get("alignedPhysicalNoNELayerCount") and physical_graph_plan.get("physicalGraphNodeCount") == alignment.get("alignedPhysicalNoNELayerCount") and physical_graph_plan.get("physicalGraphLayerCount") == int(alignment.get("inheritedPhysicalNoNELayerCount", -1)) + int(alignment.get("bankPageCount", -1)) ), "sparsePhysicalGraphIsSeparateFromDenseTraversal": ( isinstance(alignment, dict) and isinstance(physical_graph_plan, dict) and physical_graph_plan.get("pageBackedLayersReportedAsDenseLayers") is False and physical_graph_plan.get("denseResidentScienceLayerCount") == alignment.get("residentTraversalLayerCount") and physical_graph_plan.get("physicalGraphLayerCountMaximum") is None and physical_graph_plan.get("successorCompactBanksRemainAdditive") is True ), "layerGrowthIsDataDerivedAndUncapped": ( isinstance(alignment, dict) and alignment.get("layerAllocationPolicy") == ( "one_content_addressed_page_objective_per_" "logical_layer_v1" ) and alignment.get("residentRoutingDepthPolicy") == ( "additive_ceil_log2_page_objectives_per_" "functional_family_v1" ) and isinstance( alignment.get("additionalCapacityRoutingLayerSlots"), int, ) and alignment.get("additionalCapacityRoutingLayerSlots", -1) >= 0 and alignment.get("alignedPhysicalNoNELayerCount") == int(alignment.get("inheritedPhysicalNoNELayerCount", -1)) + int(alignment.get("bankPageCount", -1)) and alignment.get("physicalGraphLayerCountMaximum") is None and alignment.get("pageParameterWidthMaximum") is None and alignment.get("hiddenDimensionMaximum") is None and alignment.get("expertRankMaximum") is None and alignment.get("transferDimensionMaximum") is None and alignment.get("successorCompactBanksRemainAdditive") is True and plan.get("checks", {}).get( "capacityLayerGeometryHasNoFixedMaximum" ) is True ), } receipt_payload = { "schema": NONE_GROWTH_PLAN_CAPACITY_ALIGNMENT_RECEIPT_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(checks.values()), "sourcePlan": { "path": str(source), "sha256": source_sha256, }, "compactBank": dict(alignment) if isinstance(alignment, dict) else None, "outputPlan": { "path": str(output), "sha256": file_sha256(output), "logicalExpertPageObjectives": plan.get( "initialLogicalExpertPageObjectives" ), "additiveBankPageCount": ( physical_graph_plan.get("additiveBankPageCount") if isinstance(physical_graph_plan, dict) else None ), "physicalGraphLayerCount": ( physical_graph_plan.get("physicalGraphLayerCount") if isinstance(physical_graph_plan, dict) else None ), "physicalGraphNodeCount": ( physical_graph_plan.get("physicalGraphNodeCount") if isinstance(physical_graph_plan, dict) else None ), "denseResidentScienceLayerCount": ( physical_graph_plan.get("denseResidentScienceLayerCount") if isinstance(physical_graph_plan, dict) else None ), }, "checks": checks, "trainingClaimed": False, "promotionEligible": False, "remainingProof": [ "model_owned_scale_cohort", "immutable_page_admission", "distinct_page_gradients", "heldout_retention_and_gain", "route_and_transfer_evidence", "cold_reload", "accepted_generation_commit", ], } _atomic_json(receipt, receipt_payload) return receipt_payload def build_paged_none_growth_plan_from_existing( source_plan: Path, output_plan: Path, receipt_path: Path, ) -> dict[str, Any]: """Upgrade an immutable v1/v2 plan into a separate paged NoNE plan. The source plan is never overwritten. This is a corpus-planning boundary, not model training or promotion evidence. """ source = source_plan.resolve() output = output_plan.resolve() receipt = receipt_path.resolve() if source == output: raise ValueError("paged NoNE growth plan output must differ from source") plan = derive_paged_none_growth_plan_from_existing(source) _atomic_json(output, plan) checks = { "sourcePlanPreserved": file_sha256(source) == plan["lineage"]["sourcePlanSha256"], "outputUsesPagedSchema": plan.get("schema") == NONE_GROWTH_PLAN_SCHEMA, "logicalExpertPageObjectivesPresent": int( plan.get("initialLogicalExpertPageObjectives", 0) ) >= int(plan.get("sourceCount", 0)), "fixedResidentExpertCapAbsent": plan.get("pagingPolicy", {}).get( "hostFixedResidentExpertLimit" ) is None, "allStructuralMaximumsAbsent": all( plan.get("capacityExpansionPolicy", {}).get(field) is None for field in ( "staticPhysicalPageCeiling", "staticParameterCeiling", "staticPageParameterWidthCeiling", "staticHiddenDimensionCeiling", "staticExpertRankCeiling", "staticTransferDimensionCeiling", ) ), "modelTrainingNotClaimed": plan["lineage"][ "modelTrainingClaimed" ] is False, } receipt_payload = { "schema": NONE_GROWTH_PLAN_UPGRADE_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(checks.values()), "sourcePlan": { "path": str(source), "sha256": plan["lineage"]["sourcePlanSha256"], "schema": plan["lineage"]["sourcePlanSchema"], }, "outputPlan": { "path": str(output), "sha256": file_sha256(output), "schema": plan.get("schema"), "sourceCount": plan.get("sourceCount"), "initialLogicalExpertPageObjectives": plan.get( "initialLogicalExpertPageObjectives" ), }, "checks": checks, "promotionEligible": False, "remainingProof": [ "paged_runtime_integration", "page_training", "expert_utilization", "distinct_gradient", "causal_transfer_or_ablation_gain", "heldout_generalization", "source_retention", "cold_reload", "immutable_release_verification", ], } _atomic_json(receipt, receipt_payload) return receipt_payload def build_corpus_training_artifacts( corpus_root: Path, output_root: Path, *, tokenizer: Any, inventory_paths: list[Path] | None = None, active_train_rows: int | None = None, active_validation_rows: int | None = None, active_heldout_rows: int | None = None, payload_text_probe_bytes: int = 4096, ) -> dict[str, Any]: """Build source-sealed training, validation, heldout, and evidence JSONL. ``active_*`` values select the next training slice from the complete candidate pool for this artifact. They do not delete candidates; the full pool is written to ``candidate_pool.jsonl`` so later curriculum passes can continue over the same source-sealed denominator. """ corpus_root = corpus_root.resolve() output_root = output_root.resolve() from resynthesis.tokenizer_backend import tokenizer_boundary_receipt tokenizer_authority = tokenizer_boundary_receipt(tokenizer) if inventory_paths is None: inventory_paths = discover_corpus_inventory_paths(corpus_root) if not inventory_paths: raise FileNotFoundError("no corpus inventory JSONL files were found") records_by_source: dict[str, dict[str, Any]] = {} inventory_receipts: list[dict[str, Any]] = [] for inventory_path in inventory_paths: inventory_path = inventory_path.resolve() inventory_receipts.append( { "path": str(inventory_path), "bytes": inventory_path.stat().st_size, "sha256": file_sha256(inventory_path), } ) for record in _read_jsonl(inventory_path): source_id = str(record.get("source_id", "")).strip() if not source_id or not _admitted_for_training(record): continue prior = records_by_source.get(source_id) if prior is not None and _json_bytes(prior) != _json_bytes(record): raise ValueError(f"conflicting source record for {source_id}") records_by_source[source_id] = record if not records_by_source: raise ValueError("no training-admissible downloaded corpus sources found") candidates: list[CorpusTrainingCandidate] = [] evidence_rows: list[dict[str, Any]] = [] manifest_summaries: list[dict[str, Any]] = [] none_growth_entries: list[dict[str, Any]] = [] for source_id in sorted(records_by_source): record = records_by_source[source_id] source_sha256 = _sha256_bytes(_json_bytes(record)) manifest, manifest_sha256 = _manifest_for_record(corpus_root, record) payload_file_count = 0 payload_bytes = 0 payload_inventory: dict[str, Any] | None = None if manifest is not None: payload_inventory = _manifest_payload_inventory( manifest, record.get("payload_manifest"), ) payload_file_count = int(payload_inventory["payload_file_count"]) payload_bytes = int(payload_inventory["payload_bytes"]) evidence_text = _source_summary( record, payload_file_count=payload_file_count, payload_bytes=payload_bytes, ) none_growth_entries.append( _none_growth_entry( record, payload_file_count=payload_file_count, payload_bytes=payload_bytes, ) ) evidence_document_id = f"corpus-source:{source_id}" evidence_rows.append( { "document_id": evidence_document_id, "source_id": source_id, "source_sha256": source_sha256, "text": evidence_text, "text_sha256": _sha256_bytes(evidence_text.encode("utf-8")), "rights_disposition": "training_admissible", "license": str(record.get("license_name") or "NO_LICENSE_DECLARED"), "attribution": str(record.get("publisher") or source_id), } ) candidates.extend( _source_level_candidates( record, source_sha256=source_sha256, evidence_document_id=evidence_document_id, evidence_text=evidence_text, ) ) candidates.extend( _none_growth_candidates( record, source_sha256=source_sha256, evidence_document_id=evidence_document_id, evidence_text=evidence_text, payload_file_count=payload_file_count, payload_bytes=payload_bytes, ) ) payload_candidates, payload_evidence = _payload_candidates( corpus_root, record, manifest, source_sha256=source_sha256, payload_text_probe_bytes=payload_text_probe_bytes, ) candidates.extend(payload_candidates) evidence_rows.extend(payload_evidence) if manifest is not None: if payload_inventory is None: raise RuntimeError("payload inventory unexpectedly absent") manifest_summaries.append( { "source_id": source_id, "manifest_sha256": manifest_sha256, **payload_inventory, } ) train_candidates = _select_surface( candidates, "train", target_rows=active_train_rows, ) validation_candidates = _select_surface( candidates, "validation", target_rows=active_validation_rows, ) heldout_candidates = _select_surface( candidates, "heldout", target_rows=active_heldout_rows, ) correction_candidates = _select_surface( candidates, "correction_stress", target_rows=active_heldout_rows, ) train_rows = _rows_from_candidates(train_candidates, tokenizer=tokenizer, train=True) validation_rows = _rows_from_candidates( validation_candidates, tokenizer=tokenizer, train=False, ) heldout_rows = _rows_from_candidates( heldout_candidates, tokenizer=tokenizer, train=False, ) correction_rows = _rows_from_candidates( correction_candidates, tokenizer=tokenizer, train=False, ) from resynthesis.learn_loop import ( assert_generalization_separation, assert_heldout_separation, ) for left, right in ( (train_rows, validation_rows), (train_rows, heldout_rows), (train_rows, correction_rows), (validation_rows, heldout_rows), (validation_rows, correction_rows), (heldout_rows, correction_rows), ): assert_heldout_separation(left, right) assert_generalization_separation(left, right) candidate_pool_rows = [ { "schema": CORPUS_TRAINING_CANDIDATE_SCHEMA, "surface": candidate.surface, "family": candidate.family, "answer": candidate.answer, "domain": candidate.domain, "evidence_document_id": candidate.evidence_document_id, "evidence_text_sha256": _sha256_bytes(candidate.evidence_text.encode("utf-8")), "prompt_sha256": _sha256_bytes(candidate.prompt.encode("utf-8")), "question_id": candidate.question_id, "source_id": candidate.source_id, "source_record_id": candidate.source_record_id, "source_sha256": candidate.source_sha256, } for candidate in sorted(candidates, key=lambda item: item.question_id) ] paths = { "candidate_pool": output_root / "candidate_pool.jsonl", "train": output_root / "train.jsonl", "validation": output_root / "validation.jsonl", "heldout": output_root / "heldout.jsonl", "correction_stress": output_root / "correction_stress.jsonl", "evidence": output_root / "evidence.jsonl", } none_growth_plan_path = output_root / "none_growth_plan.json" _atomic_jsonl(paths["candidate_pool"], candidate_pool_rows) _atomic_jsonl(paths["train"], train_rows) _atomic_jsonl(paths["validation"], validation_rows) _atomic_jsonl(paths["heldout"], heldout_rows) _atomic_jsonl(paths["correction_stress"], correction_rows) _atomic_jsonl(paths["evidence"], evidence_rows) none_growth_plan = _none_growth_plan(none_growth_entries) _atomic_json(none_growth_plan_path, none_growth_plan) jsonl_artifacts = {name: _artifact_receipt(path) for name, path in paths.items()} artifacts = dict(jsonl_artifacts) artifacts["none_growth_plan"] = _json_artifact_receipt(none_growth_plan_path) train_families = {str(row.get("corpus_surface_family", "")) for row in train_rows} candidate_families = {candidate.family for candidate in candidates} none_objective_rows = [ row for row in train_rows if row.get("none_capability_objective") in NONE_OBJECTIVE_FAMILIES ] checks = { "allArtifactsNonEmpty": all( artifact["rows"] > 0 for artifact in jsonl_artifacts.values() ) and artifacts["none_growth_plan"]["bytes"] > 0, "allGeneralizationGroupsDisjoint": True, "candidatePoolPreserved": artifacts["candidate_pool"]["rows"] == len(candidate_pool_rows), "correctedPromptRuntimePrefixPreserved": all( row["corrected_prompt_ids"][: len(row["prompt_ids"])] == row["prompt_ids"] for row in train_rows ), "correctedInputRuntimeCompositionExact": all( row["corrected_input_ids"] == row["corrected_prompt_ids"] + row["answer_ids"] for row in train_rows ), "correctedTargetsRemainLossBoundaryOnly": all( row["corrected_target_ids"] == [-100] * len(row["corrected_prompt_ids"]) + row["answer_ids"] for row in train_rows ), "downloadedOpenTrainingSourcesOnly": True, "evidenceRowsHashed": all( _sha256_bytes(str(row["text"]).encode("utf-8")) == row["text_sha256"] for row in evidence_rows ), "heldoutEvidenceLocatorsExcluded": all( "training_evidence_document_id" not in row and "evidence_document_id" not in row for row in heldout_rows + validation_rows + correction_rows ), "manifestFilesRepresented": all( summary.get("fileInventoryPresent") is True and summary.get("malformedFileRows") == 0 for summary in manifest_summaries ), "manifestTopLevelCountsComplete": all( summary.get("declaredCountsPresent") is True and summary.get("declaredCountsMatch") is True for summary in manifest_summaries ), "manifestReferenceCountsMatch": all( summary.get("referenceCountsPresent") is True and summary.get("referenceCountsMatch") is True for summary in manifest_summaries ), "modelScoresNotObserved": True, "noneExpansionObjectiveFamiliesPresent": NONE_OBJECTIVE_FAMILIES.issubset( candidate_families ), "noneExpansionObjectivesScheduled": NONE_OBJECTIVE_FAMILIES.issubset( train_families ), "noneGrowthPlanWritten": artifacts["none_growth_plan"]["sha256"] == file_sha256(none_growth_plan_path), "noneLiveGeometryExpansionNotClaimed": all( row.get("geometry_growth_claimed_live") is False for row in none_objective_rows ), "noneRboTransferRotationRecursionRowsPresent": { "none_knowledge_transfer_requirement", "none_recursive_traversal_requirement", "none_rbo_rotation_requirement", }.issubset(train_families), "selectionIndependentOfGeneratedAnswers": True, "sourceRecordsHashed": True, "targetsExcludedFromForward": True, } receipt = { "schema": CORPUS_TRAINING_BUILD_SCHEMA, "passed": all(checks.values()), "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "corpusRoot": str(corpus_root), "tokenizerBackend": tokenizer_authority, "artifacts": artifacts, "checks": checks, "inventoryInputs": inventory_receipts, "sourceCount": len(records_by_source), "manifestSourceCount": len(manifest_summaries), "manifestPayloadFileCount": sum( int(summary["payload_file_count"]) for summary in manifest_summaries ), "manifestPayloadBytes": sum( int(summary["payload_bytes"]) for summary in manifest_summaries ), "manifestPayloadInventory": { "malformedFileRows": sum( int(summary["malformedFileRows"]) for summary in manifest_summaries ), "localSha256Count": sum( int(summary["localSha256Count"]) for summary in manifest_summaries ), "missingLocalSha256Count": sum( int(summary["missingLocalSha256Count"]) for summary in manifest_summaries ), "missingTopLevelCountSources": [ str(summary["source_id"]) for summary in manifest_summaries if summary.get("declaredCountsPresent") is not True ], "topLevelCountMismatchSources": [ str(summary["source_id"]) for summary in manifest_summaries if summary.get("declaredCountsPresent") is True and summary.get("declaredCountsMatch") is not True ], "referenceCountMismatchSources": [ str(summary["source_id"]) for summary in manifest_summaries if summary.get("referenceCountsMatch") is not True ], }, "candidateSurfaceCounts": { surface: sum(1 for candidate in candidates if candidate.surface == surface) for surface in ("train", "validation", "heldout", "correction_stress") }, "activeSurfaceRows": { "train": len(train_rows), "validation": len(validation_rows), "heldout": len(heldout_rows), "correction_stress": len(correction_rows), }, "correctedArmTraining": { "compositionOwner": ( "compose_runtime_evidence_prompt_ids" ), "runtimePromptPrefixPreserved": True, "evidenceSuffixTokenizedSeparately": True, "targetEnteredForward": False, }, "expertAndMhcIntendedEngagement": { "modelPath": "existing Resynthesis RBO + NoNE science_stack", "mhcSurface": "ResynthesisScienceLayerConfig.mhc_heads and glyph anchor experts participate during forward", "expertGrowthMode": "adds source-sealed NoNE objective rows for specialist pressure, transfer, recursive traversal, model-owned rotation, and MHC stability", "geometryGrowthMode": "writes none_growth_plan.json for checkpointed expert/layer migration; current shard does not claim live geometry expansion", "noneObjectiveFamilies": sorted(NONE_OBJECTIVE_FAMILIES), "noneGrowthPlanSha256": artifacts["none_growth_plan"]["sha256"], "noneGrowthPlan": { "sourceCount": none_growth_plan["sourceCount"], "proposedMinimumAdditionalExpertSlots": none_growth_plan[ "proposedMinimumAdditionalExpertSlots" ], "proposedMinimumAdditionalLayerSlots": none_growth_plan[ "proposedMinimumAdditionalLayerSlots" ], "proposedMinimumTargetGeometry": none_growth_plan[ "proposedMinimumTargetGeometry" ], "initialLogicalExpertPageObjectives": none_growth_plan[ "initialLogicalExpertPageObjectives" ], "physicalGraphPlan": none_growth_plan["physicalGraphPlan"], "functionalExpertFamilyRoadmap": none_growth_plan[ "functionalExpertFamilyRoadmap" ], "pagingPolicy": none_growth_plan["pagingPolicy"], "parameterScalePolicy": none_growth_plan[ "parameterScalePolicy" ], }, }, "scopeNote": ( "This build prepares model-update data from every admitted source record " "and every payload-manifest file entry. Text probes are bootstrap " "evidence for this pass, not a claim that every raw byte or planned " "expert page has already been optimized into weights." ), } _atomic_json(output_root / "build_receipt.json", receipt) return receipt def merge_corpus_training_artifacts( base_root: Path, additive_root: Path, output_root: Path, ) -> dict[str, Any]: """Compose two passed corpus builds without replacing either surface. This is an external I/O boundary. Rows are parsed and emitted canonically, component artifact hashes are verified before use, and train/evaluation identities are rechecked across the combined denominator. The resulting receipt records component-declared payload accounting; it does not claim that those raw bytes have already produced a retained gradient update. """ base_root = base_root.resolve() additive_root = additive_root.resolve() output_root = output_root.resolve() if base_root == additive_root or output_root in {base_root, additive_root}: raise ValueError("corpus merge roots must be distinct") if output_root.exists(): raise FileExistsError(f"refusing to overwrite corpus merge: {output_root}") required_artifacts = ( "candidate_pool", "train", "validation", "heldout", "correction_stress", "evidence", ) def validated_component(root: Path) -> tuple[Path, dict[str, Any]]: receipt_path = root / "build_receipt.json" if not receipt_path.is_file(): raise FileNotFoundError(f"corpus build receipt is absent: {root}") loaded = json.loads(receipt_path.read_text(encoding="utf-8")) if ( not isinstance(loaded, dict) or loaded.get("schema") != CORPUS_TRAINING_BUILD_SCHEMA or loaded.get("passed") is not True ): raise ValueError(f"corpus build receipt did not pass: {root}") component_checks = loaded.get("checks") if ( not isinstance(component_checks, dict) or not component_checks or any(value is not True for value in component_checks.values()) ): raise ValueError(f"corpus build checks are incomplete: {root}") component_artifacts = loaded.get("artifacts") if not isinstance(component_artifacts, dict): raise ValueError(f"corpus build has no artifact map: {root}") for name in required_artifacts: artifact = component_artifacts.get(name) if not isinstance(artifact, dict): raise ValueError(f"corpus build has no {name} artifact: {root}") path = Path(str(artifact.get("path", ""))).resolve() expected_rows = artifact.get("rows") if ( not path.is_file() or not isinstance(expected_rows, int) or isinstance(expected_rows, bool) or expected_rows < 1 or artifact.get("sha256") != file_sha256(path) ): raise ValueError(f"corpus {name} artifact identity differs: {root}") with path.open(encoding="utf-8") as handle: actual_rows = sum(1 for line in handle if line.strip()) if actual_rows != expected_rows: raise ValueError(f"corpus {name} row count differs: {root}") return receipt_path, loaded base_receipt_path, base_receipt = validated_component(base_root) additive_receipt_path, additive_receipt = validated_component(additive_root) component_receipts = { "base": base_receipt, "additive": additive_receipt, } tokenizer_authorities: dict[str, dict[str, Any]] = {} for component, component_receipt in component_receipts.items(): tokenizer_authority = component_receipt.get("tokenizerBackend") if ( not isinstance(tokenizer_authority, dict) or tokenizer_authority.get("schema") != "nnf.resynthesis.fastokens_boundary.v1" or tokenizer_authority.get("fallbackAllowed") is not False or tokenizer_authority.get("bpeIdentityPreserved") is not True or tokenizer_authority.get("vgeRemainsDownstreamAuthority") is not True ): raise ValueError( f"{component} corpus build has no valid tokenizer authority" ) tokenizer_authorities[component] = tokenizer_authority if _json_bytes(tokenizer_authorities["base"]) != _json_bytes( tokenizer_authorities["additive"] ): raise ValueError("component corpus tokenizer authorities differ") tokenizer_authority = dict(tokenizer_authorities["base"]) component_receipt_paths = { "base": base_receipt_path, "additive": additive_receipt_path, } component_artifacts = { name: receipt["artifacts"] for name, receipt in component_receipts.items() } merged_artifact_names = list(required_artifacts) if any( "record_catalog" in artifacts for artifacts in component_artifacts.values() ): merged_artifact_names.append("record_catalog") row_fingerprints: dict[str, dict[str, set[str]]] = { component: {} for component in component_receipts } row_counts: dict[str, dict[str, int]] = { component: {} for component in component_receipts } surface_identities: dict[str, dict[str, set[str]]] = { surface: { field: set() for field in ( "question_id", "record_id", "source_record_id", "prompt_sha256", "generalization_group", ) } for surface in ("train", "validation", "heldout", "correction_stress") } source_namespaces: dict[str, set[str]] = { component: set() for component in component_receipts } targets_excluded_from_forward = True heldout_evidence_locators_excluded = True corrected_prompt_runtime_prefix_preserved = True corrected_input_runtime_composition_exact = True corrected_targets_remain_loss_boundary_only = True for component, artifacts in component_artifacts.items(): corrected_arm_contract = component_receipts[component].get( "correctedArmTraining" ) requires_runtime_prefix = bool( isinstance(corrected_arm_contract, dict) and corrected_arm_contract.get("runtimePromptPrefixPreserved") is True ) for name in merged_artifact_names: artifact = artifacts.get(name) if not isinstance(artifact, dict): row_fingerprints[component][name] = set() row_counts[component][name] = 0 continue path = Path(str(artifact["path"])).resolve() fingerprints: set[str] = set() count = 0 for row in _iter_jsonl(path): count += 1 fingerprints.add(_sha256_bytes(_json_bytes(row))) if name in surface_identities: for field, values in surface_identities[name].items(): value = row.get(field) if value not in (None, ""): values.add(str(value)) targets_excluded_from_forward = bool( targets_excluded_from_forward and row.get("target_entered_forward") is not True and row.get("task_intent_targets_entered_forward") is not True ) if name in {"validation", "heldout", "correction_stress"}: heldout_evidence_locators_excluded = bool( heldout_evidence_locators_excluded and "training_evidence_document_id" not in row and "evidence_document_id" not in row ) if name == "train": prompt_ids = row.get("prompt_ids") answer_ids = row.get("answer_ids") corrected_prompt_ids = row.get("corrected_prompt_ids") corrected_input_ids = row.get("corrected_input_ids") corrected_target_ids = row.get("corrected_target_ids") if ( isinstance(prompt_ids, list) and isinstance(answer_ids, list) and isinstance(corrected_prompt_ids, list) and isinstance(corrected_input_ids, list) and isinstance(corrected_target_ids, list) ): if requires_runtime_prefix: corrected_prompt_runtime_prefix_preserved = bool( corrected_prompt_runtime_prefix_preserved and corrected_prompt_ids[: len(prompt_ids)] == prompt_ids ) corrected_input_runtime_composition_exact = bool( corrected_input_runtime_composition_exact and corrected_input_ids == corrected_prompt_ids + answer_ids ) corrected_targets_remain_loss_boundary_only = bool( corrected_targets_remain_loss_boundary_only and corrected_target_ids == [-100] * len(corrected_prompt_ids) + answer_ids ) else: if requires_runtime_prefix: corrected_prompt_runtime_prefix_preserved = False corrected_input_runtime_composition_exact = False corrected_targets_remain_loss_boundary_only = False if name == "evidence": source_id = row.get("source_id") if source_id not in (None, ""): source_namespaces[component].add(str(source_id)) row_fingerprints[component][name] = fingerprints row_counts[component][name] = count no_cross_component_duplicate_rows = all( not row_fingerprints["base"][name].intersection( row_fingerprints["additive"][name] ) for name in merged_artifact_names ) source_namespaces_disjoint = not source_namespaces["base"].intersection( source_namespaces["additive"] ) generalization_groups_disjoint = True surface_names = tuple(surface_identities) for left_index, left in enumerate(surface_names): for right in surface_names[left_index + 1 :]: for field in surface_identities[left]: if surface_identities[left][field].intersection( surface_identities[right][field] ): generalization_groups_disjoint = False output_root.mkdir(parents=True) def component_rows(name: str) -> Iterator[dict[str, Any]]: for component in ("base", "additive"): artifact = component_artifacts[component].get(name) if isinstance(artifact, dict): yield from _iter_jsonl(Path(str(artifact["path"])).resolve()) output_paths: dict[str, Path] = {} for name in merged_artifact_names: path = output_root / f"{name}.jsonl" _atomic_jsonl(path, component_rows(name)) output_paths[name] = path base_growth_artifact = component_artifacts["base"].get("none_growth_plan") if isinstance(base_growth_artifact, dict): base_growth_path = Path(str(base_growth_artifact.get("path", ""))).resolve() if ( not base_growth_path.is_file() or base_growth_artifact.get("sha256") != file_sha256(base_growth_path) ): raise ValueError("base NoNE growth-plan identity differs") growth_plan = json.loads(base_growth_path.read_text(encoding="utf-8")) if not isinstance(growth_plan, dict): raise ValueError("base NoNE growth plan is malformed") merged_growth_path = output_root / "none_growth_plan.json" _atomic_json(merged_growth_path, growth_plan) else: merged_growth_path = None artifacts = { name: _artifact_receipt(path) for name, path in output_paths.items() } if merged_growth_path is not None: artifacts["none_growth_plan"] = _json_artifact_receipt( merged_growth_path ) def summed_receipt_count(field: str, name: str) -> int: total = 0 for receipt in component_receipts.values(): counts = receipt.get(field) if not isinstance(counts, dict): raise ValueError(f"component receipt has no {field}") value = counts.get(name) if not isinstance(value, int) or isinstance(value, bool): raise ValueError(f"component receipt {field}.{name} is invalid") total += value return total active_surface_rows = { name: summed_receipt_count("activeSurfaceRows", name) for name in ("train", "validation", "heldout", "correction_stress") } candidate_surface_counts = { name: summed_receipt_count("candidateSurfaceCounts", name) for name in ("train", "validation", "heldout", "correction_stress") } exact_artifact_rows = all( int(artifacts[name]["rows"]) == row_counts["base"][name] + row_counts["additive"][name] for name in merged_artifact_names ) adaptive_selection = additive_receipt.get("sourceSelection") adaptive_selection_valid = bool( isinstance(adaptive_selection, dict) and adaptive_selection.get("selectionOwnedByObservedContent") is True and adaptive_selection.get("hardcodedReleaseYear") is False ) base_active = base_receipt["activeSurfaceRows"] additive_active = additive_receipt["activeSurfaceRows"] checks = { "componentReceiptsPassedAndHashesVerified": True, "componentTokenizerAuthoritiesMatch": True, "allArtifactsNonEmpty": all( int(artifact.get("rows", artifact.get("bytes", 0))) > 0 for artifact in artifacts.values() ), "allArtifactRowsEqualComponentSums": exact_artifact_rows, "baseActiveRowsPreserved": all( active_surface_rows[name] >= int(base_active[name]) for name in active_surface_rows ), "additiveRowsAppendedExactly": all( active_surface_rows[name] - int(base_active[name]) == int(additive_active[name]) for name in active_surface_rows ), "noCrossComponentDuplicateRows": no_cross_component_duplicate_rows, "allGeneralizationGroupsDisjoint": generalization_groups_disjoint, "componentSourceNamespacesDisjoint": source_namespaces_disjoint, "targetsExcludedFromForward": targets_excluded_from_forward, "heldoutEvidenceLocatorsExcluded": heldout_evidence_locators_excluded, "declaredCorrectedPromptRuntimePrefixesPreserved": ( corrected_prompt_runtime_prefix_preserved ), "correctedInputRuntimeCompositionExact": ( corrected_input_runtime_composition_exact ), "correctedTargetsRemainLossBoundaryOnly": ( corrected_targets_remain_loss_boundary_only ), "adaptiveSourceSelectionUsesObservedContent": adaptive_selection_valid, "hardcodedReleaseYearAbsent": adaptive_selection_valid, "modelScoresNotObserved": True, "rawPayloadOptimizationNotClaimed": True, } manifest_payload_file_count = sum( int(receipt.get("manifestPayloadFileCount", 0)) for receipt in component_receipts.values() ) manifest_payload_bytes = sum( int(receipt.get("manifestPayloadBytes", 0)) for receipt in component_receipts.values() ) base_engagement = base_receipt.get("expertAndMhcIntendedEngagement") engagement = ( dict(base_engagement) if isinstance(base_engagement, dict) else {} ) engagement["adaptiveSourceJoin"] = additive_receipt.get( "expertAndMhcIntendedEngagement" ) engagement["adaptiveSourceJoinMatrix"] = additive_receipt.get( "adaptiveSourceJoinMatrix" ) receipt = { "schema": CORPUS_TRAINING_BUILD_SCHEMA, "passed": all(checks.values()), "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "corpusRoot": str(output_root), "tokenizerBackend": tokenizer_authority, "artifacts": artifacts, "checks": checks, "componentBuildReceipts": { component: { "path": str(component_receipt_paths[component]), "sha256": file_sha256(component_receipt_paths[component]), "schema": component_receipts[component].get("schema"), } for component in component_receipts }, "sourceCount": len( source_namespaces["base"].union(source_namespaces["additive"]) ), "manifestSourceCount": sum( int(receipt_value.get("manifestSourceCount", 0)) for receipt_value in component_receipts.values() ), "manifestPayloadFileCount": manifest_payload_file_count, "manifestPayloadBytes": manifest_payload_bytes, "manifestPayloadAccounting": { "mode": "component_declared_source_namespace_sum", "componentSourceNamespacesDisjoint": source_namespaces_disjoint, "rawBytesOptimizedIntoWeights": False, "components": { component: { "sourceCount": len(source_namespaces[component]), "manifestPayloadFileCount": component_receipts[component].get( "manifestPayloadFileCount" ), "manifestPayloadBytes": component_receipts[component].get( "manifestPayloadBytes" ), } for component in component_receipts }, }, "evidenceRowCount": int(artifacts["evidence"]["rows"]), "candidateSurfaceCounts": candidate_surface_counts, "activeSurfaceRows": active_surface_rows, "sourceSelection": adaptive_selection, "adaptiveSourceJoinMatrix": additive_receipt.get( "adaptiveSourceJoinMatrix" ), "adaptiveRecordDiversity": additive_receipt.get("recordDiversity"), "correctedArmTraining": additive_receipt.get( "correctedArmTraining", base_receipt.get("correctedArmTraining"), ), "expertAndMhcIntendedEngagement": engagement, "scopeNote": ( "This immutable composition preserves every base surface and appends " "the observed-content adaptive source joins. Component-declared raw " "payload bytes remain provenance, not a claim of retained learning; " "promotion still requires gradients, heldout gain, retention, route " "participation, and cold reload." ), } _atomic_json(output_root / "build_receipt.json", receipt) return receipt