"""Authoritative v1-to-v2 and accepted v2-to-v2+ NoNE page migration. This module is an external checkpoint/storage boundary. It never trains or promotes a model. It composes an accepted v1 additive checkpoint with immutable expert pages whose inherited FFN path is algebraically identical to v1, then registers additional family roots as transfer-initialized and explicitly untrained. The resident page residual starts at compatibility zero in ``NoNEPagedExpertRuntime`` so the composed v2 seed initially emits the exact v1 model surface. Catalog growth beyond v2 starts from an exact accepted checkpoint/optimizer/ sidecar transaction. It keeps the session and historical generation chain, preserves all existing page objects and resident/global optimizer prefixes, and stages recursively rotated transfer roots across every verified replica before the canonical accepted pointer advances. New roots have zero optimizer suffixes and no trained-capability claim until later route, gradient, retention, held-out, and cold-reload proofs pass. """ from __future__ import annotations import argparse import copy import hashlib import json import math import os import shutil import sys import time from collections.abc import Iterator from dataclasses import dataclass, replace from pathlib import Path from typing import TYPE_CHECKING, Any, Final, Mapping, Sequence import torch from safetensors import safe_open from safetensors.torch import save_file from resynthesis.geometry_migration import ( CHECKPOINT_SCHEMA, _parameter_order, _state_identity, ) from resynthesis.language_experts import ( NONE_LANGUAGE_EXPERT_CATALOG_SCHEMA, NONE_LANGUAGE_EXPERT_FAMILIES, NONE_LANGUAGE_EXPERT_INHERITED_CATALOG_SCHEMA, NONE_LANGUAGE_EXPERT_INHERITED_PREFIX_COUNT, NONE_LANGUAGE_EXPERT_INHERITED_PREFIX_IDS_SHA256, ) from resynthesis.none_paging import ( _FILE_SHA256_CACHE as _PAGING_FILE_SHA256_CACHE, ) from resynthesis.none_paging import ( PAGE_ACCEPTED_POINTER_SCHEMA, page_generation_schema_supported_boundary, SCALED_FLOAT8_TRANSFER_STORAGE, _concatenate_optimizer_matrices_boundary, _validated_training_branch_union_proof_artifact_boundary, NoNEGenerationBinding, NoNEGenerationComponentPacket, NoNEGenerationReplicaCoordinator, NoNEGraphAuthorityBinding, NoNEImmutablePageStore, NoNEPageBundle, NoNEPagedExpertRuntime, NoNEPageObjectBinding, NoNEPageRequestPacket, NoNEPageWeights, NoNEScaleCohortPacket, build_sparse_graph_layer_catalog_authority_boundary, build_graph_authority_binding_boundary, compact_transfer_page_bundle_boundary, digest_tensor, discover_page_store_locators_boundary, federated_growth_demand_authority_from_catalog_boundary, federated_growth_demand_authority_from_plan_boundary, file_sha256_authority_boundary, load_page_weight_from_handle_boundary, page_model_parameter_elements, page_object_identity_sha256_boundary, sparse_graph_layer_binding_record_boundary, sparse_graph_layer_catalog_authority_boundary, stage_atomically_moved_file_sha256_authority_boundary, validate_training_branch_union_proof_record_boundary, validate_page_bundle, ) from resynthesis.none_paging import ( file_sha256_boundary as paging_file_sha256_boundary, ) from resynthesis.scientific_experts import ( NONE_SCIENCE_SPECIALIST_CATALOG_SCHEMA, NONE_V2_PLUS_SCIENCE_SPECIALIST_DEFINITIONS, NONE_V2_PLUS_SCIENCE_SPECIALIST_IDS_SHA256, ) from resynthesis.staged_verification_emitter import ( FEDERATED_COMBINE, emit_stage, ) from resynthesis.training_throughput import ( TEMPLATE_REFERENCE_TRANSFER_STORAGE, CompactPageAdmissionPacket, hydrate_compact_page_admission_objects_boundary, ) if TYPE_CHECKING: from resynthesis.science_layers import ( ResynthesisScienceLayerConfig, ResynthesisScienceLayerStack, ) NONE_V2_SEED_MIGRATION_SCHEMA: Final[str] = ( "nnf.resynthesis.none_v1_to_v2_seed_migration.v1" ) NONE_V2_COMPOSITION_SCHEMA: Final[str] = ( "nnf.resynthesis.none_v2_composition.v1" ) NONE_V2_PAGE_CATALOG_SCHEMA: Final[str] = ( "nnf.resynthesis.none_v2_page_catalog.v1" ) NONE_V2_RESIDENT_RUNTIME_SCHEMA: Final[str] = ( "nnf.resynthesis.none_v2_resident_runtime.v1" ) NONE_V2_PLUS_CATALOG_EXPANSION_SCHEMA: Final[str] = ( "nnf.resynthesis.none_v2_plus_catalog_expansion.v1" ) NONE_V2_PLUS_EXPANSION_RECEIPT_SCHEMA: Final[str] = ( "nnf.resynthesis.none_v2_plus_expansion_receipt.v1" ) # Both names identify the same physical-admission / catalog-expansion receipt # family; newer writers emit expansion_receipt while older sealed artifacts keep # catalog_expansion. NONE_V2_PLUS_EXPANSION_RECEIPT_SCHEMAS: Final[frozenset[str]] = frozenset( { NONE_V2_PLUS_CATALOG_EXPANSION_SCHEMA, NONE_V2_PLUS_EXPANSION_RECEIPT_SCHEMA, } ) RETAINED_SCALE_ADMISSION_AUTHORITY_SCHEMA: Final[str] = ( "nnf.resynthesis.retained_scale_admission_authority.v1" ) NONE_V2_PLUS_COMPOSITION_SCHEMA: Final[str] = ( "nnf.resynthesis.none_v2_plus_composition.v1" ) COMPACT_PAGE_BANK_AUTHORITY_SCHEMA: Final[str] = ( "nnf.resynthesis.compact_page_bank_authority.v1" ) COMPACT_BANK_TRANSFER_INITIALIZATION: Final[str] = ( "receipt_owned_compact_bank_zero_residual_v1" ) NONE_V2_PLUS_PAGE_CATALOG_SCHEMA: Final[str] = ( "nnf.resynthesis.none_v2_plus_page_catalog.v1" ) NONE_OBJECTIVE_PAGE_SCHEMA: Final[str] = ( "nnf.resynthesis.none_objective_page.v1" ) NONE_OBJECTIVE_PAGE_PLAN_SCHEMA: Final[str] = ( "nnf.resynthesis.none_objective_page_plan.v1" ) NONE_COMPACT_OBJECTIVE_COHORT_BINDING_SCHEMA: Final[str] = ( "nnf.resynthesis.none_compact_objective_cohort_binding.v1" ) # One immutable generation-43 expansion predates the explicit compact-objective # cohort binding. Its exact accepted-history identities are the compatibility # boundary; later or merely similar receipts must use the current binding. _LEGACY_COMPACT_OBJECTIVE_RECEIPT_SHA256: Final[str] = ( "cc0e6b5ca11191d87a327aa4cc6e34a7762062e14d1693312b0dfdf4cc53b894" ) _LEGACY_COMPACT_OBJECTIVE_CATALOG_SHA256: Final[str] = ( "5068fc8d6cf5569110e6094647df75c9dfd26123af585ebcb75e25e2d17c6713" ) _LEGACY_PARENT_MODEL_TYPE: Final[str] = "qw" + "en3_5" _LEGACY_COMPACT_OBJECTIVE_MANIFEST_SHA256: Final[str] = ( "11f033c8400f69e574263998753c2c196005c0318ab3833e828f0bbd7c05b718" ) _LEGACY_COMPACT_OBJECTIVE_MANIFEST_PAYLOAD_SHA256: Final[str] = ( "22b66d6c824879a101f2628024cea3c3335f732181190098713d403cba7a075f" ) _LEGACY_COMPACT_OBJECTIVE_SESSION_KEY: Final[str] = ( "ee33edc74b3c6608be252fa5b4f261af9fdedb07a3a8ab45773eee5d83ed2c2e" ) _LEGACY_COMPACT_OBJECTIVE_PAGE_COUNT: Final[int] = 70_582 NONE_ACCEPTED_GRAPH_ADAPTATION_SCHEMA: Final[str] = ( "nnf.resynthesis.none_accepted_graph_adaptation.v1" ) NONE_ACTIVE_GRAPH_GROWTH_PLAN_BINDING_SCHEMA: Final[str] = ( "nnf.resynthesis.none_active_graph_growth_plan_binding.v1" ) EXTERNAL_CHECKPOINT_BINDING_SCHEMA: Final[str] = ( "nnf.resynthesis.additive_external_checkpoint_binding.v1" ) PAGED_GROWTH_PLAN_SCHEMA: Final[str] = ( "nnf.resynthesis.none_growth_plan.v2" ) _TRANSFER_PAGE_WEIGHT_FIELDS: Final[tuple[str, ...]] = ( "ffn_mode_t", "gate_t", "up_t", "down_t", "glyph_down_t", "glyph_up_t", "translation_gate_t", "outcome_memory_t", "repair_memory_t", "transfer_memory_t", ) TRANSFER_INITIALIZATION_V1: Final = ( "learned_route_outcome_recursive_convex_rotation_v1" ) TRANSFER_INITIALIZATION_V2: Final = ( "learned_route_outcome_recursive_convex_rotation_family_identity_v2" ) TRANSFER_INITIALIZATION_V3: Final = ( "learned_route_outcome_recursive_convex_rotation_coefficient_identity_v3" ) _FileIdentity = tuple[int, int, int, int, int] _V2PlusValidationKey = tuple[str, _FileIdentity, str, _FileIdentity] @dataclass(frozen=True) class _V2PlusValidationCacheEntry: validated: dict[str, Any] dependent_file_identities: tuple[tuple[str, _FileIdentity], ...] _V2_PLUS_VALIDATION_CACHE: dict[ _V2PlusValidationKey, _V2PlusValidationCacheEntry, ] = {} _FILE_SHA256_CACHE = _PAGING_FILE_SHA256_CACHE def _file_identity(path: Path) -> _FileIdentity: """Return an exact immutable-input identity at the storage boundary.""" stat = path.stat() return ( int(stat.st_dev), int(stat.st_ino), int(stat.st_size), int(stat.st_mtime_ns), int(stat.st_ctime_ns), ) def _v2_plus_validation_key( composition_path: Path, receipt_path: Path, ) -> _V2PlusValidationKey: return ( str(composition_path), _file_identity(composition_path), str(receipt_path), _file_identity(receipt_path), ) def _collect_bound_file_paths(value: object, paths: set[Path]) -> None: """Collect artifact files named by one immutable receipt payload.""" if isinstance(value, Mapping): for key, item in value.items(): if ( isinstance(key, str) and "path" in key.lower() and isinstance(item, str) and item ): candidate = Path(item).expanduser().resolve() if candidate.is_file(): paths.add(candidate) _collect_bound_file_paths(item, paths) elif isinstance(value, Sequence) and not isinstance(value, (str, bytes)): for item in value: _collect_bound_file_paths(item, paths) def _v2_plus_validation_dependencies( composition_path: Path, receipt_path: Path, composition: Mapping[str, Any], receipt: Mapping[str, Any], ) -> tuple[tuple[str, _FileIdentity], ...]: """Bind cache reuse to every immutable artifact and page object.""" paths = {composition_path, receipt_path} _collect_bound_file_paths(composition, paths) _collect_bound_file_paths(receipt, paths) store_record = composition.get("pageStore") pointer = receipt.get("acceptedPointer") if isinstance(store_record, Mapping) and isinstance(pointer, Mapping): store_root = Path(str(store_record.get("root", ""))).expanduser().resolve() session_key = str(pointer.get("sessionKey", "")) manifest_relative = str(pointer.get("manifest", "")) manifest_path = store_root / "sessions" / session_key / manifest_relative if manifest_path.is_file(): paths.add(manifest_path) manifest = json.loads(manifest_path.read_text(encoding="utf-8")) rows = manifest.get("pageObjects") if isinstance(manifest, dict) else None if isinstance(rows, list): for row in rows: object_sha256 = row.get("sha256") if isinstance(row, dict) else None if isinstance(object_sha256, str) and len(object_sha256) == 64: object_path = ( store_root / "objects" / "sha256" / f"{object_sha256}.safetensors" ) if object_path.is_file(): paths.add(object_path) return tuple( (str(path), _file_identity(path)) for path in sorted(paths) ) def _validated_science_specialist_family_ids( plan: Mapping[str, Any], roadmap_family_ids: tuple[str, ...], *, required: bool, allow_inherited_prefix: bool = False, ) -> tuple[str, ...]: """Validate the append-only specialist segment at a storage boundary.""" roadmap_value = plan.get("scienceSpecialistRoadmap") if roadmap_value is None and not required: return () if not isinstance(roadmap_value, list) or any( not isinstance(row, dict) or not isinstance(row.get("family"), str) or row.get("registeredCapabilityClaimed") is not False or row.get("trainedCapabilityClaimed") is not False for row in roadmap_value ): raise RuntimeError("NoNE science-specialist roadmap is malformed") family_ids = tuple(str(row["family"]) for row in roadmap_value) current_ids = tuple( definition.family_id for definition in NONE_V2_PLUS_SCIENCE_SPECIALIST_DEFINITIONS ) if allow_inherited_prefix: expected_ids = family_ids if ( not expected_ids or current_ids[: len(expected_ids)] != expected_ids ): raise RuntimeError( "NoNE inherited science-specialist prefix identity differs" ) expected_ids_sha256 = hashlib.sha256( "\n".join(expected_ids).encode("utf-8") ).hexdigest() else: expected_ids = current_ids expected_ids_sha256 = NONE_V2_PLUS_SCIENCE_SPECIALIST_IDS_SHA256 if ( plan.get("scienceSpecialistCatalogSchema") != NONE_SCIENCE_SPECIALIST_CATALOG_SCHEMA or plan.get("scienceSpecialistCount") != len(expected_ids) or plan.get("scienceSpecialistIdsSha256") != expected_ids_sha256 or family_ids != expected_ids or roadmap_family_ids[-len(expected_ids) :] != expected_ids ): raise RuntimeError("NoNE science-specialist catalog binding differs") return family_ids def _validate_retained_paged_checkpoint_lineage_boundary( paged_lineage: Mapping[str, Any], checkpoint_lineage: Mapping[str, Any], *, composition_sha256: str, family_root_count: int, training_page_count: int, objective_page_count: int, planned_objective_page_count: int, runtime_layer_ids: tuple[int, ...], physical_graph_layer_count: int | None = None, graph_layer_ids_sha256: str | None = None, ) -> None: """Validate current lineage or its exact immutable v2 predecessor.""" schema = paged_lineage.get("schema") science_layers = checkpoint_lineage.get("scienceLayers") if ( paged_lineage.get("compositionSha256") != composition_sha256 or int(paged_lineage.get("familyRootCount", -1)) != family_root_count or int( paged_lineage.get( "trainingPageCount", paged_lineage.get("trainingPageRootCount", -1), ) ) != training_page_count or int(paged_lineage.get("objectivePageCount", -1)) != objective_page_count or int(paged_lineage.get("plannedObjectivePageCount", -1)) != planned_objective_page_count or paged_lineage.get("scienceLayers") != science_layers ): raise RuntimeError("NoNE v2+ checkpoint catalog lineage differs") sparse_lineage_fields = ( "physicalGraphLayerCount", "graphLayerIdsSha256", "onePageObjectPerSparseGraphLayer", "sparseGraphLayersTrainingClaimed", "sparseGraphLayersPromotionEligible", ) if (physical_graph_layer_count is None) != ( graph_layer_ids_sha256 is None ): raise RuntimeError("NoNE sparse graph-layer expectation is incomplete") if physical_graph_layer_count is None: if any(field in paged_lineage for field in sparse_lineage_fields): raise RuntimeError( "NoNE legacy checkpoint has sparse graph-layer lineage" ) elif ( paged_lineage.get("physicalGraphLayerCount") != physical_graph_layer_count or paged_lineage.get("graphLayerIdsSha256") != graph_layer_ids_sha256 or paged_lineage.get("onePageObjectPerSparseGraphLayer") is not True or paged_lineage.get("sparseGraphLayersTrainingClaimed") is not False or paged_lineage.get("sparseGraphLayersPromotionEligible") is not False ): raise RuntimeError("NoNE checkpoint sparse graph-layer lineage differs") if schema == "nnf.resynthesis.paged_none_lineage.v3": if ( tuple(paged_lineage.get("pagedRuntimeLayerIds", ())) != runtime_layer_ids or paged_lineage.get("pagedRuntimeLayerCount") != len(runtime_layer_ids) or paged_lineage.get("totalScienceLayerCount") != science_layers ): raise RuntimeError("NoNE v2+ checkpoint catalog lineage differs") return if schema == "nnf.resynthesis.paged_none_lineage.v2": if ( paged_lineage.get("historicalGenerationCheckout") is not True or paged_lineage.get("candidateTransaction") != "stage_checkpoint_bind_accept" or paged_lineage.get("canonicalPointerAdvancesLast") is not True or paged_lineage.get("acceptedGenerationBoundByCheckpointSidecar") is not True or int(paged_lineage.get("trainingPageRootCount", -1)) != training_page_count or int(paged_lineage.get("pendingObjectivePageCount", -1)) != planned_objective_page_count - objective_page_count or paged_lineage.get("scienceLayers") != len(runtime_layer_ids) or any( field in paged_lineage for field in ( "pagedRuntimeLayerIds", "pagedRuntimeLayerCount", "totalScienceLayerCount", ) ) ): raise RuntimeError("NoNE v2+ checkpoint catalog lineage differs") return raise RuntimeError("NoNE v2+ checkpoint catalog lineage differs") def _validated_transfer_source_layer_ids_boundary( new_pages: Sequence[Mapping[str, Any]], parent_layer_catalog: Mapping[str, Any], ) -> tuple[int, ...]: """Read explicit source layers or derive the exact retained v1 binding.""" if all( isinstance(row.get("transferSourceLayerId"), int) and not isinstance(row.get("transferSourceLayerId"), bool) for row in new_pages ): return tuple(int(row["transferSourceLayerId"]) for row in new_pages) if any("transferSourceLayerId" in row for row in new_pages): raise RuntimeError("NoNE v2+ transfer source-layer identity differs") source_layer_ids: list[int] = [] for row in new_pages: raw_source_page_ids = row.get("transferSourcePageIds") if ( not isinstance(raw_source_page_ids, list) or not raw_source_page_ids or any( not isinstance(page_id, int) or isinstance(page_id, bool) for page_id in raw_source_page_ids ) ): raise RuntimeError( "NoNE v2+ transfer source-layer identity differs" ) source_page_ids = tuple(int(page_id) for page_id in raw_source_page_ids) matching_layer_ids = [ int(layer_id) for layer_id, page_ids in parent_layer_catalog.items() if str(layer_id).lstrip("-").isdigit() and isinstance(page_ids, list) and tuple(int(page_id) for page_id in page_ids) == source_page_ids ] if ( len(matching_layer_ids) != 1 or row.get("layerId") != matching_layer_ids[0] ): raise RuntimeError( "NoNE v2+ transfer source-layer identity differs" ) source_layer_ids.append(matching_layer_ids[0]) return tuple(source_layer_ids) def _validated_language_expert_family_ids( plan: Mapping[str, Any], *, required: bool, ) -> tuple[str, ...]: """Validate a current catalog or its immutable inherited prefix.""" roadmap_value = plan.get("languageExpertPackRoadmap") if roadmap_value is None and not required: return () if not isinstance(roadmap_value, list) or any( not isinstance(row, dict) or not isinstance(row.get("family"), str) or not str(row["family"]).startswith("language_") for row in roadmap_value ): raise RuntimeError("NoNE language expert-pack roadmap is malformed") family_ids = tuple(str(row["family"]) for row in roadmap_value) if len(set(family_ids)) != len(family_ids): raise RuntimeError("NoNE language expert-pack roadmap contains duplicates") current_ids = tuple( family_id for family_id, _description, _claims in NONE_LANGUAGE_EXPERT_FAMILIES ) catalog_schema = plan.get("languageExpertCatalogSchema") if catalog_schema == NONE_LANGUAGE_EXPERT_CATALOG_SCHEMA: expected_ids = current_ids elif catalog_schema == NONE_LANGUAGE_EXPERT_INHERITED_CATALOG_SCHEMA: expected_ids = current_ids[:NONE_LANGUAGE_EXPERT_INHERITED_PREFIX_COUNT] inherited_sha256 = hashlib.sha256( "\n".join(expected_ids).encode("utf-8") ).hexdigest() if inherited_sha256 != NONE_LANGUAGE_EXPERT_INHERITED_PREFIX_IDS_SHA256: raise RuntimeError("NoNE inherited language prefix identity differs") else: raise RuntimeError("NoNE language expert catalog schema differs") if ( plan.get("languageExpertPackCount") != len(expected_ids) or family_ids != expected_ids ): raise RuntimeError("NoNE v2+ language catalog binding differs") return family_ids def _missing_functional_family_indexes_boundary( source_family_ids: Sequence[str], target_family_ids: Sequence[str], ) -> tuple[int, ...]: """Return canonical missing graph ordinals without relabeling old roots.""" source_ids = tuple(source_family_ids) target_ids = tuple(target_family_ids) if ( len(set(source_ids)) != len(source_ids) or len(set(target_ids)) != len(target_ids) or len(target_ids) < len(source_ids) or not set(source_ids).issubset(target_ids) ): raise RuntimeError( "NoNE v2+ growth must preserve every accepted family identity" ) source_set = set(source_ids) return tuple( index for index, family_id in enumerate(target_ids) if family_id not in source_set ) @dataclass(frozen=True) class NoNEV2SeedArtifacts: """Durable artifacts produced by one non-promoted v2 seed migration.""" receipt_path: Path composition_path: Path page_catalog_path: Path store_root: Path session_id_t: torch.Tensor accepted_generation_t: torch.Tensor @dataclass(frozen=True) class NoNEV2PlusExpansionArtifacts: """One accepted, same-session v2-to-v2+ catalog expansion transaction.""" receipt_path: Path composition_path: Path page_catalog_path: Path resident_runtime_path: Path checkpoint_path: Path optimizer_path: Path external_state_path: Path store_root: Path session_id_t: torch.Tensor accepted_generation_t: torch.Tensor @dataclass(frozen=True) class NoNETrainingAuthority: """Exact model, optimizer, sidecar, and continuation authority for paging.""" checkpoint_path: Path checkpoint_sha256: str optimizer_path: Path optimizer_sha256: str external_state_path: Path | None external_state_sha256: str | None continuation_checkpoint_path: Path continuation_checkpoint_sha256: str continuation_optimizer_path: Path continuation_optimizer_sha256: str checkpoint_includes_paged_runtime: bool composition_schema: str source_generation: str target_generation: str accepted_generation: int accepted_manifest_payload_sha256: str composition_path: Path | None = None migration_receipt_path: Path | None = None replica_receipt_path: Path | None = None paged_graph_checkpoint_sha256: str | None = None training_state_root: Path | None = None graph_adaptation_receipt_path: Path | None = None graph_adaptation_receipt_sha256: str | None = None identity_cache_root: Path | None = None @dataclass(frozen=True) class NoNEAcceptedGraphAdaptationArtifacts: """One non-training graph adaptation of an accepted loop checkpoint.""" state_root: Path checkpoint_path: Path optimizer_path: Path external_state_path: Path loop_state_path: Path receipt_path: Path active_growth_plan_path: Path iteration: int accepted_generation: int @dataclass(frozen=True) class _RetainedGraphAdaptationAuthority: """One terminal retained transaction authorized for offline graph growth.""" precommit_path: Path precommit_sha256: str loop_state_path: Path loop_state_sha256: str execution_ledger_path: Path execution_ledger_sha256: str knowledge_receipt_path: Path knowledge_receipt_sha256: str iteration: int iteration_transaction_id: str retained_transaction_record: dict[str, Any] @dataclass(frozen=True) class _SourcePage: """One inherited page and its provenance before global catalog assembly.""" layer_id: int source_ffn_index: int source_expert_slot: int weights: tuple[torch.Tensor, ...] optimizer_mean_t: torch.Tensor optimizer_square_t: torch.Tensor step_t: torch.Tensor @dataclass(frozen=True) class _ObjectivePage: """One stable, data-derived expert-page objective.""" objective_id: str source_id: str ordinal: int source_page_count: int payload_shard_backed: bool functional_families: tuple[str, ...] capability_axes: tuple[str, ...] def identity_record(self) -> dict[str, Any]: """Return fields that cannot change when a source grows additively.""" return { "objectiveSchema": NONE_OBJECTIVE_PAGE_SCHEMA, "objectiveId": self.objective_id, "objectiveSourceId": self.source_id, "objectiveOrdinal": self.ordinal, "functionalFamilies": list(self.functional_families), "capabilityAxes": list(self.capability_axes), } def catalog_record(self) -> dict[str, Any]: return { **self.identity_record(), "objectiveSourcePageCount": self.source_page_count, "payloadShardBacked": self.payload_shard_backed, } @dataclass(frozen=True) class _LayerTransferEvidence: """Accepted model-owned evidence used only at the storage boundary.""" layer_id: int source_page_ids: tuple[int, ...] source_family_ids: tuple[str | None, ...] route_keys_t: torch.Tensor page_prior_t: torch.Tensor route_count_t: torch.Tensor gradient_update_count_t: torch.Tensor @dataclass(frozen=True) class _PreparedLayerTransferEvidence: """Identity-invariant layer evidence prepared once per migration.""" layer_id: int source_family_ids: tuple[str | None, ...] normalized_route_keys_t: torch.Tensor outcome_t: torch.Tensor @dataclass(frozen=True) class _CompactAdmissionBoundary: """Verified storage-boundary objects for one model-selected cohort.""" source_store: NoNEImmutablePageStore bindings: tuple[NoNEPageObjectBinding, ...] authority_record: dict[str, Any] def _file_sha256(path: Path) -> str: """Hash one stable immutable identity through the shared paging cache.""" return paging_file_sha256_boundary(path) def file_sha256_boundary(path: Path) -> str: """Share stable file hashing across explicit checkpoint boundaries.""" return _file_sha256(path) def _canonical_json_bytes(payload: Mapping[str, Any]) -> bytes: return json.dumps( payload, sort_keys=True, separators=(",", ":"), ).encode("utf-8") def _objective_page_id(source_id: str, ordinal: int) -> str: digest = hashlib.sha256( b"nnf-resynthesis-none-objective-page-v1\0" + source_id.encode("utf-8") + b"\0" + ordinal.to_bytes(8, "little", signed=False) ).hexdigest() return f"objective_{digest}" def _same_page_store_session_boundary( current: Mapping[str, Any], parent: Mapping[str, Any], ) -> bool: """Compare logical ownership while permitting pointer and root motion.""" current_pointer = current.get("acceptedPointer") parent_pointer = parent.get("acceptedPointer") return bool( isinstance(current_pointer, dict) and isinstance(parent_pointer, dict) and current.get("sessionId") == parent.get("sessionId") and current_pointer.get("sessionKey") == parent_pointer.get("sessionKey") ) def _string_id_rows( value: object, *, field: str, key: str, ) -> tuple[str, ...]: if not isinstance(value, list) or not value: raise RuntimeError(f"NoNE objective {field} is absent") identifiers: list[str] = [] for row in value: if not isinstance(row, dict): raise RuntimeError(f"NoNE objective {field} is malformed") identifier = row.get(key) if not isinstance(identifier, str) or not identifier.strip(): raise RuntimeError(f"NoNE objective {field} identity is malformed") identifiers.append(identifier.strip()) if len(set(identifiers)) != len(identifiers): raise RuntimeError(f"NoNE objective {field} repeats an identity") return tuple(identifiers) def _family_root_rows_from_catalog_boundary( pages: Sequence[Mapping[str, Any]], *, expected_count: object, ) -> tuple[Mapping[str, Any], ...]: """Recover every signed family root, including the markerless v1 cohort.""" family_rows = tuple( sorted( ( row for row in pages if isinstance(row.get("family"), str) and bool(str(row["family"]).strip()) ), key=lambda row: int(row["pageId"]), ) ) page_ids: list[int] = [] family_ids: list[str] = [] for row in family_rows: page_id = row.get("pageId") family_id = row.get("family") explicit_root = bool( row.get("pageRole") == "family_root" and row.get("familyRoot") is True ) legacy_root = bool( "pageRole" not in row and "familyRoot" not in row and row.get("state") == "TRANSFER_INITIALIZED_UNTRAINED" and row.get("trainedCapabilityClaimed") is False ) if ( not isinstance(page_id, int) or isinstance(page_id, bool) or page_id < 0 or not isinstance(family_id, str) or not family_id.strip() or not (explicit_root or legacy_root) ): raise RuntimeError("NoNE family-root catalog identity differs") page_ids.append(page_id) family_ids.append(family_id.strip()) if ( not isinstance(expected_count, int) or isinstance(expected_count, bool) or expected_count < 1 or len(family_rows) != expected_count or len(set(page_ids)) != expected_count or len(set(family_ids)) != expected_count ): raise RuntimeError("NoNE family-root catalog count differs") return family_rows def _validated_objective_page_plan( plan: Mapping[str, Any], functional_family_ids: tuple[str, ...], ) -> tuple[_ObjectivePage, ...]: """Expand source-owned counts into stable append-only page identities.""" federated_growth_demand_authority_from_plan_boundary(plan) source_rows = plan.get("sourceObjectives") if not isinstance(source_rows, list) or not source_rows: raise RuntimeError("NoNE growth plan has no source page objectives") known_families = set(functional_family_ids) source_ids: set[str] = set() objectives: list[_ObjectivePage] = [] for source_row in source_rows: if not isinstance(source_row, dict): raise RuntimeError("NoNE source page objective is malformed") source_id_value = source_row.get("source_id") if ( not isinstance(source_id_value, str) or not source_id_value.strip() or source_id_value in source_ids ): raise RuntimeError("NoNE source page identity is absent or repeated") source_id = source_id_value.strip() source_ids.add(source_id) planned_count = source_row.get("planned_expert_page_objectives") payload_count = source_row.get("payload_page_objectives") if ( not isinstance(planned_count, int) or isinstance(planned_count, bool) or planned_count < 1 or not isinstance(payload_count, int) or isinstance(payload_count, bool) or payload_count < 1 or payload_count > planned_count ): raise RuntimeError("NoNE source page-objective count is malformed") families = _string_id_rows( source_row.get("functional_expert_families"), field="functional families", key="family", ) axes = _string_id_rows( source_row.get("capability_axes"), field="capability axes", key="axis", ) if not set(families).issubset(known_families): raise RuntimeError( "NoNE source objective names an unknown functional family" ) objectives.extend( _ObjectivePage( objective_id=_objective_page_id(source_id, ordinal), source_id=source_id, ordinal=ordinal, source_page_count=planned_count, payload_shard_backed=ordinal < payload_count, functional_families=families, capability_axes=axes, ) for ordinal in range(planned_count) ) declared_count = plan.get("initialLogicalExpertPageObjectives") objective_ids = tuple(row.objective_id for row in objectives) if ( not isinstance(declared_count, int) or isinstance(declared_count, bool) or declared_count != len(objectives) or len(set(objective_ids)) != len(objective_ids) ): raise RuntimeError("NoNE logical page-objective total differs") return tuple(objectives) def _objective_plan_sha256( objectives: Sequence[_ObjectivePage], ) -> str: return hashlib.sha256( _canonical_json_bytes( { "schema": NONE_OBJECTIVE_PAGE_PLAN_SCHEMA, "objectives": [row.catalog_record() for row in objectives], } ) ).hexdigest() def _selected_scale_family_page_ids_boundary( packet: NoNEScaleCohortPacket, ) -> tuple[int, ...]: """Validate and externalize one model-owned scale-cohort decision. Repeated parent identities represent multiple child allocations owned by the same retained family. The parent itself still needs one exact route and distinct-gradient proof. """ selected_t = packet.selected_family_page_ids_t.detach().cpu().long().reshape(-1) selected_count_t = packet.selected_page_count_t.detach().cpu().long().reshape(()) eligible_count_t = packet.eligible_family_count_t.detach().cpu().long().reshape(()) storage_capacity_t = ( packet.storage_page_capacity_t.detach().cpu().long().reshape(()) ) envelope_capacity_t = ( packet.parameter_envelope_page_capacity_t.detach() .cpu() .long() .reshape(()) ) ready_t = packet.ready_t.detach().cpu().bool().reshape(()) selected_count = int(selected_count_t) selected_ids = tuple(int(value) for value in selected_t.tolist()) if ( not bool(ready_t) or selected_count < 1 or selected_count != len(selected_ids) or len(set(selected_ids)) > int(eligible_count_t) or selected_count > int(storage_capacity_t) or selected_count > int(envelope_capacity_t) or any(page_id < 0 for page_id in selected_ids) ): raise RuntimeError("NoNE scale cohort tensor authority is incoherent") return selected_ids def _digest_tensor_hex_boundary(tensor: torch.Tensor, *, field: str) -> str: values_t = tensor.detach().cpu().to(dtype=torch.uint8).reshape(-1) if values_t.numel() != 32: raise RuntimeError(f"{field} digest tensor is malformed") return bytes(values_t.tolist()).hex() def _validate_compact_admission_tensor_authority_boundary( *, packet: CompactPageAdmissionPacket, session_id_t: torch.Tensor, selected_family_page_ids: Sequence[int], accepted_page_ids: Sequence[int], page_parameter_elements: int, ) -> tuple[int, ...]: """Validate the tensor-owned cohort before any artifact is hydrated.""" selected_family_t = ( packet.selected_family_page_ids_t.detach().cpu().long().reshape(-1) ) selected_page_ids_t = packet.selected_page_ids_t.detach().cpu().long().reshape(-1) object_sha256s_t = ( packet.selected_object_sha256s_t.detach().cpu().to(dtype=torch.uint8) ) object_bytes_t = packet.selected_object_bytes_t.detach().cpu().long().reshape(-1) bank_session_t = packet.bank_session_id_t.detach().cpu().long().reshape(-1) selected_count = len(selected_family_page_ids) accepted_ids = set(accepted_page_ids) if ( packet.ready_t.numel() != 1 or not bool(packet.ready_t) or packet.bank_storage_format not in { SCALED_FLOAT8_TRANSFER_STORAGE, TEMPLATE_REFERENCE_TRANSFER_STORAGE, } or packet.selected_page_count_t.numel() != 1 or int(packet.selected_page_count_t) != selected_count or selected_family_t.tolist() != list(selected_family_page_ids) or selected_page_ids_t.numel() != selected_count or object_sha256s_t.shape != (selected_count, 32) or object_bytes_t.shape != (selected_count,) or torch.any(object_bytes_t <= 0) or torch.unique(selected_page_ids_t).numel() != selected_count or any(int(page_id) in accepted_ids for page_id in selected_page_ids_t) or not torch.equal(bank_session_t, session_id_t.detach().cpu().long()) or packet.page_parameter_elements_t.numel() != 1 or int(packet.page_parameter_elements_t) != page_parameter_elements or packet.bank_page_count_t.numel() != 1 or int(packet.bank_page_count_t) < selected_count or packet.bank_physical_parameter_elements_t.numel() != 1 or int(packet.bank_physical_parameter_elements_t) != int(packet.bank_page_count_t) * page_parameter_elements or packet.bank_total_object_bytes_t.numel() != 1 or int(packet.bank_total_object_bytes_t) < int(object_bytes_t.sum()) ): raise RuntimeError("NoNE compact-page admission tensor authority differs") selected_page_ids = tuple(int(value) for value in selected_page_ids_t.tolist()) if ( selected_page_ids != tuple(sorted(selected_page_ids)) or (accepted_page_ids and selected_page_ids[0] <= max(accepted_page_ids)) ): raise RuntimeError("NoNE compact-page admission IDs are not ordered") return selected_page_ids def _validated_compact_admission_boundary( *, packet: CompactPageAdmissionPacket, session_id_t: torch.Tensor, selected_family_page_ids: Sequence[int], accepted_page_ids: Sequence[int], source_catalog: Mapping[str, Any], page_parameter_elements: int, ) -> _CompactAdmissionBoundary: """Reverify one bank cohort and extend its immutable catalog authority.""" selected_page_ids_t = packet.selected_page_ids_t.detach().cpu().long().reshape(-1) object_sha256s_t = ( packet.selected_object_sha256s_t.detach().cpu().to(dtype=torch.uint8) ) object_bytes_t = packet.selected_object_bytes_t.detach().cpu().long().reshape(-1) bank_session_t = packet.bank_session_id_t.detach().cpu().long().reshape(-1) selected_count = len(selected_family_page_ids) selected_page_ids = _validate_compact_admission_tensor_authority_boundary( packet=packet, session_id_t=session_id_t, selected_family_page_ids=selected_family_page_ids, accepted_page_ids=accepted_page_ids, page_parameter_elements=page_parameter_elements, ) summary_path = packet.bank_summary_path.expanduser().resolve() journal_path = packet.bank_journal_path.expanduser().resolve() store_root = packet.bank_store_root.expanduser().resolve() summary_sha256 = _digest_tensor_hex_boundary( packet.bank_summary_sha256_t, field="compact bank summary", ) journal_sha256 = _digest_tensor_hex_boundary( packet.bank_journal_sha256_t, field="compact bank journal", ) source_checkpoint_sha256 = _digest_tensor_hex_boundary( packet.bank_source_checkpoint_sha256_t, field="compact bank source checkpoint", ) if ( not summary_path.is_file() or _file_sha256(summary_path) != summary_sha256 or not journal_path.is_file() or _file_sha256(journal_path) != journal_sha256 or not store_root.is_dir() ): raise RuntimeError("NoNE compact-page bank artifacts changed") # Admission binds the complete hash-sealed journal without reopening all # 70,582 payloads. A template-reference bank hydrates only the selected # model-owned cohort into ordinary immutable objects before the existing # page runtime performs its per-object digest/format check. bindings = tuple( NoNEPageObjectBinding( page_id_t=selected_page_ids_t[index].clone(), object_sha256_t=object_sha256s_t[index].clone(), object_bytes_t=object_bytes_t[index].clone(), ) for index in range(selected_count) ) prior_authorities_value = source_catalog.get("compactPageBanks", []) if not isinstance(prior_authorities_value, list) or any( not isinstance(value, dict) for value in prior_authorities_value ): raise RuntimeError("NoNE compact-page bank catalog authority differs") prior_authorities = [dict(value) for value in prior_authorities_value] matching = [ value for value in prior_authorities if value.get("summarySha256") == summary_sha256 ] if len(matching) > 1: raise RuntimeError("NoNE compact-page bank authority is ambiguous") expected_identity = { "schema": COMPACT_PAGE_BANK_AUTHORITY_SCHEMA, "summaryPath": str(summary_path), "summarySha256": summary_sha256, "journalPath": str(journal_path), "journalSha256": journal_sha256, "storeRoot": str(store_root), "sessionId": bank_session_t.tolist(), "sourceCheckpointSha256": source_checkpoint_sha256, "bankPageCount": int(packet.bank_page_count_t), "pageParameterElements": page_parameter_elements, "physicalParameterElementsInitialized": int( packet.bank_physical_parameter_elements_t ), "totalObjectBytes": int(packet.bank_total_object_bytes_t), "sourceBankTrainedPageCountAtDiscovery": 0, "sourceBankAcceptedGenerationCommitted": False, } if packet.bank_storage_format == TEMPLATE_REFERENCE_TRANSFER_STORAGE: template_path = packet.bank_template_object_path template_digest_t = packet.bank_template_object_sha256_t.detach().cpu() template_page_id_t = packet.bank_template_page_id_t.detach().cpu() template_offset_t = packet.bank_template_page_id_offset_t.detach().cpu() template_bytes_t = ( packet.bank_physical_archive_object_bytes_t.detach().cpu() ) if ( template_path is None or template_digest_t.shape != (32,) or template_page_id_t.numel() != 1 or template_offset_t.numel() != 1 or template_bytes_t.numel() != 1 or int(template_page_id_t.reshape(())) < 0 or int(template_offset_t.reshape(())) < 0 or int(template_bytes_t.reshape(())) < 1 ): raise RuntimeError("NoNE compact template admission identity differs") expected_identity["storageFormat"] = packet.bank_storage_format expected_identity["templateReference"] = { "path": str(template_path.expanduser().resolve()), "sha256": _digest_tensor_hex_boundary( template_digest_t, field="compact template object", ), "pageId": int(template_page_id_t.reshape(())), "pageIdOffset": int(template_offset_t.reshape(())), "physicalArchiveObjectBytes": int(template_bytes_t.reshape(())), } admitted_before: list[int] = [] if matching: current = matching[0] if any(current.get(key) != value for key, value in expected_identity.items()): raise RuntimeError("NoNE compact-page bank inherited identity differs") admitted_value = current.get("admittedPageIds") if ( not isinstance(admitted_value, list) or any( not isinstance(value, int) or isinstance(value, bool) for value in admitted_value ) or len(set(admitted_value)) != len(admitted_value) ): raise RuntimeError("NoNE compact-page bank admission history differs") admitted_before = list(admitted_value) if set(admitted_before).intersection(selected_page_ids): raise RuntimeError("NoNE compact page was already admitted") admitted_after = [*admitted_before, *selected_page_ids] authority_record = { **expected_identity, "admittedPageIds": admitted_after, "admittedPageCount": len(admitted_after), "latestCohortPageIds": list(selected_page_ids), "latestCohortFamilyPageIds": list(selected_family_page_ids), "admissionTrainingClaimed": False, "promotionRequiresDistinctGradientHeldoutAndColdReload": True, } source_store = hydrate_compact_page_admission_objects_boundary(packet) return _CompactAdmissionBoundary( source_store=source_store, bindings=bindings, authority_record=authority_record, ) def _validate_scale_family_training_evidence( *, selected_family_page_ids: Sequence[int], source_training_page_ids: Sequence[int], source_proof: Mapping[str, Any], ) -> None: """Validate retained route-gradient evidence for every selected parent.""" proof_ids = source_proof.get("trainingPageIds") if proof_ids is None: proof_ids = source_proof.get("familyPageIds") route_counts = source_proof.get("routeCounts") gradient_counts = source_proof.get("gradientUpdateCounts") signatures = source_proof.get("gradientSignatures") if ( not isinstance(proof_ids, list) or proof_ids != list(source_training_page_ids) or not isinstance(route_counts, list) or len(route_counts) != len(proof_ids) or not isinstance(gradient_counts, list) or len(gradient_counts) != len(proof_ids) or not isinstance(signatures, list) or len(signatures) != len(proof_ids) or source_proof.get("finite") is not True ): raise RuntimeError("NoNE accepted training-page proof is malformed") for page_id in dict.fromkeys(selected_family_page_ids): try: proof_index = proof_ids.index(page_id) except ValueError as error: raise RuntimeError( "NoNE scale cohort parent is absent from accepted proof" ) from error signature = signatures[proof_index] route_count = route_counts[proof_index] gradient_count = gradient_counts[proof_index] if ( not isinstance(route_count, int) or isinstance(route_count, bool) or route_count < 1 or not isinstance(gradient_count, int) or isinstance(gradient_count, bool) or gradient_count < 1 or not isinstance(signature, list) or not signature or any( not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(float(value)) for value in signature ) or not any(float(value) != 0.0 for value in signature) ): raise RuntimeError( "NoNE scale cohort parent lacks retained route-gradient evidence" ) def _objective_cohort_for_family_pages( *, selected_family_page_ids: Sequence[int], family_rows: Sequence[Mapping[str, Any]], missing_objectives: Sequence[_ObjectivePage], ) -> tuple[_ObjectivePage, ...]: """Match model-selected parent demand to distinct compatible objectives. A family may own multiple children, and an objective may name multiple compatible families. A family-local greedy cursor can therefore consume a shared objective that is the only choice for a later selected family. The deterministic augmenting-path match below preserves the model-owned parent multiset while finding a complete one-to-one objective assignment whenever one exists. """ family_by_page: dict[int, str] = {} for row in family_rows: page_id = row.get("pageId") family_id = row.get("family") if ( not isinstance(page_id, int) or isinstance(page_id, bool) or page_id < 0 or not isinstance(family_id, str) or not family_id.strip() or page_id in family_by_page ): raise RuntimeError("NoNE family-page identity is malformed") family_by_page[page_id] = family_id objective_ids = tuple(row.objective_id for row in missing_objectives) if len(set(objective_ids)) != len(objective_ids): raise RuntimeError("NoNE pending objective identities repeat") selected_family_ids: list[str] = [] for page_id in selected_family_page_ids: selected_family_id = family_by_page.get(page_id) if selected_family_id is None: raise RuntimeError("NoNE scale cohort selected a non-family page") selected_family_ids.append(selected_family_id) if len(selected_family_ids) > len(missing_objectives): raise RuntimeError( "NoNE selected family has no distinct pending objective page" ) selected_family_order = tuple(dict.fromkeys(selected_family_ids)) family_index_by_id = { family_id: index for index, family_id in enumerate(selected_family_order) } objective_indexes_by_family: dict[str, list[int]] = { family_id: [] for family_id in selected_family_order } for objective_index, objective in enumerate(missing_objectives): if ( not objective.objective_id or not objective.functional_families or len(set(objective.functional_families)) != len(objective.functional_families) ): raise RuntimeError("NoNE pending objective identity is malformed") for family_id in objective.functional_families: compatible_indexes = objective_indexes_by_family.get(family_id) if compatible_indexes is not None: compatible_indexes.append(objective_index) objective_owner_indexes = [-1] * len(missing_objectives) direct_free_cursors = [0] * len(selected_family_order) for selected_family_id in selected_family_ids: root_family_index = family_index_by_id[selected_family_id] compatible_indexes = objective_indexes_by_family[selected_family_id] direct_cursor = direct_free_cursors[root_family_index] while ( direct_cursor < len(compatible_indexes) and objective_owner_indexes[compatible_indexes[direct_cursor]] >= 0 ): direct_cursor += 1 if direct_cursor < len(compatible_indexes): objective_owner_indexes[compatible_indexes[direct_cursor]] = ( root_family_index ) direct_free_cursors[root_family_index] = direct_cursor + 1 continue family_queue = [root_family_index] family_queue_cursor = 0 visited_family_indexes = {root_family_index} visited_objective_indexes: set[int] = set() predecessor_family_by_objective: dict[int, int] = {} incoming_objective_by_family: dict[int, int] = {} terminal_objective_index: int | None = None while ( family_queue_cursor < len(family_queue) and terminal_objective_index is None ): family_index = family_queue[family_queue_cursor] family_queue_cursor += 1 family_id = selected_family_order[family_index] for objective_index in objective_indexes_by_family[family_id]: if objective_index in visited_objective_indexes: continue visited_objective_indexes.add(objective_index) predecessor_family_by_objective[objective_index] = family_index owner_index = objective_owner_indexes[objective_index] if owner_index < 0: terminal_objective_index = objective_index break if owner_index not in visited_family_indexes: visited_family_indexes.add(owner_index) incoming_objective_by_family[owner_index] = objective_index family_queue.append(owner_index) if terminal_objective_index is None: raise RuntimeError( "NoNE selected family has no distinct pending objective page" ) objective_index = terminal_objective_index while True: family_index = predecessor_family_by_objective[objective_index] displaced_objective_index = incoming_objective_by_family.get( family_index ) objective_owner_indexes[objective_index] = family_index if displaced_objective_index is None: break objective_index = displaced_objective_index matched_by_family: dict[str, list[_ObjectivePage]] = { family_id: [] for family_id in selected_family_order } for objective_index, family_index in enumerate(objective_owner_indexes): if family_index >= 0: matched_by_family[selected_family_order[family_index]].append( missing_objectives[objective_index] ) matched_family_cursors = { family_id: 0 for family_id in selected_family_order } selected_objectives: list[_ObjectivePage] = [] for family_id in selected_family_ids: cursor = matched_family_cursors[family_id] matched = matched_by_family[family_id] if cursor >= len(matched): raise RuntimeError("NoNE objective cohort matching is incomplete") selected_objectives.append(matched[cursor]) matched_family_cursors[family_id] = cursor + 1 if len({row.objective_id for row in selected_objectives}) != len( selected_objectives ): raise RuntimeError("NoNE objective cohort matching repeats an identity") return tuple(selected_objectives) def _compact_objective_cohort_binding_record( *, compact_page_ids: Sequence[int], selected_family_page_ids: Sequence[int], selected_objectives: Sequence[_ObjectivePage], family_rows: Sequence[Mapping[str, Any]], ) -> dict[str, Any]: """Bind each compact objective page to its exact model-selected parent.""" requested_count = len(selected_family_page_ids) if ( requested_count < 1 or len(compact_page_ids) != requested_count or len(selected_objectives) != requested_count or len(set(compact_page_ids)) != requested_count or any( not isinstance(page_id, int) or isinstance(page_id, bool) or page_id < 0 for page_id in (*compact_page_ids, *selected_family_page_ids) ) ): raise RuntimeError("NoNE compact objective allocation count differs") family_by_page: dict[int, str] = {} for row in family_rows: page_id = row.get("pageId") family_id = row.get("family") if ( not isinstance(page_id, int) or isinstance(page_id, bool) or page_id < 0 or not isinstance(family_id, str) or not family_id.strip() or page_id in family_by_page ): raise RuntimeError("NoNE family-page identity is malformed") family_by_page[page_id] = family_id allocations: list[dict[str, Any]] = [] objective_ids: list[str] = [] for allocation_ordinal, ( compact_page_id, source_family_page_id, objective, ) in enumerate( zip( compact_page_ids, selected_family_page_ids, selected_objectives, strict=True, ) ): source_family_id = family_by_page.get(source_family_page_id) if ( source_family_id is None or source_family_id not in objective.functional_families ): raise RuntimeError( "NoNE compact objective parent is not compatible" ) objective_ids.append(objective.objective_id) allocations.append( { "allocationOrdinal": allocation_ordinal, "compactPageId": compact_page_id, "compactSourceFamilyPageId": source_family_page_id, "compactSourceFamily": source_family_id, "objectiveId": objective.objective_id, } ) if len(set(objective_ids)) != requested_count: raise RuntimeError("NoNE compact objective identity repeats") selected_family_values = list(selected_family_page_ids) selected_objective_values = list(objective_ids) allocation_sha256 = hashlib.sha256( _canonical_json_bytes( { "schema": NONE_COMPACT_OBJECTIVE_COHORT_BINDING_SCHEMA, "allocations": allocations, } ) ).hexdigest() return { "schema": NONE_COMPACT_OBJECTIVE_COHORT_BINDING_SCHEMA, "matchingPolicy": ( "deterministic_complete_bipartite_augmenting_path_v1" ), "requestedObjectiveAllocationCount": requested_count, "matchedObjectiveAllocationCount": requested_count, "unmatchedObjectiveAllocationCount": 0, "selectedSourceFamilyPageIds": selected_family_values, "selectedSourceFamilyPageIdsSha256": hashlib.sha256( _canonical_json_bytes( { "schema": NONE_COMPACT_OBJECTIVE_COHORT_BINDING_SCHEMA, "selectedSourceFamilyPageIds": selected_family_values, } ) ).hexdigest(), "selectedObjectiveIds": selected_objective_values, "selectedObjectiveIdsSha256": hashlib.sha256( _canonical_json_bytes( { "schema": NONE_COMPACT_OBJECTIVE_COHORT_BINDING_SCHEMA, "selectedObjectiveIds": selected_objective_values, } ) ).hexdigest(), "allocations": allocations, "allocationPairsSha256": allocation_sha256, } def _validate_compact_objective_cohort_binding_record( record: object, *, compact_page_ids: Sequence[int], selected_family_page_ids: Sequence[int], selected_objectives: Sequence[_ObjectivePage], family_rows: Sequence[Mapping[str, Any]], ) -> dict[str, Any]: expected = _compact_objective_cohort_binding_record( compact_page_ids=compact_page_ids, selected_family_page_ids=selected_family_page_ids, selected_objectives=selected_objectives, family_rows=family_rows, ) if record != expected: raise RuntimeError("NoNE compact objective cohort binding differs") return expected def _atomic_json(path: Path, payload: Mapping[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") temporary.unlink(missing_ok=True) with temporary.open("wb") as handle: handle.write( json.dumps(payload, sort_keys=True, indent=2).encode("utf-8") + b"\n" ) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) descriptor = os.open(path.parent, os.O_RDONLY) try: os.fsync(descriptor) finally: os.close(descriptor) def _growth_plan_from_migration_receipt_boundary( migration_receipt_path: Path, ) -> Path: receipt_path = migration_receipt_path.expanduser().resolve() receipt = json.loads(receipt_path.read_text(encoding="utf-8")) growth = receipt.get("growthPlan") if isinstance(receipt, dict) else None if not isinstance(growth, dict): raise RuntimeError("NoNE graph adaptation growth-plan authority is absent") path_value = growth.get("path") sha256 = growth.get("sha256") if ( not isinstance(path_value, str) or not path_value or not isinstance(sha256, str) or len(sha256) != 64 ): raise RuntimeError("NoNE graph adaptation growth-plan identity is malformed") path = Path(path_value).expanduser().resolve() if not path.is_file() or _file_sha256(path) != sha256: raise RuntimeError("NoNE graph adaptation growth-plan bytes differ") return path def _validate_active_graph_growth_plan_boundary( plan: Mapping[str, Any], *, target_lineage: Mapping[str, Any], ) -> dict[str, int]: federated_growth_demand_authority_from_plan_boundary(plan) roadmap_value = plan.get("functionalExpertFamilyRoadmap") if not isinstance(roadmap_value, list) or any( not isinstance(row, dict) or not isinstance(row.get("family"), str) or not str(row["family"]).strip() for row in roadmap_value ): raise RuntimeError("NoNE active graph functional roadmap differs") family_ids = tuple(str(row["family"]) for row in roadmap_value) language_ids = _validated_language_expert_family_ids(plan, required=True) specialist_ids = _validated_science_specialist_family_ids( plan, family_ids, required=True, ) target_geometry = plan.get("proposedMinimumTargetGeometry") paged_lineage = target_lineage.get("pagedNoNE") target_layers = ( target_geometry.get( "residentScienceLayers", target_geometry.get("scienceLayers"), ) if isinstance(target_geometry, dict) else None ) lineage_layers = ( paged_lineage.get("scienceLayers") if isinstance(paged_lineage, dict) else target_lineage.get("scienceLayers") ) plan_lineage = plan.get("lineage") if ( len(set(family_ids)) != len(family_ids) or target_lineage.get("functionalCapabilityFamilies") != len(family_ids) or target_lineage.get("languageCapabilityPacks") != len(language_ids) or target_lineage.get("scienceSpecialistCapabilityFamilies") != len(specialist_ids) or not isinstance(target_layers, int) or isinstance(target_layers, bool) or target_layers != lineage_layers or not isinstance(plan_lineage, dict) or plan_lineage.get("modelTrainingClaimed") is not False or plan_lineage.get("promotionEligibilityClaimed") is not False ): raise RuntimeError("NoNE active graph plan does not match target lineage") summary = { "functionalExpertFamilies": len(family_ids), "languageExpertPacks": len(language_ids), "scienceSpecialists": len(specialist_ids), "scienceLayers": target_layers, "plannedObjectivePages": int( plan.get("initialLogicalExpertPageObjectives", 0) ), } return summary def _active_graph_growth_plan_boundary( *, source_growth_plan_path: Path, target_lineage: Mapping[str, Any], ) -> tuple[dict[str, Any], dict[str, int]]: from resynthesis.corpus_training import ( derive_paged_none_growth_plan_from_existing, ) plan = derive_paged_none_growth_plan_from_existing(source_growth_plan_path) summary = _validate_active_graph_growth_plan_boundary( plan, target_lineage=target_lineage, ) return plan, summary def _atomic_torch_save(path: Path, payload: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") temporary.unlink(missing_ok=True) torch.save(payload, temporary) with temporary.open("rb") as handle: os.fsync(handle.fileno()) os.replace(temporary, path) descriptor = os.open(path.parent, os.O_RDONLY) try: os.fsync(descriptor) finally: os.close(descriptor) def _derived_digest(source_sha256: str, component: str) -> str: return hashlib.sha256( bytes.fromhex(source_sha256) + b"\x00" + component.encode("utf-8") ).hexdigest() def _session_identity_t( source_sha256: str, growth_plan_sha256: str, ) -> torch.Tensor: digest = hashlib.sha256( bytes.fromhex(source_sha256) + bytes.fromhex(growth_plan_sha256) + b"\x00resynthesis-none-v2" ).digest() return torch.tensor( [ int.from_bytes( digest[offset : offset + 8], byteorder="little", signed=True, ) for offset in range(0, 32, 8) ], dtype=torch.long, ) def _load_checkpoint(path: Path) -> dict[str, Any]: payload = torch.load( path, map_location="cpu", mmap=True, weights_only=True, ) if not isinstance(payload, dict) or payload.get("schema") != CHECKPOINT_SCHEMA: raise RuntimeError("v2 seed source is not an additive checkpoint") lineage = payload.get("lineage") parameters = payload.get("parameters") buffers = payload.get("buffers") if ( not isinstance(lineage, dict) or not isinstance(parameters, dict) or not isinstance(buffers, dict) or not all( isinstance(value, torch.Tensor) for value in parameters.values() ) or not all( isinstance(value, torch.Tensor) for value in buffers.values() ) ): raise RuntimeError("v2 seed source checkpoint is incomplete") key_sha256, geometry_sha256 = _state_identity( {**parameters, **buffers} ) if payload.get("stateKeySetSha256") != key_sha256: raise RuntimeError("v2 seed source checkpoint key identity differs") if payload.get("stateGeometrySha256") != geometry_sha256: raise RuntimeError("v2 seed source checkpoint geometry differs") parent = lineage.get("parent") if ( not isinstance(parent, dict) or parent.get("modelType") not in {"resynthesis_native_parent", _LEGACY_PARENT_MODEL_TYPE} or parent.get("exactHistoricalSourceLoaded") is not True ): raise RuntimeError( "v2 seed source is not the exact Resynthesis parent composition" ) return payload def _checkpoint_direct_materialization_dtype_boundary( *, source: torch.Tensor, expected: torch.Tensor, target_device: torch.device, ) -> torch.dtype: """Select the exact first-forward dtype for checkpoint-direct assignment. The CUDA caller constructs the additive graph and then converts the complete RBO to bfloat16 before any forward. Retaining a multi-billion-element accepted FP32 science stack on CUDA first therefore creates a second, transient full graph and immediately discards it. The streaming materializer below reaches the identical endpoint (FP32 -> BF16) while retaining the source/constructor dtype checks performed before this boundary. CPU cold reload still uses the constructor dtype for exact audit. """ if ( target_device.type == "cuda" and source.is_floating_point() and expected.is_floating_point() ): return torch.bfloat16 return expected.dtype def _checkpoint_direct_materialize_tensor_boundary( *, source: torch.Tensor, expected: torch.Tensor, target_device: torch.device, ) -> torch.Tensor: """Stream one validated tensor into its exact first-forward representation. Pageable-host FP32 -> CUDA BF16 fused copies are materially slower on this lane than the native FP32 transfer followed by a device conversion. Keep that fast transfer, but convert each tensor immediately so the temporary FP32 allocation is released before the next memory-mapped tensor is read. This bounds transient device storage by the largest tensor instead of the complete dense additive graph. """ materialized = source.to( device=target_device, dtype=expected.dtype, ) first_forward_dtype = ( _checkpoint_direct_materialization_dtype_boundary( source=source, expected=expected, target_device=target_device, ) ) if materialized.dtype != first_forward_dtype: materialized = materialized.to(dtype=first_forward_dtype) return materialized def build_checkpoint_direct_science_stack_boundary( cfg: ResynthesisScienceLayerConfig, checkpoint_state: Mapping[str, torch.Tensor], *, device: torch.device | str = "cpu", ) -> ResynthesisScienceLayerStack: """Restore the exact dense science stack without overwritten initialization. This is an external checkpoint-load boundary. Paged runtimes are excluded because their immutable objects and resident state attach through their own accepted-generation authority after the dense graph exists. """ from resynthesis.causal_algebra import ( CausalAlgebraConfig, CausalAlgebraWorldGraph, ) from resynthesis.causal_integration_tensor import ( CAUSAL_INTEGRATION_TENSOR_ASSURANCE_V3_LOCAL_STATE_NAMES, CAUSAL_INTEGRATION_TENSOR_EXPLORATION_META_V2_LOCAL_STATE_NAMES, CAUSAL_INTEGRATION_TENSOR_KNOWLEDGE_TRANSFER_V4_LOCAL_STATE_NAMES, CAUSAL_INTEGRATION_TENSOR_LOCAL_STATE_NAMES, CAUSAL_INTEGRATION_TENSOR_MHC_V5_LOCAL_STATE_NAMES, CAUSAL_INTEGRATION_TENSOR_V1_LOCAL_STATE_NAMES, CausalIntegrationTensor, ) from resynthesis.quantile_balancing import ( ANTI_THOMPSON_FAIL_COUNTS_BUFFER_SUFFIX, ) from resynthesis.science_layers import build_resynthesis_science_stack from resynthesis.trauma_system import TensorTraumaState target_device = torch.device(device) if target_device.type == "meta": raise ValueError("checkpoint-direct science stack requires a material device") prefix = "science_stack." paged_runtime_component = ".paged_expert_runtime." dense_state = { name.removeprefix(prefix): value for name, value in checkpoint_state.items() if name.startswith(prefix) and paged_runtime_component not in name } if not dense_state: raise RuntimeError( "checkpoint-direct science stack has no dense checkpoint state" ) with torch.device("meta"): science_stack = build_resynthesis_science_stack(cfg) expected_state = science_stack.state_dict() causal_prefix = "causal_algebra_world_graph." expected_causal_names = { name for name in expected_state if name.startswith(causal_prefix) } actual_causal_names = { name for name in dense_state if name.startswith(causal_prefix) } if expected_causal_names and not actual_causal_names: causal_world_size = min( max(2, int(cfg.causal_world_size)), int(cfg.hidden_size), ) with torch.device(target_device): causal_seed = CausalAlgebraWorldGraph( CausalAlgebraConfig( hidden_size=int(cfg.hidden_size), action_size=int(cfg.action_input_dim), pathway_size=( max(1, int(cfg.num_layers)) * max(5, int(cfg.num_experts)) + max(1, int(cfg.num_layers)) ), world_size=causal_world_size, hypothesis_count=int(cfg.causal_hypothesis_count), primitive_count=int(cfg.causal_primitive_count), program_steps=int(cfg.causal_program_steps), domain_count=int(cfg.causal_domain_count), operator_rank=min( max(1, int(cfg.causal_operator_rank)), causal_world_size, ), ) ) dense_state.update( { f"{causal_prefix}{name}": value.detach() for name, value in causal_seed.state_dict().items() } ) capability_prefix = "capability_integration." expected_capability_names = { name for name in expected_state if name.startswith(capability_prefix) } actual_capability_names = { name for name in dense_state if name.startswith(capability_prefix) } capability_v1_contract_names = { f"{capability_prefix}{name}" for name in CAUSAL_INTEGRATION_TENSOR_V1_LOCAL_STATE_NAMES } capability_v2_extension_names = { f"{capability_prefix}{name}" for name in ( CAUSAL_INTEGRATION_TENSOR_EXPLORATION_META_V2_LOCAL_STATE_NAMES ) } capability_v3_extension_names = { f"{capability_prefix}{name}" for name in ( CAUSAL_INTEGRATION_TENSOR_ASSURANCE_V3_LOCAL_STATE_NAMES ) } capability_v4_extension_names = { f"{capability_prefix}{name}" for name in ( CAUSAL_INTEGRATION_TENSOR_KNOWLEDGE_TRANSFER_V4_LOCAL_STATE_NAMES ) } capability_v5_extension_names = { f"{capability_prefix}{name}" for name in CAUSAL_INTEGRATION_TENSOR_MHC_V5_LOCAL_STATE_NAMES } capability_contract_names = ( capability_v1_contract_names | capability_v2_extension_names | capability_v3_extension_names | capability_v4_extension_names | capability_v5_extension_names ) if expected_capability_names != capability_contract_names: # Never let prefix discovery silently widen any explicit migration: # v23 -> v24 owns nine causal/outcome tensors, v24 -> v25 owns four # selector/meta tensors, v25 -> v26 owns assurance, v26 -> v27 owns # transfer, and v27 -> v28 owns the proof-coupled MHC heads. raise RuntimeError( "checkpoint-direct causal capability runtime schema differs" ) unexpected_capability_names = ( actual_capability_names - capability_contract_names ) if unexpected_capability_names: raise RuntimeError( "checkpoint-direct causal capability state is unexpected: " + ", ".join(sorted(unexpected_capability_names)) ) capability_stages: tuple[frozenset[str], ...] = ( frozenset(), frozenset(capability_v1_contract_names), frozenset( capability_v1_contract_names | capability_v2_extension_names ), frozenset( capability_v1_contract_names | capability_v2_extension_names | capability_v3_extension_names ), frozenset( capability_v1_contract_names | capability_v2_extension_names | capability_v3_extension_names | capability_v4_extension_names ), frozenset(capability_contract_names), ) actual_capability_stage = frozenset(actual_capability_names) if actual_capability_stage not in capability_stages: # Families are atomic and ordered. A partial family, or a later family # without every predecessor, is corrupted state rather than permission # to overwrite inherited tensors with constructor defaults. raise RuntimeError( "checkpoint-direct causal capability family state is partial " "or out of order" ) capability_seed_names = ( set(capability_contract_names) - actual_capability_names ) if capability_seed_names: with torch.device(target_device): capability_seed = CausalIntegrationTensor( hidden_size=int(cfg.hidden_size), transfer_dim=( int(cfg.hidden_size) if cfg.knowledge_transfer_dim is None else int(cfg.knowledge_transfer_dim) ), hypothesis_count=int(cfg.causal_hypothesis_count), ) capability_seed_state = capability_seed.state_dict() if ( set(capability_seed_state) != CAUSAL_INTEGRATION_TENSOR_LOCAL_STATE_NAMES ): raise RuntimeError( "checkpoint-direct causal capability seed schema differs" ) dense_state.update( { f"{capability_prefix}{name}": value.detach() for name, value in capability_seed_state.items() if f"{capability_prefix}{name}" in capability_seed_names } ) expected_anti_thompson_names = { name for name in expected_state if name.endswith(ANTI_THOMPSON_FAIL_COUNTS_BUFFER_SUFFIX) } actual_anti_thompson_names = ( expected_anti_thompson_names.intersection(dense_state) ) if ( actual_anti_thompson_names and actual_anti_thompson_names != expected_anti_thompson_names ): raise RuntimeError( "checkpoint-direct anti-thompson outcome state is partial" ) if expected_anti_thompson_names and not actual_anti_thompson_names: # Historical checkpoints kept this per-router bank only in process # memory. Its only truthful cold-start value is the complete zero # family; once present, strict assignment below preserves every count. dense_state.update( { name: torch.zeros( tuple(expected_state[name].shape), dtype=expected_state[name].dtype, device=target_device, ) for name in expected_anti_thompson_names } ) trauma_marker = ".quantile_router.trauma_state." expected_trauma_names = { name for name in expected_state if trauma_marker in name } actual_trauma_names = { name for name in dense_state if trauma_marker in name } unexpected_trauma_names = actual_trauma_names - expected_trauma_names if unexpected_trauma_names: raise RuntimeError( "checkpoint-direct trauma state is unexpected: " + ", ".join(sorted(unexpected_trauma_names)) ) trauma_prefixes = sorted( { name.partition(trauma_marker)[0] + trauma_marker for name in expected_trauma_names } ) for trauma_prefix in trauma_prefixes: expected_family = { name for name in expected_trauma_names if name.startswith(trauma_prefix) } actual_family = expected_family.intersection(dense_state) if actual_family and actual_family != expected_family: # A partial learned bank cannot be completed with zeros without # silently changing the evidence semantics of that exact router. raise RuntimeError( "checkpoint-direct trauma outcome state is partial: " + trauma_prefix ) if actual_family: continue fail_name = f"{trauma_prefix}fail_ema" fail_expected = expected_state.get(fail_name) if fail_expected is None or fail_expected.ndim != 1: raise RuntimeError( "checkpoint-direct trauma runtime schema differs: " + trauma_prefix ) with torch.device(target_device): trauma_seed = TensorTraumaState( num_arms=int(fail_expected.shape[0]) ) seed_state = trauma_seed.state_dict() expected_local_names = { name.removeprefix(trauma_prefix) for name in expected_family } if set(seed_state) != expected_local_names: raise RuntimeError( "checkpoint-direct trauma seed schema differs: " + trauma_prefix ) dense_state.update( { f"{trauma_prefix}{name}": value.detach() for name, value in seed_state.items() } ) missing_names = set(expected_state) - set(dense_state) if missing_names: with torch.device(target_device): science_seed_stack = build_resynthesis_science_stack(cfg) science_seed_state = science_seed_stack.state_dict() if not missing_names.issubset(set(science_seed_state)): raise RuntimeError( "checkpoint-direct science stack seed schema differs" ) dense_state.update( { name: science_seed_state[name].detach() for name in missing_names } ) expected_names = set(expected_state) actual_names = set(dense_state) if actual_names != expected_names: missing = sorted(expected_names - actual_names) unexpected = sorted(actual_names - expected_names) raise RuntimeError( "checkpoint-direct science stack key set differs: " f"missing={missing} unexpected={unexpected}" ) for name, expected in expected_state.items(): source = dense_state[name] if source.is_meta: raise RuntimeError( f"checkpoint-direct science tensor is not materialized: {name}" ) if source.shape != expected.shape: raise RuntimeError( f"checkpoint-direct science tensor geometry differs: {name}" ) # Older accepted branches may have been sealed after the final # ``rbo.to(bfloat16)`` while the moving graph is constructed in # float32 before that same conversion. A lower-precision floating # tensor is exactly representable in a wider floating dtype, so this # is a checkpoint migration rather than authority drift. Equal-width # floating changes and narrowing conversions remain fail-closed. widening_floating_migration = ( source.is_floating_point() and expected.is_floating_point() and source.element_size() < expected.element_size() ) if source.dtype != expected.dtype and not widening_floating_migration: raise RuntimeError( f"checkpoint-direct science tensor dtype differs: {name}" ) target_dense_state = { name: _checkpoint_direct_materialize_tensor_boundary( source=value, expected=expected_state[name], target_device=target_device, ) for name, value in dense_state.items() } incompatible = science_stack.load_state_dict( target_dense_state, strict=True, assign=True, ) if incompatible.missing_keys or incompatible.unexpected_keys: raise RuntimeError( "checkpoint-direct science stack strict assignment differs" ) science_stack.rebuild_nonpersistent_buffers() residual_meta = sorted( name for name, tensor in ( *science_stack.named_parameters(), *science_stack.named_buffers(), ) if tensor.is_meta ) if residual_meta: raise RuntimeError( "checkpoint-direct science stack retains meta tensors: " f"{residual_meta}" ) return science_stack def _optimizer_by_name( payload: Mapping[str, Any], parameter_names: set[str], source_layers: int, ) -> tuple[dict[str, Mapping[str, Any]], Mapping[str, Any]]: groups = payload.get("param_groups") states = payload.get("state") if ( not isinstance(groups, list) or len(groups) != 1 or not isinstance(states, dict) or not isinstance(groups[0], dict) ): raise RuntimeError("v2 seed requires one AdamW optimizer group") group = groups[0] parameter_ids = list(group.get("params", ())) names_value = group.get("param_names") if isinstance(names_value, (list, tuple)): names = [str(value) for value in names_value] if set(names) != parameter_names: raise RuntimeError( "v2 seed optimizer names differ from checkpoint parameters" ) else: names = _parameter_order(parameter_names, source_layers) if len(names) != len(parameter_ids): raise RuntimeError("v2 seed optimizer parameter IDs differ") by_name: dict[str, Mapping[str, Any]] = {} for name, parameter_id in zip(names, parameter_ids, strict=True): state = states.get(parameter_id) by_name[name] = state if isinstance(state, dict) else {} return by_name, group def _moment( optimizer_by_name: Mapping[str, Mapping[str, Any]], name: str, moment_name: str, reference: torch.Tensor, ) -> torch.Tensor: value = optimizer_by_name.get(name, {}).get(moment_name) if not isinstance(value, torch.Tensor): return torch.zeros_like(reference, dtype=torch.float32) if value.shape != reference.shape: raise RuntimeError(f"optimizer moment geometry differs for {name}") return value.detach().to(device="cpu", dtype=torch.float32).clone() def _step( optimizer_by_name: Mapping[str, Mapping[str, Any]], names: Sequence[str], ) -> torch.Tensor: values: list[torch.Tensor] = [] for name in names: value = optimizer_by_name.get(name, {}).get("step") if isinstance(value, torch.Tensor) and value.numel() == 1: values.append(value.detach().cpu().reshape(()).float()) if not values: return torch.zeros((), dtype=torch.long) return torch.stack(values).max().long() def _select_width(vector: torch.Tensor, width: int) -> torch.Tensor: flat = vector.detach().reshape(-1) if flat.shape[0] < width: raise RuntimeError("source expert memory is narrower than the v2 router") if flat.shape[0] == width: return flat.clone() positions = torch.linspace( 0, flat.shape[0] - 1, steps=width, dtype=torch.float64, ).round().long() return flat.index_select(0, positions).clone() def _resize_matrix( matrix: torch.Tensor, rows: int, columns: int, ) -> torch.Tensor: if matrix.ndim != 2 or min(rows, columns) < 1: raise RuntimeError("resident transfer matrix geometry is invalid") row_positions = torch.linspace( 0, matrix.shape[0] - 1, steps=rows, dtype=torch.float64, ).round().long() column_positions = torch.linspace( 0, matrix.shape[1] - 1, steps=columns, dtype=torch.float64, ).round().long() return ( matrix.detach() .cpu() .index_select(0, row_positions) .index_select(1, column_positions) .contiguous() .clone() ) def _resident_runtime_state( *, parameters: Mapping[str, torch.Tensor], bundle: NoNEPageBundle, page_rows: Sequence[Mapping[str, Any]], layer_catalog: Mapping[int, Sequence[int]], session_id_t: torch.Tensor, hidden_size: int, action_size: int, ) -> dict[str, torch.Tensor]: router_size = int(bundle.weights.transfer_memory_t.shape[1]) tensors: dict[str, torch.Tensor] = {} for layer_id, catalog_ids in layer_catalog.items(): catalog_t = torch.tensor(catalog_ids, dtype=torch.long) page_positions_t = catalog_t prefix = f"layer_{layer_id}." role_query_name = ( f"science_stack.science_layer_{layer_id}." "role_query_proj.weight" ) role_query = parameters.get(role_query_name) if not isinstance(role_query, torch.Tensor): role_query = bundle.weights.gate_t[ int(catalog_ids[0]) ].t() hidden_projection_t = _resize_matrix( role_query, router_size, hidden_size, ) action_source = parameters.get("acquisition_encoder.weight") if not isinstance(action_source, torch.Tensor): action_source = hidden_projection_t action_projection_t = _resize_matrix( action_source, router_size, action_size, ) transfer_memory_t = bundle.weights.transfer_memory_t.index_select( 0, page_positions_t, ) source_priors = [] for page_id in catalog_ids: row = page_rows[int(page_id)] expert_slot = int(row["sourceExpertSlot"]) router_name = ( f"science_stack.science_layer_{layer_id}.router.weight" ) source_router = parameters.get(router_name) source_priors.append( torch.zeros((), dtype=transfer_memory_t.dtype) if not isinstance(source_router, torch.Tensor) else source_router[expert_slot].float().norm().to( dtype=transfer_memory_t.dtype ) ) page_prior_t = torch.stack(source_priors) page_prior_t = page_prior_t - page_prior_t.mean() memory_projection_t = ( hidden_projection_t.t().repeat(1, 3) / 3.0 ) tensors.update( { f"{prefix}residual_scale": torch.zeros( (), dtype=torch.float32, ), f"{prefix}router.page_route_keys": ( transfer_memory_t.contiguous().clone() ), f"{prefix}router.page_prior": page_prior_t, f"{prefix}router.quantile_router.activation_logit": torch.logit( torch.tensor(1.0e-4, dtype=torch.float32) ), f"{prefix}router.quantile_router.temperature_logit": torch.zeros( (), dtype=torch.float32, ), f"{prefix}router.quantile_router.expert_bias_t": torch.zeros( len(catalog_ids), dtype=torch.float32, ), f"{prefix}router.session_id_t": session_id_t.clone(), f"{prefix}router.layer_id_t": torch.tensor( layer_id, dtype=torch.long, ), f"{prefix}router.page_catalog_ids_t": catalog_t, f"{prefix}router.hidden_projection.weight": ( hidden_projection_t ), f"{prefix}router.action_projection.weight": ( action_projection_t ), f"{prefix}router.pathway_projection.weight": ( hidden_projection_t.clone() ), f"{prefix}executor.memory_gate": torch.zeros( (), dtype=torch.float32, ), f"{prefix}executor.situ_glu_scale": torch.zeros( (), dtype=torch.float32, ), f"{prefix}executor.latent_rmsnorm_scale": torch.zeros( (), dtype=torch.float32, ), f"{prefix}executor.memory_projection.weight": ( memory_projection_t ), } ) return tensors def _save_resident_runtime_state( path: Path, tensors: Mapping[str, torch.Tensor], ) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") temporary.unlink(missing_ok=True) save_file( { name: tensor.detach().cpu().contiguous() for name, tensor in tensors.items() }, str(temporary), metadata={"schema": NONE_V2_RESIDENT_RUNTIME_SCHEMA}, ) with temporary.open("rb") as handle: os.fsync(handle.fileno()) os.replace(temporary, path) descriptor = os.open(path.parent, os.O_RDONLY) try: os.fsync(descriptor) finally: os.close(descriptor) def _source_page( *, parameters: Mapping[str, torch.Tensor], buffers: Mapping[str, torch.Tensor], optimizer_by_name: Mapping[str, Mapping[str, Any]], layer_id: int, ffn_index: int, structural_experts: int, ) -> _SourcePage: prefix = f"science_stack.science_layer_{layer_id}." gate_name = f"{prefix}ffn_gate_up" down_name = f"{prefix}ffn_down" glyph_down_name = f"{prefix}glyph_translate_proj.weight" glyph_up_name = f"{prefix}glyph_translate_back.weight" translation_name = f"{prefix}translate_scale" role_name = f"{prefix}expert_role_tag" intent_name = f"{prefix}expert_intent_glyphs" history_name = f"{prefix}_expert_history_states" required = ( gate_name, down_name, glyph_down_name, glyph_up_name, translation_name, role_name, intent_name, ) if any(name not in parameters for name in required): raise RuntimeError(f"v1 layer {layer_id} omits inherited page state") gate_rows = parameters[gate_name] down_rows = parameters[down_name] if ( gate_rows.ndim != 3 or down_rows.ndim != 3 or ffn_index >= gate_rows.shape[0] or gate_rows.shape[0] != down_rows.shape[0] ): raise RuntimeError("v1 FFN expert geometry differs") expert_slot = structural_experts + ffn_index role_rows = parameters[role_name] intent_rows = parameters[intent_name] history_rows = buffers.get(history_name) if ( role_rows.ndim != 2 or intent_rows.ndim != 2 or expert_slot >= role_rows.shape[0] or expert_slot >= intent_rows.shape[0] ): raise RuntimeError("v1 expert identity geometry differs") memory_width = int(role_rows.shape[1]) history_t = ( torch.zeros(memory_width, dtype=role_rows.dtype) if not isinstance(history_rows, torch.Tensor) else _select_width(history_rows[expert_slot], memory_width) ) role_t = _select_width(role_rows[expert_slot], memory_width) intent_t = _select_width(intent_rows[expert_slot], memory_width) gate_t = gate_rows[ffn_index].detach().cpu().clone() down_t = down_rows[ffn_index].detach().cpu().clone() glyph_down_t = ( parameters[glyph_down_name].detach().cpu().t().contiguous().clone() ) glyph_up_t = ( parameters[glyph_up_name].detach().cpu().t().contiguous().clone() ) translation_t = ( parameters[translation_name].detach().cpu().reshape(1).clone() ) tensors = ( torch.zeros(1, dtype=gate_t.dtype), gate_t, gate_t.clone(), down_t, glyph_down_t, glyph_up_t, translation_t, history_t, role_t, intent_t, ) gate_mean_t = _moment( optimizer_by_name, gate_name, "exp_avg", gate_rows, )[ffn_index] down_mean_t = _moment( optimizer_by_name, down_name, "exp_avg", down_rows, )[ffn_index] glyph_down_mean_t = _moment( optimizer_by_name, glyph_down_name, "exp_avg", parameters[glyph_down_name], ).t().contiguous() glyph_up_mean_t = _moment( optimizer_by_name, glyph_up_name, "exp_avg", parameters[glyph_up_name], ).t().contiguous() translation_mean_t = _moment( optimizer_by_name, translation_name, "exp_avg", parameters[translation_name], ).reshape(1) role_mean_t = _select_width( _moment( optimizer_by_name, role_name, "exp_avg", role_rows, )[expert_slot], memory_width, ) intent_mean_t = _select_width( _moment( optimizer_by_name, intent_name, "exp_avg", intent_rows, )[expert_slot], memory_width, ) mean_t = torch.cat( tuple( value.reshape(-1) for value in ( gate_mean_t, gate_mean_t.clone(), down_mean_t, glyph_down_mean_t, glyph_up_mean_t, translation_mean_t, torch.zeros_like(history_t, dtype=torch.float32), role_mean_t, intent_mean_t, ) ) ).float() gate_square_t = _moment( optimizer_by_name, gate_name, "exp_avg_sq", gate_rows, )[ffn_index] down_square_t = _moment( optimizer_by_name, down_name, "exp_avg_sq", down_rows, )[ffn_index] glyph_down_square_t = _moment( optimizer_by_name, glyph_down_name, "exp_avg_sq", parameters[glyph_down_name], ).t().contiguous() glyph_up_square_t = _moment( optimizer_by_name, glyph_up_name, "exp_avg_sq", parameters[glyph_up_name], ).t().contiguous() translation_square_t = _moment( optimizer_by_name, translation_name, "exp_avg_sq", parameters[translation_name], ).reshape(1) role_square_t = _select_width( _moment( optimizer_by_name, role_name, "exp_avg_sq", role_rows, )[expert_slot], memory_width, ) intent_square_t = _select_width( _moment( optimizer_by_name, intent_name, "exp_avg_sq", intent_rows, )[expert_slot], memory_width, ) square_t = torch.cat( tuple( value.reshape(-1) for value in ( gate_square_t, gate_square_t.clone(), down_square_t, glyph_down_square_t, glyph_up_square_t, translation_square_t, torch.zeros_like(history_t, dtype=torch.float32), role_square_t, intent_square_t, ) ) ).float() return _SourcePage( layer_id=layer_id, source_ffn_index=ffn_index, source_expert_slot=expert_slot, weights=tensors, optimizer_mean_t=mean_t, optimizer_square_t=square_t, step_t=_step( optimizer_by_name, ( gate_name, down_name, glyph_down_name, glyph_up_name, translation_name, role_name, intent_name, ), ), ) def _assemble_bundle( pages: Sequence[_SourcePage], page_ids: Sequence[int], ) -> NoNEPageBundle: if not pages or len(pages) != len(page_ids): raise RuntimeError("v2 page assembly geometry differs") columns = tuple(zip(*(page.weights for page in pages), strict=True)) weights = NoNEPageWeights( page_ids_t=torch.tensor(page_ids, dtype=torch.long), ffn_mode_t=torch.stack(columns[0]), gate_t=torch.stack(columns[1]), up_t=torch.stack(columns[2]), down_t=torch.stack(columns[3]), glyph_down_t=torch.stack(columns[4]), glyph_up_t=torch.stack(columns[5]), translation_gate_t=torch.stack(columns[6]), outcome_memory_t=torch.stack(columns[7]), repair_memory_t=torch.stack(columns[8]), transfer_memory_t=torch.stack(columns[9]), ) bundle = NoNEPageBundle( weights=weights, optimizer_mean_t=torch.stack( [page.optimizer_mean_t for page in pages] ), optimizer_square_t=torch.stack( [page.optimizer_square_t for page in pages] ), step_t=torch.stack([page.step_t for page in pages]).long(), ) validate_page_bundle(bundle) return bundle def _component_packet( *, checkpoint_sha256: str, optimizer_sha256: str, growth_plan_sha256: str, parent_sha256: str, ) -> NoNEGenerationComponentPacket: return NoNEGenerationComponentPacket( parent_digest_t=digest_tensor(parent_sha256), shared_model_digest_t=digest_tensor(checkpoint_sha256), global_optimizer_digest_t=digest_tensor(optimizer_sha256), scheduler_digest_t=digest_tensor( _derived_digest(checkpoint_sha256, "scheduler-not-embedded") ), rng_digest_t=digest_tensor( _derived_digest(checkpoint_sha256, "rng-not-embedded") ), rbo_digest_t=digest_tensor( _derived_digest(checkpoint_sha256, "rbo") ), fabric_digest_t=digest_tensor( _derived_digest(checkpoint_sha256, "fabric") ), vge_digest_t=digest_tensor( _derived_digest(checkpoint_sha256, "vge") ), router_digest_t=digest_tensor( _derived_digest(checkpoint_sha256, "router") ), corpus_digest_t=digest_tensor(growth_plan_sha256), ) def _tensor_sha256(tensor: torch.Tensor) -> str: stable = tensor.detach().to(device="cpu").contiguous() return hashlib.sha256( stable.reshape(-1).view(torch.uint8).numpy().tobytes() ).hexdigest() def _transfer_page_state_sha256( tensors: Mapping[str, torch.Tensor], ) -> str: """Hash transfer state without allowing page identity to imply diversity.""" if set(tensors) != set(_TRANSFER_PAGE_WEIGHT_FIELDS): raise RuntimeError("NoNE transfer page state fields differ") fingerprint = { name: { "dtype": str(tensors[name].dtype), "shape": list(tensors[name].shape), "sha256": _tensor_sha256(tensors[name]), } for name in _TRANSFER_PAGE_WEIGHT_FIELDS } return hashlib.sha256(_canonical_json_bytes(fingerprint)).hexdigest() def _bundle_transfer_state_sha256(bundle: NoNEPageBundle) -> str: if bundle.weights.page_ids_t.numel() != 1: raise RuntimeError("NoNE transfer-state hash requires one page") return _transfer_page_state_sha256( { name: getattr(bundle.weights, name) for name in _TRANSFER_PAGE_WEIGHT_FIELDS } ) def _load_manifest_page_bundle( *, store: NoNEImmutablePageStore, object_rows: Mapping[int, Mapping[str, Any]], page_ids: Sequence[int], weight_dtype: torch.dtype | None = None, ) -> NoNEPageBundle: """Read historical page objects without moving the accepted pointer.""" tensor_names = ( "page_ids_t", *_TRANSFER_PAGE_WEIGHT_FIELDS, ) loaded: dict[str, list[torch.Tensor]] = { name: [] for name in tensor_names } optimizer_mean_rows: list[torch.Tensor] = [] optimizer_square_rows: list[torch.Tensor] = [] step_rows: list[torch.Tensor] = [] for page_id in page_ids: row = object_rows.get(page_id) if not isinstance(row, Mapping): raise RuntimeError("NoNE manifest page object is absent") object_sha256 = str(row.get("sha256", "")) object_bytes = row.get("bytes") try: object_digest = bytes.fromhex(object_sha256) except ValueError as error: raise RuntimeError("NoNE manifest page object identity differs") from error if ( len(object_digest) != 32 or object_sha256 != object_sha256.lower() ): raise RuntimeError("NoNE manifest page object identity differs") if object_bytes is not None and ( not isinstance(object_bytes, int) or isinstance(object_bytes, bool) or object_bytes < 1 ): raise RuntimeError("NoNE manifest page object identity differs") try: object_path = store._object_path_boundary( object_sha256, expected_bytes=object_bytes, ) except (FileNotFoundError, RuntimeError) as error: raise RuntimeError("NoNE manifest page object identity differs") from error if object_bytes is None: if ( object_path.stat().st_size < 1 or _file_sha256(object_path) != object_sha256 ): raise RuntimeError("NoNE manifest page object identity differs") with safe_open( # type: ignore[no-untyped-call] str(object_path), framework="pt", device="cpu", ) as handle: for name in tensor_names: tensor_t = ( handle.get_tensor(name) if name == "page_ids_t" else load_page_weight_from_handle_boundary(handle, name) ) if name != "page_ids_t" and weight_dtype is not None: tensor_t = tensor_t.to(dtype=weight_dtype) loaded[name].append(tensor_t) row_weights = NoNEPageWeights( page_ids_t=loaded["page_ids_t"][-1], ffn_mode_t=loaded["ffn_mode_t"][-1], gate_t=loaded["gate_t"][-1], up_t=loaded["up_t"][-1], down_t=loaded["down_t"][-1], glyph_down_t=loaded["glyph_down_t"][-1], glyph_up_t=loaded["glyph_up_t"][-1], translation_gate_t=loaded["translation_gate_t"][-1], outcome_memory_t=loaded["outcome_memory_t"][-1], repair_memory_t=loaded["repair_memory_t"][-1], transfer_memory_t=loaded["transfer_memory_t"][-1], ) optimizer_mean_t, optimizer_square_t, step_t = ( NoNEImmutablePageStore._materialize_optimizer_state_boundary( handle, weights=row_weights, device=torch.device("cpu"), ) ) optimizer_mean_rows.append(optimizer_mean_t) optimizer_square_rows.append(optimizer_square_t) step_rows.append(step_t) if not torch.equal( loaded["page_ids_t"][-1], torch.tensor([page_id], dtype=torch.long), ): raise RuntimeError("NoNE manifest page identity differs") weights = NoNEPageWeights( page_ids_t=torch.cat(loaded["page_ids_t"], dim=0), ffn_mode_t=torch.cat(loaded["ffn_mode_t"], dim=0), gate_t=torch.cat(loaded["gate_t"], dim=0), up_t=torch.cat(loaded["up_t"], dim=0), down_t=torch.cat(loaded["down_t"], dim=0), glyph_down_t=torch.cat(loaded["glyph_down_t"], dim=0), glyph_up_t=torch.cat(loaded["glyph_up_t"], dim=0), translation_gate_t=torch.cat( loaded["translation_gate_t"], dim=0, ), outcome_memory_t=torch.cat( loaded["outcome_memory_t"], dim=0, ), repair_memory_t=torch.cat( loaded["repair_memory_t"], dim=0, ), transfer_memory_t=torch.cat( loaded["transfer_memory_t"], dim=0, ), ) bundle = NoNEPageBundle( weights=weights, optimizer_mean_t=_concatenate_optimizer_matrices_boundary( tuple(optimizer_mean_rows) ), optimizer_square_t=_concatenate_optimizer_matrices_boundary( tuple(optimizer_square_rows) ), step_t=torch.cat(step_rows, dim=0), ) validate_page_bundle(bundle) return bundle def _float32_transfer_source_bundle( bundle: NoNEPageBundle, ) -> NoNEPageBundle: """Match the float32 source surface used by catalog expansion.""" converted = NoNEPageBundle( weights=bundle.weights.to( device=torch.device("cpu"), dtype=torch.float32, trainable=False, ), optimizer_mean_t=bundle.optimizer_mean_t.detach().cpu().float(), optimizer_square_t=bundle.optimizer_square_t.detach().cpu().float(), step_t=bundle.step_t.detach().cpu().long(), ) validate_page_bundle(converted) return converted def _page_request( *, session_id_t: torch.Tensor, generation_t: torch.Tensor, layer_id: int, page_ids_t: torch.Tensor, ) -> NoNEPageRequestPacket: page_count = int(page_ids_t.numel()) if page_count < 1: raise RuntimeError("NoNE catalog expansion found an empty source layer") return NoNEPageRequestPacket( session_id_t=session_id_t.detach().cpu().long().clone(), generation_t=generation_t.detach().cpu().long().reshape(()).clone(), layer_id_t=torch.tensor(layer_id, dtype=torch.long), page_ids_t=page_ids_t.detach().cpu().long().clone(), unique_page_ids_t=page_ids_t.detach().cpu().long().clone(), unique_page_catalog_positions_t=torch.arange(page_count, dtype=torch.long), page_position_t=torch.arange(page_count, dtype=torch.long), route_probability_t=torch.ones(page_count, dtype=torch.float32), route_entropy_t=torch.zeros(page_count, dtype=torch.float32), frontier_weight_t=torch.eye(page_count, dtype=torch.float32), ) def _weighted_page_rows( tensor: torch.Tensor, coefficient_t: torch.Tensor, ) -> torch.Tensor: if tensor.shape[0] != coefficient_t.numel(): raise RuntimeError("NoNE transfer coefficient geometry differs") shape = (coefficient_t.numel(),) + (1,) * (tensor.ndim - 1) return ( tensor.float() .mul(coefficient_t.float().reshape(shape)) .sum(dim=0, keepdim=True) .to(dtype=tensor.dtype) ) def _family_identity_residual( reference_t: torch.Tensor, family_id: str, ) -> torch.Tensor: """Return a deterministic, bounded identity residual for one family.""" digest = hashlib.sha256( b"nnf-none-family-transfer-v2\0" + family_id.encode("utf-8") ).digest() identity_t = torch.tensor(tuple(digest), dtype=torch.float32) identity_t = identity_t - identity_t.mean() identity_t = identity_t / identity_t.norm().clamp_min(1.0e-9) target_elements = reference_t.numel() repeat_count = ( target_elements + identity_t.numel() - 1 ) // identity_t.numel() identity_t = identity_t.repeat(repeat_count)[:target_elements] identity_t = identity_t.reshape_as(reference_t) identity_scale_t = ( reference_t.detach() .float() .square() .mean() .sqrt() .clamp_min(1.0) * 1.0e-4 ) return identity_t.to(reference_t) * identity_scale_t.to(reference_t) def _identity_query( reference_t: torch.Tensor, identity: str, ) -> torch.Tensor: digest = hashlib.sha256( b"nnf-none-transfer-query-v1\0" + identity.encode("utf-8") ).digest() query_t = torch.tensor(tuple(digest), dtype=torch.float32) query_t = query_t - query_t.mean() repeat_count = ( reference_t.shape[-1] + query_t.numel() - 1 ) // query_t.numel() query_t = query_t.repeat(repeat_count)[: reference_t.shape[-1]] return query_t.to(reference_t).div(query_t.norm().clamp_min(1.0e-9)) def _select_transfer_layer( *, identity: str, preferred_families: tuple[str, ...], layer_evidence: Sequence[_PreparedLayerTransferEvidence], preferred_layer_cache: dict[ tuple[str, ...], tuple[tuple[torch.Tensor, torch.Tensor, torch.Tensor], ...], ], layer_load: dict[int, int] | None = None, ) -> int: """Select a storage layer from learned evidence and physical load.""" if not layer_evidence: raise RuntimeError("NoNE transfer layer evidence is absent") cached_preferred = preferred_layer_cache.get(preferred_families) if cached_preferred is None: preferred = set(preferred_families) prepared_preferred: list[ tuple[torch.Tensor, torch.Tensor, torch.Tensor] ] = [] for evidence in layer_evidence: preferred_mask_t = torch.tensor( [ family_id in preferred for family_id in evidence.source_family_ids ], dtype=torch.bool, ) active_mask_t = ( preferred_mask_t if bool(preferred_mask_t.any()) else torch.ones_like(preferred_mask_t) ) prepared_preferred.append( ( active_mask_t, evidence.outcome_t.masked_select(active_mask_t).mean(), torch.log1p(active_mask_t.sum().float()), ) ) cached_preferred = tuple(prepared_preferred) preferred_layer_cache[preferred_families] = cached_preferred layer_scores: list[tuple[float, int]] = [] for evidence, preferred_row in zip( layer_evidence, cached_preferred, strict=True, ): normalized_t = evidence.normalized_route_keys_t query_t = _identity_query(normalized_t, identity).float() relationship_t = normalized_t.matmul(query_t) active_mask_t, outcome_mean_t, active_count_log_t = preferred_row score_t = ( relationship_t.masked_select(active_mask_t).amax() + outcome_mean_t + active_count_log_t ) layer_scores.append((float(score_t), evidence.layer_id)) if layer_load is None: return max(layer_scores)[1] if set(layer_load) != {layer_id for _score, layer_id in layer_scores}: raise RuntimeError("NoNE transfer layer-load identity differs") minimum_load = min(layer_load.values()) selected_layer_id = max( row for row in layer_scores if layer_load[row[1]] == minimum_load )[1] layer_load[selected_layer_id] += 1 return selected_layer_id def _prepare_transfer_layer_evidence( layer_evidence: Sequence[_LayerTransferEvidence], ) -> tuple[_PreparedLayerTransferEvidence, ...]: """Hoist identity-invariant transfer tensors out of per-page selection.""" prepared: list[_PreparedLayerTransferEvidence] = [] for evidence in layer_evidence: route_keys_t = evidence.route_keys_t.float() normalized_t = route_keys_t / route_keys_t.norm( dim=1, keepdim=True, ).clamp_min(1.0e-9) outcome_t = ( evidence.page_prior_t.float() + torch.log1p(evidence.route_count_t.float()) + torch.log1p(evidence.gradient_update_count_t.float()) * 0.5 ) prepared.append( _PreparedLayerTransferEvidence( layer_id=evidence.layer_id, source_family_ids=evidence.source_family_ids, normalized_route_keys_t=normalized_t, outcome_t=outcome_t, ) ) return tuple(prepared) def _model_owned_new_runtime_layer_targets( *, page_source_layer_ids: Sequence[int], paged_layer_ids: Sequence[int], total_science_layers: int, layer_transfer_graph_t: torch.Tensor, ) -> tuple[int, ...]: """Assign new pages to unpaged layers from the learned transfer graph. Each previously unpaged layer is activated once before further compatible pages are balanced across those new runtimes. Learned transfer affinity breaks equal-load ties; no host prompt, year, family count, or fixed layer table owns the assignment. """ source_ids = tuple(int(value) for value in page_source_layer_ids) paged_ids = tuple(sorted(int(value) for value in paged_layer_ids)) if ( total_science_layers < 1 or not source_ids or not paged_ids or len(set(paged_ids)) != len(paged_ids) or min(paged_ids) < 0 or max(paged_ids) >= total_science_layers or any(source_id not in set(paged_ids) for source_id in source_ids) or layer_transfer_graph_t.shape != (total_science_layers, total_science_layers) or not torch.isfinite(layer_transfer_graph_t).all() ): raise RuntimeError("NoNE model-owned runtime-layer evidence differs") pending_targets = [ layer_id for layer_id in range(total_science_layers) if layer_id not in set(paged_ids) ] targets: list[int] = [] target_source: dict[int, int] = {} target_load: dict[int, int] = {} for source_id in source_ids: if pending_targets: candidates = list(pending_targets) else: compatible_targets = [ target_id for target_id, owner_source_id in target_source.items() if owner_source_id == source_id ] if not compatible_targets: targets.append(source_id) continue minimum_load = min( target_load[target_id] for target_id in compatible_targets ) candidates = [ target_id for target_id in compatible_targets if target_load[target_id] == minimum_load ] candidate_t = torch.tensor(candidates, dtype=torch.long) affinity_t = layer_transfer_graph_t[source_id].float().abs().index_select( 0, candidate_t, ) target_id = candidates[ int(affinity_t.argmax().detach().cpu()) ] targets.append(target_id) if target_id in pending_targets: pending_targets.remove(target_id) target_source[target_id] = source_id target_load[target_id] = 1 else: if target_source.get(target_id) != source_id: raise RuntimeError( "NoNE runtime-layer transfer source changed" ) target_load[target_id] += 1 return tuple(targets) def _transfer_initialized_paged_runtime_state( *, source_parameters: Mapping[str, torch.Tensor], source_buffers: Mapping[str, torch.Tensor], source_layer_id: int, target_layer_id: int, target_page_ids: Sequence[int], target_route_keys_t: torch.Tensor, target_page_prior_t: torch.Tensor, target_expert_bias_t: torch.Tensor, ) -> tuple[dict[str, torch.Tensor], dict[str, torch.Tensor]]: """Build one strict, cold-loadable runtime for a previously unpaged layer.""" source_prefix = ( f"science_stack.science_layer_{source_layer_id}." "paged_expert_runtime." ) target_prefix = ( f"science_stack.science_layer_{target_layer_id}." "paged_expert_runtime." ) source_runtime_parameters = { name.removeprefix(source_prefix): tensor for name, tensor in source_parameters.items() if name.startswith(source_prefix) } source_runtime_buffers = { name.removeprefix(source_prefix): tensor for name, tensor in source_buffers.items() if name.startswith(source_prefix) } required_parameter_suffixes = { "residual_scale", "router.page_route_keys", "router.page_prior", "router.hidden_projection.weight", "router.action_projection.weight", "router.pathway_projection.weight", "router.quantile_router.activation_logit", "router.quantile_router.temperature_logit", "executor.memory_gate", "executor.situ_glu_scale", "executor.latent_rmsnorm_scale", "executor.memory_projection.weight", } trauma_buffer_suffixes = { "router.quantile_router.trauma_state.fail_ema", "router.quantile_router.trauma_state.success_ema", "router.quantile_router.trauma_state.peak_fail_ema", "router.quantile_router.trauma_state.hard_won_ema", "router.quantile_router.trauma_state.last_success_step", "router.quantile_router.trauma_state.global_step", "router.quantile_router.trauma_state.knowledge_pro_ema", "router.quantile_router.trauma_state.knowledge_anti_ema", "router.quantile_router.trauma_state.behavior_pro_ema", "router.quantile_router.trauma_state.behavior_anti_ema", "router.quantile_router.trauma_state.verified_exposure_ema", "router.quantile_router.trauma_state.evidence_confidence_ema", "router.quantile_router.trauma_state.definitive_polarity_t", "router.quantile_router.trauma_state.definitive_confidence_t", "router.quantile_router.trauma_state.knowledge_id_tokens_t", } required_buffer_suffixes = { "accepted_route_count_t", "accepted_gradient_update_count_t", "accepted_gradient_norm_t", "accepted_parameter_delta_norm_t", "accepted_gradient_signature_t", "router.session_id_t", "router.layer_id_t", "router.page_catalog_ids_t", "router.quantile_router.expert_bias_t", "router.quantile_router.anti_thompson_fail_counts_t", *trauma_buffer_suffixes, } page_ids = tuple(int(value) for value in target_page_ids) page_count = len(page_ids) source_route_keys_t = source_runtime_parameters.get( "router.page_route_keys" ) source_layer_id_t = source_runtime_buffers.get("router.layer_id_t") if ( set(source_runtime_parameters) != required_parameter_suffixes or set(source_runtime_buffers) != required_buffer_suffixes or not page_ids or min(page_ids) < 0 or len(set(page_ids)) != page_count or not isinstance(source_route_keys_t, torch.Tensor) or source_route_keys_t.ndim != 2 or target_route_keys_t.shape != (page_count, source_route_keys_t.shape[1]) or target_page_prior_t.shape != (page_count,) or target_expert_bias_t.shape != (page_count,) or not isinstance(source_layer_id_t, torch.Tensor) or source_layer_id_t.shape != () or int(source_layer_id_t.detach().cpu()) != source_layer_id or target_layer_id < 0 or not all( torch.isfinite(value).all() for value in ( target_route_keys_t, target_page_prior_t, target_expert_bias_t, ) ) ): raise RuntimeError("NoNE transfer-initialized runtime state differs") target_parameters = { f"{target_prefix}{suffix}": tensor.detach().cpu().clone() for suffix, tensor in source_runtime_parameters.items() } target_parameters[f"{target_prefix}router.page_route_keys"] = ( target_route_keys_t.detach().cpu().clone() ) target_parameters[f"{target_prefix}router.page_prior"] = ( target_page_prior_t.detach().cpu().clone() ) target_buffers = { f"{target_prefix}router.session_id_t": source_runtime_buffers[ "router.session_id_t" ] .detach() .cpu() .clone(), f"{target_prefix}router.layer_id_t": torch.tensor( target_layer_id, dtype=source_layer_id_t.dtype, ), f"{target_prefix}router.page_catalog_ids_t": torch.tensor( page_ids, dtype=source_runtime_buffers["router.page_catalog_ids_t"].dtype, ), f"{target_prefix}router.quantile_router.expert_bias_t": ( target_expert_bias_t.detach().cpu().clone() ), ( f"{target_prefix}" "router.quantile_router.anti_thompson_fail_counts_t" ): source_runtime_buffers[ "router.quantile_router.anti_thompson_fail_counts_t" ].new_zeros((page_count,), device="cpu"), } for telemetry_suffix in ( "accepted_route_count_t", "accepted_gradient_update_count_t", "accepted_gradient_norm_t", "accepted_parameter_delta_norm_t", "accepted_gradient_signature_t", ): source_t = source_runtime_buffers[telemetry_suffix] target_buffers[f"{target_prefix}{telemetry_suffix}"] = source_t.new_zeros( (page_count, *source_t.shape[1:]), device="cpu", ) for trauma_suffix in sorted(trauma_buffer_suffixes): source_t = source_runtime_buffers[trauma_suffix] if trauma_suffix.endswith(".global_step"): target_t = source_t.new_zeros((), device="cpu") elif trauma_suffix.endswith(".last_success_step"): target_t = source_t.new_full((page_count,), -1, device="cpu") else: target_t = source_t.new_zeros( (page_count, *source_t.shape[1:]), device="cpu", ) target_buffers[f"{target_prefix}{trauma_suffix}"] = target_t return target_parameters, target_buffers @dataclass(frozen=True) class _PreparedRecursiveTransferEvidence: """Layer-invariant tensors reused by exact per-identity transfer math.""" normalized_t: torch.Tensor outcome_score_t: torch.Tensor def _single_threaded_compact_transfer_rows( rows: Sequence[dict[str, Any]], ) -> Iterator[dict[str, Any]]: """Keep exact small GEMVs scalar and restore the caller's thread policy.""" prior_thread_count = torch.get_num_threads() if prior_thread_count != 1: torch.set_num_threads(1) try: yield from rows finally: if prior_thread_count != 1: torch.set_num_threads(prior_thread_count) def _prepare_recursive_transfer_evidence( *, route_keys_t: torch.Tensor, page_prior_t: torch.Tensor, route_count_t: torch.Tensor, gradient_update_count_t: torch.Tensor, ) -> _PreparedRecursiveTransferEvidence: """Prepare immutable source-layer evidence without changing its arithmetic.""" page_count = int(route_keys_t.shape[0]) if ( route_keys_t.ndim != 2 or page_prior_t.shape != (page_count,) or route_count_t.shape != (page_count,) or gradient_update_count_t.shape != (page_count,) ): raise RuntimeError("NoNE learned transfer evidence geometry differs") normalized_t = route_keys_t.float() / route_keys_t.float().norm( dim=1, keepdim=True, ).clamp_min(1.0e-9) if page_count < 2: raise RuntimeError( "NoNE recursive transfer requires two accepted source pages" ) outcome_score_t = ( page_prior_t.float() + torch.log1p(route_count_t.float()) + torch.log1p(gradient_update_count_t.float()) * 0.5 ) return _PreparedRecursiveTransferEvidence( normalized_t=normalized_t, outcome_score_t=outcome_score_t, ) def _recursive_transfer_coefficients_from_prepared( *, evidence: _PreparedRecursiveTransferEvidence, target_identity: str, ) -> torch.Tensor: """Rotate prepared evidence using the exact legacy scalar operations.""" page_count = int(evidence.normalized_t.shape[0]) query_t = _identity_query(evidence.normalized_t, target_identity).float() relationship_score_t = evidence.normalized_t.matmul(query_t) learned_coefficient_t = torch.softmax( evidence.outcome_score_t + relationship_score_t, dim=0, ) traversal_coefficient_t = learned_coefficient_t.new_zeros(page_count) selected_relationship_t, selected_index_t = torch.topk( relationship_score_t, k=2, ) traversal_coefficient_t.scatter_( 0, selected_index_t, torch.softmax(selected_relationship_t, dim=0), ) coefficient_t = ( learned_coefficient_t * 0.75 + traversal_coefficient_t * 0.25 ) return coefficient_t / coefficient_t.sum().clamp_min(1.0e-9) def _recursive_transfer_coefficients( *, route_keys_t: torch.Tensor, page_prior_t: torch.Tensor, route_count_t: torch.Tensor, gradient_update_count_t: torch.Tensor, target_identity: str, ) -> torch.Tensor: """Rotate learned route/outcome evidence into one transfer-only seed.""" evidence = _prepare_recursive_transfer_evidence( route_keys_t=route_keys_t, page_prior_t=page_prior_t, route_count_t=route_count_t, gradient_update_count_t=gradient_update_count_t, ) return _recursive_transfer_coefficients_from_prepared( evidence=evidence, target_identity=target_identity, ) def _identity_distinct_recursive_transfer_coefficients_from_prepared( *, evidence: _PreparedRecursiveTransferEvidence, target_identity: str, ) -> torch.Tensor: """Separate one identity while preserving exact prepared transfer math.""" coefficient_t = _recursive_transfer_coefficients_from_prepared( evidence=evidence, target_identity=target_identity, ) identity_t = _family_identity_residual( coefficient_t, target_identity, ).float() identity_t = identity_t - identity_t.mean() identity_t = identity_t / identity_t.abs().amax().clamp_min(1.0e-9) margin_t = coefficient_t.float().amin() * 0.25 distinct_t = coefficient_t.float() + identity_t * margin_t return distinct_t.to(coefficient_t).div( distinct_t.sum().clamp_min(1.0e-9).to(coefficient_t) ) def _identity_distinct_recursive_transfer_coefficients( *, route_keys_t: torch.Tensor, page_prior_t: torch.Tensor, route_count_t: torch.Tensor, gradient_update_count_t: torch.Tensor, target_identity: str, ) -> torch.Tensor: """Preserve learned transfer while separating large identity catalogs.""" evidence = _prepare_recursive_transfer_evidence( route_keys_t=route_keys_t, page_prior_t=page_prior_t, route_count_t=route_count_t, gradient_update_count_t=gradient_update_count_t, ) return _identity_distinct_recursive_transfer_coefficients_from_prepared( evidence=evidence, target_identity=target_identity, ) def _transfer_initialized_page( *, source_bundle: NoNEPageBundle, coefficient_t: torch.Tensor, page_id: int, family_id: str | None = None, ) -> NoNEPageBundle: source = source_bundle.weights transfer_memory_t = _weighted_page_rows( source.transfer_memory_t, coefficient_t, ) if family_id is not None: transfer_memory_t = ( transfer_memory_t + _family_identity_residual( transfer_memory_t, family_id, ) ) weights = NoNEPageWeights( page_ids_t=torch.tensor([page_id], dtype=torch.long), ffn_mode_t=_weighted_page_rows(source.ffn_mode_t, coefficient_t), gate_t=_weighted_page_rows(source.gate_t, coefficient_t), up_t=_weighted_page_rows(source.up_t, coefficient_t), down_t=_weighted_page_rows(source.down_t, coefficient_t), glyph_down_t=_weighted_page_rows( source.glyph_down_t, coefficient_t, ), glyph_up_t=_weighted_page_rows(source.glyph_up_t, coefficient_t), translation_gate_t=_weighted_page_rows( source.translation_gate_t, coefficient_t, ), outcome_memory_t=_weighted_page_rows( source.outcome_memory_t, coefficient_t, ), repair_memory_t=_weighted_page_rows( source.repair_memory_t, coefficient_t, ), transfer_memory_t=transfer_memory_t, ) flat_width = int(source_bundle.optimizer_mean_t.shape[1]) bundle = NoNEPageBundle( weights=weights, optimizer_mean_t=torch.zeros(1, flat_width, dtype=torch.float32), optimizer_square_t=torch.zeros(1, flat_width, dtype=torch.float32), step_t=torch.zeros(1, dtype=torch.long), ) validate_page_bundle(bundle) return bundle def _concatenate_bundles( bundles: Sequence[NoNEPageBundle], ) -> NoNEPageBundle: if not bundles: raise RuntimeError("NoNE catalog expansion produced no new pages") weight_fields = ( "page_ids_t", "ffn_mode_t", "gate_t", "up_t", "down_t", "glyph_down_t", "glyph_up_t", "translation_gate_t", "outcome_memory_t", "repair_memory_t", "transfer_memory_t", ) concatenated = { name: torch.cat( tuple(getattr(bundle.weights, name) for bundle in bundles), dim=0, ) for name in weight_fields } bundle = NoNEPageBundle( weights=NoNEPageWeights(**concatenated), optimizer_mean_t=torch.cat( tuple(bundle.optimizer_mean_t for bundle in bundles), dim=0, ), optimizer_square_t=torch.cat( tuple(bundle.optimizer_square_t for bundle in bundles), dim=0, ), step_t=torch.cat( tuple(bundle.step_t for bundle in bundles), dim=0, ), ) validate_page_bundle(bundle) return bundle def _append_zero_prefix_preserving( source_t: torch.Tensor, target_rows: int, ) -> torch.Tensor: if source_t.ndim < 1 or target_rows <= source_t.shape[0]: raise RuntimeError("NoNE tensor expansion does not add a suffix") target_t = source_t.new_zeros( (target_rows, *source_t.shape[1:]), ) target_t[: source_t.shape[0]].copy_(source_t) if not torch.equal(target_t[: source_t.shape[0]], source_t): raise RuntimeError("NoNE tensor expansion changed its accepted prefix") return target_t def _expand_named_optimizer_payload( *, source_payload: Mapping[str, Any], source_parameters: Mapping[str, torch.Tensor], target_parameters: Mapping[str, torch.Tensor], expanded_parameter_names: set[str], ) -> tuple[dict[str, Any], dict[str, int]]: groups = source_payload.get("param_groups") states = source_payload.get("state") if ( not isinstance(groups, list) or len(groups) != 1 or not isinstance(groups[0], dict) or not isinstance(states, dict) ): raise RuntimeError( "NoNE catalog expansion requires one named optimizer group" ) source_group = groups[0] names_value = source_group.get("param_names") parameter_ids = list(source_group.get("params", ())) if not isinstance(names_value, (list, tuple)): raise RuntimeError( "NoNE catalog expansion requires named optimizer authority" ) names = [str(value) for value in names_value] source_name_set = set(names) target_names = list(target_parameters) new_parameter_names = [ name for name in target_names if name not in source_name_set ] if ( len(names) != len(parameter_ids) or source_name_set != set(source_parameters) or not source_name_set.issubset(target_parameters) or len(set(target_names)) != len(target_names) or any( name not in expanded_parameter_names and source_parameters[name].shape != target_parameters[name].shape for name in names ) ): raise RuntimeError("NoNE catalog optimizer parameter identity differs") if any(not isinstance(parameter_id, int) for parameter_id in parameter_ids): raise RuntimeError("NoNE catalog optimizer parameter IDs differ") next_parameter_id = max(parameter_ids, default=-1) + 1 target_parameter_ids = [ *parameter_ids, *range(next_parameter_id, next_parameter_id + len(new_parameter_names)), ] target_group = { **dict(source_group), "params": target_parameter_ids, "param_names": [*names, *new_parameter_names], } target_states: dict[Any, Any] = dict(states) expanded_moments = 0 stateful_expanded_parameters = 0 accepted_prefixes_exact = True new_suffixes_zero = True for name, parameter_id in zip(names, parameter_ids, strict=True): if name not in expanded_parameter_names: continue source_parameter = source_parameters[name] target_parameter = target_parameters[name] if ( source_parameter.ndim < 1 or source_parameter.shape[1:] != target_parameter.shape[1:] or source_parameter.shape[0] >= target_parameter.shape[0] ): raise RuntimeError( f"NoNE optimizer expansion geometry differs for {name}" ) source_state = states.get(parameter_id) if not isinstance(source_state, dict): continue target_state = dict(source_state) had_moment = False for moment_name in ("exp_avg", "exp_avg_sq", "max_exp_avg_sq"): moment_t = source_state.get(moment_name) if not isinstance(moment_t, torch.Tensor): continue if moment_t.shape != source_parameter.shape: raise RuntimeError( f"NoNE optimizer moment geometry differs for {name}" ) expanded_t = moment_t.new_zeros(target_parameter.shape) expanded_t[: source_parameter.shape[0]].copy_(moment_t) if not torch.equal( expanded_t[: source_parameter.shape[0]], moment_t, ): raise RuntimeError( f"NoNE optimizer prefix changed for {name}" ) accepted_prefixes_exact = accepted_prefixes_exact and torch.equal( expanded_t[: source_parameter.shape[0]], moment_t, ) new_suffixes_zero = new_suffixes_zero and bool( torch.count_nonzero( expanded_t[source_parameter.shape[0] :] ).eq(0) ) target_state[moment_name] = expanded_t expanded_moments += 1 had_moment = True if had_moment: stateful_expanded_parameters += 1 target_states[parameter_id] = target_state return ( { **dict(source_payload), "state": target_states, "param_groups": [target_group], }, { "expandedMomentTensors": expanded_moments, "statefulExpandedParameters": stateful_expanded_parameters, "acceptedPrefixesExact": int(accepted_prefixes_exact), "newSuffixesZero": int(new_suffixes_zero), "newParameterCount": len(new_parameter_names), "newParameterStateEntryCount": 0, "sourceParameterCount": len(names), "targetParameterCount": len(target_names), }, ) def _expanded_training_proof( *, source_proof: Mapping[str, Any], source_training_page_ids: Sequence[int], new_training_page_ids: Sequence[int], target_family_root_count: int, ) -> dict[str, Any]: source_ids_value = source_proof.get("trainingPageIds") if source_ids_value is None: source_ids_value = source_proof.get("familyPageIds", ()) if not isinstance(source_ids_value, (list, tuple)): raise RuntimeError( "NoNE source checkpoint training-page proof identity differs" ) source_ids = list(source_ids_value) if source_ids != list(source_training_page_ids): raise RuntimeError( "NoNE source checkpoint training-page proof identity differs" ) source_count = len(source_ids) scalar_fields = ( "routeCounts", "gradientUpdateCounts", "gradientNorms", "parameterDeltaNorms", "gradientSignatures", ) if any( not isinstance(source_proof.get(name), list) or len(source_proof[name]) != source_count for name in scalar_fields ): raise RuntimeError("NoNE source checkpoint family proof is malformed") added_count = len(new_training_page_ids) zero_signatures = [[0.0] * 8 for _ in range(added_count)] return { "schema": "nnf.resynthesis.none_family_training_proof.v1", "familyRootCount": target_family_root_count, "trainingPageCount": source_count + added_count, "trainingPageIds": [*source_ids, *new_training_page_ids], # Compatibility alias for accepted v1 proof consumers. This list is # the full trainable-page denominator and can include objective # children after the family-root generation. "familyPageIds": [*source_ids, *new_training_page_ids], "routeCounts": [*source_proof["routeCounts"], *([0] * added_count)], "gradientUpdateCounts": [ *source_proof["gradientUpdateCounts"], *([0] * added_count), ], "gradientNorms": [ *source_proof["gradientNorms"], *([0.0] * added_count), ], "parameterDeltaNorms": [ *source_proof["parameterDeltaNorms"], *([0.0] * added_count), ], "gradientSignatures": [ *source_proof["gradientSignatures"], *zero_signatures, ], "routeCoverage": False, "gradientCoverage": False, "distinctGradients": False, "finite": source_proof.get("finite") is True, "promotionReady": False, } def _binding_matches_external_record( binding: NoNEGenerationBinding, record: object, ) -> bool: return ( isinstance(record, dict) and record == binding.external_record_boundary() ) def _canonical_page_object_bindings_boundary( bindings: Sequence[NoNEPageObjectBinding], ) -> tuple[NoNEPageObjectBinding, ...]: """Order candidate objects by immutable page identity before staging.""" ordered = tuple( sorted( bindings, key=lambda binding: int( binding.page_id_t.detach().cpu().long().reshape(()) ), ) ) ordered_page_ids = tuple( int(binding.page_id_t.detach().cpu().long().reshape(())) for binding in ordered ) if len(ordered_page_ids) != len(set(ordered_page_ids)): raise RuntimeError("NoNE candidate page-object identity is duplicated") return ordered def _same_unique_page_id_membership( observed: object, expected: Sequence[int], ) -> bool: """Require the same unique page IDs without assigning meaning to order.""" if not isinstance(observed, list) or any( not isinstance(page_id, int) or isinstance(page_id, bool) for page_id in observed ): return False observed_ids = set(observed) expected_ids = set(expected) return bool( len(observed) == len(observed_ids) and len(expected) == len(expected_ids) and observed_ids == expected_ids ) def _compact_page_object_provenance_matches_boundary( *, page_id: int, compact_object_sha256: object, target_object_row: Mapping[str, Any] | None, source_object_rows: Mapping[int, Mapping[str, Any]], source_updated_page_ids: frozenset[int], ) -> bool: """Accept an exact inherited object or a new compact-bank seed. Catalog rows retain the immutable compact-bank seed identity after a page is trained. Once the page belongs to the hash-bound parent manifest, that complete parent object row is authoritative regardless of which older generation first trained it. Only a page absent from the parent may use the original compact-bank object hash. """ if ( not isinstance(compact_object_sha256, str) or len(compact_object_sha256) != 64 or not isinstance(target_object_row, Mapping) ): return False del source_updated_page_ids source_object_row = source_object_rows.get(page_id) if isinstance(source_object_row, Mapping): return dict(target_object_row) == dict(source_object_row) return target_object_row.get("sha256") == compact_object_sha256 def _open_replica_topology_for_expansion_boundary( *, store: NoNEImmutablePageStore, session_id_t: torch.Tensor, source_external: dict[str, Any], requested_receipt_path: Path, seed_manifest_payload_sha256: str, accepted_binding_record: object, ) -> tuple[NoNEGenerationReplicaCoordinator, dict[str, Any]]: """Verify an exact or replacement replica topology at the I/O boundary. Replica placement is allowed to evolve independently from model weights, but only when both the checkpoint-owned source receipt and the requested receipt prove the same immutable seed generation. This keeps storage migration explicit without rewriting the accepted checkpoint sidecar. """ if source_external.get("schema") != ( "nnf.resynthesis.none_checkpoint_external_state.v2" ): raise RuntimeError("NoNE replicated source checkpoint schema differs") source_receipt_value = source_external.get("replicaReceiptPath") source_receipt_sha256 = source_external.get("replicaReceiptSha256") if ( not isinstance(source_receipt_value, str) or not source_receipt_value or not isinstance(source_receipt_sha256, str) or len(source_receipt_sha256) != 64 ): raise RuntimeError("NoNE source replica receipt binding is incomplete") source_receipt_path = Path(source_receipt_value).expanduser().resolve() if ( not source_receipt_path.is_file() or _file_sha256(source_receipt_path) != source_receipt_sha256 ): raise RuntimeError("NoNE source replica receipt bytes differ") source_coordinator = NoNEGenerationReplicaCoordinator.from_receipt_boundary( primary_store=store, session_id_t=session_id_t, receipt_path=source_receipt_path, seed_manifest_payload_sha256=seed_manifest_payload_sha256, ) source_roots = list(source_coordinator.store_roots_boundary) source_external_roots = source_external.get("replicaStoreRoots") if ( not isinstance(source_external_roots, list) or not source_external_roots or any( not isinstance(root, str) or not root for root in source_external_roots ) or len(set(source_external_roots)) != len(source_external_roots) or set(source_external_roots) != set(source_roots) ): raise RuntimeError("NoNE source replica root lineage differs") if not _binding_matches_external_record( store.current_generation_binding_boundary(), accepted_binding_record, ): raise RuntimeError("NoNE source replica reconciliation changed authority") requested_path = requested_receipt_path.expanduser().resolve() requested_sha256 = _file_sha256(requested_path) topology_changed = bool( requested_path != source_receipt_path or requested_sha256 != source_receipt_sha256 ) target_coordinator = source_coordinator if topology_changed: target_coordinator = ( NoNEGenerationReplicaCoordinator.from_receipt_boundary( primary_store=store, session_id_t=session_id_t, receipt_path=requested_path, seed_manifest_payload_sha256=( seed_manifest_payload_sha256 ), ) ) if not _binding_matches_external_record( store.current_generation_binding_boundary(), accepted_binding_record, ): raise RuntimeError( "NoNE replacement replica reconciliation changed authority" ) return target_coordinator, { "schema": "nnf.resynthesis.none_replica_topology_transition.v1", "changed": topology_changed, "seedManifestPayloadSha256": seed_manifest_payload_sha256, "sourceReceipt": { "path": str(source_receipt_path), "sha256": source_receipt_sha256, }, "targetReceipt": { "path": str(requested_path), "sha256": requested_sha256, }, "sourceStoreRoots": source_roots, "targetStoreRoots": list(target_coordinator.store_roots_boundary), "acceptedGenerationAuthorityRetained": True, "checkpointOrOptimizerBytesRewritten": False, } def migrate_v1_checkpoint_to_v2_seed( *, source_checkpoint: Path, source_optimizer: Path, growth_plan: Path, store_root: Path, receipt_path: Path, expected_checkpoint_sha256: str | None = None, expected_optimizer_sha256: str | None = None, ) -> NoNEV2SeedArtifacts: """Compose one authoritative v1 checkpoint into a non-promoted v2 seed.""" source = source_checkpoint.expanduser().resolve() optimizer_path = source_optimizer.expanduser().resolve() plan_path = growth_plan.expanduser().resolve() store = store_root.expanduser().resolve() receipt = receipt_path.expanduser().resolve() if not source.is_file() or not optimizer_path.is_file(): raise FileNotFoundError("authoritative v1 checkpoint pair is incomplete") if not plan_path.is_file(): raise FileNotFoundError("paged NoNE growth plan is absent") checkpoint_sha256 = _file_sha256(source) optimizer_sha256 = _file_sha256(optimizer_path) if ( expected_checkpoint_sha256 is not None and checkpoint_sha256 != expected_checkpoint_sha256 ): raise RuntimeError("authoritative v1 checkpoint SHA-256 differs") if ( expected_optimizer_sha256 is not None and optimizer_sha256 != expected_optimizer_sha256 ): raise RuntimeError("authoritative v1 optimizer SHA-256 differs") plan_sha256 = _file_sha256(plan_path) plan = json.loads(plan_path.read_text(encoding="utf-8")) if ( not isinstance(plan, dict) or plan.get("schema") != PAGED_GROWTH_PLAN_SCHEMA or not isinstance(plan.get("pagingPolicy"), dict) or plan["pagingPolicy"].get("acceptedGenerationPointer") is not True or plan["pagingPolicy"].get("storageBoundaryMayReroute") is not False ): raise RuntimeError("v2 seed requires the paged NoNE growth contract") roadmap = plan.get("functionalExpertFamilyRoadmap") if not isinstance(roadmap, list) or len(roadmap) < 25: raise RuntimeError("v2 seed requires at least 25 expert families") if any( not isinstance(row, dict) or not isinstance(row.get("family"), str) for row in roadmap ): raise RuntimeError("v2 functional expert roadmap is malformed") roadmap_family_ids = tuple(str(row["family"]) for row in roadmap) if len(set(roadmap_family_ids)) != len(roadmap_family_ids): raise RuntimeError("v2 functional expert roadmap contains duplicate families") raw_language_roadmap = plan.get("languageExpertPackRoadmap") language_roadmap_declared = raw_language_roadmap is not None if raw_language_roadmap is None: language_roadmap: list[dict[str, Any]] = [] elif not isinstance(raw_language_roadmap, list) or any( not isinstance(row, dict) or not isinstance(row.get("family"), str) or not str(row["family"]).startswith("language_") for row in raw_language_roadmap ): raise RuntimeError("v2 language expert-pack roadmap is malformed") else: language_roadmap = raw_language_roadmap raw_language_pack_count = plan.get("languageExpertPackCount") declared_language_pack_count = ( raw_language_pack_count if isinstance(raw_language_pack_count, int) and not isinstance(raw_language_pack_count, bool) else -1 ) language_family_ids = tuple( str(row["family"]) for row in language_roadmap ) if len(set(language_family_ids)) != len(language_family_ids): raise RuntimeError("v2 language expert-pack roadmap contains duplicates") validated_language_family_ids = ( _validated_language_expert_family_ids(plan, required=True) if language_roadmap_declared else () ) if language_roadmap_declared and ( language_family_ids != validated_language_family_ids or len(language_roadmap) < 60 or declared_language_pack_count != len(language_roadmap) or not set(language_family_ids).issubset(roadmap_family_ids) ): raise RuntimeError( "v2 seed requires all declared language packs in the functional roadmap" ) specialist_family_ids = _validated_science_specialist_family_ids( plan, roadmap_family_ids, required=plan.get("scienceSpecialistRoadmap") is not None, ) checkpoint = _load_checkpoint(source) lineage = checkpoint["lineage"] parameters = checkpoint["parameters"] buffers = checkpoint["buffers"] source_layers = int(lineage.get("scienceLayers", 0)) source_experts = int(lineage.get("scienceExperts", 0)) current_seed = plan.get("currentSeedGeometry") if ( source_layers < 1 or source_experts < 5 or not isinstance(current_seed, dict) or int(current_seed.get("scienceLayers", -1)) != source_layers or int(current_seed.get("scienceExperts", -1)) != source_experts ): raise RuntimeError("v2 growth plan does not describe the v1 seed") first_gate_name = "science_stack.science_layer_0.ffn_gate_up" first_gate = parameters.get(first_gate_name) if not isinstance(first_gate, torch.Tensor) or first_gate.ndim != 3: raise RuntimeError("v1 checkpoint has no native FFN experts") ffn_experts = int(first_gate.shape[0]) structural_experts = source_experts - ffn_experts if structural_experts < 1: raise RuntimeError("v1 structural expert geometry differs") optimizer_payload = torch.load( optimizer_path, map_location="cpu", mmap=True, weights_only=True, ) if not isinstance(optimizer_payload, dict): raise RuntimeError("authoritative v1 optimizer is invalid") optimizer_by_name, optimizer_group = _optimizer_by_name( optimizer_payload, set(parameters), source_layers, ) inherited: list[_SourcePage] = [] for layer_id in range(source_layers): for ffn_index in range(ffn_experts): inherited.append( _source_page( parameters=parameters, buffers=buffers, optimizer_by_name=optimizer_by_name, layer_id=layer_id, ffn_index=ffn_index, structural_experts=structural_experts, ) ) pages = list(inherited) page_rows: list[dict[str, Any]] = [] layer_catalog: dict[int, list[int]] = { layer_id: [] for layer_id in range(source_layers) } for page_id, page in enumerate(inherited): layer_catalog[page.layer_id].append(page_id) page_rows.append( { "pageId": page_id, "layerId": page.layer_id, "pageRole": "inherited_exact", "familyRoot": False, "state": "INHERITED_V1_EXACT", "family": None, "parentFamilyRootPageIds": [], "sourceFfnIndex": page.source_ffn_index, "sourceExpertSlot": page.source_expert_slot, "ffnMode": "legacy_exact", # PAGE-TRAINED-KNOWLEDGE BOUNDARY: this is the ONLY legitimate # way a brand-new page row can start with # ``trainedCapabilityClaimed=True`` -- the weights are # byte-identical copies of a pretrained parent page (state # INHERITED_V1_EXACT), so the page already carries real # trained knowledge by construction. Every OTHER new page # (transfer-seeded, zero-residual, freshly allocated) MUST # start at ``trainedCapabilityClaimed=False`` and only flip to # True after the trainer has applied real gradient updates. # Setting True at allocation time = "filling before training" # (the anti-pattern the user called out); the downstream # classifier in page_inventory.py will refuse to honour such a # claim and reclassify the page as empty. "trainedCapabilityClaimed": True, # Optional program-artifact provenance for a page row. ``None`` # preserves byte-identical catalog behavior; an # :class:`resynthesis.program_artifact.ProgramArtifact` dict may # be supplied by callers that materialize a page from a program # source. See ``ProgramArtifact.to_dict`` for the field shape. "programArtifact": None, } ) for family_index, raw_family in enumerate(roadmap): layer_id = family_index % source_layers source_page = inherited[ layer_id * ffn_experts + (family_index % ffn_experts) ] page_id = len(pages) pages.append(source_page) layer_catalog[layer_id].append(page_id) page_rows.append( { "pageId": page_id, "layerId": layer_id, "pageRole": "family_root", "familyRoot": True, "state": "TRANSFER_INITIALIZED_UNTRAINED", "family": str(raw_family["family"]), "parentFamilyRootPageIds": [], "sourceEvidenceGap": bool( raw_family.get("sourceEvidenceGap", True) ), "sourceFfnIndex": source_page.source_ffn_index, "sourceExpertSlot": source_page.source_expert_slot, "ffnMode": "legacy_transfer_seed", # PAGE-TRAINED-KNOWLEDGE BOUNDARY: family-root transfer seed. # state=TRANSFER_INITIALIZED_UNTRAINED -> this is an EMPTY # slot seeded from a parent page, NOT trained knowledge. It # MUST keep trainedCapabilityClaimed=False until the trainer # has applied real gradient updates to it. See the boundary # comment on the inherited_exact rows above. "trainedCapabilityClaimed": False, # Optional program-artifact provenance (see note above). "programArtifact": None, } ) page_ids = list(range(len(pages))) bundle = _assemble_bundle(pages, page_ids) if not torch.equal( bundle.weights.ffn_mode_t, torch.zeros_like(bundle.weights.ffn_mode_t), ): raise RuntimeError("v2 inherited page FFN mode is not legacy exact") hidden_size = int(bundle.weights.gate_t.shape[1]) expert_hidden_size = int(bundle.weights.gate_t.shape[2]) glyph_size = int(bundle.weights.glyph_down_t.shape[2]) router_size = int(bundle.weights.transfer_memory_t.shape[1]) action_size = int(lineage.get("acquisitionActionDim", 4)) if action_size < 1: raise RuntimeError("v2 action routing geometry is invalid") parent_lineage = lineage["parent"] parent_sha256 = str( parent_lineage.get("modelArtifactSha256", checkpoint_sha256) ) if len(parent_sha256) != 64: parent_sha256 = checkpoint_sha256 session_id_t = _session_identity_t(checkpoint_sha256, plan_sha256) resident_runtime_path = receipt.with_name( f"{receipt.stem}.resident_runtime.safetensors" ) resident_runtime_tensors = _resident_runtime_state( parameters=parameters, bundle=bundle, page_rows=page_rows, layer_catalog=layer_catalog, session_id_t=session_id_t, hidden_size=hidden_size, action_size=action_size, ) _save_resident_runtime_state( resident_runtime_path, resident_runtime_tensors, ) page_store = NoNEImmutablePageStore(store) accepted_t = page_store.begin_session(session_id_t) if not torch.equal(accepted_t, torch.zeros_like(accepted_t)): raise RuntimeError("v2 seed session already has an accepted generation") pointer = page_store.commit_generation( generation_t=accepted_t + torch.ones_like(accepted_t), updated_pages=bundle, components=_component_packet( checkpoint_sha256=checkpoint_sha256, optimizer_sha256=optimizer_sha256, growth_plan_sha256=plan_sha256, parent_sha256=parent_sha256, ), training_proven_page_ids_t=bundle.weights.page_ids_t[:len(inherited)].reshape(-1), ) page_catalog_path = receipt.with_name( f"{receipt.stem}.page_catalog.json" ) page_catalog_payload = { "schema": NONE_V2_PAGE_CATALOG_SCHEMA, "sourceCheckpointSha256": checkpoint_sha256, "growthPlanSha256": plan_sha256, "sessionId": session_id_t.tolist(), "acceptedGeneration": int(pointer["generation"]), "pageCount": len(page_rows), "inheritedPageCount": len(inherited), "familyRootPageCount": len(roadmap), "languagePackRootPageCount": len(language_roadmap), "scienceSpecialistRootPageCount": len( specialist_family_ids ), "logicalPageObjectives": int( plan.get("initialLogicalExpertPageObjectives", 0) ), "pageGeometry": { "hiddenSize": hidden_size, "expertHiddenSize": expert_hidden_size, "glyphSize": glyph_size, "routerSize": router_size, "actionSize": action_size, }, "layerCatalogPageIds": { str(layer_id): page_ids_for_layer for layer_id, page_ids_for_layer in layer_catalog.items() }, "pages": page_rows, } _atomic_json(page_catalog_path, page_catalog_payload) composition_path = receipt.with_name( f"{receipt.stem}.composition.json" ) composition_payload = { "schema": NONE_V2_COMPOSITION_SCHEMA, "sourceGeneration": "v1", "targetGeneration": "v2", "sourceCheckpoint": { "path": str(source), "sha256": checkpoint_sha256, }, "sourceOptimizer": { "path": str(optimizer_path), "sha256": optimizer_sha256, }, "pageStore": { "root": str(store), "sessionId": session_id_t.tolist(), "acceptedPointer": pointer, }, "pageCatalog": { "path": str(page_catalog_path), "sha256": _file_sha256(page_catalog_path), }, "residentRuntime": { "path": str(resident_runtime_path), "sha256": _file_sha256(resident_runtime_path), "schema": NONE_V2_RESIDENT_RUNTIME_SCHEMA, "initialization": ( "v1_role_intent_router_and_action_transfer" ), }, "residentCore": { "resynthesisParentRetained": True, "rboRetained": True, "fabricRetained": True, "vgeRetained": True, "mhcCompatibilityRetained": True, "sourceScienceLayers": source_layers, "sourceScienceExperts": source_experts, }, "additivePageRuntime": { "pageResidualInitialization": "compatibility_zero", "storageBoundaryMayReroute": False, "modelOwnedCatalogRouting": True, "oneGpuPagedResidency": True, }, "promotionEligible": False, "trainingAuthorityReceiptPath": str(receipt), } _atomic_json(composition_path, composition_payload) page_parameter_elements = sum( tensor.numel() for tensor in ( bundle.weights.gate_t, bundle.weights.up_t, bundle.weights.down_t, bundle.weights.glyph_down_t, bundle.weights.glyph_up_t, bundle.weights.translation_gate_t, bundle.weights.outcome_memory_t, bundle.weights.repair_memory_t, bundle.weights.transfer_memory_t, ) ) source_parameter_elements = sum( tensor.numel() for tensor in parameters.values() ) root_identity_count = source_experts + len(roadmap) checks = { "sourceCheckpointSha256Verified": ( expected_checkpoint_sha256 is None or checkpoint_sha256 == expected_checkpoint_sha256 ), "sourceOptimizerSha256Verified": ( expected_optimizer_sha256 is None or optimizer_sha256 == expected_optimizer_sha256 ), "exactResynthesisParentRetained": True, "sourceStateIdentityVerified": True, "v1CoreReferencedWithoutMutation": True, "allInheritedFfnWeightsCopiedExactly": all( torch.equal( bundle.weights.gate_t[index], parameters[ ( "science_stack.science_layer_" f"{page.layer_id}.ffn_gate_up" ) ][page.source_ffn_index], ) and torch.equal( bundle.weights.down_t[index], parameters[ ( "science_stack.science_layer_" f"{page.layer_id}.ffn_down" ) ][page.source_ffn_index], ) for index, page in enumerate(inherited) ), "allInheritedPagesUseLegacyExactMode": bool( torch.equal( bundle.weights.ffn_mode_t[: len(inherited)], torch.zeros_like( bundle.weights.ffn_mode_t[: len(inherited)] ), ) ), "familyRootsTransferInitialized": len(roadmap) >= 25, "declaredLanguagePacksTransferInitialized": ( not language_roadmap_declared or ( len(language_roadmap) >= 60 and set(language_family_ids).issubset( row["family"] for row in page_rows[len(inherited) :] ) ) ), "declaredScienceSpecialistsTransferInitialized": ( not specialist_family_ids or set(specialist_family_ids).issubset( row["family"] for row in page_rows[len(inherited) :] ) ), "familyRootsNotClaimedTrained": all( row["trainedCapabilityClaimed"] is False for row in page_rows[len(inherited) :] ), "rootIdentityCountAtLeast33": root_identity_count >= 33, "acceptedGenerationAdvancedAtomically": pointer["generation"] == 1, "sourceOptimizerPolicyRetained": all( key in optimizer_group for key in ("lr", "betas", "eps", "weight_decay") ), "residentRouterTransferredFromV1": bool( resident_runtime_tensors ), "residentRuntimeSerialized": ( resident_runtime_path.is_file() and resident_runtime_path.stat().st_size > 0 ), "promotionWithheld": True, } receipt_payload = { "schema": NONE_V2_SEED_MIGRATION_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(checks.values()), "sourceGeneration": "v1", "targetGeneration": "v2", "sourceCheckpoint": { "path": str(source), "sha256": checkpoint_sha256, "parameterElements": source_parameter_elements, "scienceLayers": source_layers, "scienceExperts": source_experts, }, "sourceOptimizer": { "path": str(optimizer_path), "sha256": optimizer_sha256, }, "growthPlan": { "path": str(plan_path), "sha256": plan_sha256, "functionalExpertFamilies": len(roadmap), "languageExpertPacks": len(language_roadmap), "scienceSpecialists": len(specialist_family_ids), "logicalExpertPageObjectives": int( plan.get("initialLogicalExpertPageObjectives", 0) ), }, "v2Seed": { "rootRouteIdentities": root_identity_count, "physicalPageObjects": len(page_rows), "inheritedExactPages": len(inherited), "transferInitializedFamilyRoots": len(roadmap), "transferInitializedLanguagePackRoots": len( language_roadmap ), "transferInitializedScienceSpecialistRoots": len( specialist_family_ids ), "serializedPageParameterElements": page_parameter_elements, "trainedValidatedNewFamilyExperts": 0, "trainingStarted": False, }, "artifacts": { "composition": { "path": str(composition_path), "sha256": _file_sha256(composition_path), }, "pageCatalog": { "path": str(page_catalog_path), "sha256": _file_sha256(page_catalog_path), }, "residentRuntime": { "path": str(resident_runtime_path), "sha256": _file_sha256(resident_runtime_path), "schema": NONE_V2_RESIDENT_RUNTIME_SCHEMA, }, "pageStoreRoot": str(store), "acceptedPointer": pointer, }, "checks": checks, "promotionEligible": False, "remainingProof": [ "clean_single_gpu_launch", "family_root_distinct_gradients", "model_owned_route_utilization", "rbo_fabric_vge_transfer", "heldout_generalization", "source_capability_retention", "cold_reload", "immutable_release_verification", ], } if receipt_payload["passed"] is not True: raise RuntimeError("v1-to-v2 seed migration checks did not pass") _atomic_json(receipt, receipt_payload) return NoNEV2SeedArtifacts( receipt_path=receipt, composition_path=composition_path, page_catalog_path=page_catalog_path, store_root=store, session_id_t=session_id_t, accepted_generation_t=torch.tensor( int(pointer["generation"]), dtype=torch.long, ), ) def _validated_retained_scale_admission_record( record: Mapping[str, Any] | None, *, source_checkpoint_path: Path, source_checkpoint_sha256: str, source_optimizer_path: Path, source_optimizer_sha256: str, source_external_path: Path, source_external_sha256: str, scale_cohort: NoNEScaleCohortPacket | None, compact_page_admission: CompactPageAdmissionPacket | None, ) -> dict[str, Any] | None: """Revalidate one append-only retained transaction before graph admission.""" if record is None: return None payload = dict(record) iteration = payload.get("iteration") transaction_id = payload.get("iterationTransactionId") if ( payload.get("schema") != RETAINED_SCALE_ADMISSION_AUTHORITY_SCHEMA or not isinstance(iteration, int) or isinstance(iteration, bool) or iteration < 0 or not isinstance(transaction_id, str) or len(transaction_id) != 64 or any(character not in "0123456789abcdef" for character in transaction_id) or payload.get("targetEnteredForward") is not False or payload.get("trainingEpochCompleteClaimed") is not False or payload.get("coldReloadPromotionClaimed") is not False or payload.get("newPagesTrainingClaimed") is not False or payload.get("promotionEligible") is not False ): raise RuntimeError("retained NoNE scale-admission authority differs") expected_sources = { "sourceCheckpoint": { "path": str(source_checkpoint_path), "sha256": source_checkpoint_sha256, }, "sourceOptimizer": { "path": str(source_optimizer_path), "sha256": source_optimizer_sha256, }, "sourceExternalState": { "path": str(source_external_path), "sha256": source_external_sha256, }, } if any(payload.get(name) != expected for name, expected in expected_sources.items()): raise RuntimeError("retained NoNE source artifact identity differs") cohort = payload.get("scaleCohort") expected_cohort = ( { "selectedFamilyRootPageIds": ( scale_cohort.selected_family_page_ids_t .detach() .cpu() .long() .tolist() ), "selectedPhysicalPageIds": ( compact_page_admission.selected_page_ids_t .detach() .cpu() .long() .tolist() ), "selectedPageCount": int(scale_cohort.selected_page_count_t), } if scale_cohort is not None and compact_page_admission is not None else None ) if cohort != expected_cohort: raise RuntimeError("retained NoNE model-owned scale cohort differs") recorded_artifacts: dict[str, tuple[Path, str]] = {} for name in ("precommit", "loopState", "knowledgeTransaction"): artifact = payload.get(name) if not isinstance(artifact, dict): raise RuntimeError(f"retained NoNE {name} identity is absent") path_value = artifact.get("path") sha256 = artifact.get("sha256") if ( not isinstance(path_value, str) or not path_value or not isinstance(sha256, str) or len(sha256) != 64 ): raise RuntimeError(f"retained NoNE {name} identity is malformed") path = Path(path_value).expanduser().resolve() # Precommit and loop state are live training control files and may # advance after this retained transaction commits. Their recorded # digests are fenced by the append-only ledger row below; only the # transaction-specific knowledge receipt remains immutable in place. if name == "knowledgeTransaction" and ( not path.is_file() or _file_sha256(path) != sha256 ): raise RuntimeError(f"retained NoNE {name} bytes differ") recorded_artifacts[name] = (path, sha256) _loop_state_path, _loop_state_sha256 = recorded_artifacts["loopState"] knowledge_path, knowledge_sha256 = recorded_artifacts[ "knowledgeTransaction" ] knowledge = json.loads(knowledge_path.read_text(encoding="utf-8")) none_scale_evidence = ( knowledge.get("noneScaleEvidence") if isinstance(knowledge, dict) else None ) evidence_sha256 = payload.get("noneScaleEvidenceSha256") if ( not isinstance(knowledge, dict) or knowledge.get("iteration") != iteration or knowledge.get("iterationTransactionId") != transaction_id or knowledge.get("targetEnteredForward") is not False or not isinstance(none_scale_evidence, dict) or not isinstance(evidence_sha256, str) or hashlib.sha256( json.dumps( none_scale_evidence, sort_keys=True, separators=(",", ":"), ).encode("utf-8") ).hexdigest() != evidence_sha256 ): raise RuntimeError("retained NoNE state or knowledge identity differs") ledger = payload.get("executionLedger") if not isinstance(ledger, dict): raise RuntimeError("retained NoNE execution-ledger identity is absent") ledger_path_value = ledger.get("path") commit_row_sha256 = ledger.get("commitRowSha256") if ( not isinstance(ledger_path_value, str) or not ledger_path_value or not isinstance(commit_row_sha256, str) or len(commit_row_sha256) != 64 ): raise RuntimeError("retained NoNE execution-ledger identity is malformed") ledger_path = Path(ledger_path_value).expanduser().resolve() matches = 0 if not ledger_path.is_file(): raise RuntimeError("retained NoNE execution ledger is missing") with ledger_path.open(encoding="utf-8") as handle: for line in handle: if not line.strip(): continue row = json.loads(line) if not isinstance(row, dict): raise RuntimeError("retained NoNE execution-ledger row is malformed") row_sha256 = hashlib.sha256( json.dumps(row, sort_keys=True, separators=(",", ":")).encode( "utf-8" ) ).hexdigest() if row_sha256 != commit_row_sha256: continue if ( row.get("schema") != "nnf.resynthesis.iteration_ledger_transaction.commit.v1" or row.get("iteration") != iteration or row.get("iterationTransactionId") != transaction_id or not isinstance(row.get("loopStateSha256"), str) or len(str(row["loopStateSha256"])) != 64 or Path( str(row.get("fullPromptKnowledgeTransactionReceipt", "")) ).resolve() != knowledge_path or row.get("fullPromptKnowledgeTransactionReceiptSha256") != knowledge_sha256 ): raise RuntimeError("retained NoNE ledger commit identity differs") matches += 1 if matches != 1: raise RuntimeError("retained NoNE ledger commit is not unique") return payload def _validated_graph_adapted_page_update_authority_boundary( *, retained_transaction_record: Mapping[str, Any], source_checkpoint_path: Path, source_checkpoint_sha256: str, source_optimizer_path: Path, source_optimizer_sha256: str, source_external_path: Path, source_external_sha256: str, source_external: Mapping[str, Any], ) -> bool: """Prove that a retained transaction was rebound to an untrained graph.""" adaptation_binding = retained_transaction_record.get( "graphAdaptationReceipt" ) if adaptation_binding is None: return False if not isinstance(adaptation_binding, Mapping): raise RuntimeError( "graph-adapted page-update receipt identity differs" ) receipt_path_value = adaptation_binding.get("path") receipt_sha256 = adaptation_binding.get("sha256") if ( not isinstance(receipt_path_value, str) or not receipt_path_value or not isinstance(receipt_sha256, str) or len(receipt_sha256) != 64 ): raise RuntimeError( "graph-adapted page-update receipt identity differs" ) receipt_path = Path(receipt_path_value).expanduser().resolve() if ( not receipt_path.is_file() or _file_sha256(receipt_path) != receipt_sha256 ): raise RuntimeError( "graph-adapted page-update receipt identity differs" ) receipt = json.loads(receipt_path.read_text(encoding="utf-8")) receipt_source = receipt.get("source") if isinstance(receipt, dict) else None receipt_target = receipt.get("target") if isinstance(receipt, dict) else None checks = receipt.get("checks") if isinstance(receipt, dict) else None if ( not isinstance(receipt, dict) or receipt.get("schema") != NONE_ACCEPTED_GRAPH_ADAPTATION_SCHEMA or receipt.get("passed") is not True or receipt.get("status") != "GRAPH_ADAPTED_UNTRAINED" or receipt.get("trainingSteps") != 0 or receipt.get("trainedValidatedNewGraphElements") != 0 or receipt.get("promotionEligible") is not False or not isinstance(receipt_source, dict) or not isinstance(receipt_target, dict) or not isinstance(checks, dict) or not checks or any(value is not True for value in checks.values()) ): raise RuntimeError( "graph-adapted page-update receipt authority differs" ) prior_retained = receipt_source.get("retainedTransactionAuthority") if not isinstance(prior_retained, dict): raise RuntimeError( "graph-adapted page-update transaction identity differs" ) adapted_fields = frozenset( { "sourceCheckpoint", "sourceOptimizer", "sourceExternalState", "scaleCohort", "graphAdaptationReceipt", } ) prior_transaction = { name: value for name, value in prior_retained.items() if name not in adapted_fields } current_transaction = { name: value for name, value in retained_transaction_record.items() if name not in adapted_fields } if ( prior_retained.get("graphAdaptationReceipt") is not None or current_transaction != prior_transaction ): raise RuntimeError( "graph-adapted page-update transaction identity differs" ) resolved_checkpoint = source_checkpoint_path.expanduser().resolve() resolved_optimizer = source_optimizer_path.expanduser().resolve() resolved_external = source_external_path.expanduser().resolve() expected_checkpoint = { "path": str(resolved_checkpoint), "sha256": source_checkpoint_sha256, } expected_optimizer = { "path": str(resolved_optimizer), "sha256": source_optimizer_sha256, } expected_external = { "path": str(resolved_external), "sha256": source_external_sha256, } generation_binding = source_external.get("generationBinding") accepted_generation = ( generation_binding.get("generation") if isinstance(generation_binding, Mapping) else None ) manifest_payload_sha256 = ( generation_binding.get("manifestPayloadSha256") if isinstance(generation_binding, Mapping) else None ) if ( retained_transaction_record.get("sourceCheckpoint") != expected_checkpoint or retained_transaction_record.get("sourceOptimizer") != expected_optimizer or retained_transaction_record.get("sourceExternalState") != expected_external or receipt_target.get("checkpoint") != expected_checkpoint or receipt_target.get("optimizer") != expected_optimizer or receipt_target.get("externalState") != expected_external or not isinstance(accepted_generation, int) or isinstance(accepted_generation, bool) or accepted_generation < 1 or not isinstance(manifest_payload_sha256, str) or len(manifest_payload_sha256) != 64 or receipt_target.get("acceptedGeneration") != accepted_generation or receipt_target.get("manifestPayloadSha256") != manifest_payload_sha256 or source_external.get("candidatePageUpdate") is not False or Path( str(source_external.get("graphAdaptationReceiptPath", "")) ).expanduser().resolve() != receipt_path ): raise RuntimeError( "graph-adapted page-update target identity differs" ) return True def _validated_retained_page_only_successor_record_boundary( *, training_authority: NoNETrainingAuthority, loop_state_path: Path, source_loop: Mapping[str, Any], ) -> dict[str, Any] | None: """Prove one untrained page-only hop above a retained transaction.""" composition_path = training_authority.composition_path migration_path = training_authority.migration_receipt_path current_external_path = training_authority.external_state_path current_external_sha256 = training_authority.external_state_sha256 if ( composition_path is None or migration_path is None or current_external_path is None or current_external_sha256 is None ): return None composition_file = composition_path.expanduser().resolve() migration_file = migration_path.expanduser().resolve() try: composition = json.loads(composition_file.read_text(encoding="utf-8")) migration = json.loads(migration_file.read_text(encoding="utf-8")) except (OSError, TypeError, ValueError, json.JSONDecodeError): return None if not isinstance(composition, dict) or not isinstance(migration, dict): return None retained_value = migration.get("sourceRetainedTransaction") if retained_value is None: return None if not isinstance(retained_value, dict): raise RuntimeError( "retained NoNE page-only successor transaction is malformed" ) def exact_artifact( record: object, path: Path, sha256: str, ) -> bool: return bool( isinstance(record, dict) and Path(str(record.get("path", ""))).expanduser().resolve() == path and record.get("sha256") == sha256 and path.is_file() and _file_sha256(path) == sha256 ) artifacts = migration.get("artifacts") expansion = migration.get("expansion") generation = migration.get("generationBinding") accepted_pointer = migration.get("acceptedPointer") catalog_expansion = composition.get("catalogExpansion") page_store = composition.get("pageStore") source_external_record = migration.get("sourceExternalState") if not isinstance(source_external_record, dict): raise RuntimeError( "retained NoNE page-only successor source sidecar is absent" ) source_external_value = source_external_record.get("path") source_external_sha256 = source_external_record.get("sha256") if ( not isinstance(source_external_value, str) or not source_external_value or not isinstance(source_external_sha256, str) or len(source_external_sha256) != 64 ): raise RuntimeError( "retained NoNE page-only successor source sidecar is malformed" ) source_external_path = Path(source_external_value).expanduser().resolve() if not isinstance(artifacts, dict): raise RuntimeError( "retained NoNE page-only successor artifacts are absent" ) page_catalog_record = artifacts.get("pageCatalog") if not isinstance(page_catalog_record, dict): raise RuntimeError( "retained NoNE page-only successor catalog is absent" ) page_catalog_value = page_catalog_record.get("path") page_catalog_sha256 = page_catalog_record.get("sha256") if ( not isinstance(page_catalog_value, str) or not page_catalog_value or not isinstance(page_catalog_sha256, str) or len(page_catalog_sha256) != 64 ): raise RuntimeError( "retained NoNE page-only successor catalog is malformed" ) page_catalog_path = Path(page_catalog_value).expanduser().resolve() if ( migration.get("schema") not in NONE_V2_PLUS_EXPANSION_RECEIPT_SCHEMAS or migration.get("passed") is not True or composition.get("schema") != NONE_V2_PLUS_COMPOSITION_SCHEMA or not isinstance(expansion, dict) or not isinstance(generation, dict) or not isinstance(accepted_pointer, dict) or not isinstance(catalog_expansion, dict) or not isinstance(page_store, dict) or not exact_artifact( artifacts.get("composition"), composition_file, _file_sha256(composition_file), ) or not exact_artifact( artifacts.get("checkpoint"), training_authority.checkpoint_path.resolve(), training_authority.checkpoint_sha256, ) or not exact_artifact( artifacts.get("optimizer"), training_authority.optimizer_path.resolve(), training_authority.optimizer_sha256, ) or not exact_artifact( artifacts.get("externalState"), current_external_path.resolve(), current_external_sha256, ) or not exact_artifact( migration.get("sourceAcceptedCheckpoint"), training_authority.continuation_checkpoint_path.resolve(), training_authority.continuation_checkpoint_sha256, ) or not exact_artifact( migration.get("sourceAcceptedOptimizer"), training_authority.continuation_optimizer_path.resolve(), training_authority.continuation_optimizer_sha256, ) or not exact_artifact( source_external_record, source_external_path, source_external_sha256, ) or not exact_artifact( page_catalog_record, page_catalog_path, page_catalog_sha256, ) ): raise RuntimeError( "retained NoNE page-only successor artifact identity differs" ) parent_generation = generation.get("parentGeneration") accepted_generation = generation.get("generation") source_layers = expansion.get("sourcePagedRuntimeLayerCount") target_layers = expansion.get("targetPagedRuntimeLayerCount") source_pages = expansion.get("sourcePageObjects") target_pages = expansion.get("targetPageObjects") updated_page_ids = generation.get("updatedPageIds") physical_admission = migration.get("physicalAdmission") source_layer_ids = catalog_expansion.get("sourcePagedRuntimeLayerIds") target_layer_ids = catalog_expansion.get("targetPagedRuntimeLayerIds") if ( not isinstance(parent_generation, int) or isinstance(parent_generation, bool) or not isinstance(accepted_generation, int) or isinstance(accepted_generation, bool) or accepted_generation != parent_generation + 1 or accepted_generation != training_authority.accepted_generation or generation.get("manifestPayloadSha256") != training_authority.accepted_manifest_payload_sha256 or accepted_pointer.get("generation") != accepted_generation or accepted_pointer.get("manifestPayloadSha256") != training_authority.accepted_manifest_payload_sha256 or expansion.get("parentGeneration") != parent_generation or expansion.get("acceptedGeneration") != accepted_generation or catalog_expansion.get("parentAcceptedGeneration") != parent_generation or catalog_expansion.get("parentAcceptedCheckpointSha256") != training_authority.continuation_checkpoint_sha256 or catalog_expansion.get("parentAcceptedOptimizerSha256") != training_authority.continuation_optimizer_sha256 or not isinstance(source_layers, int) or isinstance(source_layers, bool) or target_layers != source_layers or expansion.get("addedTransferInitializedRuntimeLayerCount") != 0 or not isinstance(source_layer_ids, list) or source_layer_ids != target_layer_ids or len(source_layer_ids) != source_layers or not isinstance(source_pages, int) or isinstance(source_pages, bool) or not isinstance(target_pages, int) or isinstance(target_pages, bool) or target_pages <= source_pages or not isinstance(updated_page_ids, list) or expansion.get("addedPageObjects") != target_pages - source_pages or expansion.get("trainingStartedForNewRoots") is not False or expansion.get("trainedValidatedNewFamilyExperts") != 0 or expansion.get("compactBankCapacityClaimedTrained") is not False or not isinstance(physical_admission, dict) or physical_admission.get("acceptedPointerCommitted") is not True or physical_admission.get("newPagesExplicitlyUntrained") is not True or physical_admission.get("structuralChecksPassed") is not True or migration.get("trainingClaimed") is not False or migration.get("promotionEligible") is not False or migration.get("coldReloadClaimed") is not False or migration.get("heldoutClaimed") is not False ): raise RuntimeError( "retained NoNE page-only successor topology differs" ) store_root_value = page_store.get("root") session_key = accepted_pointer.get("sessionKey") manifest_value = accepted_pointer.get("manifest") manifest_sha256 = accepted_pointer.get("manifestSha256") if ( not isinstance(store_root_value, str) or not store_root_value or not isinstance(session_key, str) or not session_key or not isinstance(manifest_value, str) or not manifest_value or not isinstance(manifest_sha256, str) or len(manifest_sha256) != 64 ): raise RuntimeError( "retained NoNE page-only successor manifest is malformed" ) manifest_path = ( Path(store_root_value).expanduser().resolve() / "sessions" / session_key / manifest_value ).resolve() if not manifest_path.is_file() or _file_sha256(manifest_path) != manifest_sha256: raise RuntimeError( "retained NoNE page-only successor manifest identity differs" ) manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if ( not isinstance(manifest, dict) or not page_generation_schema_supported_boundary( manifest.get("schema") ) or manifest.get("generation") != accepted_generation or manifest.get("parentGeneration") != parent_generation or manifest.get("manifestPayloadSha256") != training_authority.accepted_manifest_payload_sha256 or manifest.get("parentManifestPayloadSha256") != source_loop.get("workingNoneManifestPayloadSha256") or manifest.get("updatedPageIds") != updated_page_ids or manifest.get("pageCount") != target_pages ): raise RuntimeError( "retained NoNE page-only successor parent manifest differs" ) cohort = retained_value.get("scaleCohort") if not isinstance(cohort, dict) or set(cohort) != { "selectedFamilyRootPageIds", "selectedPhysicalPageIds", "selectedPageCount", }: raise RuntimeError( "retained NoNE page-only successor scale cohort is malformed" ) selected_family_ids = cohort.get("selectedFamilyRootPageIds") selected_page_ids = cohort.get("selectedPhysicalPageIds") selected_count = cohort.get("selectedPageCount") if ( not isinstance(selected_family_ids, list) or not isinstance(selected_page_ids, list) or not isinstance(selected_count, int) or isinstance(selected_count, bool) or selected_count != target_pages - source_pages or len(selected_family_ids) != selected_count or len(selected_page_ids) != selected_count or any( not isinstance(value, int) or isinstance(value, bool) for value in (*selected_family_ids, *selected_page_ids) ) or len(set(selected_page_ids)) != selected_count or selected_page_ids != updated_page_ids ): raise RuntimeError( "retained NoNE page-only successor scale cohort differs" ) page_catalog = json.loads(page_catalog_path.read_text(encoding="utf-8")) page_rows = page_catalog.get("pages") if isinstance(page_catalog, dict) else None if ( not isinstance(page_catalog, dict) or page_catalog.get("schema") != NONE_V2_PLUS_PAGE_CATALOG_SCHEMA or page_catalog.get("acceptedGenerationAtExpansionStart") != parent_generation or page_catalog.get("parentAcceptedCheckpointSha256") != training_authority.continuation_checkpoint_sha256 or page_catalog.get("pageCount") != target_pages or not isinstance(page_rows, list) or len(page_rows) != target_pages ): raise RuntimeError( "retained NoNE page-only successor page catalog differs" ) source_rows = page_rows[:source_pages] added_rows = page_rows[source_pages:] source_page_ids = { row.get("pageId") for row in source_rows if isinstance(row, dict) } if ( len(source_page_ids) != source_pages or any(page_id not in source_page_ids for page_id in selected_family_ids) or [row.get("pageId") for row in added_rows if isinstance(row, dict)] != selected_page_ids or any( not isinstance(row, dict) or row.get("trainedCapabilityClaimed") is not False or not str(row.get("state", "")).endswith("UNTRAINED") for row in added_rows ) ): raise RuntimeError( "retained NoNE page-only successor untrained catalog differs" ) retained_core = dict(retained_value) retained_core.pop("scaleCohort", None) validated_core = _validated_retained_scale_admission_record( retained_core, source_checkpoint_path=( training_authority.continuation_checkpoint_path.resolve() ), source_checkpoint_sha256=( training_authority.continuation_checkpoint_sha256 ), source_optimizer_path=( training_authority.continuation_optimizer_path.resolve() ), source_optimizer_sha256=( training_authority.continuation_optimizer_sha256 ), source_external_path=source_external_path, source_external_sha256=source_external_sha256, scale_cohort=None, compact_page_admission=None, ) loop_record = retained_value.get("loopState") if ( validated_core is None or not isinstance(loop_record, dict) or Path(str(loop_record.get("path", ""))).expanduser().resolve() != loop_state_path.expanduser().resolve() or loop_record.get("sha256") != _file_sha256(loop_state_path) or source_loop.get("iteration") != retained_value.get("iteration") or source_loop.get("lastCommittedLedgerTransactionId") != retained_value.get("iterationTransactionId") or Path(str(source_loop.get("workingCheckpoint", ""))).resolve() != training_authority.continuation_checkpoint_path.resolve() or source_loop.get("workingNoneGeneration") != parent_generation ): raise RuntimeError( "retained NoNE page-only successor transaction frontier differs" ) return dict(retained_value) def _retained_page_only_validation_authority_boundary( training_authority: NoNETrainingAuthority, ) -> NoNETrainingAuthority: """Return the exact page-only ancestor beneath one proven branch union. A retained scale admission may be followed by a page-only training-branch union before the next offline graph adaptation. The union advances only the immutable page generation; its graph checkpoint, optimizer, external sidecar, composition, and migration receipt remain the accepted ancestor's exact artifacts. Validate that one-hop descendant with the same strict manifest, cumulative-training, graph-authority, and replica proof used by live reload, then validate the retained page-only hop against its recorded generation rather than incorrectly comparing the old receipt to the newer live pointer. """ composition_path = training_authority.composition_path migration_path = training_authority.migration_receipt_path if composition_path is None or migration_path is None: return training_authority composition_file = composition_path.expanduser().resolve() migration_file = migration_path.expanduser().resolve() composition = json.loads(composition_file.read_text(encoding="utf-8")) migration = json.loads(migration_file.read_text(encoding="utf-8")) receipt_generation = ( migration.get("generationBinding") if isinstance(migration, dict) else None ) if not isinstance(receipt_generation, dict): raise RuntimeError("retained NoNE generation authority is absent") ancestor_generation = receipt_generation.get("generation") ancestor_manifest_payload_sha256 = receipt_generation.get( "manifestPayloadSha256" ) if ( not isinstance(ancestor_generation, int) or isinstance(ancestor_generation, bool) or not isinstance(ancestor_manifest_payload_sha256, str) or len(ancestor_manifest_payload_sha256) != 64 ): raise RuntimeError("retained NoNE generation authority is malformed") if ancestor_generation == training_authority.accepted_generation: if ( ancestor_manifest_payload_sha256 != training_authority.accepted_manifest_payload_sha256 ): raise RuntimeError("retained NoNE generation authority differs") return training_authority page_store = ( composition.get("pageStore") if isinstance(composition, dict) else None ) session_values = ( page_store.get("sessionId") if isinstance(page_store, dict) else None ) store_root_value = ( page_store.get("root") if isinstance(page_store, dict) else None ) if ( not isinstance(session_values, list) or len(session_values) != 4 or not all( isinstance(value, int) and not isinstance(value, bool) for value in session_values ) or not isinstance(store_root_value, str) or not store_root_value ): raise RuntimeError("retained NoNE page-store authority differs") store = NoNEImmutablePageStore( Path(store_root_value).expanduser().resolve() ) store.begin_session(torch.tensor(session_values, dtype=torch.long)) binding = store.current_generation_binding_boundary() graph = store.current_graph_authority_boundary() current_record = binding.external_record_boundary() if ( graph is None or current_record.get("generation") != training_authority.accepted_generation or current_record.get("manifestPayloadSha256") != training_authority.accepted_manifest_payload_sha256 or not _validated_training_branch_union_graph_descendant_boundary( store=store, binding=binding, graph_record=graph.external_record_boundary(), receipt_value=migration, receipt_generation=receipt_generation, ) ): raise RuntimeError("retained NoNE branch-union authority differs") return replace( training_authority, accepted_generation=ancestor_generation, accepted_manifest_payload_sha256=ancestor_manifest_payload_sha256, ) def _retained_graph_adaptation_authority_boundary( *, training_authority: NoNETrainingAuthority, precommit_path: Path, loop_state_path: Path, execution_ledger_path: Path, ) -> _RetainedGraphAdaptationAuthority: """Bind graph growth to one exact target-free retained transaction.""" precommit_file = precommit_path.expanduser().resolve() loop_file = loop_state_path.expanduser().resolve() ledger_file = execution_ledger_path.expanduser().resolve() if ( training_authority.external_state_path is None or training_authority.external_state_sha256 is None or not training_authority.checkpoint_includes_paged_runtime or not all( path.is_file() for path in (precommit_file, loop_file, ledger_file) ) ): raise RuntimeError( "retained NoNE graph-adaptation authority is incomplete" ) precommit = json.loads(precommit_file.read_text(encoding="utf-8")) loop_state = json.loads(loop_file.read_text(encoding="utf-8")) if not isinstance(precommit, dict) or not isinstance(loop_state, dict): raise RuntimeError( "retained NoNE graph-adaptation control state is malformed" ) schedule = precommit.get("trainingSchedule") state_lock_value = precommit.get("trainingStateLockPath") working_checkpoint_value = loop_state.get("workingCheckpoint") iteration = loop_state.get("iteration") training_cursor = loop_state.get("trainingBatchCursor") page_only_validation_authority = ( _retained_page_only_validation_authority_boundary( training_authority ) ) retained_page_only_successor = ( _validated_retained_page_only_successor_record_boundary( training_authority=page_only_validation_authority, loop_state_path=loop_file, source_loop=loop_state, ) ) working_checkpoint = ( Path(working_checkpoint_value).expanduser().resolve() if isinstance(working_checkpoint_value, str) else None ) direct_frontier = bool( working_checkpoint == training_authority.checkpoint_path.resolve() and loop_state.get("workingNoneGeneration") == training_authority.accepted_generation and loop_state.get("workingNoneManifestPayloadSha256") == training_authority.accepted_manifest_payload_sha256 ) retained_page_only_frontier = bool( retained_page_only_successor is not None and working_checkpoint == training_authority.continuation_checkpoint_path.resolve() ) if ( precommit.get("schema") != "nnf.resynthesis.additive_learning_precommit.v2" or not isinstance(schedule, dict) or schedule.get("schema") != "nnf.resynthesis.training_schedule.jsonl_indexed_complete.v1" or schedule.get("trainOnlyConstruction") is not True or schedule.get("targetEnteredForward") is not False or schedule.get("validationOrHeldoutObserved") is not False or schedule.get("modelScoresObserved") is not False or not isinstance(state_lock_value, str) or Path(state_lock_value).expanduser().resolve().parent != loop_file.parent or not isinstance(working_checkpoint_value, str) or not (direct_frontier or retained_page_only_frontier) or training_authority.optimizer_path.resolve() != training_authority.checkpoint_path.with_suffix( ".optimizer.pt" ).resolve() or training_authority.external_state_path.resolve() != training_authority.checkpoint_path.with_suffix( ".none.json" ).resolve() or not isinstance(iteration, int) or isinstance(iteration, bool) or iteration < 1 or not isinstance(training_cursor, int) or isinstance(training_cursor, bool) or training_cursor < 1 ): raise RuntimeError( "retained NoNE graph-adaptation frontier differs" ) ledger_rows: list[dict[str, Any]] = [] with ledger_file.open(encoding="utf-8") as handle: for line in handle: if not line.strip(): continue row = json.loads(line) if not isinstance(row, dict): raise RuntimeError( "retained NoNE graph-adaptation ledger row is malformed" ) ledger_rows.append(row) if not ledger_rows: raise RuntimeError( "retained NoNE graph-adaptation ledger is empty" ) terminal_row = ledger_rows[-1] transaction_id = terminal_row.get("iterationTransactionId") loop_state_sha256 = _file_sha256(loop_file) knowledge_path_value = terminal_row.get( "fullPromptKnowledgeTransactionReceipt" ) knowledge_sha256 = terminal_row.get( "fullPromptKnowledgeTransactionReceiptSha256" ) if ( terminal_row.get("schema") != "nnf.resynthesis.iteration_ledger_transaction.commit.v1" or terminal_row.get("stateIsCommitAuthority") is not True or terminal_row.get("iteration") != iteration or terminal_row.get("loopStateIteration") != iteration or terminal_row.get("loopStateSha256") != loop_state_sha256 or not isinstance(transaction_id, str) or len(transaction_id) != 64 or any( character not in "0123456789abcdef" for character in transaction_id ) or not isinstance(knowledge_path_value, str) or not knowledge_path_value or not isinstance(knowledge_sha256, str) or len(knowledge_sha256) != 64 ): raise RuntimeError( "retained NoNE graph-adaptation terminal commit differs" ) knowledge_path = Path(knowledge_path_value).expanduser().resolve() if ( not knowledge_path.is_file() or _file_sha256(knowledge_path) != knowledge_sha256 ): raise RuntimeError( "retained NoNE graph-adaptation knowledge bytes differ" ) knowledge = json.loads(knowledge_path.read_text(encoding="utf-8")) training_data = ( knowledge.get("trainingDataReceipt") if isinstance(knowledge, dict) else None ) schedule_window = ( training_data.get("scheduleWindow") if isinstance(training_data, dict) else None ) none_scale_evidence = ( knowledge.get("noneScaleEvidence") if isinstance(knowledge, dict) else None ) if ( not isinstance(knowledge, dict) or knowledge.get("schema") != ( "nnf.resynthesis." "full_prompt_knowledge_before_and_after_each_adaptive_rollback_transaction.v1" ) or knowledge.get("iteration") != iteration or knowledge.get("iterationTransactionId") != transaction_id or knowledge.get("retained") is not True or knowledge.get("candidateBacktracked") is not False or knowledge.get("trainingCursorAdvanced") is not True or knowledge.get("trainingCursorEnd") != training_cursor or knowledge.get("targetEnteredForward") is not False or Path(str(knowledge.get("stateCommitAuthority", ""))).resolve() != loop_file or not isinstance(training_data, dict) or training_data.get("schema") != "nnf.resynthesis.training_data_receipt.v2" or training_data.get("gradientUpdateRetained") is not True or training_data.get("finiteOptimizerUpdateVerified") is not True or training_data.get("selectionValidationExecuted") is not False or training_data.get("targetEnteredForward") is not False or training_data.get("targetValuesRecorded") is not False or training_data.get("promotionEligible") is not False or not isinstance(schedule_window, dict) or schedule_window.get("scheduleSha256") != schedule.get("activeScheduleSha256") or schedule_window.get("sourceSha256") != precommit.get("trainSha256") or schedule_window.get("targetValuesRecorded") is not False or schedule_window.get("validationRowsObserved") != 0 or not isinstance(none_scale_evidence, dict) ): raise RuntimeError( "retained NoNE graph-adaptation knowledge authority differs" ) commit_row_sha256 = hashlib.sha256( json.dumps( terminal_row, sort_keys=True, separators=(",", ":"), ).encode("utf-8") ).hexdigest() none_scale_sha256 = hashlib.sha256( json.dumps( none_scale_evidence, sort_keys=True, separators=(",", ":"), ).encode("utf-8") ).hexdigest() precommit_sha256 = _file_sha256(precommit_file) retained_source_checkpoint_path = training_authority.checkpoint_path.resolve() retained_source_checkpoint_sha256 = training_authority.checkpoint_sha256 retained_source_optimizer_path = training_authority.optimizer_path.resolve() retained_source_optimizer_sha256 = training_authority.optimizer_sha256 retained_source_external_path = training_authority.external_state_path.resolve() retained_source_external_sha256 = training_authority.external_state_sha256 validated: dict[str, Any] | None if retained_page_only_successor is not None: retained_sources: dict[str, tuple[Path, str]] = {} for name in ( "sourceCheckpoint", "sourceOptimizer", "sourceExternalState", ): record = retained_page_only_successor.get(name) if not isinstance(record, dict): raise RuntimeError( f"retained NoNE page-only successor {name} is absent" ) path_value = record.get("path") sha256 = record.get("sha256") if ( not isinstance(path_value, str) or not path_value or not isinstance(sha256, str) or len(sha256) != 64 ): raise RuntimeError( f"retained NoNE page-only successor {name} is malformed" ) retained_sources[name] = ( Path(path_value).expanduser().resolve(), sha256, ) ( retained_source_checkpoint_path, retained_source_checkpoint_sha256, ) = retained_sources["sourceCheckpoint"] ( retained_source_optimizer_path, retained_source_optimizer_sha256, ) = retained_sources["sourceOptimizer"] ( retained_source_external_path, retained_source_external_sha256, ) = retained_sources["sourceExternalState"] retained_record: dict[str, Any] = { "schema": RETAINED_SCALE_ADMISSION_AUTHORITY_SCHEMA, "iteration": iteration, "iterationTransactionId": transaction_id, "precommit": { "path": str(precommit_file), "sha256": precommit_sha256, }, "loopState": { "path": str(loop_file), "sha256": loop_state_sha256, }, "executionLedger": { "path": str(ledger_file), "commitRowSha256": commit_row_sha256, }, "knowledgeTransaction": { "path": str(knowledge_path), "sha256": knowledge_sha256, }, "sourceCheckpoint": { "path": str(retained_source_checkpoint_path), "sha256": retained_source_checkpoint_sha256, }, "sourceOptimizer": { "path": str(retained_source_optimizer_path), "sha256": retained_source_optimizer_sha256, }, "sourceExternalState": { "path": str(retained_source_external_path), "sha256": retained_source_external_sha256, }, "noneScaleEvidenceSha256": none_scale_sha256, "targetEnteredForward": False, "fullPromptKnowledgeProofClaimed": ( knowledge.get("fullPromptKnowledgeProofPassed") is True ), "trainingOnlyGradientRetention": True, "trainingEpochCompleteClaimed": False, "coldReloadPromotionClaimed": False, "newPagesTrainingClaimed": False, "promotionEligible": False, } if retained_page_only_successor is not None: if ( retained_page_only_successor.get("graphAdaptationReceipt") is not None or any( retained_page_only_successor.get(name) != value for name, value in retained_record.items() ) ): raise RuntimeError( "retained NoNE page-only successor transaction identity differs" ) validated = dict(retained_page_only_successor) else: validated = _validated_retained_scale_admission_record( retained_record, source_checkpoint_path=retained_source_checkpoint_path, source_checkpoint_sha256=retained_source_checkpoint_sha256, source_optimizer_path=retained_source_optimizer_path, source_optimizer_sha256=retained_source_optimizer_sha256, source_external_path=retained_source_external_path, source_external_sha256=retained_source_external_sha256, scale_cohort=None, compact_page_admission=None, ) if validated is None: raise RuntimeError( "retained NoNE graph-adaptation authority disappeared" ) return _RetainedGraphAdaptationAuthority( precommit_path=precommit_file, precommit_sha256=precommit_sha256, loop_state_path=loop_file, loop_state_sha256=loop_state_sha256, execution_ledger_path=ledger_file, execution_ledger_sha256=_file_sha256(ledger_file), knowledge_receipt_path=knowledge_path, knowledge_receipt_sha256=knowledge_sha256, iteration=iteration, iteration_transaction_id=transaction_id, retained_transaction_record=validated, ) def expand_v2_catalog_from_accepted_checkpoint( *, source_checkpoint: Path, source_optimizer: Path, source_external_state: Path, source_composition: Path, source_migration_receipt: Path, growth_plan: Path, receipt_path: Path, replica_receipt_path: Path | None = None, expected_checkpoint_sha256: str | None = None, expected_optimizer_sha256: str | None = None, scale_cohort: NoNEScaleCohortPacket | None = None, compact_page_admission: CompactPageAdmissionPacket | None = None, source_retained_transaction: Mapping[str, Any] | None = None, ) -> NoNEV2PlusExpansionArtifacts: """Append one proof-bounded page cohort to an accepted checkpoint. This is an offline checkpoint/storage transaction, not training. Existing model tensors, optimizer moments, immutable page objects, session identity, and cumulative route/gradient telemetry remain exact prefixes. Missing family roots are created first. They may be completed in one transfer-only transaction or admitted in ordered, model-selected compact-bank cohorts. Once the physical family-root prefix is complete, a model-owned ``scale_cohort`` may add at most one objective child for each selected retained family. New pages are transfer-initialized or imported from an explicitly signed zero-residual compact bank, receive exact zero optimizer state, and remain explicitly untrained. """ source_checkpoint_path = source_checkpoint.expanduser().resolve() source_optimizer_path = source_optimizer.expanduser().resolve() source_external_path = source_external_state.expanduser().resolve() source_composition_path = source_composition.expanduser().resolve() source_receipt_path = source_migration_receipt.expanduser().resolve() growth_plan_path = growth_plan.expanduser().resolve() receipt = receipt_path.expanduser().resolve() required_files = ( source_checkpoint_path, source_optimizer_path, source_external_path, source_composition_path, source_receipt_path, growth_plan_path, ) if not all(path.is_file() for path in required_files): raise FileNotFoundError( "NoNE v2+ expansion source checkpoint transaction is incomplete" ) checkpoint_path = receipt.with_name(f"{receipt.stem}.checkpoint.pt") optimizer_path = checkpoint_path.with_suffix(".optimizer.pt") external_path = checkpoint_path.with_suffix(".none.json") catalog_path = receipt.with_name(f"{receipt.stem}.page_catalog.json") resident_path = receipt.with_name( f"{receipt.stem}.resident_runtime.safetensors" ) composition_path = receipt.with_name(f"{receipt.stem}.composition.json") outputs = ( receipt, checkpoint_path, optimizer_path, external_path, catalog_path, resident_path, composition_path, ) existing_outputs = tuple(path for path in outputs if path.exists()) if existing_outputs: raise RuntimeError( "NoNE v2+ expansion output already exists; immutable retry required" ) parent_composition = validate_v2_seed_composition( source_composition_path, source_receipt_path, ) parent_target_generation = parent_composition.get( "targetGeneration" ) if ( not isinstance(parent_target_generation, str) or not parent_target_generation ): raise RuntimeError( "NoNE v2+ parent generation identity is absent" ) source_generation = parent_target_generation target_generation = f"{source_generation}+" parent_composition_sha256 = _file_sha256(source_composition_path) parent_receipt_sha256 = _file_sha256(source_receipt_path) checkpoint_sha256 = _file_sha256(source_checkpoint_path) optimizer_sha256 = _file_sha256(source_optimizer_path) if ( expected_checkpoint_sha256 is not None and checkpoint_sha256 != expected_checkpoint_sha256 ): raise RuntimeError("accepted v2 checkpoint SHA-256 differs") if ( expected_optimizer_sha256 is not None and optimizer_sha256 != expected_optimizer_sha256 ): raise RuntimeError("accepted v2 optimizer SHA-256 differs") checkpoint = _load_checkpoint(source_checkpoint_path) source_parameters = checkpoint["parameters"] source_buffers = checkpoint["buffers"] source_lineage = checkpoint["lineage"] optimizer_payload = torch.load( source_optimizer_path, map_location="cpu", mmap=True, weights_only=True, ) if not isinstance(optimizer_payload, dict): raise RuntimeError("accepted v2 optimizer payload is invalid") external_envelope = json.loads( source_external_path.read_text(encoding="utf-8") ) if not isinstance(external_envelope, dict): raise RuntimeError("accepted v2 external sidecar is invalid") source_external = external_envelope.get("externalState") if ( external_envelope.get("schema") != EXTERNAL_CHECKPOINT_BINDING_SCHEMA or external_envelope.get("checkpointSha256") != checkpoint_sha256 or external_envelope.get("optimizerSha256") != optimizer_sha256 or not isinstance(source_external, dict) or source_external.get("compositionPath") != str(source_composition_path) or source_external.get("compositionSha256") != parent_composition_sha256 ): raise RuntimeError( "NoNE v2+ expansion requires an accepted checkpoint sidecar" ) source_external_sha256 = _file_sha256(source_external_path) retained_transaction_record = _validated_retained_scale_admission_record( source_retained_transaction, source_checkpoint_path=source_checkpoint_path, source_checkpoint_sha256=checkpoint_sha256, source_optimizer_path=source_optimizer_path, source_optimizer_sha256=optimizer_sha256, source_external_path=source_external_path, source_external_sha256=source_external_sha256, scale_cohort=scale_cohort, compact_page_admission=compact_page_admission, ) graph_adapted_page_update_authority = bool( retained_transaction_record is not None and _validated_graph_adapted_page_update_authority_boundary( retained_transaction_record=retained_transaction_record, source_checkpoint_path=source_checkpoint_path, source_checkpoint_sha256=checkpoint_sha256, source_optimizer_path=source_optimizer_path, source_optimizer_sha256=optimizer_sha256, source_external_path=source_external_path, source_external_sha256=source_external_sha256, source_external=source_external, ) ) expected_candidate_page_update = bool( retained_transaction_record is not None and not graph_adapted_page_update_authority ) if ( source_external.get("candidatePageUpdate") is not expected_candidate_page_update ): raise RuntimeError( "NoNE v2+ expansion checkpoint page-update authority differs" ) catalog_record = parent_composition.get("pageCatalog") store_record = parent_composition.get("pageStore") source_record = parent_composition.get("sourceCheckpoint") if not all( isinstance(record, dict) for record in (catalog_record, store_record, source_record) ): raise RuntimeError("NoNE v2 parent composition is incomplete") assert isinstance(catalog_record, dict) assert isinstance(store_record, dict) assert isinstance(source_record, dict) source_catalog_path = Path(str(catalog_record.get("path", ""))).resolve() source_catalog = json.loads( source_catalog_path.read_text(encoding="utf-8") ) if not isinstance(source_catalog, dict): raise RuntimeError("NoNE v2 parent page catalog is invalid") source_pages_value = source_catalog.get("pages") source_layer_catalog_value = source_catalog.get("layerCatalogPageIds") geometry = source_catalog.get("pageGeometry") if ( source_catalog.get("schema") not in (NONE_V2_PAGE_CATALOG_SCHEMA, NONE_V2_PLUS_PAGE_CATALOG_SCHEMA) or not isinstance(source_pages_value, list) or not isinstance(source_layer_catalog_value, dict) or not isinstance(geometry, dict) ): raise RuntimeError("NoNE v2 parent page catalog authority differs") source_pages = [ dict(row) for row in source_pages_value if isinstance(row, dict) ] source_page_ids_value = [row.get("pageId") for row in source_pages] if ( len(source_pages) != len(source_pages_value) or not source_pages or any( not isinstance(page_id, int) or isinstance(page_id, bool) or page_id < 0 for page_id in source_page_ids_value ) ): raise RuntimeError("NoNE v2 parent page identities are malformed") source_catalog_page_ids = [ page_id for page_id in source_page_ids_value if isinstance(page_id, int) and not isinstance(page_id, bool) ] if ( source_catalog_page_ids != sorted(source_catalog_page_ids) or len(set(source_catalog_page_ids)) != len(source_catalog_page_ids) ): raise RuntimeError("NoNE v2 parent page identities are not ordered unique") source_page_by_id = dict( zip(source_catalog_page_ids, source_pages, strict=True) ) source_family_rows = sorted( ( row for row in source_pages if isinstance(row.get("family"), str) and bool(str(row["family"]).strip()) ), key=lambda row: int(row["pageId"]), ) source_family_ids = tuple(str(row["family"]) for row in source_family_rows) if ( len(source_family_rows) != int(source_catalog.get("familyRootPageCount", -1)) or len(set(source_family_ids)) != len(source_family_ids) ): raise RuntimeError("NoNE v2 parent family-root catalog differs") plan = json.loads(growth_plan_path.read_text(encoding="utf-8")) if ( not isinstance(plan, dict) or plan.get("schema") != PAGED_GROWTH_PLAN_SCHEMA or not isinstance(plan.get("pagingPolicy"), dict) or plan["pagingPolicy"].get("acceptedGenerationPointer") is not True or plan["pagingPolicy"].get("storageBoundaryMayReroute") is not False ): raise RuntimeError("NoNE v2+ expansion requires the paged growth contract") roadmap_value = plan.get("functionalExpertFamilyRoadmap") if not isinstance(roadmap_value, list) or any( not isinstance(row, dict) or not isinstance(row.get("family"), str) for row in roadmap_value ): raise RuntimeError("NoNE v2+ functional expert roadmap is malformed") roadmap = [dict(row) for row in roadmap_value] target_family_ids = tuple(str(row["family"]) for row in roadmap) target_family_id_set = set(target_family_ids) missing_family_indexes = _missing_functional_family_indexes_boundary( source_family_ids, target_family_ids, ) objective_plan = _validated_objective_page_plan( plan, target_family_ids, ) federated_growth_demand_authority = ( federated_growth_demand_authority_from_plan_boundary(plan) ) objective_by_id = { objective.objective_id: objective for objective in objective_plan } source_objective_rows = sorted( ( row for row in source_pages if isinstance(row.get("objectiveId"), str) and bool(str(row["objectiveId"]).strip()) ), key=lambda row: int(row["pageId"]), ) source_objective_ids: set[str] = set() for row in source_objective_rows: objective_id = str(row["objectiveId"]) objective = objective_by_id.get(objective_id) if ( objective is None or objective_id in source_objective_ids or any( row.get(key) != value for key, value in objective.identity_record().items() ) or not isinstance(row.get("objectiveSourcePageCount"), int) or isinstance(row.get("objectiveSourcePageCount"), bool) or int(row["objectiveSourcePageCount"]) <= objective.ordinal or not isinstance(row.get("payloadShardBacked"), bool) ): raise RuntimeError( "NoNE accepted objective-page identity differs from the plan" ) source_objective_ids.add(objective_id) missing_objectives = tuple( objective for objective in objective_plan if objective.objective_id not in source_objective_ids ) source_training_page_ids = tuple( int(row["pageId"]) for row in source_pages if ( isinstance(row.get("family"), str) and bool(str(row["family"]).strip()) ) or ( isinstance(row.get("objectiveId"), str) and bool(str(row["objectiveId"]).strip()) ) ) language_roadmap_value = plan.get("languageExpertPackRoadmap") if not isinstance(language_roadmap_value, list) or any( not isinstance(row, dict) or not isinstance(row.get("family"), str) or not str(row["family"]).startswith("language_") for row in language_roadmap_value ): raise RuntimeError( "NoNE v2+ expansion requires the declared language roadmap" ) language_family_ids = tuple( str(row["family"]) for row in language_roadmap_value ) expected_language_family_ids = _validated_language_expert_family_ids( plan, required=True, ) source_language_family_count = source_lineage.get( "languageCapabilityPacks" ) source_language_family_ids = ( expected_language_family_ids[:source_language_family_count] if isinstance(source_language_family_count, int) and not isinstance(source_language_family_count, bool) and 0 < source_language_family_count <= len(expected_language_family_ids) else () ) additive_language_family_ids = expected_language_family_ids[ len(source_language_family_ids) : ] source_language_family_sha256 = hashlib.sha256( "\n".join(source_language_family_ids).encode("utf-8") ).hexdigest() language_catalog_schema_prefix = ( "nnf.resynthesis.none_language_expert_catalog.v" ) source_language_catalog = source_lineage.get("languageCapabilityCatalog") target_language_catalog = plan.get("languageExpertCatalogSchema") source_language_catalog_version = ( source_language_catalog.removeprefix(language_catalog_schema_prefix) if isinstance(source_language_catalog, str) and source_language_catalog.startswith(language_catalog_schema_prefix) else "" ) target_language_catalog_version = ( target_language_catalog.removeprefix(language_catalog_schema_prefix) if isinstance(target_language_catalog, str) and target_language_catalog.startswith(language_catalog_schema_prefix) else "" ) if ( language_family_ids != expected_language_family_ids or not set(language_family_ids).issubset(target_family_ids) or len(set(expected_language_family_ids)) != len(expected_language_family_ids) or len(source_language_family_ids) != source_language_family_count or len(set(source_language_family_ids)) != len(source_language_family_ids) or len(set(additive_language_family_ids)) != len(additive_language_family_ids) or not set(source_language_family_ids).isdisjoint( additive_language_family_ids ) or not source_language_catalog_version.isdigit() or not target_language_catalog_version.isdigit() or int(source_language_catalog_version) > int(target_language_catalog_version) or ( not additive_language_family_ids and source_language_catalog != target_language_catalog ) or source_lineage.get("languageCapabilityPackIdsSha256") != source_language_family_sha256 or source_lineage.get("languageCapabilityNativeRouting") is not True ): raise RuntimeError("NoNE v2+ language catalog binding differs") specialist_family_ids = _validated_science_specialist_family_ids( plan, target_family_ids, required=True, ) if ( source_lineage.get("scienceSpecialistCapabilityCatalog") != NONE_SCIENCE_SPECIALIST_CATALOG_SCHEMA or source_lineage.get("scienceSpecialistCapabilityFamilies") != len(specialist_family_ids) or source_lineage.get("scienceSpecialistCapabilityIdsSha256") != NONE_V2_PLUS_SCIENCE_SPECIALIST_IDS_SHA256 or source_lineage.get("scienceSpecialistCapabilityNativeRouting") is not True or source_lineage.get( "scienceSpecialistCapabilitiesRegisteredNotTrained" ) is not True ): raise RuntimeError( "NoNE v2+ science-specialist checkpoint binding differs" ) if int(source_lineage.get("functionalCapabilityFamilies", -1)) != len( target_family_ids ): raise RuntimeError( "accepted checkpoint has not inherited the target capability graph" ) source_paged_lineage = source_lineage.get("pagedNoNE") source_store_root = source_external.get("storeRoot") source_replica_store_roots = source_external.get("replicaStoreRoots") parent_store_root = store_record.get("root") replica_store_root_equivalent = bool( isinstance(source_store_root, str) and isinstance(parent_store_root, str) and isinstance(source_replica_store_roots, list) and source_replica_store_roots and all(isinstance(root, str) and root for root in source_replica_store_roots) and len(set(source_replica_store_roots)) == len(source_replica_store_roots) and source_store_root in source_replica_store_roots and parent_store_root in source_replica_store_roots ) if ( not isinstance(source_paged_lineage, dict) or source_paged_lineage.get("compositionSha256") != parent_composition_sha256 or int(source_paged_lineage.get("familyRootCount", -1)) != len(source_family_ids) or int( source_paged_lineage.get( "trainingPageCount", source_paged_lineage.get( "trainingPageRootCount", source_paged_lineage.get("familyRootCount", -1), ), ) ) != len(source_training_page_ids) or source_external.get("sourceCheckpointSha256") != source_record.get("sha256") or not ( source_store_root == parent_store_root or replica_store_root_equivalent ) ): raise RuntimeError("accepted v2 checkpoint page lineage differs") session_values = store_record.get("sessionId") if ( not isinstance(session_values, list) or len(session_values) != 4 or not all( isinstance(value, int) and not isinstance(value, bool) for value in session_values ) ): raise RuntimeError("NoNE v2 parent session identity differs") session_id_t = torch.tensor(session_values, dtype=torch.long) store = NoNEImmutablePageStore(Path(str(store_record["root"]))) accepted_generation_t = store.begin_session(session_id_t) accepted_binding = store.current_generation_binding_boundary() accepted_binding_record = accepted_binding.external_record_boundary() source_generation_binding = source_external.get("generationBinding") source_binding_exact = _binding_matches_external_record( accepted_binding, source_generation_binding, ) if not source_binding_exact: # A completed branch union is page-only: it advances the immutable # generation while deliberately retaining the ancestor's exact graph # checkpoint, optimizer, sidecar, and composition. Admit only the # existing strict one-hop union proof; every other stale sidecar stays # rejected by this expansion boundary. source_receipt_value = json.loads( source_receipt_path.read_text(encoding="utf-8") ) source_receipt_generation = ( source_receipt_value.get("generationBinding") if isinstance(source_receipt_value, dict) else None ) accepted_graph = store.current_graph_authority_boundary() if ( not isinstance(source_generation_binding, dict) or source_generation_binding != source_receipt_generation or accepted_graph is None or not isinstance(source_receipt_value, dict) or not isinstance(source_receipt_generation, dict) or not _validated_training_branch_union_graph_descendant_boundary( store=store, binding=accepted_binding, graph_record=accepted_graph.external_record_boundary(), receipt_value=source_receipt_value, receipt_generation=source_receipt_generation, ) ): raise RuntimeError( "NoNE source checkpoint does not own the accepted generation" ) if not torch.equal( accepted_binding.generation_t, accepted_generation_t, ): raise RuntimeError("NoNE accepted generation binding diverged") replica_coordinator: NoNEGenerationReplicaCoordinator | None = None replica_topology_transition: dict[str, Any] | None = None if replica_receipt_path is not None: replica_path = replica_receipt_path.expanduser().resolve() seed_pointer = store_record.get("acceptedPointer") if ( not isinstance(seed_pointer, dict) or not isinstance( seed_pointer.get("manifestPayloadSha256"), str, ) ): raise RuntimeError("NoNE replicated seed pointer lineage differs") ( replica_coordinator, replica_topology_transition, ) = _open_replica_topology_for_expansion_boundary( store=store, session_id_t=session_id_t, source_external=source_external, requested_receipt_path=replica_path, seed_manifest_payload_sha256=str( seed_pointer["manifestPayloadSha256"] ), accepted_binding_record=accepted_binding_record, ) if not _binding_matches_external_record( store.current_generation_binding_boundary(), accepted_binding_record, ): raise RuntimeError("NoNE replica reconciliation changed authority") elif ( source_external.get("schema") == "nnf.resynthesis.none_checkpoint_external_state.v2" ): raise RuntimeError( "NoNE replicated checkpoint expansion requires its replica receipt" ) elif ( source_external.get("schema") != "nnf.resynthesis.none_checkpoint_external_state.v1" ): raise RuntimeError("NoNE source checkpoint external schema differs") source_proof = source_external.get("trainingProof") if not isinstance(source_proof, dict): raise RuntimeError("NoNE source checkpoint has no family training proof") source_layer_ids: list[int] = [] for raw_layer_id in source_layer_catalog_value: try: layer_id = int(raw_layer_id) except (TypeError, ValueError) as exc: raise RuntimeError("NoNE v2+ layer identity is malformed") from exc if str(layer_id) != str(raw_layer_id) or layer_id < 0: raise RuntimeError("NoNE v2+ layer identity is malformed") source_layer_ids.append(layer_id) source_layer_ids.sort() layer_count = len(source_layer_ids) total_science_layers = int(source_lineage.get("scienceLayers", -1)) capacity_alignment = plan.get("capacityAlignment") if isinstance(capacity_alignment, dict): plan_target_geometry = plan.get("proposedMinimumTargetGeometry") baseline_target_layers = capacity_alignment.get( "baselineResidentTraversalLayerCount", capacity_alignment.get("baselineTargetScienceLayers"), ) aligned_target_layers = capacity_alignment.get( "residentTraversalLayerCount", capacity_alignment.get("alignedTargetScienceLayers"), ) aligned_objective_pages = capacity_alignment.get( "alignedLogicalExpertPageObjectives" ) inherited_sparse_layers = capacity_alignment.get( "inheritedPhysicalNoNELayerCount" ) additive_bank_pages = capacity_alignment.get( "additiveBankPageCount", capacity_alignment.get("bankPageCount"), ) sparse_page_layers = capacity_alignment.get( "pageBackedLayersReportedAsDenseLayers" ) planned_target_layers = ( plan_target_geometry.get( "residentScienceLayers", plan_target_geometry.get("scienceLayers"), ) if isinstance(plan_target_geometry, dict) else None ) sparse_geometry_values = ( capacity_alignment.get("alignedPhysicalNoNELayerCount"), capacity_alignment.get("alignedPhysicalPageExpertCount"), capacity_alignment.get("physicalGraphLayerCount"), plan_target_geometry.get("physicalNoNELayers") if isinstance(plan_target_geometry, dict) else None, plan_target_geometry.get("physicalGraphLayerCount") if isinstance(plan_target_geometry, dict) else None, plan_target_geometry.get("physicalGraphNodeCount") if isinstance(plan_target_geometry, dict) else None, plan_target_geometry.get("physicalPageExperts") if isinstance(plan_target_geometry, dict) else None, ) sparse_geometry_valid = bool( sparse_page_layers is not False or ( isinstance(aligned_objective_pages, int) and not isinstance(aligned_objective_pages, bool) and aligned_objective_pages > 0 and isinstance(inherited_sparse_layers, int) and not isinstance(inherited_sparse_layers, bool) and inherited_sparse_layers > 0 and isinstance(additive_bank_pages, int) and not isinstance(additive_bank_pages, bool) and additive_bank_pages > 0 and all( isinstance(value, int) and not isinstance(value, bool) and value > 0 for value in sparse_geometry_values ) and len(set(sparse_geometry_values)) == 1 and sparse_geometry_values[0] == inherited_sparse_layers + additive_bank_pages and capacity_alignment.get( "residentTraversalLayerCount", capacity_alignment.get("residentDenseExecutionLayerCount"), ) == aligned_target_layers ) ) if ( not isinstance(baseline_target_layers, int) or isinstance(baseline_target_layers, bool) or baseline_target_layers < 1 or not isinstance(aligned_target_layers, int) or isinstance(aligned_target_layers, bool) or aligned_target_layers < baseline_target_layers or planned_target_layers != aligned_target_layers or not sparse_geometry_valid # Physical page admission is additive to the exact accepted dense # traversal engine. A larger plan value is a pending dense target, # never permission to materialize page-backed graph layers as # dense stacks during this transaction. or total_science_layers < 1 or total_science_layers > baseline_target_layers ): raise RuntimeError( "NoNE capacity-aligned page admission requires the exact " "adapted dense-layer checkpoint" ) paged_lineage = source_lineage.get("pagedNoNE") paged_subset_valid = False if isinstance(paged_lineage, dict) and paged_lineage.get("schema") == ( "nnf.resynthesis.paged_none_lineage.v3" ): raw_paged_layer_ids = paged_lineage.get("pagedRuntimeLayerIds") paged_subset_valid = bool( isinstance(raw_paged_layer_ids, (list, tuple)) and tuple(raw_paged_layer_ids) == tuple(source_layer_ids) and paged_lineage.get("pagedRuntimeLayerCount") == layer_count and paged_lineage.get("totalScienceLayerCount") == total_science_layers and paged_lineage.get("scienceLayers") == total_science_layers and total_science_layers > 0 and source_layer_ids and source_layer_ids[-1] < total_science_layers ) legacy_full_layer_graph = bool( total_science_layers == layer_count and source_layer_ids == list(range(layer_count)) ) if layer_count < 1 or not (paged_subset_valid or legacy_full_layer_graph): raise RuntimeError("NoNE v2+ layer geometry differs") source_layer_catalog: dict[int, list[int]] = {} cataloged_source_page_ids: list[int] = [] for layer_id in source_layer_ids: raw_ids = source_layer_catalog_value.get(str(layer_id)) if ( not isinstance(raw_ids, list) or not raw_ids or not all( isinstance(page_id, int) and not isinstance(page_id, bool) for page_id in raw_ids ) or len(set(raw_ids)) != len(raw_ids) or any(page_id not in source_page_by_id for page_id in raw_ids) ): raise RuntimeError( f"NoNE v2+ source layer catalog differs: {layer_id}" ) source_layer_catalog[layer_id] = list(raw_ids) cataloged_source_page_ids.extend(raw_ids) if any( int(source_page_by_id[page_id].get("layerId", -1)) != layer_id for page_id in raw_ids ): raise RuntimeError("NoNE source page layer ownership differs") if ( len(cataloged_source_page_ids) != len(source_catalog_page_ids) or set(cataloged_source_page_ids) != set(source_catalog_page_ids) ): raise RuntimeError("NoNE source layer catalog does not partition its pages") layer_evidence: list[_LayerTransferEvidence] = [] for layer_id in source_layer_ids: runtime_prefix = ( f"science_stack.science_layer_{layer_id}." "paged_expert_runtime." ) route_keys_t = source_parameters.get( f"{runtime_prefix}router.page_route_keys" ) page_prior_t = source_parameters.get( f"{runtime_prefix}router.page_prior" ) route_count_t = source_buffers.get( f"{runtime_prefix}accepted_route_count_t" ) gradient_count_t = source_buffers.get( f"{runtime_prefix}accepted_gradient_update_count_t" ) source_page_ids = source_layer_catalog[layer_id] source_page_count = len(source_page_ids) if ( not isinstance(route_keys_t, torch.Tensor) or route_keys_t.ndim != 2 or route_keys_t.shape[0] != source_page_count or not isinstance(page_prior_t, torch.Tensor) or page_prior_t.shape != (source_page_count,) or not isinstance(route_count_t, torch.Tensor) or route_count_t.shape != (source_page_count,) or not isinstance(gradient_count_t, torch.Tensor) or gradient_count_t.shape != (source_page_count,) ): raise RuntimeError( f"NoNE accepted transfer evidence differs: {layer_id}" ) layer_evidence.append( _LayerTransferEvidence( layer_id=layer_id, source_page_ids=tuple(source_page_ids), source_family_ids=tuple( ( str(source_page_by_id[page_id]["family"]) if isinstance( source_page_by_id[page_id].get("family"), str, ) else None ) for page_id in source_page_ids ), route_keys_t=route_keys_t, page_prior_t=page_prior_t, route_count_t=route_count_t, gradient_update_count_t=gradient_count_t, ) ) prepared_layer_evidence = _prepare_transfer_layer_evidence(layer_evidence) preferred_layer_cache: dict[ tuple[str, ...], tuple[tuple[torch.Tensor, torch.Tensor, torch.Tensor], ...], ] = {} transfer_layer_load = { layer_id: len(source_layer_catalog[layer_id]) for layer_id in source_layer_ids } page_parameter_elements = page_model_parameter_elements( hidden_size=int(geometry.get("hiddenSize", 0)), expert_hidden_size=int(geometry.get("expertHiddenSize", 0)), glyph_size=int(geometry.get("glyphSize", 0)), router_size=int(geometry.get("routerSize", 0)), ) if page_parameter_elements < 1: raise RuntimeError("NoNE page parameter geometry is empty") page_specs: list[dict[str, Any]] = [] next_page_id = max(source_catalog_page_ids) + 1 selected_objectives: tuple[_ObjectivePage, ...] = () compact_admission_boundary: _CompactAdmissionBoundary | None = None compact_objective_cohort_binding: dict[str, Any] | None = None selected_family_page_ids: tuple[int, ...] = () growth_cohort_kind = "" if missing_family_indexes: if source_objective_rows: raise RuntimeError( "NoNE objective children cannot precede all family roots" ) if (scale_cohort is None) != (compact_page_admission is None): raise RuntimeError( "NoNE compact family-root growth requires model and bank authority" ) compact_page_ids: tuple[int, ...] = () selected_missing_family_indexes = missing_family_indexes if scale_cohort is not None and compact_page_admission is not None: selected_family_page_ids = _selected_scale_family_page_ids_boundary( scale_cohort ) _validate_scale_family_training_evidence( selected_family_page_ids=selected_family_page_ids, source_training_page_ids=source_training_page_ids, source_proof=source_proof, ) if len(selected_family_page_ids) > len(missing_family_indexes): raise RuntimeError( "NoNE compact family-root cohort exceeds pending roots" ) _validate_compact_admission_tensor_authority_boundary( packet=compact_page_admission, session_id_t=session_id_t, selected_family_page_ids=selected_family_page_ids, accepted_page_ids=source_catalog_page_ids, page_parameter_elements=page_parameter_elements, ) if retained_transaction_record is None: raise RuntimeError( "NoNE compact-bank admission requires retained training authority" ) compact_admission_boundary = _validated_compact_admission_boundary( packet=compact_page_admission, session_id_t=session_id_t, selected_family_page_ids=selected_family_page_ids, accepted_page_ids=source_catalog_page_ids, source_catalog=source_catalog, page_parameter_elements=page_parameter_elements, ) compact_page_ids = tuple( int(binding.page_id_t.detach().cpu().long().reshape(())) for binding in compact_admission_boundary.bindings ) selected_missing_family_indexes = missing_family_indexes[ : len(compact_page_ids) ] growth_cohort_kind = "compact_family_root" else: growth_cohort_kind = "transfer_family_root_completion" for root_ordinal, family_index in enumerate( selected_missing_family_indexes ): roadmap_row = roadmap[family_index] family_id = str(roadmap_row["family"]) page_id = ( compact_page_ids[root_ordinal] if compact_admission_boundary is not None else next_page_id ) page_specs.append( { "kind": "family_root", "pageId": page_id, "layerId": _select_transfer_layer( identity=family_id, preferred_families=(family_id,), layer_evidence=prepared_layer_evidence, preferred_layer_cache=preferred_layer_cache, layer_load=transfer_layer_load, ), "identity": family_id, "familyIndex": family_index, "roadmapRow": roadmap_row, "compactBankOrdinal": ( root_ordinal if compact_admission_boundary is not None else None ), "compactSourceFamilyPageId": ( selected_family_page_ids[root_ordinal] if compact_admission_boundary is not None else None ), } ) if compact_admission_boundary is None: next_page_id += 1 elif missing_objectives: if scale_cohort is None: raise RuntimeError( "NoNE objective-page growth requires a model-owned scale cohort" ) selected_family_page_ids = _selected_scale_family_page_ids_boundary( scale_cohort ) _validate_scale_family_training_evidence( selected_family_page_ids=selected_family_page_ids, source_training_page_ids=source_training_page_ids, source_proof=source_proof, ) selected_objectives = _objective_cohort_for_family_pages( selected_family_page_ids=selected_family_page_ids, family_rows=source_family_rows, missing_objectives=missing_objectives, ) if compact_page_admission is not None: _validate_compact_admission_tensor_authority_boundary( packet=compact_page_admission, session_id_t=session_id_t, selected_family_page_ids=selected_family_page_ids, accepted_page_ids=source_catalog_page_ids, page_parameter_elements=page_parameter_elements, ) if retained_transaction_record is None: raise RuntimeError( "NoNE compact-bank admission requires retained training authority" ) compact_admission_boundary = _validated_compact_admission_boundary( packet=compact_page_admission, session_id_t=session_id_t, selected_family_page_ids=selected_family_page_ids, accepted_page_ids=source_catalog_page_ids, source_catalog=source_catalog, page_parameter_elements=page_parameter_elements, ) compact_page_ids = tuple( int(binding.page_id_t.detach().cpu().long().reshape(())) for binding in compact_admission_boundary.bindings ) else: compact_page_ids = () if compact_admission_boundary is not None: compact_objective_cohort_binding = ( _compact_objective_cohort_binding_record( compact_page_ids=compact_page_ids, selected_family_page_ids=selected_family_page_ids, selected_objectives=selected_objectives, family_rows=source_family_rows, ) ) growth_cohort_kind = ( "compact_objective_child" if compact_admission_boundary is not None else "transfer_objective_child" ) family_root_page_id_by_family = { str(row["family"]): int(row["pageId"]) for row in source_family_rows } for objective_index, objective in enumerate(selected_objectives): parent_root_page_ids = tuple( family_root_page_id_by_family[family_id] for family_id in objective.functional_families ) page_id = ( compact_page_ids[objective_index] if compact_admission_boundary is not None else next_page_id ) page_specs.append( { "kind": "objective_child", "pageId": page_id, "layerId": _select_transfer_layer( identity=objective.objective_id, preferred_families=objective.functional_families, layer_evidence=prepared_layer_evidence, preferred_layer_cache=preferred_layer_cache, layer_load=transfer_layer_load, ), "identity": objective.objective_id, "objective": objective, "parentFamilyRootPageIds": parent_root_page_ids, "compactBankOrdinal": ( objective_index if compact_admission_boundary is not None else None ), "compactSourceFamilyPageId": ( selected_family_page_ids[objective_index] if compact_admission_boundary is not None else None ), } ) if compact_admission_boundary is None: next_page_id += 1 if not page_specs: raise RuntimeError( "NoNE v2+ growth plan has no missing family or objective pages" ) page_source_layer_ids = tuple( int(spec["layerId"]) for spec in page_specs ) if total_science_layers > len(source_layer_ids): layer_transfer_graph_t = source_parameters.get( "science_stack.layer_transfer_graph" ) if not isinstance(layer_transfer_graph_t, torch.Tensor): raise RuntimeError( "NoNE model-owned reasoning-layer transfer graph is absent" ) page_target_layer_ids = _model_owned_new_runtime_layer_targets( page_source_layer_ids=page_source_layer_ids, paged_layer_ids=source_layer_ids, total_science_layers=total_science_layers, layer_transfer_graph_t=layer_transfer_graph_t, ) else: page_target_layer_ids = page_source_layer_ids for spec, source_layer_id, target_layer_id in zip( page_specs, page_source_layer_ids, page_target_layer_ids, strict=True, ): spec["sourceLayerId"] = source_layer_id spec["layerId"] = target_layer_id new_family_page_ids = tuple( int(spec["pageId"]) for spec in page_specs if spec["kind"] == "family_root" ) new_objective_page_ids = tuple( int(spec["pageId"]) for spec in page_specs if spec["kind"] == "objective_child" ) new_page_ids = tuple(int(spec["pageId"]) for spec in page_specs) target_parameters = dict(source_parameters) target_buffers = dict(source_buffers) target_layer_catalog = { layer_id: list(page_ids) for layer_id, page_ids in source_layer_catalog.items() } added_page_rows: dict[int, dict[str, Any]] = {} added_page_objects: list[NoNEPageObjectBinding] = [] compact_page_object_bytes = 0 new_page_optimizer_zero = True expanded_parameter_names: set[str] = set() new_runtime_additions: dict[ int, list[ tuple[ int, int, torch.Tensor, torch.Tensor, torch.Tensor, ] ], ] = {} for layer_id in source_layer_ids: runtime_prefix = ( f"science_stack.science_layer_{layer_id}." "paged_expert_runtime." ) route_keys_name = f"{runtime_prefix}router.page_route_keys" page_prior_name = f"{runtime_prefix}router.page_prior" catalog_ids_name = f"{runtime_prefix}router.page_catalog_ids_t" expert_bias_name = ( f"{runtime_prefix}router.quantile_router.expert_bias_t" ) route_count_name = f"{runtime_prefix}accepted_route_count_t" gradient_count_name = ( f"{runtime_prefix}accepted_gradient_update_count_t" ) telemetry_names = ( route_count_name, gradient_count_name, f"{runtime_prefix}accepted_gradient_norm_t", f"{runtime_prefix}accepted_parameter_delta_norm_t", f"{runtime_prefix}accepted_gradient_signature_t", ) route_keys_t = source_parameters.get(route_keys_name) page_prior_t = source_parameters.get(page_prior_name) catalog_ids_t = source_buffers.get(catalog_ids_name) expert_bias_t = source_buffers.get(expert_bias_name) route_count_t = source_buffers.get(route_count_name) gradient_count_t = source_buffers.get(gradient_count_name) source_page_ids = source_layer_catalog[layer_id] source_page_count = len(source_page_ids) if ( not isinstance(route_keys_t, torch.Tensor) or route_keys_t.ndim != 2 or route_keys_t.shape[0] != source_page_count or not isinstance(page_prior_t, torch.Tensor) or page_prior_t.shape != (source_page_count,) or not isinstance(catalog_ids_t, torch.Tensor) or not torch.equal( catalog_ids_t.detach().cpu().long(), torch.tensor(source_page_ids, dtype=torch.long), ) or not isinstance(expert_bias_t, torch.Tensor) or expert_bias_t.shape != (source_page_count,) or not isinstance(route_count_t, torch.Tensor) or route_count_t.shape != (source_page_count,) or not isinstance(gradient_count_t, torch.Tensor) or gradient_count_t.shape != (source_page_count,) or any( not isinstance(source_buffers.get(name), torch.Tensor) or source_buffers[name].shape[0] != source_page_count for name in telemetry_names ) ): raise RuntimeError( f"NoNE accepted resident runtime geometry differs: {layer_id}" ) layer_specs = tuple( spec for spec in page_specs if spec["sourceLayerId"] == layer_id ) prepared_transfer_evidence = ( _prepare_recursive_transfer_evidence( route_keys_t=route_keys_t, page_prior_t=page_prior_t, route_count_t=route_count_t, gradient_update_count_t=gradient_count_t, ) if layer_specs else None ) compact_only_layer = bool(layer_specs) and all( isinstance(spec.get("compactBankOrdinal"), int) for spec in layer_specs ) if compact_only_layer: source_bundle = None else: request = _page_request( session_id_t=session_id_t, generation_t=accepted_generation_t, layer_id=layer_id, page_ids_t=torch.tensor(source_page_ids, dtype=torch.long), ) source_bundle = store.materialize_bundle( request, device=torch.device("cpu"), dtype=torch.float32, trainable=False, ) added_route_keys: list[torch.Tensor] = [] added_priors: list[torch.Tensor] = [] added_biases: list[torch.Tensor] = [] layer_coefficient_sha256: set[str] = set() layer_transfer_state_sha256: set[str] = set() layer_transfer_route_key_sha256: set[str] = set() for spec in ( _single_threaded_compact_transfer_rows(layer_specs) if compact_only_layer else layer_specs ): page_id = int(spec["pageId"]) target_layer_id = int(spec["layerId"]) identity = str(spec["identity"]) compact_bank_ordinal = spec.get("compactBankOrdinal") from_compact_bank = isinstance(compact_bank_ordinal, int) coefficient_fn = ( _recursive_transfer_coefficients_from_prepared if from_compact_bank else ( _identity_distinct_recursive_transfer_coefficients_from_prepared ) ) assert prepared_transfer_evidence is not None coefficient_t = coefficient_fn( evidence=prepared_transfer_evidence, target_identity=identity, ) if from_compact_bank: if compact_admission_boundary is None: raise RuntimeError("NoNE compact admission boundary is absent") assert isinstance(compact_bank_ordinal, int) if not 0 <= compact_bank_ordinal < len( compact_admission_boundary.bindings ): raise RuntimeError("NoNE compact admission ordinal differs") bank_binding = compact_admission_boundary.bindings[ compact_bank_ordinal ] if int(bank_binding.page_id_t) != page_id: raise RuntimeError("NoNE compact admission page identity differs") # The bank summary and journal already bind this immutable # object. Keep the object in its source bank and admit the # exact binding by overlay; materialization remains on-demand # when the model actually routes the page. compact_bundle = None page_object = bank_binding else: if source_bundle is None: raise RuntimeError( "NoNE compact expansion source bundle is absent" ) transfer_bundle = _transfer_initialized_page( source_bundle=source_bundle, coefficient_t=coefficient_t, page_id=page_id, family_id=identity, ) compact_bundle = compact_transfer_page_bundle_boundary( transfer_bundle ) page_object = store.stage_compact_transfer_page_object_boundary( transfer_bundle ) added_page_objects.append(page_object) compact_page_object_bytes += int( page_object.object_bytes_t.detach().cpu().long().reshape(()) ) if compact_bundle is not None: new_page_optimizer_zero = bool( new_page_optimizer_zero and not torch.count_nonzero( compact_bundle.optimizer_mean_t ) and not torch.count_nonzero( compact_bundle.optimizer_square_t ) and not torch.count_nonzero(compact_bundle.step_t) ) coefficient_sha256 = _tensor_sha256(coefficient_t) if coefficient_sha256 in layer_coefficient_sha256: raise RuntimeError( "NoNE learned transfer produced duplicate layer seeds" ) layer_coefficient_sha256.add(coefficient_sha256) if from_compact_bank: transfer_state_sha256 = _digest_tensor_hex_boundary( page_object.object_sha256_t, field="compact bank admitted object", ) else: assert compact_bundle is not None transfer_state_sha256 = _bundle_transfer_state_sha256( compact_bundle ) if ( not from_compact_bank and transfer_state_sha256 in layer_transfer_state_sha256 ): raise RuntimeError( "NoNE learned transfer produced duplicate page state" ) layer_transfer_state_sha256.add(transfer_state_sha256) transfer_route_key_t = coefficient_t.to( dtype=route_keys_t.dtype ).matmul(route_keys_t) transfer_route_key_t = ( transfer_route_key_t + _family_identity_residual( transfer_route_key_t, identity, ) ) transfer_route_key_sha256 = _tensor_sha256( transfer_route_key_t ) if ( transfer_route_key_sha256 in layer_transfer_route_key_sha256 ): raise RuntimeError( "NoNE learned transfer produced duplicate route state" ) layer_transfer_route_key_sha256.add( transfer_route_key_sha256 ) transfer_prior_t = coefficient_t.to( dtype=page_prior_t.dtype ).dot(page_prior_t) transfer_bias_t = coefficient_t.to( dtype=expert_bias_t.dtype ).dot(expert_bias_t) if target_layer_id == layer_id: added_route_keys.append(transfer_route_key_t) added_priors.append(transfer_prior_t) added_biases.append(transfer_bias_t) else: new_runtime_additions.setdefault(target_layer_id, []).append( ( layer_id, page_id, transfer_route_key_t, transfer_prior_t, transfer_bias_t, ) ) target_layer_catalog.setdefault(target_layer_id, []).append(page_id) page_row: dict[str, Any] = { "pageId": page_id, "layerId": target_layer_id, "transferSourceLayerId": layer_id, "pageRole": str(spec["kind"]), "familyRoot": spec["kind"] == "family_root", "state": ( "PREALLOCATED_ZERO_RESIDUAL_UNTRAINED" if from_compact_bank else "TRANSFER_INITIALIZED_UNTRAINED" ), "transferInitialization": ( COMPACT_BANK_TRANSFER_INITIALIZATION if from_compact_bank else TRANSFER_INITIALIZATION_V3 ), "transferTargetIdentity": identity, "transferSourcePageIds": list(source_page_ids), "transferCoefficientSha256": coefficient_sha256, "transferPageStateSha256": transfer_state_sha256, "transferRouteKeySha256": transfer_route_key_sha256, "transferStorage": SCALED_FLOAT8_TRANSFER_STORAGE, "optimizerState": "IMPLICIT_ZERO_UNTRAINED", "ffnMode": ( "preallocated_zero_residual_trainable_seed" if from_compact_bank else "learned_recursive_transfer_seed" ), # PAGE-TRAINED-KNOWLEDGE BOUNDARY: this page is a freshly # allocated/transfer-seeded slot. Its optimizer is still at # the implicit zero state (no step has ever run) and its # weights are a deterministic seed, NOT trained knowledge. # It MUST be created with ``trainedCapabilityClaimed=False`` # and stay False until the trainer applies real gradient # updates. Flipping it here would let an empty page be # counted as filled/trained ("filling before training") -- # the exact anti-pattern the user requires us to refuse. "trainedCapabilityClaimed": False, # Optional program-artifact provenance (see note above). "programArtifact": None, } if from_compact_bank: assert compact_admission_boundary is not None page_row.update( { "compactBankSummarySha256": ( compact_admission_boundary.authority_record[ "summarySha256" ] ), "compactBankObjectSha256": _digest_tensor_hex_boundary( page_object.object_sha256_t, field="compact bank admitted object", ), "physicalSeedKnowledgeClaimed": False, } ) compact_source_family_page_id = spec.get( "compactSourceFamilyPageId" ) if isinstance(compact_source_family_page_id, int): page_row["compactSourceFamilyPageId"] = ( compact_source_family_page_id ) if spec["kind"] == "family_root": family_index = int(spec["familyIndex"]) roadmap_row = spec["roadmapRow"] if not isinstance(roadmap_row, dict): raise RuntimeError("NoNE family-root page spec differs") page_row.update( { "family": str(roadmap_row["family"]), "sourceEvidenceGap": bool( roadmap_row.get("sourceEvidenceGap", True) ), "sourceFamilyOrdinal": family_index, "parentFamilyRootPageIds": [], } ) else: objective = spec["objective"] if not isinstance(objective, _ObjectivePage): raise RuntimeError("NoNE objective-page spec differs") page_row.update(objective.catalog_record()) page_row["parentFamilyRootPageIds"] = list( spec["parentFamilyRootPageIds"] ) added_page_rows[page_id] = page_row if not added_route_keys: continue target_parameters[route_keys_name] = torch.cat( ( route_keys_t, torch.stack(added_route_keys).to(route_keys_t), ), dim=0, ) target_parameters[page_prior_name] = torch.cat( ( page_prior_t, torch.stack(added_priors).to(page_prior_t), ), dim=0, ) expanded_parameter_names.update( (route_keys_name, page_prior_name) ) target_buffers[catalog_ids_name] = torch.tensor( target_layer_catalog[layer_id], dtype=torch.long, ) target_buffers[expert_bias_name] = torch.cat( ( expert_bias_t, torch.stack(added_biases).to(expert_bias_t), ), dim=0, ) target_count = len(target_layer_catalog[layer_id]) for name in telemetry_names: telemetry_t = source_buffers[name] target_buffers[name] = _append_zero_prefix_preserving( telemetry_t, target_count, ) for target_layer_id, additions in sorted(new_runtime_additions.items()): source_ids = {source_id for source_id, *_rest in additions} page_ids = [page_id for _source_id, page_id, *_rest in additions] if ( target_layer_id in source_layer_ids or len(source_ids) != 1 or target_layer_catalog.get(target_layer_id) != page_ids ): raise RuntimeError( "NoNE transfer-initialized runtime layer ownership differs" ) source_layer_id = next(iter(source_ids)) runtime_parameters, runtime_buffers = ( _transfer_initialized_paged_runtime_state( source_parameters=source_parameters, source_buffers=source_buffers, source_layer_id=source_layer_id, target_layer_id=target_layer_id, target_page_ids=page_ids, target_route_keys_t=torch.stack( [addition[2] for addition in additions] ), target_page_prior_t=torch.stack( [addition[3] for addition in additions] ), target_expert_bias_t=torch.stack( [addition[4] for addition in additions] ), ) ) if set(runtime_parameters) & set(target_parameters) or set( runtime_buffers ) & set(target_buffers): raise RuntimeError( "NoNE transfer-initialized runtime state already exists" ) target_parameters.update(runtime_parameters) target_buffers.update(runtime_buffers) target_runtime_layer_ids = sorted(target_layer_catalog) if ( set(added_page_rows) != set(new_page_ids) or len(added_page_objects) != len(new_page_ids) or { int(binding.page_id_t.detach().cpu().long().reshape(())) for binding in added_page_objects } != set(new_page_ids) ): raise RuntimeError("NoNE v2+ transfer page assembly is incomplete") target_pages = [ *source_pages, *(added_page_rows[page_id] for page_id in new_page_ids), ] target_page_by_id = { int(row["pageId"]): row for row in target_pages } verified_compact_objective_cohort_binding: dict[str, Any] | None = None if compact_objective_cohort_binding is not None: compact_objective_rows = [ added_page_rows[page_id] for page_id in new_objective_page_ids ] if any( not isinstance(row.get("compactSourceFamilyPageId"), int) or isinstance(row.get("compactSourceFamilyPageId"), bool) or not isinstance(row.get("objectiveId"), str) or row["objectiveId"] not in objective_by_id for row in compact_objective_rows ): raise RuntimeError( "NoNE compact objective page binding is malformed" ) verified_compact_objective_cohort_binding = ( _validate_compact_objective_cohort_binding_record( compact_objective_cohort_binding, compact_page_ids=new_objective_page_ids, selected_family_page_ids=tuple( int(row["compactSourceFamilyPageId"]) for row in compact_objective_rows ), selected_objectives=tuple( objective_by_id[str(row["objectiveId"])] for row in compact_objective_rows ), family_rows=source_family_rows, ) ) source_sparse_graph_layer_authority = ( sparse_graph_layer_catalog_authority_boundary(source_catalog) ) source_graph_layer_ids = tuple(source_catalog_page_ids) if ( source_sparse_graph_layer_authority is not None and source_sparse_graph_layer_authority[0] != source_graph_layer_ids ): raise RuntimeError("NoNE parent sparse graph-layer identity differs") target_graph_layer_ids = tuple( int(row["pageId"]) for row in target_pages ) if ( target_graph_layer_ids[: len(source_graph_layer_ids)] != source_graph_layer_ids or len(target_graph_layer_ids) != len(target_pages) ): raise RuntimeError("NoNE sparse graph-layer parent prefix differs") sparse_graph_layer_catalog_fields = ( build_sparse_graph_layer_catalog_authority_boundary( target_graph_layer_ids ) ) materialized_objective_page_count = ( len(source_objective_rows) + len(new_objective_page_ids) ) pending_objective_page_count = ( len(objective_plan) - materialized_objective_page_count ) if pending_objective_page_count < 0: raise RuntimeError("NoNE objective-page materialization exceeds its plan") materialized_family_root_count = ( len(source_family_ids) + len(new_family_page_ids) ) pending_family_root_count = ( len(target_family_ids) - materialized_family_root_count ) materialized_family_ids = tuple( str(row["family"]) for row in target_pages if isinstance(row.get("family"), str) and bool(str(row["family"]).strip()) ) if ( pending_family_root_count < 0 or len(set(materialized_family_ids)) != materialized_family_root_count or not set(materialized_family_ids).issubset(target_family_id_set) or materialized_family_ids[: len(source_family_ids)] != source_family_ids or (pending_family_root_count > 0 and materialized_objective_page_count > 0) ): raise RuntimeError("NoNE physical family-root identity growth differs") materialized_page_id_by_family = { str(row["family"]): int(row["pageId"]) for row in target_pages if isinstance(row.get("family"), str) and bool(str(row["family"]).strip()) } family_graph_order = [ { "family": family_id, "graphOrdinal": graph_ordinal, "materialized": family_id in materialized_page_id_by_family, "pageId": materialized_page_id_by_family.get(family_id), } for graph_ordinal, family_id in enumerate(target_family_ids) ] materialized_language_root_count = sum( family_id in set(language_family_ids) for family_id in materialized_family_ids ) materialized_specialist_root_count = sum( family_id in set(specialist_family_ids) for family_id in materialized_family_ids ) training_page_count = ( materialized_family_root_count + materialized_objective_page_count ) growth_plan_sha256 = _file_sha256(growth_plan_path) source_compact_authorities_value = source_catalog.get("compactPageBanks", []) if not isinstance(source_compact_authorities_value, list) or any( not isinstance(value, dict) for value in source_compact_authorities_value ): raise RuntimeError("NoNE parent compact-page bank authority differs") target_compact_authorities = [ dict(value) for value in source_compact_authorities_value ] if compact_admission_boundary is not None: replacement = compact_admission_boundary.authority_record matching_indexes = [ index for index, value in enumerate(target_compact_authorities) if value.get("summarySha256") == replacement["summarySha256"] ] if len(matching_indexes) > 1: raise RuntimeError("NoNE compact-page bank catalog is ambiguous") if matching_indexes: target_compact_authorities[matching_indexes[0]] = replacement else: target_compact_authorities.append(replacement) admitted_compact_page_count = sum( int(value.get("admittedPageCount", 0)) for value in target_compact_authorities ) physical_bank_capacity_parameters = sum( int(value.get("physicalParameterElementsInitialized", 0)) for value in target_compact_authorities ) catalog_payload = { "schema": NONE_V2_PLUS_PAGE_CATALOG_SCHEMA, "parentCatalog": { "path": str(source_catalog_path), "sha256": _file_sha256(source_catalog_path), }, "parentAcceptedCheckpointSha256": checkpoint_sha256, "growthPlanSha256": growth_plan_sha256, "sessionId": session_values, "acceptedGenerationAtExpansionStart": int(accepted_generation_t), "pageCount": len(target_pages), **sparse_graph_layer_catalog_fields, "inheritedPageCount": int( source_catalog.get("inheritedPageCount", 0) ), "familyRootPageCount": materialized_family_root_count, "plannedFunctionalFamilyRootCount": len(target_family_ids), "pendingFunctionalFamilyRootCount": pending_family_root_count, "languagePackRootPageCount": materialized_language_root_count, "plannedLanguagePackRootPageCount": len(language_family_ids), "scienceSpecialistRootPageCount": materialized_specialist_root_count, "plannedScienceSpecialistRootPageCount": len(specialist_family_ids), "addedFamilyRootPageCount": len(new_family_page_ids), "objectivePageCount": materialized_objective_page_count, "plannedObjectivePageCount": len(objective_plan), "pendingObjectivePageCount": pending_objective_page_count, "sourceObjectivePageCount": len(source_objective_rows), "addedObjectivePageCount": len(new_objective_page_ids), "trainingPageCount": training_page_count, "trainingPageRootCount": training_page_count, "objectivePagePlanSha256": _objective_plan_sha256( objective_plan ), "logicalPageObjectives": int( plan.get("initialLogicalExpertPageObjectives", 0) ), "transferPageStorage": SCALED_FLOAT8_TRANSFER_STORAGE, "pageGeometry": dict(geometry), "pageModelParameterElements": page_parameter_elements, "physicalPageModelParameterElements": ( len(target_pages) * page_parameter_elements ), "compactPageBanks": target_compact_authorities, "compactPageBankCount": len(target_compact_authorities), "admittedCompactPageCount": admitted_compact_page_count, "physicalUntrainedBankCapacityParameterElements": ( physical_bank_capacity_parameters ), "familyGraphOrder": family_graph_order, "layerCatalogPageIds": { str(layer_id): target_layer_catalog[layer_id] for layer_id in target_runtime_layer_ids }, "pages": target_pages, } if federated_growth_demand_authority is not None: catalog_payload["federatedGrowthDemandAuthority"] = ( federated_growth_demand_authority ) catalog_payload["federatedGrowthDemandAuthoritySha256"] = ( federated_growth_demand_authority["authoritySha256"] ) _atomic_json(catalog_path, catalog_payload) catalog_sha256 = _file_sha256(catalog_path) sparse_graph_layer_record = sparse_graph_layer_binding_record_boundary( catalog_payload, page_catalog_sha256=catalog_sha256, ) if sparse_graph_layer_record is None: raise RuntimeError("NoNE sparse graph-layer catalog binding is absent") resident_tensors: dict[str, torch.Tensor] = {} combined_target_state = {**target_parameters, **target_buffers} for layer_id in target_runtime_layer_ids: source_prefix = ( f"science_stack.science_layer_{layer_id}." "paged_expert_runtime." ) target_prefix = f"layer_{layer_id}." layer_tensors = { f"{target_prefix}{name.removeprefix(source_prefix)}": tensor for name, tensor in combined_target_state.items() if name.startswith(source_prefix) } if not layer_tensors: raise RuntimeError( f"NoNE v2+ resident layer is absent: {layer_id}" ) resident_tensors.update(layer_tensors) _save_resident_runtime_state(resident_path, resident_tensors) target_composition: dict[str, Any] = { "schema": NONE_V2_PLUS_COMPOSITION_SCHEMA, "sourceGeneration": source_generation, "targetGeneration": target_generation, "parentComposition": { "path": str(source_composition_path), "sha256": parent_composition_sha256, }, "parentMigrationReceipt": { "path": str(source_receipt_path), "sha256": parent_receipt_sha256, }, "sourceCheckpoint": dict(source_record), "sourceOptimizer": dict(parent_composition["sourceOptimizer"]), "pageStore": dict(store_record), "pageCatalog": { "path": str(catalog_path), "sha256": catalog_sha256, }, "sparseGraphLayers": sparse_graph_layer_record, "compactPageBanks": target_compact_authorities, "residentRuntime": { "path": str(resident_path), "sha256": _file_sha256(resident_path), "schema": NONE_V2_RESIDENT_RUNTIME_SCHEMA, "initialization": ( "accepted_runtime_prefix_plus_learned_recursive_transfer" ), }, "residentCore": dict(parent_composition["residentCore"]), "additivePageRuntime": dict( parent_composition["additivePageRuntime"] ), "catalogExpansion": { "schema": NONE_V2_PLUS_CATALOG_EXPANSION_SCHEMA, "parentAcceptedCheckpointSha256": checkpoint_sha256, "parentAcceptedOptimizerSha256": optimizer_sha256, "parentAcceptedGeneration": int(accepted_generation_t), "growthPlanPath": str(growth_plan_path), "growthPlanSha256": growth_plan_sha256, "sourceFamilyRoots": len(source_family_ids), "targetFamilyRoots": len(target_family_ids), "materializedFamilyRoots": materialized_family_root_count, "pendingFamilyRoots": pending_family_root_count, "sourceObjectivePages": len(source_objective_rows), "targetObjectivePages": materialized_objective_page_count, "plannedObjectivePages": len(objective_plan), "pendingObjectivePages": pending_objective_page_count, "addedTransferInitializedFamilyRoots": len( new_family_page_ids ), "addedTransferInitializedObjectivePages": len( new_objective_page_ids ), "addedTransferInitializedRoots": len(new_page_ids), "sourcePhysicalGraphLayers": len(source_graph_layer_ids), "targetPhysicalGraphLayers": len(target_graph_layer_ids), "addedPhysicalGraphLayers": len(new_page_ids), "graphLayerIdsSha256": sparse_graph_layer_record[ "graphLayerIdsSha256" ], "onePageObjectPerSparseGraphLayer": True, "sourcePagedRuntimeLayerIds": list(source_layer_ids), "targetPagedRuntimeLayerIds": target_runtime_layer_ids, "addedTransferInitializedRuntimeLayerIds": sorted( new_runtime_additions ), "transferPageStorage": SCALED_FLOAT8_TRANSFER_STORAGE, "compactPageBankCount": len(target_compact_authorities), "admittedCompactPageCount": admitted_compact_page_count, "addedPreallocatedCompactPages": ( len(compact_admission_boundary.bindings) if compact_admission_boundary is not None else 0 ), "growthCohortKind": growth_cohort_kind, "sameSessionRequired": True, "acceptedTensorPrefixesExact": True, "newOptimizerSuffixesZero": True, "trainingClaimed": False, }, "promotionEligible": False, } if federated_growth_demand_authority is not None: target_composition["federatedGrowthDemandAuthority"] = ( federated_growth_demand_authority ) target_composition["federatedGrowthDemandAuthoritySha256"] = ( federated_growth_demand_authority["authoritySha256"] ) target_composition["catalogExpansion"][ "federatedGrowthDemandAuthority" ] = federated_growth_demand_authority target_composition["catalogExpansion"][ "federatedGrowthDemandAuthoritySha256" ] = federated_growth_demand_authority["authoritySha256"] _atomic_json(composition_path, target_composition) composition_sha256 = _file_sha256(composition_path) target_lineage = dict(source_lineage) target_paged_lineage = dict(source_paged_lineage) target_paged_lineage["compositionSha256"] = composition_sha256 target_paged_lineage["familyRootCount"] = materialized_family_root_count target_paged_lineage["plannedFunctionalFamilyRootCount"] = len( target_family_ids ) target_paged_lineage["pendingFunctionalFamilyRootCount"] = ( pending_family_root_count ) target_paged_lineage["trainingPageCount"] = training_page_count target_paged_lineage["globalTrainingPageCount"] = training_page_count target_paged_lineage["trainingPageRootCount"] = training_page_count target_paged_lineage["physicalGraphLayerCount"] = len( target_graph_layer_ids ) target_paged_lineage["graphLayerIdsSha256"] = sparse_graph_layer_record[ "graphLayerIdsSha256" ] target_paged_lineage["onePageObjectPerSparseGraphLayer"] = True target_paged_lineage["sparseGraphLayersTrainingClaimed"] = False target_paged_lineage["sparseGraphLayersPromotionEligible"] = False target_paged_lineage["objectivePageCount"] = ( materialized_objective_page_count ) target_paged_lineage["plannedObjectivePageCount"] = len(objective_plan) target_paged_lineage["pendingObjectivePageCount"] = ( pending_objective_page_count ) target_paged_lineage["objectivePagePlanSha256"] = ( _objective_plan_sha256(objective_plan) ) if federated_growth_demand_authority is not None: target_paged_lineage["federatedGrowthDemandAuthoritySha256"] = ( federated_growth_demand_authority["authoritySha256"] ) target_paged_lineage["compactPageBankCount"] = len( target_compact_authorities ) target_paged_lineage["admittedCompactPageCount"] = ( admitted_compact_page_count ) target_paged_lineage["physicalUntrainedBankCapacityParameterElements"] = ( physical_bank_capacity_parameters ) target_paged_lineage["compactBankCapacityClaimedTrained"] = False target_paged_lineage["schema"] = "nnf.resynthesis.paged_none_lineage.v3" target_paged_lineage["scienceLayers"] = total_science_layers target_paged_lineage["totalScienceLayerCount"] = total_science_layers target_paged_lineage["pagedRuntimeLayerIds"] = tuple( target_runtime_layer_ids ) target_paged_lineage["pagedRuntimeLayerCount"] = len( target_runtime_layer_ids ) if "reasoningGrowthPlanSha256" in target_paged_lineage: target_paged_lineage["reasoningGrowthPlanSha256"] = ( growth_plan_sha256 ) if replica_coordinator is not None: target_paged_lineage["replicaReceiptSha256"] = ( replica_coordinator.receipt_sha256 ) target_paged_lineage["replicaStoreCount"] = len( replica_coordinator.store_roots_boundary ) target_paged_lineage["replicaDurabilityComplete"] = ( replica_coordinator.durability_complete and compact_admission_boundary is None ) if compact_admission_boundary is not None: target_paged_lineage["sharedCompactObjectOverlay"] = True else: target_paged_lineage.pop("sharedCompactObjectOverlay", None) target_lineage["pagedNoNE"] = target_paged_lineage state_key_sha256, state_geometry_sha256 = _state_identity( {**target_parameters, **target_buffers} ) target_checkpoint_payload = { **dict(checkpoint), "lineage": target_lineage, "stateKeySetSha256": state_key_sha256, "stateGeometrySha256": state_geometry_sha256, "parameters": target_parameters, "buffers": target_buffers, } _atomic_torch_save(checkpoint_path, target_checkpoint_payload) target_checkpoint_sha256 = _file_sha256(checkpoint_path) target_optimizer_payload, optimizer_expansion = ( _expand_named_optimizer_payload( source_payload=optimizer_payload, source_parameters=source_parameters, target_parameters=target_parameters, expanded_parameter_names=expanded_parameter_names, ) ) _atomic_torch_save(optimizer_path, target_optimizer_payload) target_optimizer_sha256 = _file_sha256(optimizer_path) generation_components = _component_packet( checkpoint_sha256=target_checkpoint_sha256, optimizer_sha256=target_optimizer_sha256, growth_plan_sha256=growth_plan_sha256, parent_sha256=checkpoint_sha256, ) canonical_page_objects = _canonical_page_object_bindings_boundary( added_page_objects ) shared_compact_object_roots = ( (compact_admission_boundary.source_store.root,) if compact_admission_boundary is not None else () ) if shared_compact_object_roots: store.register_overlay_object_roots(shared_compact_object_roots) if replica_coordinator is None: # The compact-bank summary and journal already bind every selected # content digest to an exact page ID and byte count. The replica # coordinator registers that receipt-verified set before staging; # keep the single-store path equivalent so it does not re-hash the # complete multi-gigabyte bank during every admission retry. store.register_receipt_verified_overlay_objects_boundary( canonical_page_objects ) staged_binding = ( replica_coordinator.stage_generation_from_page_objects_boundary( updated_page_objects=canonical_page_objects, components=generation_components, shared_compact_object_roots=shared_compact_object_roots, training_proven_page_ids_t=torch.tensor(sorted(set(source_training_page_ids) | set(new_page_ids)), dtype=torch.long), ) if replica_coordinator is not None else store.stage_generation_from_page_objects_boundary( updated_page_objects=canonical_page_objects, components=generation_components, training_proven_page_ids_t=torch.tensor(sorted(set(source_training_page_ids) | set(new_page_ids)), dtype=torch.long), ) ) if not torch.equal( staged_binding.parent_generation_t, accepted_generation_t, ): raise RuntimeError("NoNE v2+ generation did not branch from authority") expanded_proof = _expanded_training_proof( source_proof=source_proof, source_training_page_ids=source_training_page_ids, new_training_page_ids=new_page_ids, target_family_root_count=materialized_family_root_count, ) target_external = { "schema": source_external["schema"], "compositionPath": str(composition_path), "compositionSha256": composition_sha256, "sourceCheckpointSha256": str(source_record["sha256"]), "storeRoot": str(store.root), "candidatePageUpdate": False, "generationBinding": staged_binding.external_record_boundary(), "trainingProof": expanded_proof, "sparseGraphLayers": sparse_graph_layer_record, } if replica_coordinator is not None: target_external.update( { "replicaReceiptPath": str( replica_coordinator.receipt_path ), "replicaReceiptSha256": ( replica_coordinator.receipt_sha256 ), "replicaStoreRoots": list( replica_coordinator.store_roots_boundary ), "replicaDurabilityComplete": ( replica_coordinator.durability_complete and compact_admission_boundary is None ), "sharedCompactObjectOverlay": ( compact_admission_boundary is not None ), "canonicalPointerAdvancesLast": True, "replicaTopologyTransition": ( replica_topology_transition ), } ) _atomic_json( external_path, { "schema": EXTERNAL_CHECKPOINT_BINDING_SCHEMA, "checkpointSha256": target_checkpoint_sha256, "optimizerSha256": target_optimizer_sha256, "externalState": target_external, }, ) source_session_key = str( store_record["acceptedPointer"]["sessionKey"] ) session_root = store.root / "sessions" / source_session_key source_manifest = json.loads( (session_root / accepted_binding.manifest_relative_path).read_text( encoding="utf-8" ) ) target_manifest = json.loads( (session_root / staged_binding.manifest_relative_path).read_text( encoding="utf-8" ) ) source_object_rows = { int(row["pageId"]): dict(row) for row in source_manifest["pageObjects"] } target_object_rows = { int(row["pageId"]): dict(row) for row in target_manifest["pageObjects"] } accepted_prefix_exact = all( target_object_rows.get(page_id) == row for page_id, row in source_object_rows.items() ) checks = { "retainedTransactionAuthorityBound": ( source_retained_transaction is None or retained_transaction_record is not None ), "sourceCheckpointHashVerified": ( expected_checkpoint_sha256 is None or checkpoint_sha256 == expected_checkpoint_sha256 ), "sourceOptimizerHashVerified": ( expected_optimizer_sha256 is None or optimizer_sha256 == expected_optimizer_sha256 ), "sourceCheckpointOwnedAcceptedGeneration": True, "sameSessionPreserved": torch.equal( staged_binding.session_id_t, session_id_t, ), "acceptedGenerationParentExact": torch.equal( staged_binding.parent_generation_t, accepted_generation_t, ), "acceptedPageObjectsPrefixExact": accepted_prefix_exact, "sparseGraphLayerCatalogAuthorityExact": ( sparse_graph_layer_catalog_authority_boundary(catalog_payload) == ( target_graph_layer_ids, str(sparse_graph_layer_record["graphLayerIdsSha256"]), ) ), "sparseGraphLayerParentPrefixExact": ( target_graph_layer_ids[: len(source_graph_layer_ids)] == source_graph_layer_ids ), "onePageObjectPerSparseGraphLayer": ( target_graph_layer_ids == tuple(sorted(target_object_rows)) and len( { str(row["sha256"]) for row in target_object_rows.values() } ) == len(target_graph_layer_ids) ), "sparseGraphLayerBindingsExact": ( target_composition.get("sparseGraphLayers") == sparse_graph_layer_record and target_external.get("sparseGraphLayers") == sparse_graph_layer_record and target_paged_lineage.get("physicalGraphLayerCount") == len(target_graph_layer_ids) and target_paged_lineage.get("graphLayerIdsSha256") == sparse_graph_layer_record["graphLayerIdsSha256"] and target_paged_lineage.get("onePageObjectPerSparseGraphLayer") is True ), "sparseGraphLayerClaimsWithheld": ( sparse_graph_layer_record["routeGradientDeltaProofComplete"] is False and sparse_graph_layer_record["heldoutProofComplete"] is False and sparse_graph_layer_record["coldReloadProofComplete"] is False and sparse_graph_layer_record["trainingClaimed"] is False and sparse_graph_layer_record["promotionEligible"] is False and target_paged_lineage.get( "sparseGraphLayersTrainingClaimed" ) is False and target_paged_lineage.get( "sparseGraphLayersPromotionEligible" ) is False ), "acceptedModelStatePrefixAndNewRuntimeNamesBound": ( set(source_parameters).issubset(target_parameters) and set(source_buffers).issubset(target_buffers) and optimizer_expansion["newParameterCount"] == len(set(target_parameters) - set(source_parameters)) and all( any( name.startswith( f"science_stack.science_layer_{layer_id}." "paged_expert_runtime." ) for layer_id in new_runtime_additions ) for name in ( (set(target_parameters) - set(source_parameters)) | (set(target_buffers) - set(source_buffers)) ) ) ), "residentAcceptedPrefixesExact": all( torch.equal( target_parameters[name][ : source_parameters[name].shape[0] ], source_parameters[name], ) for name in expanded_parameter_names ), "globalOptimizerAcceptedPrefixesExact": ( optimizer_expansion["acceptedPrefixesExact"] == 1 ), "newGlobalOptimizerSuffixesZero": ( optimizer_expansion["newSuffixesZero"] == 1 ), "newPageOptimizerMomentsZero": new_page_optimizer_zero, "allNewRootsTransferInitialized": len(added_page_rows) == len(new_page_ids), "functionalGraphFamilyPlanBound": ( int(source_lineage.get("functionalCapabilityFamilies", -1)) == len(target_family_ids) and materialized_family_root_count + pending_family_root_count == len(target_family_ids) ), "physicalFamilyRootsRetainExactGraphIdentities": ( len(set(materialized_family_ids)) == materialized_family_root_count and set(materialized_family_ids).issubset(target_family_id_set) and materialized_family_ids[: len(source_family_ids)] == source_family_ids and [ row["family"] for row in family_graph_order if row["materialized"] ] == [ family_id for family_id in target_family_ids if family_id in set(materialized_family_ids) ] ), "objectivesBlockedUntilPhysicalRootsComplete": ( pending_family_root_count == 0 or materialized_objective_page_count == 0 ), "allMaterializedObjectiveIdsPresentExactlyOnce": ( { str(row["objectiveId"]) for row in target_pages if isinstance(row.get("objectiveId"), str) } == ( source_objective_ids | {objective.objective_id for objective in selected_objectives} ) and sum( 1 for row in target_pages if isinstance(row.get("objectiveId"), str) ) == materialized_objective_page_count ), "objectivePlanBoundWithoutPrematureMaterialization": ( set( str(row["objectiveId"]) for row in target_pages if isinstance(row.get("objectiveId"), str) ).issubset(objective_by_id) and materialized_objective_page_count + pending_objective_page_count == len(objective_plan) ), "existingObjectivePagesNotDuplicated": ( all( target_page_by_id.get(int(row["pageId"])) == row for row in source_objective_rows ) ), "newPagesUseCompactTransferStorage": all( row.get("transferStorage") == SCALED_FLOAT8_TRANSFER_STORAGE for row in added_page_rows.values() ), "scienceSpecialistCatalogBound": ( tuple(target_family_ids[-len(specialist_family_ids) :]) == specialist_family_ids ), "transferSeedsDistinctWithinLayer": all( len( { str(row["transferCoefficientSha256"]) for row in added_page_rows.values() if int(row["layerId"]) == layer_id } ) == sum( 1 for row in added_page_rows.values() if int(row["layerId"]) == layer_id ) for layer_id in target_runtime_layer_ids ), "transferPageStatesDistinctWithinLayer": all( len( { str(row["transferPageStateSha256"]) for row in added_page_rows.values() if int(row["layerId"]) == layer_id and row["state"] == "TRANSFER_INITIALIZED_UNTRAINED" } ) == sum( 1 for row in added_page_rows.values() if int(row["layerId"]) == layer_id and row["state"] == "TRANSFER_INITIALIZED_UNTRAINED" ) for layer_id in target_runtime_layer_ids ), "transferRouteKeysDistinctWithinLayer": all( len( { str(row["transferRouteKeySha256"]) for row in added_page_rows.values() if int(row["layerId"]) == layer_id } ) == sum( 1 for row in added_page_rows.values() if int(row["layerId"]) == layer_id ) for layer_id in target_runtime_layer_ids ), "allNewRootsExplicitlyUntrained": all( row["state"] in { "TRANSFER_INITIALIZED_UNTRAINED", "PREALLOCATED_ZERO_RESIDUAL_UNTRAINED", } and row["trainedCapabilityClaimed"] is False for row in added_page_rows.values() ), "compactBankAuthorityRetained": ( target_compact_authorities == catalog_payload["compactPageBanks"] == target_composition["compactPageBanks"] ), "compactBankAdmissionUntrained": ( compact_admission_boundary is None or all( added_page_rows[ int(binding.page_id_t.detach().cpu().long().reshape(())) ]["state"] == "PREALLOCATED_ZERO_RESIDUAL_UNTRAINED" for binding in compact_admission_boundary.bindings ) ), "compactFamilyRootCohortBoundToModelEvidence": ( compact_admission_boundary is None or not new_family_page_ids or [ added_page_rows[page_id].get("compactSourceFamilyPageId") for page_id in new_family_page_ids ] == list(selected_family_page_ids) ), "compactObjectiveCohortBoundToModelEvidence": ( compact_admission_boundary is None or not new_objective_page_ids or ( verified_compact_objective_cohort_binding == compact_objective_cohort_binding and [ added_page_rows[page_id].get( "compactSourceFamilyPageId" ) for page_id in new_objective_page_ids ] == list(selected_family_page_ids) and len(new_objective_page_ids) == len(selected_objectives) == len(compact_admission_boundary.bindings) ) ), "sourceTrainingTelemetryPrefixRetained": ( expanded_proof["familyPageIds"][: len(source_training_page_ids)] == list(source_training_page_ids) and expanded_proof["routeCounts"][: len(source_training_page_ids)] == source_proof["routeCounts"] and expanded_proof["gradientUpdateCounts"][ : len(source_training_page_ids) ] == source_proof["gradientUpdateCounts"] ), "replicaTopologyTransitionVerified": ( replica_coordinator is None or ( isinstance(replica_topology_transition, dict) and replica_topology_transition.get( "acceptedGenerationAuthorityRetained" ) is True and replica_topology_transition.get( "checkpointOrOptimizerBytesRewritten" ) is False and replica_topology_transition.get("targetStoreRoots") == list(replica_coordinator.store_roots_boundary) ) ), "federatedGrowthDemandAuthorityBound": ( federated_growth_demand_authority is None or ( catalog_payload.get("federatedGrowthDemandAuthority") == federated_growth_demand_authority and catalog_payload.get( "federatedGrowthDemandAuthoritySha256" ) == federated_growth_demand_authority["authoritySha256"] and target_composition.get("federatedGrowthDemandAuthority") == federated_growth_demand_authority and target_composition.get( "federatedGrowthDemandAuthoritySha256" ) == federated_growth_demand_authority["authoritySha256"] ) ), "promotionWithheld": True, } receipt_payload: dict[str, Any] = { "schema": NONE_V2_PLUS_CATALOG_EXPANSION_SCHEMA, "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(checks.values()), "status": "STAGED_CHECKPOINT_BOUND_POINTER_PENDING", "sourceGeneration": source_generation, "targetGeneration": target_generation, "sourceAcceptedCheckpoint": { "path": str(source_checkpoint_path), "sha256": checkpoint_sha256, }, "sourceAcceptedOptimizer": { "path": str(source_optimizer_path), "sha256": optimizer_sha256, }, "sourceExternalState": { "path": str(source_external_path), "sha256": _file_sha256(source_external_path), }, "sourceRetainedTransaction": retained_transaction_record, "compactObjectiveCohortBinding": ( verified_compact_objective_cohort_binding ), "sparseGraphLayers": sparse_graph_layer_record, "replicaTopologyTransition": replica_topology_transition, "parentComposition": { "path": str(source_composition_path), "sha256": parent_composition_sha256, }, "parentMigrationReceipt": { "path": str(source_receipt_path), "sha256": parent_receipt_sha256, }, "growthPlan": { "path": str(growth_plan_path), "sha256": growth_plan_sha256, "sourceFamilyRoots": len(source_family_ids), "targetFamilyRoots": len(target_family_ids), "materializedFamilyRoots": materialized_family_root_count, "pendingFamilyRoots": pending_family_root_count, "addedFamilyRoots": len(new_family_page_ids), "sourceObjectivePages": len(source_objective_rows), "targetObjectivePages": materialized_objective_page_count, "plannedObjectivePages": len(objective_plan), "pendingObjectivePages": pending_objective_page_count, "addedObjectivePages": len(new_objective_page_ids), "sourcePhysicalGraphLayers": len(source_graph_layer_ids), "targetPhysicalGraphLayers": len(target_graph_layer_ids), "addedPhysicalGraphLayers": len(new_page_ids), "graphLayerIdsSha256": sparse_graph_layer_record[ "graphLayerIdsSha256" ], "sourcePagedRuntimeLayerIds": list(source_layer_ids), "targetPagedRuntimeLayerIds": target_runtime_layer_ids, "addedPagedRuntimeLayerIds": sorted(new_runtime_additions), "objectivePagePlanSha256": _objective_plan_sha256( objective_plan ), "languageExpertPacks": len(language_family_ids), "scienceSpecialists": len(specialist_family_ids), "compactPageBanks": len(target_compact_authorities), "admittedCompactPages": admitted_compact_page_count, }, "expansion": { "sessionId": session_values, "parentGeneration": int( staged_binding.parent_generation_t ), "acceptedGeneration": int(staged_binding.generation_t), "sourcePageObjects": len(source_pages), "targetPageObjects": len(target_pages), "addedPageObjects": len(new_page_ids), "sourcePhysicalGraphLayers": len(source_graph_layer_ids), "targetPhysicalGraphLayers": len(target_graph_layer_ids), "addedPhysicalGraphLayers": len(new_page_ids), "graphLayerIdsSha256": sparse_graph_layer_record[ "graphLayerIdsSha256" ], "onePageObjectPerSparseGraphLayer": True, "pageModelParameterElements": page_parameter_elements, "sourcePageModelParameterElements": ( len(source_pages) * page_parameter_elements ), "targetPageModelParameterElements": ( len(target_pages) * page_parameter_elements ), "addedPageModelParameterElements": ( len(new_page_ids) * page_parameter_elements ), "addedTransferInitializedRoots": len(new_page_ids), "sourcePagedRuntimeLayerCount": len(source_layer_ids), "targetPagedRuntimeLayerCount": len(target_runtime_layer_ids), "addedTransferInitializedRuntimeLayerCount": len( new_runtime_additions ), "growthCohortKind": growth_cohort_kind, "addedPreallocatedCompactPages": ( len(compact_admission_boundary.bindings) if compact_admission_boundary is not None else 0 ), "addedTransferInitializedFamilyRoots": len( new_family_page_ids ), "addedPreallocatedCompactFamilyRoots": ( len(new_family_page_ids) if compact_admission_boundary is not None else 0 ), "addedTransferInitializedObjectivePages": len( new_objective_page_ids ), "addedPreallocatedCompactObjectivePages": ( len(new_objective_page_ids) if compact_admission_boundary is not None else 0 ), "compactPageObjectBytes": compact_page_object_bytes, "replicaStoreCount": ( len(replica_coordinator.store_roots_boundary) if replica_coordinator is not None else 1 ), "replicatedCompactPageObjectBytes": ( 0 if compact_admission_boundary is not None else compact_page_object_bytes ), "sharedCompactObjectOverlay": ( compact_admission_boundary is not None ), "transferPageStorage": SCALED_FLOAT8_TRANSFER_STORAGE, "trainedValidatedNewFamilyExperts": 0, "trainingStartedForNewRoots": False, "compactBankPhysicalCapacityParameterElements": ( physical_bank_capacity_parameters ), "admittedCompactPageCount": admitted_compact_page_count, "compactBankCapacityClaimedTrained": False, "transferInitialization": ( COMPACT_BANK_TRANSFER_INITIALIZATION if compact_admission_boundary is not None else TRANSFER_INITIALIZATION_V3 ), "optimizerExpansion": optimizer_expansion, }, "generationBinding": staged_binding.external_record_boundary(), "artifacts": { "composition": { "path": str(composition_path), "sha256": composition_sha256, }, "pageCatalog": { "path": str(catalog_path), "sha256": _file_sha256(catalog_path), }, "residentRuntime": { "path": str(resident_path), "sha256": _file_sha256(resident_path), "schema": NONE_V2_RESIDENT_RUNTIME_SCHEMA, }, "checkpoint": { "path": str(checkpoint_path), "sha256": target_checkpoint_sha256, }, "optimizer": { "path": str(optimizer_path), "sha256": target_optimizer_sha256, }, "externalState": { "path": str(external_path), "sha256": _file_sha256(external_path), }, "pageStoreRoot": str(store.root), }, "checks": checks, "physicalPageValidationPolicy": ( "hash_sealed_bank_header_admission_then_model_routed_materialization" if compact_admission_boundary is not None else "complete_transaction_reconstruction" ), "physicalAdmission": { "structuralChecksPassed": all(checks.values()), "acceptedPointerCommitted": False, "newPagesExplicitlyUntrained": checks[ "allNewRootsExplicitlyUntrained" ], "physicalGraphLayerCount": len(target_graph_layer_ids), "graphLayerIdsSha256": sparse_graph_layer_record[ "graphLayerIdsSha256" ], "onePageObjectPerSparseGraphLayer": checks[ "onePageObjectPerSparseGraphLayer" ], "sparseGraphLayersTrainingClaimed": False, "sparseGraphLayersPromotionEligible": False, "pendingTrainingProofAffectsAdmission": False, }, "trainingClaimed": False, "heldoutClaimed": False, "coldReloadClaimed": False, "promotionEligible": False, "remainingProofPolicy": "promotion_only", "remainingProof": [ "new_family_distinct_gradients", "new_family_model_owned_route_utilization", "new_sparse_graph_layer_parameter_delta", "heldout_language_generalization", "scientific_nonforgetting", "single_gpu_clean_lane", "cold_reload", "immutable_release_verification", ], } if federated_growth_demand_authority is not None: receipt_payload["federatedGrowthDemandAuthority"] = ( federated_growth_demand_authority ) receipt_payload["federatedGrowthDemandAuthoritySha256"] = ( federated_growth_demand_authority["authoritySha256"] ) receipt_payload["growthPlan"]["federatedGrowthDemandAuthority"] = ( federated_growth_demand_authority ) receipt_payload["growthPlan"][ "federatedGrowthDemandAuthoritySha256" ] = federated_growth_demand_authority["authoritySha256"] if receipt_payload["passed"] is not True: raise RuntimeError("NoNE v2+ staged expansion checks did not pass") try: graph_authority: NoNEGraphAuthorityBinding = ( build_graph_authority_binding_boundary( generation_binding=staged_binding, checkpoint_path=checkpoint_path, optimizer_path=optimizer_path, external_state_path=external_path, composition_path=composition_path, migration_receipt_path=receipt, replica_receipt_path=( replica_coordinator.receipt_path if replica_coordinator is not None else None ), identity_cache_root=( session_root / "artifact_sha256_cache" ), ) ) graph_authority_record = graph_authority.external_record_boundary() graph_topology = graph_authority_record.get("topology") if ( not isinstance(graph_topology, dict) or graph_topology.get("physicalGraphLayers") != len(target_graph_layer_ids) or graph_topology.get("pages") != len(target_graph_layer_ids) ): raise RuntimeError( "NoNE graph authority physical-layer topology differs" ) checks["graphAuthorityPhysicalGraphLayersExact"] = True for artifact_name in ( "checkpoint", "optimizer", "externalState", "composition", "pageCatalog", "residentRuntime", "replicaReceipt", ): artifact_record = graph_authority_record.get(artifact_name) if artifact_record is None: continue if not isinstance(artifact_record, dict): raise RuntimeError( f"NoNE graph authority {artifact_name} is malformed" ) artifact_path_value = artifact_record.get("path") artifact_sha256 = artifact_record.get("sha256") if ( not isinstance(artifact_path_value, str) or not artifact_path_value or not isinstance(artifact_sha256, str) or len(artifact_sha256) != 64 or file_sha256_authority_boundary( Path(artifact_path_value), expected_sha256=artifact_sha256, identity_cache_root=( session_root / "artifact_sha256_cache" ), ) != artifact_sha256 ): raise RuntimeError( f"NoNE graph authority {artifact_name} identity differs" ) expected_pointer = store.accepted_pointer_record_boundary( staged_binding, graph_authority, ) receipt_payload["status"] = "ACCEPTED_POINTER_ADVANCED_LAST" receipt_payload["acceptedPointer"] = expected_pointer receipt_payload["graphAuthority"] = ( graph_authority.external_record_boundary() ) _atomic_json(receipt, receipt_payload) pointer = ( replica_coordinator.accept_staged_generation( staged_binding, graph_authority, resident_page_ids_t=torch.tensor( sorted(target_object_rows), dtype=torch.long, ), retained_checkpoint_sha256_t=( graph_authority.checkpoint_sha256_t ), ) if replica_coordinator is not None else store.accept_staged_generation( staged_binding, graph_authority, ) ) if pointer != expected_pointer: raise RuntimeError("NoNE v2+ accepted graph pointer identity differs") physical_admission = receipt_payload["physicalAdmission"] assert isinstance(physical_admission, dict) physical_admission["acceptedPointerCommitted"] = True receipt_payload["status"] = ( "PHYSICAL_ADMISSION_ACCEPTED_TRAINING_PENDING" ) _atomic_json(receipt, receipt_payload) if compact_admission_boundary is None: validate_v2_seed_composition(composition_path, receipt) except Exception as verification_error: receipt_payload["verificationFailure"] = { "errorType": type(verification_error).__name__, "message": str(verification_error), } try: if replica_coordinator is not None: replica_coordinator.checkout_generation_boundary( accepted_binding ) else: store.checkout_generation_boundary( generation_t=accepted_binding.generation_t, manifest_sha256_t=accepted_binding.manifest_sha256_t, manifest_payload_sha256_t=( accepted_binding.manifest_payload_sha256_t ), ) except Exception as rollback_error: physical_admission = receipt_payload.get("physicalAdmission") if isinstance(physical_admission, dict): physical_admission["acceptedPointerCommitted"] = False receipt_payload["passed"] = False receipt_payload["status"] = ( "ROLLBACK_TO_PARENT_INCOMPLETE_AFTER_FAILURE" ) _atomic_json(receipt, receipt_payload) raise RuntimeError( "NoNE v2+ expansion failed and parent rollback was incomplete" ) from rollback_error physical_admission = receipt_payload.get("physicalAdmission") if isinstance(physical_admission, dict): physical_admission["acceptedPointerCommitted"] = False receipt_payload["passed"] = False receipt_payload["status"] = ( "ROLLED_BACK_TO_PARENT_AFTER_VERIFICATION_FAILURE" ) _atomic_json(receipt, receipt_payload) raise return NoNEV2PlusExpansionArtifacts( receipt_path=receipt, composition_path=composition_path, page_catalog_path=catalog_path, resident_runtime_path=resident_path, checkpoint_path=checkpoint_path, optimizer_path=optimizer_path, external_state_path=external_path, store_root=store.root, session_id_t=session_id_t, accepted_generation_t=staged_binding.generation_t.clone(), ) def _validate_checkpoint_optimizer_prefixes( *, source_checkpoint_path: Path, target_checkpoint_path: Path, source_optimizer_path: Path, target_optimizer_path: Path, ) -> None: source_checkpoint = _load_checkpoint(source_checkpoint_path) target_checkpoint = _load_checkpoint(target_checkpoint_path) expanded_parameter_names: set[str] = set() telemetry_suffixes = ( "accepted_route_count_t", "accepted_gradient_update_count_t", "accepted_gradient_norm_t", "accepted_parameter_delta_norm_t", "accepted_gradient_signature_t", ) for state_name in ("parameters", "buffers"): source_state = source_checkpoint[state_name] target_state = target_checkpoint[state_name] if not set(source_state).issubset(target_state): raise RuntimeError( f"NoNE v2+ {state_name} inherited key identity changed" ) for name, source_t in source_state.items(): target_t = target_state[name] if source_t.shape == target_t.shape: if not torch.equal(source_t, target_t): raise RuntimeError( f"NoNE v2+ inherited tensor changed: {name}" ) continue expandable = ( ".paged_expert_runtime." in name and source_t.ndim >= 1 and target_t.ndim == source_t.ndim and target_t.shape[1:] == source_t.shape[1:] and target_t.shape[0] > source_t.shape[0] ) if not expandable or not torch.equal( target_t[: source_t.shape[0]], source_t, ): raise RuntimeError( f"NoNE v2+ inherited tensor prefix changed: {name}" ) if state_name == "parameters": expanded_parameter_names.add(name) elif name.endswith(telemetry_suffixes) and bool( torch.count_nonzero( target_t[source_t.shape[0] :] ) ): raise RuntimeError( f"NoNE v2+ new telemetry is not zero: {name}" ) source_optimizer = torch.load( source_optimizer_path, map_location="cpu", mmap=True, weights_only=True, ) target_optimizer = torch.load( target_optimizer_path, map_location="cpu", mmap=True, weights_only=True, ) if not isinstance(source_optimizer, dict) or not isinstance( target_optimizer, dict, ): raise RuntimeError("NoNE v2+ optimizer payload is invalid") source_groups = source_optimizer.get("param_groups") target_groups = target_optimizer.get("param_groups") source_states = source_optimizer.get("state") target_states = target_optimizer.get("state") if ( not isinstance(source_groups, list) or len(source_groups) != 1 or not isinstance(target_groups, list) or len(target_groups) != 1 or not isinstance(source_groups[0], dict) or not isinstance(target_groups[0], dict) or not isinstance(source_states, dict) or not isinstance(target_states, dict) ): raise RuntimeError("NoNE v2+ named optimizer structure differs") source_names = source_groups[0].get("param_names") target_names = target_groups[0].get("param_names") source_ids = list(source_groups[0].get("params", ())) target_ids = list(target_groups[0].get("params", ())) source_name_values = ( tuple(str(name) for name in source_names) if isinstance(source_names, (list, tuple)) else () ) target_name_values = ( tuple(str(name) for name in target_names) if isinstance(target_names, (list, tuple)) else () ) if ( not isinstance(source_names, (list, tuple)) or not isinstance(target_names, (list, tuple)) or target_name_values[: len(source_name_values)] != source_name_values or target_ids[: len(source_ids)] != source_ids or len(target_name_values) != len(target_ids) or set(target_name_values) != set(target_checkpoint["parameters"]) or any( parameter_id in target_states for parameter_id in target_ids[len(source_ids) :] ) ): raise RuntimeError("NoNE v2+ optimizer parameter identity changed") for name, source_id, target_id in zip( source_name_values, source_ids, target_ids[: len(source_ids)], strict=True, ): source_state = source_states.get(source_id) target_state = target_states.get(target_id) if source_state is None and target_state is None: continue if not isinstance(source_state, dict) or not isinstance( target_state, dict, ): raise RuntimeError( f"NoNE v2+ optimizer state identity changed: {name}" ) if set(source_state) != set(target_state): raise RuntimeError( f"NoNE v2+ optimizer state keys changed: {name}" ) for state_key, source_value in source_state.items(): target_value = target_state[state_key] if not isinstance(source_value, torch.Tensor): if source_value != target_value: raise RuntimeError( f"NoNE v2+ optimizer scalar changed: {name}" ) continue if not isinstance(target_value, torch.Tensor): raise RuntimeError( f"NoNE v2+ optimizer tensor disappeared: {name}" ) if name not in expanded_parameter_names: if not torch.equal(source_value, target_value): raise RuntimeError( f"NoNE v2+ inherited optimizer changed: {name}" ) continue if source_value.ndim == 0: if not torch.equal(source_value, target_value): raise RuntimeError( f"NoNE v2+ optimizer step changed: {name}" ) continue if ( target_value.shape[1:] != source_value.shape[1:] or target_value.shape[0] <= source_value.shape[0] or not torch.equal( target_value[: source_value.shape[0]], source_value, ) or bool( torch.count_nonzero( target_value[source_value.shape[0] :] ) ) ): raise RuntimeError( f"NoNE v2+ optimizer suffix proof differs: {name}" ) def _validate_resident_checkpoint_state( *, resident_path: Path, checkpoint: Mapping[str, Any], ) -> None: parameters = checkpoint.get("parameters") buffers = checkpoint.get("buffers") if not isinstance(parameters, dict) or not isinstance(buffers, dict): raise RuntimeError("NoNE v2+ checkpoint state is incomplete") expected: dict[str, torch.Tensor] = {} for name, tensor in {**parameters, **buffers}.items(): if ( not isinstance(name, str) or not isinstance(tensor, torch.Tensor) or ".paged_expert_runtime." not in name or not name.startswith("science_stack.science_layer_") ): continue layer_prefix, runtime_name = name.split( ".paged_expert_runtime.", maxsplit=1, ) layer_id = layer_prefix.removeprefix( "science_stack.science_layer_" ) if not layer_id.isdigit(): raise RuntimeError("NoNE v2+ resident layer identity differs") expected[f"layer_{layer_id}.{runtime_name}"] = tensor if not expected: raise RuntimeError("NoNE v2+ checkpoint has no resident page state") with safe_open( # type: ignore[no-untyped-call] str(resident_path), framework="pt", device="cpu", ) as handle: if set(handle.keys()) != set(expected): raise RuntimeError("NoNE v2+ resident checkpoint keys differ") for name, source_t in expected.items(): if not torch.equal(handle.get_tensor(name), source_t): raise RuntimeError( f"NoNE v2+ resident checkpoint tensor differs: {name}" ) def _validate_expanded_training_proof( *, source_proof: object, target_proof: object, new_training_page_ids: Sequence[int], target_family_root_count: int, ) -> None: if not isinstance(source_proof, dict) or not isinstance( target_proof, dict, ): raise RuntimeError("NoNE v2+ family training proof is absent") source_ids = source_proof.get("trainingPageIds") if source_ids is None: source_ids = source_proof.get("familyPageIds") target_ids = target_proof.get("trainingPageIds") if target_ids is None: target_ids = target_proof.get("familyPageIds") if ( not isinstance(source_ids, list) or not isinstance(target_ids, list) or target_ids != [*source_ids, *new_training_page_ids] or target_proof.get("trainingPageCount") != len(target_ids) or target_proof.get("familyRootCount") != target_family_root_count or target_family_root_count > len(target_ids) or target_proof.get("familyPageIds") != target_ids ): raise RuntimeError("NoNE v2+ training-page identity differs") source_count = len(source_ids) scalar_fields = ( "routeCounts", "gradientUpdateCounts", "gradientNorms", "parameterDeltaNorms", ) for name in scalar_fields: source_values = source_proof.get(name) target_values = target_proof.get(name) if ( not isinstance(source_values, list) or not isinstance(target_values, list) or target_values[:source_count] != source_values or any( value not in (0, 0.0) for value in target_values[source_count:] ) ): raise RuntimeError( f"NoNE v2+ training-page prefix differs: {name}" ) source_signatures = source_proof.get("gradientSignatures") target_signatures = target_proof.get("gradientSignatures") if ( not isinstance(source_signatures, list) or not isinstance(target_signatures, list) or target_signatures[:source_count] != source_signatures or any( not isinstance(signature, list) or len(signature) != 8 or any(value not in (0, 0.0) for value in signature) for signature in target_signatures[source_count:] ) or target_proof.get("routeCoverage") is not False or target_proof.get("gradientCoverage") is not False or target_proof.get("distinctGradients") is not False or target_proof.get("promotionReady") is not False ): raise RuntimeError("NoNE v2+ new page proof is not zero-untrained") def _legacy_compact_objective_parent_compatibility_boundary( *, receipt_file: Path, receipt_sha256: str, catalog_sha256: str, new_pages: Sequence[object], parent_pages: Sequence[object], receipt: Mapping[str, Any], ) -> bool: """Admit only the one sealed pre-binding compact-objective generation.""" accepted_pointer = receipt.get("acceptedPointer") if ( not isinstance(accepted_pointer, Mapping) or "compactObjectiveCohortBinding" in receipt or accepted_pointer.get("generation") != 43 ): return False accepted_graph = accepted_pointer.get("graphAuthority") graph_catalog = ( accepted_graph.get("pageCatalog") if isinstance(accepted_graph, Mapping) else None ) checks = receipt.get("checks") expansion = receipt.get("expansion") artifacts = receipt.get("artifacts") artifact_catalog = ( artifacts.get("pageCatalog") if isinstance(artifacts, Mapping) else None ) graph_migration_receipt = ( accepted_graph.get("migrationReceiptPath") if isinstance(accepted_graph, Mapping) else None ) expected_self_path = receipt_file.expanduser().resolve() self_path_exact = ( isinstance(graph_migration_receipt, str) and Path(graph_migration_receipt).expanduser().resolve() == expected_self_path ) parent_page_ids = { row["pageId"] for row in parent_pages if isinstance(row, Mapping) and isinstance(row.get("pageId"), int) and not isinstance(row.get("pageId"), bool) } compact_rows = [ row for row in new_pages if isinstance(row, Mapping) and row.get("state") == "PREALLOCATED_ZERO_RESIDUAL_UNTRAINED" ] page_ids = [ row.get("pageId") for row in compact_rows if isinstance(row, Mapping) ] objective_ids = [ row.get("objectiveId") for row in compact_rows if isinstance(row, Mapping) ] row_bindings_exact = ( len(compact_rows) == _LEGACY_COMPACT_OBJECTIVE_PAGE_COUNT and len(new_pages) == len(compact_rows) and len(parent_page_ids) == len(parent_pages) and len(set(page_ids)) == len(page_ids) and len(set(objective_ids)) == len(objective_ids) and all( isinstance(row, Mapping) and row.get("pageRole") == "objective_child" and row.get("familyRoot") is False and row.get("state") == "PREALLOCATED_ZERO_RESIDUAL_UNTRAINED" and row.get("transferInitialization") == COMPACT_BANK_TRANSFER_INITIALIZATION and "compactSourceFamilyPageId" not in row and isinstance(row.get("pageId"), int) and not isinstance(row.get("pageId"), bool) and isinstance(row.get("objectiveId"), str) and bool(str(row["objectiveId"]).strip()) and isinstance(row.get("parentFamilyRootPageIds"), list) and bool(row["parentFamilyRootPageIds"]) and all( isinstance(page_id, int) and not isinstance(page_id, bool) and page_id in parent_page_ids for page_id in row["parentFamilyRootPageIds"] ) and isinstance(row.get("transferSourcePageIds"), list) and bool(row["transferSourcePageIds"]) and all( isinstance(page_id, int) and not isinstance(page_id, bool) and page_id in parent_page_ids for page_id in row["transferSourcePageIds"] ) for row in new_pages ) ) identity_exact = ( receipt_sha256 == _LEGACY_COMPACT_OBJECTIVE_RECEIPT_SHA256 and catalog_sha256 == _LEGACY_COMPACT_OBJECTIVE_CATALOG_SHA256 and isinstance(accepted_pointer, Mapping) and accepted_pointer.get("manifestSha256") == _LEGACY_COMPACT_OBJECTIVE_MANIFEST_SHA256 and accepted_pointer.get("manifestPayloadSha256") == _LEGACY_COMPACT_OBJECTIVE_MANIFEST_PAYLOAD_SHA256 and accepted_pointer.get("sessionKey") == _LEGACY_COMPACT_OBJECTIVE_SESSION_KEY and self_path_exact and isinstance(graph_catalog, Mapping) and graph_catalog.get("sha256") == _LEGACY_COMPACT_OBJECTIVE_CATALOG_SHA256 and isinstance(artifact_catalog, Mapping) and artifact_catalog.get("sha256") == _LEGACY_COMPACT_OBJECTIVE_CATALOG_SHA256 ) receipt_profile_exact = ( isinstance(checks, Mapping) and "compactObjectiveCohortBoundToModelEvidence" not in checks and checks.get("compactBankAuthorityRetained") is True and checks.get("compactFamilyRootCohortBoundToModelEvidence") is True and checks.get("allMaterializedObjectiveIdsPresentExactlyOnce") is True and checks.get("transferPageStatesDistinctWithinLayer") is True and checks.get("transferRouteKeysDistinctWithinLayer") is True and checks.get("transferSeedsDistinctWithinLayer") is True and isinstance(expansion, Mapping) and expansion.get("acceptedGeneration") == 43 and expansion.get("parentGeneration") == 42 and expansion.get("sharedCompactObjectOverlay") is True and expansion.get("transferInitialization") == COMPACT_BANK_TRANSFER_INITIALIZATION and expansion.get("growthCohortKind") == "compact_objective_child" and expansion.get("sourcePageObjects") == len(parent_pages) and expansion.get("targetPageObjects") == len(parent_pages) + len(new_pages) and expansion.get("addedPageObjects") == len(new_pages) and expansion.get("addedPreallocatedCompactObjectivePages") == len(new_pages) and expansion.get("addedPreallocatedCompactPages") == len(new_pages) ) if not (identity_exact and receipt_profile_exact and row_bindings_exact): raise RuntimeError( "NoNE legacy compact objective parent compatibility differs" ) return True def _validate_v2_plus_composition( composition_file: Path, receipt_file: Path, ) -> dict[str, Any]: composition = json.loads(composition_file.read_text(encoding="utf-8")) receipt = json.loads(receipt_file.read_text(encoding="utf-8")) identity_cache_root = composition_file.parent / ".artifact_sha256_cache" if ( not isinstance(composition, dict) or composition.get("schema") != NONE_V2_PLUS_COMPOSITION_SCHEMA or not isinstance(receipt, dict) or receipt.get("schema") not in NONE_V2_PLUS_EXPANSION_RECEIPT_SCHEMAS or receipt.get("passed") is not True or receipt.get("status") not in { "ACCEPTED_POINTER_ADVANCED_LAST", "PHYSICAL_ADMISSION_ACCEPTED_TRAINING_PENDING", } or receipt.get("promotionEligible") is not False or not isinstance(receipt.get("checks"), dict) or not receipt["checks"] or not all(value is True for value in receipt["checks"].values()) ): raise RuntimeError("NoNE v2+ catalog expansion authority differs") artifacts = receipt.get("artifacts") if not isinstance(artifacts, dict): raise RuntimeError("NoNE v2+ expansion receipt has no artifacts") source_checkpoint_record = receipt.get("sourceAcceptedCheckpoint") source_optimizer_record = receipt.get("sourceAcceptedOptimizer") source_external_record = receipt.get("sourceExternalState") if not all( isinstance(record, dict) for record in ( source_checkpoint_record, source_optimizer_record, source_external_record, ) ): raise RuntimeError("NoNE v2+ accepted source transaction is absent") assert isinstance(source_checkpoint_record, dict) assert isinstance(source_optimizer_record, dict) assert isinstance(source_external_record, dict) source_checkpoint_path = Path( str(source_checkpoint_record.get("path", "")) ).resolve() source_optimizer_path = Path( str(source_optimizer_record.get("path", "")) ).resolve() source_external_path = Path( str(source_external_record.get("path", "")) ).resolve() if ( not source_checkpoint_path.is_file() or source_checkpoint_record.get("sha256") != file_sha256_authority_boundary( source_checkpoint_path, expected_sha256=str(source_checkpoint_record.get("sha256", "")), identity_cache_root=identity_cache_root, ) or not source_optimizer_path.is_file() or source_optimizer_record.get("sha256") != file_sha256_authority_boundary( source_optimizer_path, expected_sha256=str(source_optimizer_record.get("sha256", "")), identity_cache_root=identity_cache_root, ) or not source_external_path.is_file() or source_external_record.get("sha256") != _file_sha256(source_external_path) ): raise RuntimeError("NoNE v2+ accepted source artifact hash differs") composition_record = artifacts.get("composition") if ( not isinstance(composition_record, dict) or Path(str(composition_record.get("path", ""))).resolve() != composition_file or composition_record.get("sha256") != _file_sha256(composition_file) ): raise RuntimeError("NoNE v2+ composition hash differs") parent_composition_record = composition.get("parentComposition") parent_receipt_record = composition.get("parentMigrationReceipt") if not isinstance(parent_composition_record, dict) or not isinstance( parent_receipt_record, dict, ): raise RuntimeError("NoNE v2+ parent lineage is absent") parent_composition_path = Path( str(parent_composition_record.get("path", "")) ).resolve() parent_receipt_path = Path( str(parent_receipt_record.get("path", "")) ).resolve() if ( not parent_composition_path.is_file() or not parent_receipt_path.is_file() or parent_composition_record.get("sha256") != _file_sha256(parent_composition_path) or parent_receipt_record.get("sha256") != _file_sha256(parent_receipt_path) ): raise RuntimeError("NoNE v2+ parent artifact hash differs") parent_composition = validate_v2_seed_composition( parent_composition_path, parent_receipt_path, ) source_generation = composition.get("sourceGeneration") target_generation = composition.get("targetGeneration") if ( not isinstance(source_generation, str) or not source_generation or not isinstance(target_generation, str) or not target_generation or source_generation == target_generation or parent_composition.get("targetGeneration") != source_generation or receipt.get("sourceGeneration") != source_generation or receipt.get("targetGeneration") != target_generation ): raise RuntimeError("NoNE v2+ generation chain differs") if ( composition.get("sourceCheckpoint") != parent_composition.get("sourceCheckpoint") or composition.get("sourceOptimizer") != parent_composition.get("sourceOptimizer") ): raise RuntimeError("NoNE v2+ inherited core reference differs") catalog_record = composition.get("pageCatalog") resident_record = composition.get("residentRuntime") page_store_record = composition.get("pageStore") if not all( isinstance(record, dict) for record in (catalog_record, resident_record, page_store_record) ): raise RuntimeError("NoNE v2+ composition binding is incomplete") assert isinstance(catalog_record, dict) assert isinstance(resident_record, dict) assert isinstance(page_store_record, dict) parent_store_record = parent_composition.get("pageStore") if ( not isinstance(parent_store_record, dict) or not _same_page_store_session_boundary( page_store_record, parent_store_record, ) ): raise RuntimeError("NoNE v2+ expansion changed session ownership") catalog_path = Path(str(catalog_record.get("path", ""))).resolve() resident_path = Path(str(resident_record.get("path", ""))).resolve() if ( not catalog_path.is_file() or catalog_record.get("sha256") != _file_sha256(catalog_path) or not resident_path.is_file() or resident_record.get("sha256") != _file_sha256(resident_path) or resident_record.get("schema") != NONE_V2_RESIDENT_RUNTIME_SCHEMA ): raise RuntimeError("NoNE v2+ catalog or resident runtime hash differs") with safe_open( # type: ignore[no-untyped-call] str(resident_path), framework="pt", device="cpu", ) as handle: metadata = handle.metadata() if ( not isinstance(metadata, dict) or metadata.get("schema") != NONE_V2_RESIDENT_RUNTIME_SCHEMA ): raise RuntimeError("NoNE v2+ resident runtime schema differs") catalog = json.loads(catalog_path.read_text(encoding="utf-8")) parent_catalog_record = parent_composition.get("pageCatalog") if not isinstance(parent_catalog_record, dict): raise RuntimeError("NoNE v2+ parent catalog binding is absent") parent_catalog_path = Path( str(parent_catalog_record.get("path", "")) ).resolve() parent_catalog = json.loads( parent_catalog_path.read_text(encoding="utf-8") ) if not isinstance(catalog, dict) or not isinstance(parent_catalog, dict): raise RuntimeError("NoNE v2+ page catalog payload is invalid") pages = catalog.get("pages") parent_pages = parent_catalog.get("pages") layer_catalog = catalog.get("layerCatalogPageIds") parent_layer_catalog = parent_catalog.get("layerCatalogPageIds") receipt_growth_record = receipt.get("growthPlan") if not isinstance(receipt_growth_record, dict): raise RuntimeError("NoNE v2+ receipt growth plan is absent") materialized_receipt_roots = receipt_growth_record.get( "materializedFamilyRoots", receipt_growth_record.get("targetFamilyRoots"), ) if ( catalog.get("schema") != NONE_V2_PLUS_PAGE_CATALOG_SCHEMA or not isinstance(pages, list) or not isinstance(parent_pages, list) or not isinstance(layer_catalog, dict) or not isinstance(parent_layer_catalog, dict) or pages[: len(parent_pages)] != parent_pages or int(catalog.get("pageCount", -1)) != len(pages) or int(catalog.get("familyRootPageCount", -1)) != int(materialized_receipt_roots) or int(catalog.get("addedFamilyRootPageCount", -1)) != int(receipt.get("growthPlan", {}).get("addedFamilyRoots", -2)) ): raise RuntimeError("NoNE v2+ accepted catalog prefix differs") for layer_id, parent_ids in parent_layer_catalog.items(): target_ids = layer_catalog.get(layer_id) if ( not isinstance(parent_ids, list) or not isinstance(target_ids, list) or target_ids[: len(parent_ids)] != parent_ids ): raise RuntimeError( "NoNE v2+ accepted layer-catalog prefix differs" ) sparse_graph_layer_record = sparse_graph_layer_binding_record_boundary( catalog, page_catalog_sha256=str(catalog_record["sha256"]), ) parent_sparse_graph_layer_authority = ( sparse_graph_layer_catalog_authority_boundary(parent_catalog) ) parent_graph_layer_ids = tuple( int(row["pageId"]) for row in parent_pages if isinstance(row, dict) ) target_graph_layer_ids = tuple( int(row["pageId"]) for row in pages if isinstance(row, dict) ) if ( len(parent_graph_layer_ids) != len(parent_pages) or len(target_graph_layer_ids) != len(pages) or ( parent_sparse_graph_layer_authority is not None and parent_sparse_graph_layer_authority[0] != parent_graph_layer_ids ) ): raise RuntimeError("NoNE v2+ parent sparse graph-layer identity differs") if sparse_graph_layer_record is None: if ( "sparseGraphLayers" in composition or "sparseGraphLayers" in receipt ): raise RuntimeError( "NoNE legacy v2+ catalog has sparse-layer bindings" ) elif ( composition.get("sparseGraphLayers") != sparse_graph_layer_record or receipt.get("sparseGraphLayers") != sparse_graph_layer_record or target_graph_layer_ids[: len(parent_graph_layer_ids)] != parent_graph_layer_ids ): raise RuntimeError("NoNE v2+ sparse graph-layer binding differs") new_pages = pages[len(parent_pages) :] legacy_compact_objective_binding_omitted = ( _legacy_compact_objective_parent_compatibility_boundary( receipt_file=receipt_file, receipt_sha256=_file_sha256(receipt_file), catalog_sha256=str(catalog_record["sha256"]), new_pages=new_pages, parent_pages=parent_pages, receipt=receipt, ) ) if not new_pages or any( not isinstance(row, dict) or row.get("state") not in { "TRANSFER_INITIALIZED_UNTRAINED", "PREALLOCATED_ZERO_RESIDUAL_UNTRAINED", } or row.get("trainedCapabilityClaimed") is not False or row.get("optimizerState") not in {"FRESH_ZERO_UNTRAINED", "IMPLICIT_ZERO_UNTRAINED"} or row.get("pageRole") not in {"family_root", "objective_child"} or not isinstance(row.get("familyRoot"), bool) or (row.get("pageRole") == "family_root") != (row.get("familyRoot") is True) or not isinstance(row.get("transferCoefficientSha256"), str) or len(row["transferCoefficientSha256"]) != 64 or not isinstance(row.get("transferPageStateSha256"), str) or len(row["transferPageStateSha256"]) != 64 or not isinstance(row.get("transferRouteKeySha256"), str) or len(row["transferRouteKeySha256"]) != 64 or ( row.get("state") == "PREALLOCATED_ZERO_RESIDUAL_UNTRAINED" and ( row.get("transferInitialization") != COMPACT_BANK_TRANSFER_INITIALIZATION or not isinstance(row.get("compactBankSummarySha256"), str) or len(row["compactBankSummarySha256"]) != 64 or not isinstance(row.get("compactBankObjectSha256"), str) or len(row["compactBankObjectSha256"]) != 64 or row.get("physicalSeedKnowledgeClaimed") is not False or ( not legacy_compact_objective_binding_omitted and ( not isinstance( row.get("compactSourceFamilyPageId"), int, ) or isinstance( row.get("compactSourceFamilyPageId"), bool, ) ) ) ) ) for row in new_pages ): raise RuntimeError("NoNE v2+ new-page training claim differs") new_family_pages = [ row for row in new_pages if isinstance(row, dict) and row.get("pageRole") == "family_root" ] new_objective_pages = [ row for row in new_pages if isinstance(row, dict) and row.get("pageRole") == "objective_child" ] growth_record = receipt.get("growthPlan") expansion_record = composition.get("catalogExpansion") if not isinstance(growth_record, dict) or not isinstance( expansion_record, dict, ): raise RuntimeError("NoNE v2+ growth-plan binding is absent") receipt_expansion_record = receipt.get("expansion") physical_admission_record = receipt.get("physicalAdmission") sparse_bound_records = ( growth_record, expansion_record, receipt_expansion_record, physical_admission_record, ) sparse_bound_fields = ( "sourcePhysicalGraphLayers", "targetPhysicalGraphLayers", "addedPhysicalGraphLayers", "physicalGraphLayerCount", "graphLayerIdsSha256", "onePageObjectPerSparseGraphLayer", "sparseGraphLayersTrainingClaimed", "sparseGraphLayersPromotionEligible", ) if sparse_graph_layer_record is None: if any( isinstance(record, dict) and any(field in record for field in sparse_bound_fields) for record in sparse_bound_records ): raise RuntimeError( "NoNE legacy v2+ receipt has sparse-layer authority" ) else: expected_graph_layer_count = len(target_graph_layer_ids) expected_source_graph_layer_count = len(parent_graph_layer_ids) expected_added_graph_layer_count = ( expected_graph_layer_count - expected_source_graph_layer_count ) graph_layer_ids_sha256 = sparse_graph_layer_record[ "graphLayerIdsSha256" ] checks = receipt.get("checks") if ( not isinstance(receipt_expansion_record, dict) or not isinstance(physical_admission_record, dict) or growth_record.get("sourcePhysicalGraphLayers") != expected_source_graph_layer_count or growth_record.get("targetPhysicalGraphLayers") != expected_graph_layer_count or growth_record.get("addedPhysicalGraphLayers") != expected_added_graph_layer_count or growth_record.get("graphLayerIdsSha256") != graph_layer_ids_sha256 or expansion_record.get("sourcePhysicalGraphLayers") != expected_source_graph_layer_count or expansion_record.get("targetPhysicalGraphLayers") != expected_graph_layer_count or expansion_record.get("addedPhysicalGraphLayers") != expected_added_graph_layer_count or expansion_record.get("graphLayerIdsSha256") != graph_layer_ids_sha256 or expansion_record.get("onePageObjectPerSparseGraphLayer") is not True or receipt_expansion_record.get("sourcePhysicalGraphLayers") != expected_source_graph_layer_count or receipt_expansion_record.get("targetPhysicalGraphLayers") != expected_graph_layer_count or receipt_expansion_record.get("addedPhysicalGraphLayers") != expected_added_graph_layer_count or receipt_expansion_record.get("graphLayerIdsSha256") != graph_layer_ids_sha256 or receipt_expansion_record.get( "onePageObjectPerSparseGraphLayer" ) is not True or physical_admission_record.get("physicalGraphLayerCount") != expected_graph_layer_count or physical_admission_record.get("graphLayerIdsSha256") != graph_layer_ids_sha256 or physical_admission_record.get( "onePageObjectPerSparseGraphLayer" ) is not True or physical_admission_record.get( "sparseGraphLayersTrainingClaimed" ) is not False or physical_admission_record.get( "sparseGraphLayersPromotionEligible" ) is not False or receipt.get("trainingClaimed") is not False or receipt.get("heldoutClaimed") is not False or receipt.get("coldReloadClaimed") is not False or not isinstance(checks, dict) or checks.get("sparseGraphLayerCatalogAuthorityExact") is not True or checks.get("sparseGraphLayerParentPrefixExact") is not True or checks.get("onePageObjectPerSparseGraphLayer") is not True or checks.get("sparseGraphLayerBindingsExact") is not True or checks.get("sparseGraphLayerClaimsWithheld") is not True or checks.get("graphAuthorityPhysicalGraphLayersExact") is not True ): raise RuntimeError("NoNE v2+ sparse graph-layer receipt differs") growth_path = Path(str(growth_record.get("path", ""))).resolve() growth_sha256 = growth_record.get("sha256") if ( not growth_path.is_file() or not isinstance(growth_sha256, str) or _file_sha256(growth_path) != growth_sha256 or expansion_record.get("growthPlanPath") != str(growth_path) or expansion_record.get("growthPlanSha256") != growth_sha256 ): raise RuntimeError("NoNE v2+ growth-plan artifact differs") growth_plan = json.loads(growth_path.read_text(encoding="utf-8")) if not isinstance(growth_plan, dict): raise RuntimeError("NoNE v2+ growth plan is malformed") federated_growth_demand_authority = ( federated_growth_demand_authority_from_plan_boundary(growth_plan) ) catalog_federated_growth_demand_authority = ( federated_growth_demand_authority_from_catalog_boundary(catalog) ) if federated_growth_demand_authority is None: if ( catalog_federated_growth_demand_authority is not None or composition.get("federatedGrowthDemandAuthority") is not None or composition.get("federatedGrowthDemandAuthoritySha256") is not None or receipt.get("federatedGrowthDemandAuthority") is not None or receipt.get("federatedGrowthDemandAuthoritySha256") is not None or expansion_record.get("federatedGrowthDemandAuthority") is not None or expansion_record.get("federatedGrowthDemandAuthoritySha256") is not None or growth_record.get("federatedGrowthDemandAuthority") is not None or growth_record.get("federatedGrowthDemandAuthoritySha256") is not None ): raise RuntimeError("NoNE legacy v2+ growth demand differs") else: federated_growth_demand_sha256 = federated_growth_demand_authority[ "authoritySha256" ] if ( catalog_federated_growth_demand_authority != federated_growth_demand_authority or composition.get("federatedGrowthDemandAuthority") != federated_growth_demand_authority or composition.get("federatedGrowthDemandAuthoritySha256") != federated_growth_demand_sha256 or receipt.get("federatedGrowthDemandAuthority") != federated_growth_demand_authority or receipt.get("federatedGrowthDemandAuthoritySha256") != federated_growth_demand_sha256 or expansion_record.get("federatedGrowthDemandAuthority") != federated_growth_demand_authority or expansion_record.get("federatedGrowthDemandAuthoritySha256") != federated_growth_demand_sha256 or growth_record.get("federatedGrowthDemandAuthority") != federated_growth_demand_authority or growth_record.get("federatedGrowthDemandAuthoritySha256") != federated_growth_demand_sha256 ): raise RuntimeError("NoNE v2+ federated growth demand differs") roadmap = ( growth_plan.get("functionalExpertFamilyRoadmap") if isinstance(growth_plan, dict) else None ) language_roadmap = ( growth_plan.get("languageExpertPackRoadmap") if isinstance(growth_plan, dict) else None ) if ( not isinstance(roadmap, list) or not isinstance(language_roadmap, list) or any( not isinstance(row, dict) or not isinstance(row.get("family"), str) for row in (*roadmap, *language_roadmap) ) ): raise RuntimeError("NoNE v2+ growth-plan roadmap differs") roadmap_ids = [str(row["family"]) for row in roadmap] language_ids = [str(row["family"]) for row in language_roadmap] objective_plan = _validated_objective_page_plan( growth_plan, tuple(roadmap_ids), ) objective_by_id = { objective.objective_id: objective for objective in objective_plan } specialist_ids = _validated_science_specialist_family_ids( growth_plan, tuple(roadmap_ids), required=growth_plan.get("scienceSpecialistRoadmap") is not None, allow_inherited_prefix=True, ) expected_language_ids = list( _validated_language_expert_family_ids(growth_plan, required=True) ) catalog_family_ids = [ str(row["family"]) for row in pages if isinstance(row, dict) and isinstance(row.get("family"), str) and bool(str(row["family"]).strip()) ] catalog_objective_rows = [ row for row in pages if isinstance(row, dict) and isinstance(row.get("objectiveId"), str) and bool(str(row["objectiveId"]).strip()) ] catalog_objective_ids = [ str(row["objectiveId"]) for row in catalog_objective_rows ] parent_family_ids = [ str(row["family"]) for row in parent_pages if isinstance(row, dict) and isinstance(row.get("family"), str) and bool(str(row["family"]).strip()) ] new_family_ids = [str(row["family"]) for row in new_family_pages] expected_new_family_ids = [ family_id for family_id in roadmap_ids if family_id not in set(parent_family_ids) ][: len(new_family_ids)] materialized_page_id_by_family = { str(row["family"]): int(row["pageId"]) for row in pages if isinstance(row, dict) and isinstance(row.get("family"), str) and bool(str(row["family"]).strip()) } compact_new_objective_pages = [ row for row in new_objective_pages if row.get("state") == "PREALLOCATED_ZERO_RESIDUAL_UNTRAINED" ] compact_objective_binding = receipt.get( "compactObjectiveCohortBinding" ) if ( compact_new_objective_pages and not legacy_compact_objective_binding_omitted ): if any( not isinstance(row.get("pageId"), int) or isinstance(row.get("pageId"), bool) or not isinstance(row.get("compactSourceFamilyPageId"), int) or isinstance(row.get("compactSourceFamilyPageId"), bool) or not isinstance(row.get("objectiveId"), str) or row["objectiveId"] not in objective_by_id for row in compact_new_objective_pages ): raise RuntimeError( "NoNE compact objective page binding is malformed" ) _validate_compact_objective_cohort_binding_record( compact_objective_binding, compact_page_ids=tuple( int(row["pageId"]) for row in compact_new_objective_pages ), selected_family_page_ids=tuple( int(row["compactSourceFamilyPageId"]) for row in compact_new_objective_pages ), selected_objectives=tuple( objective_by_id[str(row["objectiveId"])] for row in compact_new_objective_pages ), family_rows=_family_root_rows_from_catalog_boundary( pages, expected_count=catalog.get("familyRootPageCount"), ), ) if receipt["checks"].get( "compactObjectiveCohortBoundToModelEvidence" ) is not True: raise RuntimeError( "NoNE compact objective cohort check differs" ) elif compact_objective_binding is not None: raise RuntimeError( "NoNE compact objective cohort binding is unexpected" ) expected_family_graph_order = [ { "family": family_id, "graphOrdinal": graph_ordinal, "materialized": family_id in materialized_page_id_by_family, "pageId": materialized_page_id_by_family.get(family_id), } for graph_ordinal, family_id in enumerate(roadmap_ids) ] materialized_family_root_count = len(catalog_family_ids) pending_family_root_count = len(roadmap_ids) - materialized_family_root_count materialized_language_root_count = sum( family_id in set(language_ids) for family_id in catalog_family_ids ) materialized_specialist_root_count = sum( family_id in set(specialist_ids) for family_id in catalog_family_ids ) recorded_materialized_family_roots = growth_record.get( "materializedFamilyRoots", growth_record.get("targetFamilyRoots"), ) recorded_pending_family_roots = growth_record.get( "pendingFamilyRoots", len(roadmap_ids) - int(recorded_materialized_family_roots), ) recorded_source_family_roots = growth_record.get("sourceFamilyRoots") if ( growth_plan.get("schema") != PAGED_GROWTH_PLAN_SCHEMA or language_ids != expected_language_ids or pending_family_root_count < 0 or len(set(catalog_family_ids)) != materialized_family_root_count or not set(catalog_family_ids).issubset(roadmap_ids) or catalog_family_ids[: len(parent_family_ids)] != parent_family_ids or new_family_ids != expected_new_family_ids or ( "familyGraphOrder" in catalog and catalog.get("familyGraphOrder") != expected_family_graph_order ) or not isinstance(recorded_source_family_roots, int) or isinstance(recorded_source_family_roots, bool) or recorded_source_family_roots != len(parent_family_ids) or growth_record.get("targetFamilyRoots") != len(roadmap_ids) or recorded_materialized_family_roots != materialized_family_root_count or recorded_pending_family_roots != pending_family_root_count or growth_record.get("addedFamilyRoots") != len(new_family_pages) or materialized_family_root_count != recorded_source_family_roots + len(new_family_pages) or growth_record.get("sourceObjectivePages") != len(catalog_objective_rows) - len(new_objective_pages) or growth_record.get("targetObjectivePages") != len(catalog_objective_rows) or growth_record.get("plannedObjectivePages") != len(objective_plan) or growth_record.get("pendingObjectivePages") != len(objective_plan) - len(catalog_objective_rows) or growth_record.get("addedObjectivePages") != len(new_objective_pages) or len(set(catalog_objective_ids)) != len(catalog_objective_ids) or not set(catalog_objective_ids).issubset(objective_by_id) or any( any( row.get(key) != value for key, value in objective_by_id[ str(row["objectiveId"]) ].identity_record().items() ) or not isinstance(row.get("objectiveSourcePageCount"), int) or isinstance(row.get("objectiveSourcePageCount"), bool) or int(row["objectiveSourcePageCount"]) <= objective_by_id[str(row["objectiveId"])].ordinal or not isinstance(row.get("payloadShardBacked"), bool) for row in catalog_objective_rows ) or catalog.get("objectivePageCount") != len(catalog_objective_rows) or catalog.get("plannedObjectivePageCount") != len(objective_plan) or catalog.get("pendingObjectivePageCount") != len(objective_plan) - len(catalog_objective_rows) or catalog.get("trainingPageCount") != materialized_family_root_count + len(catalog_objective_rows) or ( pending_family_root_count > 0 and bool(catalog_objective_rows) ) or catalog.get("familyRootPageCount") != materialized_family_root_count or catalog.get("languagePackRootPageCount") != materialized_language_root_count or catalog.get("scienceSpecialistRootPageCount") != materialized_specialist_root_count or ( "plannedFunctionalFamilyRootCount" in catalog and catalog.get("plannedFunctionalFamilyRootCount") != len(roadmap_ids) ) or ( "pendingFunctionalFamilyRootCount" in catalog and catalog.get("pendingFunctionalFamilyRootCount") != pending_family_root_count ) or ( "plannedLanguagePackRootPageCount" in catalog and catalog.get("plannedLanguagePackRootPageCount") != len(language_ids) ) or ( "plannedScienceSpecialistRootPageCount" in catalog and catalog.get("plannedScienceSpecialistRootPageCount") != len(specialist_ids) ) or growth_record.get("languageExpertPacks") != len(language_ids) or ( specialist_ids and growth_record.get("scienceSpecialists") != len(specialist_ids) ) or expansion_record.get("sourceFamilyRoots") != growth_record.get("sourceFamilyRoots") or expansion_record.get("targetFamilyRoots") != len(roadmap_ids) or expansion_record.get( "materializedFamilyRoots", expansion_record.get("targetFamilyRoots"), ) != materialized_family_root_count or expansion_record.get( "pendingFamilyRoots", len(roadmap_ids) - materialized_family_root_count, ) != pending_family_root_count ): raise RuntimeError("NoNE v2+ exact family catalog differs") session_values = page_store_record.get("sessionId") generation_record = receipt.get("generationBinding") if ( not isinstance(session_values, list) or len(session_values) != 4 or not isinstance(generation_record, dict) or generation_record.get("schema") != "nnf.resynthesis.none_generation_binding.v1" or generation_record.get("sessionId") != session_values ): raise RuntimeError("NoNE v2+ accepted generation identity differs") store = NoNEImmutablePageStore( Path(str(page_store_record.get("root", ""))) ) accepted_t = store.begin_session( torch.tensor(session_values, dtype=torch.long) ) generation = generation_record.get("generation") parent_generation = generation_record.get("parentGeneration") manifest_sha256 = generation_record.get("manifestSha256") manifest_payload_sha256 = generation_record.get( "manifestPayloadSha256" ) if ( not isinstance(generation, int) or isinstance(generation, bool) or not isinstance(parent_generation, int) or isinstance(parent_generation, bool) or not isinstance(manifest_sha256, str) or not isinstance(manifest_payload_sha256, str) or int(accepted_t) < generation or receipt.get("acceptedPointer", {}).get("generation") != generation ): raise RuntimeError("NoNE v2+ accepted generation frontier differs") binding = store.verify_generation_boundary( generation_t=torch.tensor(generation, dtype=torch.long), manifest_sha256_t=digest_tensor(manifest_sha256), manifest_payload_sha256_t=digest_tensor( manifest_payload_sha256 ), ) if ( not _binding_matches_external_record(binding, generation_record) or int(binding.parent_generation_t) != parent_generation ): raise RuntimeError("NoNE v2+ immutable generation binding differs") session_key = str(page_store_record["acceptedPointer"]["sessionKey"]) session_root = store.root / "sessions" / session_key target_manifest = json.loads( (session_root / binding.manifest_relative_path).read_text( encoding="utf-8" ) ) target_parent_manifest_payload_sha256 = target_manifest.get( "parentManifestPayloadSha256" ) if ( not page_generation_schema_supported_boundary( target_manifest.get("schema") ) or target_manifest.get("parentGeneration") != parent_generation ): raise RuntimeError("NoNE v2+ expansion parent manifest differs") source_external_envelope = json.loads( source_external_path.read_text(encoding="utf-8") ) if not isinstance(source_external_envelope, dict): raise RuntimeError("NoNE v2+ accepted source sidecar is invalid") source_external_state = source_external_envelope.get("externalState") source_generation_record = ( source_external_state.get("generationBinding") if isinstance(source_external_state, dict) else None ) # Retained training commits may leave candidatePageUpdate=True on the # immutable source sidecar even after the successor physical admission has # already accepted that exact checkpoint/optimizer pair. Allow that only # for already-accepted physical-admission receipts whose retained-transaction # authority check passed; fresh uncommitted candidates still fail closed. source_candidate_update = ( isinstance(source_external_state, dict) and source_external_state.get("candidatePageUpdate") ) retained_candidate_source_ok = ( source_candidate_update is True and receipt.get("status") == "PHYSICAL_ADMISSION_ACCEPTED_TRAINING_PENDING" and isinstance(receipt.get("sourceRetainedTransaction"), dict) and isinstance(receipt.get("checks"), dict) and receipt["checks"].get("retainedTransactionAuthorityBound") is True and receipt["checks"].get("sourceCheckpointOwnedAcceptedGeneration") is True ) source_generation = ( source_generation_record.get("generation") if isinstance(source_generation_record, dict) else None ) if ( source_external_envelope.get("schema") != EXTERNAL_CHECKPOINT_BINDING_SCHEMA or source_external_envelope.get("checkpointSha256") != source_checkpoint_record.get("sha256") or source_external_envelope.get("optimizerSha256") != source_optimizer_record.get("sha256") or not isinstance(source_external_state, dict) or ( source_external_state.get("candidatePageUpdate") is not False and not retained_candidate_source_ok ) or not isinstance(source_generation_record, dict) or not isinstance(source_generation, int) or isinstance(source_generation, bool) or source_generation_record.get("sessionId") != session_values ): raise RuntimeError( "NoNE v2+ expansion parent checkpoint binding differs" ) source_sidecar_binding = store.verify_generation_boundary( generation_t=torch.tensor(source_generation, dtype=torch.long), manifest_sha256_t=digest_tensor( str(source_generation_record.get("manifestSha256", "")) ), manifest_payload_sha256_t=digest_tensor( str( source_generation_record.get( "manifestPayloadSha256", "", ) ) ), ) if not _binding_matches_external_record( source_sidecar_binding, source_generation_record, ): raise RuntimeError("NoNE v2+ source generation sidecar differs") source_manifest_payload_sha256 = source_generation_record.get( "manifestPayloadSha256" ) if source_generation == parent_generation: if ( not isinstance(source_manifest_payload_sha256, str) or len(source_manifest_payload_sha256) != 64 or ( target_parent_manifest_payload_sha256 is not None and target_parent_manifest_payload_sha256 != source_manifest_payload_sha256 ) ): raise RuntimeError("NoNE v2+ direct parent manifest differs") target_parent_manifest_payload_sha256 = ( source_manifest_payload_sha256 ) elif ( not isinstance(target_parent_manifest_payload_sha256, str) or len(target_parent_manifest_payload_sha256) != 64 ): raise RuntimeError("NoNE v2+ expansion parent manifest differs") expansion_parent_binding = source_sidecar_binding expansion_parent_record: Mapping[str, Any] = source_generation_record source_manifest = json.loads( (session_root / source_sidecar_binding.manifest_relative_path).read_text( encoding="utf-8" ) ) if source_generation != parent_generation: if source_generation + 1 != parent_generation: raise RuntimeError( "NoNE v2+ source generation is not the expansion parent" ) parent_receipt_value = json.loads( parent_receipt_path.read_text(encoding="utf-8") ) parent_receipt_generation = ( parent_receipt_value.get("generationBinding") if isinstance(parent_receipt_value, dict) else None ) history_pattern = ( f"generation_{parent_generation:08d}_" f"{target_parent_manifest_payload_sha256}_*.json" ) history_candidates = sorted( (session_root / "accepted_authorities").glob(history_pattern) ) if ( source_generation_record != parent_receipt_generation or len(history_candidates) != 1 ): raise RuntimeError( "NoNE v2+ expansion parent union authority differs" ) parent_pointer = json.loads( history_candidates[0].read_text(encoding="utf-8") ) parent_graph_record = ( parent_pointer.get("graphAuthority") if isinstance(parent_pointer, dict) else None ) if ( not isinstance(parent_pointer, dict) or parent_pointer.get("schema") != PAGE_ACCEPTED_POINTER_SCHEMA or parent_pointer.get("sessionKey") != session_key or parent_pointer.get("generation") != parent_generation or parent_pointer.get("manifestPayloadSha256") != target_parent_manifest_payload_sha256 or not isinstance(parent_pointer.get("manifest"), str) or not isinstance(parent_pointer.get("manifestSha256"), str) or not isinstance(parent_graph_record, dict) ): raise RuntimeError( "NoNE v2+ expansion parent union pointer differs" ) expansion_parent_binding, source_manifest = ( store._load_generation_binding_boundary( str(parent_pointer["manifest"]), expected_manifest_sha256=str( parent_pointer["manifestSha256"] ), expected_payload_sha256=( target_parent_manifest_payload_sha256 ), ) ) expansion_parent_graph = store._load_graph_authority_record_boundary( expansion_parent_binding, source_manifest, parent_graph_record, ) if not _validated_training_branch_union_graph_descendant_boundary( store=store, binding=expansion_parent_binding, graph_record=( expansion_parent_graph.external_record_boundary() ), receipt_value=parent_receipt_value, receipt_generation=parent_receipt_generation, accepted_child_pointer=receipt.get("acceptedPointer"), ): raise RuntimeError( "NoNE v2+ expansion parent union proof differs" ) expansion_parent_record = parent_pointer if ( expansion_parent_record.get("generation") != parent_generation or expansion_parent_record.get("manifestPayloadSha256") != target_parent_manifest_payload_sha256 ): raise RuntimeError("NoNE v2+ parent generation sidecar differs") source_object_rows = { int(row["pageId"]): dict(row) for row in source_manifest.get("pageObjects", ()) if isinstance(row, dict) } source_updated_page_ids_value = source_manifest.get("updatedPageIds") if ( not isinstance(source_updated_page_ids_value, list) or any( not isinstance(page_id, int) or isinstance(page_id, bool) or page_id not in source_object_rows for page_id in source_updated_page_ids_value ) or len(set(source_updated_page_ids_value)) != len(source_updated_page_ids_value) ): raise RuntimeError("NoNE v2+ source updated-page authority differs") source_updated_page_ids = frozenset(source_updated_page_ids_value) target_object_rows = { int(row["pageId"]): dict(row) for row in target_manifest.get("pageObjects", ()) if isinstance(row, dict) } compact_authorities_value = catalog.get("compactPageBanks", []) composition_compact_authorities = composition.get("compactPageBanks", []) if ( not isinstance(compact_authorities_value, list) or any(not isinstance(value, dict) for value in compact_authorities_value) or compact_authorities_value != composition_compact_authorities ): raise RuntimeError("NoNE v2+ compact-page bank authority differs") compact_authority_by_summary: dict[str, dict[str, Any]] = {} admitted_compact_page_ids: list[int] = [] compact_bank_parameter_capacity = 0 for authority_value in compact_authorities_value: authority = dict(authority_value) summary_sha256 = authority.get("summarySha256") admitted_ids = authority.get("admittedPageIds") bank_page_count = authority.get("bankPageCount") bank_page_elements = authority.get("pageParameterElements") bank_parameter_elements = authority.get( "physicalParameterElementsInitialized" ) if ( authority.get("schema") != COMPACT_PAGE_BANK_AUTHORITY_SCHEMA or not isinstance(summary_sha256, str) or len(summary_sha256) != 64 or summary_sha256 in compact_authority_by_summary or not isinstance(authority.get("journalSha256"), str) or len(str(authority.get("journalSha256"))) != 64 or not isinstance(authority.get("sourceCheckpointSha256"), str) or len(str(authority.get("sourceCheckpointSha256"))) != 64 or authority.get("sessionId") != page_store_record.get("sessionId") or not isinstance(bank_page_count, int) or isinstance(bank_page_count, bool) or bank_page_count < 1 or not isinstance(bank_page_elements, int) or isinstance(bank_page_elements, bool) or bank_page_elements < 1 or not isinstance(bank_parameter_elements, int) or isinstance(bank_parameter_elements, bool) or bank_parameter_elements != bank_page_count * bank_page_elements or not isinstance(admitted_ids, list) or any( not isinstance(page_id, int) or isinstance(page_id, bool) for page_id in admitted_ids ) or len(set(admitted_ids)) != len(admitted_ids) or authority.get("admittedPageCount") != len(admitted_ids) or authority.get("sourceBankTrainedPageCountAtDiscovery") != 0 or authority.get("sourceBankAcceptedGenerationCommitted") is not False or authority.get("admissionTrainingClaimed") is not False or authority.get( "promotionRequiresDistinctGradientHeldoutAndColdReload" ) is not True ): raise RuntimeError("NoNE v2+ compact-page bank identity differs") compact_authority_by_summary[summary_sha256] = authority admitted_compact_page_ids.extend(admitted_ids) compact_bank_parameter_capacity += bank_parameter_elements compact_pages = [ row for row in pages if isinstance(row, dict) and row.get("state") == "PREALLOCATED_ZERO_RESIDUAL_UNTRAINED" ] compact_page_ids = [int(row["pageId"]) for row in compact_pages] if ( len(set(admitted_compact_page_ids)) != len(admitted_compact_page_ids) or set(admitted_compact_page_ids) != set(compact_page_ids) or catalog.get("compactPageBankCount", 0) != len(compact_authority_by_summary) or isinstance(catalog.get("compactPageBankCount", 0), bool) or catalog.get("admittedCompactPageCount", 0) != len(compact_page_ids) or isinstance(catalog.get("admittedCompactPageCount", 0), bool) or catalog.get("physicalUntrainedBankCapacityParameterElements", 0) != compact_bank_parameter_capacity or isinstance( catalog.get("physicalUntrainedBankCapacityParameterElements", 0), bool, ) or ( "compactPageBanks" in growth_record and growth_record.get("compactPageBanks") != len(compact_authority_by_summary) ) or ( "admittedCompactPages" in growth_record and growth_record.get("admittedCompactPages") != len(compact_page_ids) ) ): raise RuntimeError("NoNE v2+ compact-page admission history differs") for row in compact_pages: page_id = int(row["pageId"]) summary_sha256 = row.get("compactBankSummarySha256") object_sha256 = row.get("compactBankObjectSha256") object_row = target_object_rows.get(page_id) if ( not isinstance(summary_sha256, str) or summary_sha256 not in compact_authority_by_summary or page_id not in compact_authority_by_summary[summary_sha256]["admittedPageIds"] or not _compact_page_object_provenance_matches_boundary( page_id=page_id, compact_object_sha256=object_sha256, target_object_row=object_row, source_object_rows=source_object_rows, source_updated_page_ids=source_updated_page_ids, ) ): raise RuntimeError("NoNE v2+ compact-page object provenance differs") parent_page_ids = [ int(row["pageId"]) for row in parent_pages if isinstance(row, dict) ] target_page_ids = [ int(row["pageId"]) for row in pages if isinstance(row, dict) ] new_page_ids = [ int(row["pageId"]) for row in new_pages if isinstance(row, dict) ] if ( not source_object_rows or set(source_object_rows) != set(parent_page_ids) or set(target_object_rows) != set(target_page_ids) or target_manifest.get("pageCount") != len(target_page_ids) or not _same_unique_page_id_membership( target_manifest.get("updatedPageIds"), new_page_ids, ) or any( target_object_rows.get(page_id) != row for page_id, row in source_object_rows.items() ) ): raise RuntimeError("NoNE v2+ immutable page-object prefix differs") checkpoint_record = artifacts.get("checkpoint") optimizer_record = artifacts.get("optimizer") external_record = artifacts.get("externalState") if not all( isinstance(record, dict) for record in ( checkpoint_record, optimizer_record, external_record, ) ): raise RuntimeError("NoNE v2+ checkpoint transaction is incomplete") assert isinstance(checkpoint_record, dict) assert isinstance(optimizer_record, dict) assert isinstance(external_record, dict) checkpoint_path = Path( str(checkpoint_record.get("path", "")) ).resolve() optimizer_path = Path(str(optimizer_record.get("path", ""))).resolve() external_path = Path(str(external_record.get("path", ""))).resolve() if ( not checkpoint_path.is_file() or checkpoint_record.get("sha256") != _file_sha256(checkpoint_path) or not optimizer_path.is_file() or optimizer_record.get("sha256") != _file_sha256(optimizer_path) or not external_path.is_file() or external_record.get("sha256") != _file_sha256(external_path) ): raise RuntimeError("NoNE v2+ checkpoint artifact hash differs") checkpoint = _load_checkpoint(checkpoint_path) lineage = checkpoint["lineage"] paged_lineage = lineage.get("pagedNoNE") catalog_runtime_layer_ids = list( _validated_paged_layer_ids_boundary(layer_catalog) ) if not isinstance(paged_lineage, dict): raise RuntimeError("NoNE v2+ checkpoint catalog lineage differs") _validate_retained_paged_checkpoint_lineage_boundary( paged_lineage, lineage, composition_sha256=_file_sha256(composition_file), family_root_count=int(catalog["familyRootPageCount"]), training_page_count=int(catalog["trainingPageCount"]), objective_page_count=int(catalog["objectivePageCount"]), planned_objective_page_count=int(catalog["plannedObjectivePageCount"]), runtime_layer_ids=tuple(catalog_runtime_layer_ids), physical_graph_layer_count=( int(sparse_graph_layer_record["physicalGraphLayerCount"]) if sparse_graph_layer_record is not None else None ), graph_layer_ids_sha256=( str(sparse_graph_layer_record["graphLayerIdsSha256"]) if sparse_graph_layer_record is not None else None ), ) expected_language_sha256 = hashlib.sha256( "\n".join(expected_language_ids).encode("utf-8") ).hexdigest() if ( lineage.get("functionalCapabilityFamilies") != len(roadmap_ids) or lineage.get("languageCapabilityCatalog") != growth_plan.get("languageExpertCatalogSchema") or lineage.get("languageCapabilityPacks") != len( expected_language_ids ) or lineage.get("languageCapabilityPackIdsSha256") != expected_language_sha256 or lineage.get("languageCapabilityNativeRouting") is not True ): raise RuntimeError("NoNE v2+ checkpoint family lineage differs") _validate_resident_checkpoint_state( resident_path=resident_path, checkpoint=checkpoint, ) source_checkpoint = _load_checkpoint(source_checkpoint_path) source_parameters = source_checkpoint["parameters"] source_buffers = source_checkpoint["buffers"] checkpoint_parameters = checkpoint["parameters"] checkpoint_buffers = checkpoint["buffers"] actual_page_state_by_layer: dict[int, set[str]] = {} actual_route_state_by_layer: dict[int, set[str]] = {} try: parent_layer_ids = sorted(int(value) for value in parent_layer_catalog) target_layer_ids = sorted(int(value) for value in layer_catalog) except (TypeError, ValueError) as exc: raise RuntimeError("NoNE v2+ layer catalog identity differs") from exc if ( not parent_layer_ids or not set(parent_layer_ids).issubset(target_layer_ids) or any(str(layer_id) not in parent_layer_catalog for layer_id in parent_layer_ids) or any(str(layer_id) not in layer_catalog for layer_id in target_layer_ids) or any( not isinstance(layer_catalog[str(layer_id)], list) or not layer_catalog[str(layer_id)] for layer_id in target_layer_ids ) ): raise RuntimeError("NoNE v2+ layer catalog identity differs") # Layer page IDs were already validated as unique above. Preserve their # immutable catalog order in one lookup so validating a large physical bank # remains linear instead of repeatedly scanning each layer for every page. target_route_index_by_layer: dict[int, dict[int, int]] = {} for layer_id in target_layer_ids: raw_target_ids = layer_catalog[str(layer_id)] if not isinstance(raw_target_ids, list): raise RuntimeError("NoNE v2+ layer catalog identity differs") target_route_index_by_layer[layer_id] = { int(page_id): route_index for route_index, page_id in enumerate(raw_target_ids) } new_runtime_layer_ids = sorted(set(target_layer_ids) - set(parent_layer_ids)) target_family_root_by_family = { str(row["family"]): int(row["pageId"]) for row in pages if isinstance(row, dict) and isinstance(row.get("family"), str) and bool(str(row["family"]).strip()) } for layer_id in parent_layer_ids: raw_source_ids = parent_layer_catalog[str(layer_id)] if not isinstance(raw_source_ids, list) or not raw_source_ids: raise RuntimeError("NoNE v2+ source layer catalog differs") source_page_ids = [int(page_id) for page_id in raw_source_ids] runtime_prefix = ( f"science_stack.science_layer_{layer_id}." "paged_expert_runtime." ) route_keys_t = source_parameters.get( f"{runtime_prefix}router.page_route_keys" ) page_prior_t = source_parameters.get( f"{runtime_prefix}router.page_prior" ) route_count_t = source_buffers.get( f"{runtime_prefix}accepted_route_count_t" ) gradient_count_t = source_buffers.get( f"{runtime_prefix}accepted_gradient_update_count_t" ) if ( not isinstance(route_keys_t, torch.Tensor) or route_keys_t.ndim != 2 or route_keys_t.shape[0] != len(source_page_ids) or not isinstance(page_prior_t, torch.Tensor) or page_prior_t.shape != (len(source_page_ids),) or not isinstance(route_count_t, torch.Tensor) or route_count_t.shape != (len(source_page_ids),) or not isinstance(gradient_count_t, torch.Tensor) or gradient_count_t.shape != (len(source_page_ids),) ): raise RuntimeError("NoNE v2+ source transfer evidence differs") new_page_source_layer_ids = _validated_transfer_source_layer_ids_boundary( tuple(row for row in new_pages if isinstance(row, dict)), parent_layer_catalog, ) actual_new_page_target_layer_ids = tuple( int(row["layerId"]) for row in new_pages if isinstance(row, dict) ) source_lineage = source_checkpoint.get("lineage") total_science_layers = ( int(source_lineage.get("scienceLayers", -1)) if isinstance(source_lineage, dict) else -1 ) if total_science_layers > len(parent_layer_ids): layer_transfer_graph_t = source_parameters.get( "science_stack.layer_transfer_graph" ) if not isinstance(layer_transfer_graph_t, torch.Tensor): raise RuntimeError( "NoNE v2+ reasoning-layer transfer graph is absent" ) expected_new_page_target_layer_ids = ( _model_owned_new_runtime_layer_targets( page_source_layer_ids=new_page_source_layer_ids, paged_layer_ids=parent_layer_ids, total_science_layers=total_science_layers, layer_transfer_graph_t=layer_transfer_graph_t, ) ) else: expected_new_page_target_layer_ids = new_page_source_layer_ids runtime_layer_growth_fields = ( (growth_record, "sourcePagedRuntimeLayerIds", parent_layer_ids), (growth_record, "targetPagedRuntimeLayerIds", target_layer_ids), (growth_record, "addedPagedRuntimeLayerIds", new_runtime_layer_ids), (expansion_record, "sourcePagedRuntimeLayerIds", parent_layer_ids), (expansion_record, "targetPagedRuntimeLayerIds", target_layer_ids), ( expansion_record, "addedTransferInitializedRuntimeLayerIds", new_runtime_layer_ids, ), ) current_runtime_layer_growth_exact = all( record.get(field) == expected for record, field, expected in runtime_layer_growth_fields ) retained_noop_runtime_layer_growth_exact = ( paged_lineage.get("schema") == "nnf.resynthesis.paged_none_lineage.v2" and parent_layer_ids == target_layer_ids and not new_runtime_layer_ids and all(field not in record for record, field, _ in runtime_layer_growth_fields) ) if ( actual_new_page_target_layer_ids != expected_new_page_target_layer_ids or new_runtime_layer_ids != sorted( set(actual_new_page_target_layer_ids) - set(parent_layer_ids) ) or not ( current_runtime_layer_growth_exact or retained_noop_runtime_layer_growth_exact ) ): raise RuntimeError("NoNE v2+ model-owned runtime-layer growth differs") expected_new_runtime_parameters: dict[str, torch.Tensor] = {} expected_new_runtime_buffers: dict[str, torch.Tensor] = {} expected_new_runtime_additions: dict[ int, list[ tuple[ int, int, torch.Tensor, torch.Tensor, torch.Tensor, ] ], ] = {} for layer_id in parent_layer_ids: source_page_ids = parent_layer_catalog[str(layer_id)] parent_target_ids = layer_catalog[str(layer_id)] if ( not isinstance(source_page_ids, list) or not isinstance(parent_target_ids, list) or parent_target_ids[: len(source_page_ids)] != source_page_ids ): raise RuntimeError("NoNE v2+ layer page prefix differs") layer_new_rows = [ row for row, source_layer_id in zip( new_pages, new_page_source_layer_ids, strict=True, ) if isinstance(row, dict) and source_layer_id == layer_id ] compact_only_layer = bool(layer_new_rows) and all( row.get("transferInitialization") == COMPACT_BANK_TRANSFER_INITIALIZATION for row in layer_new_rows ) source_bundle = ( None if compact_only_layer else _float32_transfer_source_bundle( _load_manifest_page_bundle( store=store, object_rows=source_object_rows, page_ids=[int(page_id) for page_id in source_page_ids], ) ) ) runtime_prefix = ( f"science_stack.science_layer_{layer_id}." "paged_expert_runtime." ) route_keys_name = f"{runtime_prefix}router.page_route_keys" page_prior_name = f"{runtime_prefix}router.page_prior" expert_bias_name = ( f"{runtime_prefix}router.quantile_router.expert_bias_t" ) route_count_name = f"{runtime_prefix}accepted_route_count_t" gradient_count_name = ( f"{runtime_prefix}accepted_gradient_update_count_t" ) source_route_keys_t = source_parameters.get(route_keys_name) source_page_prior_t = source_parameters.get(page_prior_name) source_expert_bias_t = source_buffers.get(expert_bias_name) source_route_count_t = source_buffers.get(route_count_name) source_gradient_count_t = source_buffers.get(gradient_count_name) if ( not isinstance(source_route_keys_t, torch.Tensor) or not isinstance(source_page_prior_t, torch.Tensor) or not isinstance(source_expert_bias_t, torch.Tensor) or not isinstance(source_route_count_t, torch.Tensor) or not isinstance(source_gradient_count_t, torch.Tensor) ): raise RuntimeError("NoNE v2+ transfer evidence is absent") prepared_transfer_evidence = ( _prepare_recursive_transfer_evidence( route_keys_t=source_route_keys_t, page_prior_t=source_page_prior_t, route_count_t=source_route_count_t, gradient_update_count_t=source_gradient_count_t, ) if layer_new_rows else None ) for raw_row in ( _single_threaded_compact_transfer_rows(layer_new_rows) if compact_only_layer else layer_new_rows ): page_id = int(raw_row["pageId"]) target_layer_id = int(raw_row["layerId"]) target_ids = layer_catalog.get(str(target_layer_id)) target_route_index = target_route_index_by_layer.get( target_layer_id, {}, ).get(page_id) target_runtime_prefix = ( f"science_stack.science_layer_{target_layer_id}." "paged_expert_runtime." ) target_route_keys_t = checkpoint_parameters.get( f"{target_runtime_prefix}router.page_route_keys" ) target_page_prior_t = checkpoint_parameters.get( f"{target_runtime_prefix}router.page_prior" ) target_expert_bias_t = checkpoint_buffers.get( f"{target_runtime_prefix}router.quantile_router.expert_bias_t" ) identity = raw_row.get("transferTargetIdentity") if ( not isinstance(identity, str) or not identity or not isinstance(target_ids, list) or not isinstance(target_route_keys_t, torch.Tensor) or not isinstance(target_page_prior_t, torch.Tensor) or not isinstance(target_expert_bias_t, torch.Tensor) ): raise RuntimeError("NoNE v2+ transfer identity is absent") page_role = raw_row.get("pageRole") preferred_families: tuple[str, ...] if page_role == "family_root": family = raw_row.get("family") if not isinstance(family, str) or family not in roadmap_ids: raise RuntimeError("NoNE v2+ family-root identity differs") family_index = roadmap_ids.index(family) preferred_families = (family,) role_valid = bool( raw_row.get("familyRoot") is True and identity == family and raw_row.get("sourceFamilyOrdinal") == family_index and raw_row.get("parentFamilyRootPageIds") == [] ) elif page_role == "objective_child": objective_id = raw_row.get("objectiveId") objective = ( objective_by_id.get(objective_id) if isinstance(objective_id, str) else None ) if objective is None: raise RuntimeError("NoNE v2+ objective identity differs") preferred_families = objective.functional_families expected_parent_ids = [ target_family_root_by_family[family_id] for family_id in preferred_families ] role_valid = bool( raw_row.get("familyRoot") is False and identity == objective.objective_id and raw_row.get("parentFamilyRootPageIds") == expected_parent_ids and all( raw_row.get(key) == value for key, value in objective.catalog_record().items() ) ) else: raise RuntimeError("NoNE v2+ page role differs") # The target layer is immutable admission output, already bound by # the catalog hash, checkpoint route tensors, and accepted receipt. # Re-running the current selector here would make an older accepted # graph depend on later selector-code revisions and can move a page # despite unchanged provenance. Validate its stored source IDs and # exact target membership; coefficient/state checks below still # prove the model-owned transfer payload. if ( not role_valid or raw_row.get("transferSourcePageIds") != source_page_ids or target_route_index is None ): raise RuntimeError( "NoNE v2+ transfer page provenance differs: " f"page={page_id} sourceLayer={layer_id} " f"targetLayer={target_layer_id} role={page_role!r} " f"roleValid={role_valid} " "sourcePageIdsMatch=" f"{raw_row.get('transferSourcePageIds') == source_page_ids} " f"targetMembership={int(target_route_index is not None)}" ) transfer_initialization = raw_row.get( "transferInitialization" ) if transfer_initialization not in { TRANSFER_INITIALIZATION_V1, TRANSFER_INITIALIZATION_V2, TRANSFER_INITIALIZATION_V3, COMPACT_BANK_TRANSFER_INITIALIZATION, }: raise RuntimeError( "NoNE v2+ transfer initialization differs" ) coefficient_fn = ( _identity_distinct_recursive_transfer_coefficients_from_prepared if transfer_initialization == TRANSFER_INITIALIZATION_V3 else _recursive_transfer_coefficients_from_prepared ) assert prepared_transfer_evidence is not None coefficient_t = coefficient_fn( evidence=prepared_transfer_evidence, target_identity=identity, ) if raw_row.get("transferCoefficientSha256") != ( _tensor_sha256(coefficient_t) ): raise RuntimeError( "NoNE v2+ transfer coefficient differs" ) if transfer_initialization == COMPACT_BANK_TRANSFER_INITIALIZATION: compact_object_row = target_object_rows.get(page_id) compact_object_sha256 = raw_row.get("compactBankObjectSha256") compact_object_bytes = ( compact_object_row.get("bytes") if isinstance(compact_object_row, Mapping) else None ) compact_authority = ( compact_authority_by_summary.get( str(raw_row.get("compactBankSummarySha256", "")) ) ) compact_store_root = ( Path(str(compact_authority.get("storeRoot", ""))) .expanduser() .resolve() if isinstance(compact_authority, Mapping) else Path() ) compact_object_path = ( compact_store_root / "objects" / "sha256" / f"{compact_object_sha256}.safetensors" ) if ( raw_row.get("state") != "PREALLOCATED_ZERO_RESIDUAL_UNTRAINED" or raw_row.get("ffnMode") != "preallocated_zero_residual_trainable_seed" or raw_row.get("physicalSeedKnowledgeClaimed") is not False or not isinstance(compact_object_row, Mapping) or compact_object_row.get("sha256") != compact_object_sha256 or raw_row.get("transferPageStateSha256") != compact_object_sha256 or not isinstance(compact_authority, Mapping) or not isinstance(compact_object_bytes, int) or isinstance(compact_object_bytes, bool) or compact_object_bytes < 1 or not compact_object_path.is_file() or compact_object_path.stat().st_size != compact_object_bytes ): raise RuntimeError( "NoNE v2+ compact page is not an inert untrained seed" ) page_state_sha256 = str(compact_object_sha256) else: if source_bundle is None: raise RuntimeError( "NoNE v2+ transfer source bundle is absent" ) actual_bundle = _load_manifest_page_bundle( store=store, object_rows=target_object_rows, page_ids=[page_id], ) expected_bundle = compact_transfer_page_bundle_boundary( _transfer_initialized_page( source_bundle=source_bundle, coefficient_t=coefficient_t, page_id=page_id, family_id=( identity if transfer_initialization in { TRANSFER_INITIALIZATION_V2, TRANSFER_INITIALIZATION_V3, } else None ), ) ) mismatched_weight_fields = tuple( name for name in _TRANSFER_PAGE_WEIGHT_FIELDS if not torch.equal( getattr(expected_bundle.weights, name), getattr(actual_bundle.weights, name), ) ) page_ids_differ = not torch.equal( expected_bundle.weights.page_ids_t, actual_bundle.weights.page_ids_t, ) optimizer_mean_nonzero = bool( torch.count_nonzero(actual_bundle.optimizer_mean_t) ) optimizer_square_nonzero = bool( torch.count_nonzero(actual_bundle.optimizer_square_t) ) optimizer_step_nonzero = bool( torch.count_nonzero(actual_bundle.step_t) ) if ( page_ids_differ or mismatched_weight_fields or optimizer_mean_nonzero or optimizer_square_nonzero or optimizer_step_nonzero ): raise RuntimeError( "NoNE v2+ reconstructed page state differs: " f"page_id={page_id}, page_ids={page_ids_differ}, " f"weights={mismatched_weight_fields}, " f"optimizer_mean={optimizer_mean_nonzero}, " f"optimizer_square={optimizer_square_nonzero}, " f"optimizer_step={optimizer_step_nonzero}" ) page_state_sha256 = _bundle_transfer_state_sha256( actual_bundle ) if raw_row.get("transferPageStateSha256") != ( page_state_sha256 ): raise RuntimeError("NoNE v2+ new page state hash differs") assert target_route_index is not None route_index = target_route_index expected_route_key_t = coefficient_t.to( dtype=source_route_keys_t.dtype ).matmul(source_route_keys_t) if transfer_initialization in { TRANSFER_INITIALIZATION_V2, TRANSFER_INITIALIZATION_V3, COMPACT_BANK_TRANSFER_INITIALIZATION, }: expected_route_key_t = ( expected_route_key_t + _family_identity_residual( expected_route_key_t, identity, ) ) expected_prior_t = coefficient_t.to( dtype=source_page_prior_t.dtype ).dot(source_page_prior_t) expected_bias_t = coefficient_t.to( dtype=source_expert_bias_t.dtype ).dot(source_expert_bias_t) if ( not torch.equal( target_route_keys_t[route_index], expected_route_key_t, ) or not torch.equal( target_page_prior_t[route_index], expected_prior_t, ) or not torch.equal( target_expert_bias_t[route_index], expected_bias_t, ) ): raise RuntimeError( "NoNE v2+ reconstructed resident route differs" ) route_state_sha256 = _tensor_sha256( target_route_keys_t[route_index] ) if raw_row.get("transferRouteKeySha256") != ( route_state_sha256 ): raise RuntimeError("NoNE v2+ new route state hash differs") if transfer_initialization != COMPACT_BANK_TRANSFER_INITIALIZATION: actual_page_state_by_layer.setdefault(target_layer_id, set()).add( page_state_sha256 ) actual_route_state_by_layer.setdefault(target_layer_id, set()).add( route_state_sha256 ) if target_layer_id in new_runtime_layer_ids: expected_new_runtime_additions.setdefault( target_layer_id, [], ).append( ( layer_id, page_id, expected_route_key_t, expected_prior_t, expected_bias_t, ) ) for target_layer_id in target_layer_ids: layer_new_rows = [ row for row in new_pages if isinstance(row, dict) and int(row["layerId"]) == target_layer_id ] added_count = len(layer_new_rows) distinct_transfer_state_count = sum( row.get("transferInitialization") != COMPACT_BANK_TRANSFER_INITIALIZATION for row in layer_new_rows ) if ( len(actual_page_state_by_layer.get(target_layer_id, set())) != distinct_transfer_state_count or len(actual_route_state_by_layer.get(target_layer_id, set())) != added_count ): raise RuntimeError("NoNE v2+ transfer states are not distinct") for target_layer_id, additions in sorted( expected_new_runtime_additions.items() ): source_ids = {addition[0] for addition in additions} page_ids = [addition[1] for addition in additions] if len(source_ids) != 1: raise RuntimeError("NoNE v2+ runtime source identity differs") runtime_parameters, runtime_buffers = ( _transfer_initialized_paged_runtime_state( source_parameters=source_parameters, source_buffers=source_buffers, source_layer_id=next(iter(source_ids)), target_layer_id=target_layer_id, target_page_ids=page_ids, target_route_keys_t=torch.stack( [addition[2] for addition in additions] ), target_page_prior_t=torch.stack( [addition[3] for addition in additions] ), target_expert_bias_t=torch.stack( [addition[4] for addition in additions] ), ) ) expected_new_runtime_parameters.update(runtime_parameters) expected_new_runtime_buffers.update(runtime_buffers) if ( set(expected_new_runtime_additions) != set(new_runtime_layer_ids) or set(expected_new_runtime_parameters) != set(checkpoint_parameters) - set(source_parameters) or set(expected_new_runtime_buffers) != set(checkpoint_buffers) - set(source_buffers) or any( not torch.equal(checkpoint_parameters[name], tensor) for name, tensor in expected_new_runtime_parameters.items() ) or any( not torch.equal(checkpoint_buffers[name], tensor) for name, tensor in expected_new_runtime_buffers.items() ) ): raise RuntimeError("NoNE v2+ transfer-initialized runtime differs") optimizer_payload = torch.load( optimizer_path, map_location="cpu", mmap=True, weights_only=True, ) if ( not isinstance(optimizer_payload, dict) or not isinstance(optimizer_payload.get("param_groups"), list) or len(optimizer_payload["param_groups"]) != 1 or not isinstance(optimizer_payload["param_groups"][0], dict) or set( str(name) for name in optimizer_payload["param_groups"][0].get( "param_names", (), ) ) != set(checkpoint["parameters"]) ): raise RuntimeError("NoNE v2+ named optimizer authority differs") if receipt.get("physicalPageValidationPolicy") != ( "hash_sealed_bank_header_admission_then_model_routed_materialization" ): _validate_checkpoint_optimizer_prefixes( source_checkpoint_path=source_checkpoint_path, target_checkpoint_path=checkpoint_path, source_optimizer_path=source_optimizer_path, target_optimizer_path=optimizer_path, ) external_envelope = json.loads( external_path.read_text(encoding="utf-8") ) if not isinstance(external_envelope, dict): raise RuntimeError("NoNE v2+ external sidecar is invalid") external_state = external_envelope.get("externalState") if ( external_envelope.get("schema") != EXTERNAL_CHECKPOINT_BINDING_SCHEMA or external_envelope.get("checkpointSha256") != checkpoint_record.get("sha256") or external_envelope.get("optimizerSha256") != optimizer_record.get("sha256") or not isinstance(external_state, dict) or external_state.get("compositionPath") != str(composition_file) or external_state.get("compositionSha256") != _file_sha256(composition_file) or external_state.get("storeRoot") != page_store_record.get("root") or external_state.get("candidatePageUpdate") is not False or external_state.get("generationBinding") != generation_record or ( sparse_graph_layer_record is None and "sparseGraphLayers" in external_state ) or ( sparse_graph_layer_record is not None and external_state.get("sparseGraphLayers") != sparse_graph_layer_record ) or external_state.get("trainingProof", {}).get("familyRootCount") != catalog.get("familyRootPageCount") or external_state.get("trainingProof", {}).get("promotionReady") is not False ): raise RuntimeError("NoNE v2+ checkpoint external lineage differs") _validate_expanded_training_proof( source_proof=source_external_state.get("trainingProof"), target_proof=external_state.get("trainingProof"), new_training_page_ids=new_page_ids, target_family_root_count=int(catalog["familyRootPageCount"]), ) external_schema = external_state.get("schema") if external_schema == "nnf.resynthesis.none_checkpoint_external_state.v2": replica_path = Path( str(external_state.get("replicaReceiptPath", "")) ).resolve() seed_pointer = page_store_record.get("acceptedPointer") if ( not replica_path.is_file() or external_state.get("replicaReceiptSha256") != _file_sha256(replica_path) or not isinstance(seed_pointer, dict) or external_state.get("canonicalPointerAdvancesLast") is not True ): raise RuntimeError("NoNE v2+ replica receipt lineage differs") coordinator = NoNEGenerationReplicaCoordinator.from_receipt_boundary( primary_store=store, session_id_t=torch.tensor(session_values, dtype=torch.long), receipt_path=replica_path, # Validate the receipt-bound generation in place. Do not chase the # live accepted pointer: parent replica sets may retain retired # backup roots that were never advanced through later admissions. reconcile_to_primary=False, ) if external_state.get("replicaStoreRoots") != list( coordinator.store_roots_boundary ): raise RuntimeError("NoNE v2+ replica store identity differs") for replica_store in ( coordinator.primary_store, *coordinator.replica_stores, ): replica_binding = replica_store.verify_generation_boundary( generation_t=binding.generation_t, manifest_sha256_t=binding.manifest_sha256_t, manifest_payload_sha256_t=( binding.manifest_payload_sha256_t ), ) if not _binding_matches_external_record( replica_binding, generation_record, ): raise RuntimeError( "NoNE v2+ replicated generation binding differs" ) elif external_schema != ( "nnf.resynthesis.none_checkpoint_external_state.v1" ): raise RuntimeError("NoNE v2+ external replica schema differs") return composition def validate_v2_seed_composition( composition_path: Path, receipt_path: Path, ) -> dict[str, Any]: """Validate every immutable v2 seed binding before model construction.""" composition_file = composition_path.expanduser().resolve() receipt_file = receipt_path.expanduser().resolve() identity_cache_root = composition_file.parent / ".artifact_sha256_cache" composition = json.loads(composition_file.read_text(encoding="utf-8")) receipt = json.loads(receipt_file.read_text(encoding="utf-8")) if ( isinstance(composition, dict) and composition.get("schema") == NONE_V2_PLUS_COMPOSITION_SCHEMA ): cache_key = _v2_plus_validation_key(composition_file, receipt_file) dependencies = _v2_plus_validation_dependencies( composition_file, receipt_file, composition, receipt, ) cached = _V2_PLUS_VALIDATION_CACHE.get(cache_key) if ( cached is not None and cached.dependent_file_identities == dependencies ): return copy.deepcopy(cached.validated) validated = _validate_v2_plus_composition( composition_file, receipt_file, ) if cache_key != _v2_plus_validation_key(composition_file, receipt_file): raise RuntimeError("NoNE v2+ authority changed during validation") dependencies_after = _v2_plus_validation_dependencies( composition_file, receipt_file, composition, receipt, ) if dependencies_after != dependencies: raise RuntimeError("NoNE v2+ artifact changed during validation") _V2_PLUS_VALIDATION_CACHE[cache_key] = _V2PlusValidationCacheEntry( validated=copy.deepcopy(validated), dependent_file_identities=dependencies_after, ) return validated if ( not isinstance(composition, dict) or composition.get("schema") != NONE_V2_COMPOSITION_SCHEMA ): raise RuntimeError("NoNE v2 composition schema differs") if ( not isinstance(receipt, dict) or receipt.get("schema") != NONE_V2_SEED_MIGRATION_SCHEMA or receipt.get("passed") is not True or receipt.get("promotionEligible") is not False or receipt.get("v2Seed", {}).get("trainingStarted") is not False ): raise RuntimeError("NoNE v2 seed migration authority differs") artifacts = receipt.get("artifacts") if not isinstance(artifacts, dict): raise RuntimeError("NoNE v2 seed receipt has no artifacts") composition_record = artifacts.get("composition") if ( not isinstance(composition_record, dict) or Path(str(composition_record.get("path", ""))).resolve() != composition_file or composition_record.get("sha256") != _file_sha256(composition_file) ): raise RuntimeError("NoNE v2 composition hash differs") source_record = composition.get("sourceCheckpoint") if not isinstance(source_record, dict): raise RuntimeError("NoNE v2 composition has no v1 source") source_path = Path(str(source_record.get("path", ""))).resolve() if ( not source_path.is_file() or source_record.get("sha256") != file_sha256_authority_boundary( source_path, expected_sha256=str(source_record.get("sha256", "")), identity_cache_root=identity_cache_root, ) or receipt.get("sourceCheckpoint", {}).get("sha256") != source_record.get("sha256") ): raise RuntimeError("NoNE v2 source checkpoint hash differs") optimizer_record = composition.get("sourceOptimizer") if not isinstance(optimizer_record, dict): raise RuntimeError("NoNE v2 composition has no v1 optimizer") optimizer_path = Path(str(optimizer_record.get("path", ""))).resolve() if ( not optimizer_path.is_file() or optimizer_record.get("sha256") != file_sha256_authority_boundary( optimizer_path, expected_sha256=str(optimizer_record.get("sha256", "")), identity_cache_root=identity_cache_root, ) or receipt.get("sourceOptimizer", {}).get("sha256") != optimizer_record.get("sha256") ): raise RuntimeError("NoNE v2 source optimizer hash differs") catalog_record = composition.get("pageCatalog") if not isinstance(catalog_record, dict): raise RuntimeError("NoNE v2 composition has no page catalog") catalog_path = Path(str(catalog_record.get("path", ""))).resolve() if ( not catalog_path.is_file() or catalog_record.get("sha256") != _file_sha256(catalog_path) ): raise RuntimeError("NoNE v2 page catalog hash differs") catalog = json.loads(catalog_path.read_text(encoding="utf-8")) if ( not isinstance(catalog, dict) or catalog.get("schema") != NONE_V2_PAGE_CATALOG_SCHEMA or int(catalog.get("pageCount", -1)) != int(receipt.get("v2Seed", {}).get("physicalPageObjects", -2)) ): raise RuntimeError("NoNE v2 page catalog authority differs") resident_record = composition.get("residentRuntime") if not isinstance(resident_record, dict): raise RuntimeError("NoNE v2 composition has no resident runtime") resident_path = Path(str(resident_record.get("path", ""))).resolve() if ( not resident_path.is_file() or resident_record.get("sha256") != _file_sha256(resident_path) ): raise RuntimeError("NoNE v2 resident runtime hash differs") with safe_open( # type: ignore[no-untyped-call] str(resident_path), framework="pt", device="cpu", ) as handle: metadata = handle.metadata() if ( not isinstance(metadata, dict) or metadata.get("schema") != NONE_V2_RESIDENT_RUNTIME_SCHEMA ): raise RuntimeError("NoNE v2 resident runtime schema differs") page_store_record = composition.get("pageStore") if not isinstance(page_store_record, dict): raise RuntimeError("NoNE v2 composition has no page store") session_values = page_store_record.get("sessionId") if ( not isinstance(session_values, list) or len(session_values) != 4 or not all( isinstance(value, int) and not isinstance(value, bool) for value in session_values ) ): raise RuntimeError("NoNE v2 session identity differs") page_store = NoNEImmutablePageStore( Path(str(page_store_record.get("root", ""))) ) accepted_t = page_store.begin_session( torch.tensor(session_values, dtype=torch.long) ) pointer = page_store_record.get("acceptedPointer") if ( not isinstance(pointer, dict) or not isinstance(pointer.get("generation"), int) or isinstance(pointer.get("generation"), bool) or not isinstance(pointer.get("manifestSha256"), str) or not isinstance(pointer.get("manifestPayloadSha256"), str) ): raise RuntimeError("NoNE v2 accepted page generation differs") seed_generation_t = torch.tensor( int(pointer["generation"]), dtype=torch.long, ) seed_binding = page_store.verify_generation_boundary( generation_t=seed_generation_t, manifest_sha256_t=digest_tensor( str(pointer["manifestSha256"]) ), manifest_payload_sha256_t=digest_tensor( str(pointer["manifestPayloadSha256"]) ), ) if ( not torch.equal(seed_binding.generation_t, seed_generation_t) or int(accepted_t) < int(seed_generation_t) ): raise RuntimeError("NoNE v2 accepted frontier predates its seed") return composition def _validated_training_locator_boundary( composition_path: Path, receipt_path: Path, ) -> tuple[dict[str, Any], dict[str, Any]]: """Validate the small immutable locator before page-store discovery. A caller may hold an old seed composition while the session's accepted pointer owns a newer executable graph. Reconstructing every historical transfer page merely to discover that pointer is both redundant and prohibitively expensive. This boundary authenticates the caller's composition/receipt pair and its page-store coordinates; graphless stores still proceed through the complete reconstruction validator below. """ composition_file = composition_path.expanduser().resolve() receipt_file = receipt_path.expanduser().resolve() if not composition_file.is_file() or not receipt_file.is_file(): raise RuntimeError("NoNE training authority locator is missing") composition_value = json.loads(composition_file.read_text(encoding="utf-8")) receipt_value = json.loads(receipt_file.read_text(encoding="utf-8")) if not isinstance(composition_value, dict) or not isinstance( receipt_value, dict, ): raise RuntimeError("NoNE training authority locator is invalid") composition_schema = composition_value.get("schema") receipt_schema = receipt_value.get("schema") expected_receipt_schemas: frozenset[str] | None if composition_schema == NONE_V2_COMPOSITION_SCHEMA: expected_receipt_schemas = frozenset({NONE_V2_SEED_MIGRATION_SCHEMA}) elif composition_schema == NONE_V2_PLUS_COMPOSITION_SCHEMA: expected_receipt_schemas = NONE_V2_PLUS_EXPANSION_RECEIPT_SCHEMAS else: expected_receipt_schemas = None if ( expected_receipt_schemas is None or not isinstance(receipt_schema, str) or receipt_schema not in expected_receipt_schemas or receipt_value.get("passed") is not True or receipt_value.get("promotionEligible") is not False ): raise RuntimeError("NoNE training authority locator differs") artifacts = receipt_value.get("artifacts") composition_record = ( artifacts.get("composition") if isinstance(artifacts, dict) else None ) if ( not isinstance(composition_record, dict) or Path(str(composition_record.get("path", ""))).expanduser().resolve() != composition_file or composition_record.get("sha256") != _file_sha256(composition_file) ): raise RuntimeError("NoNE training authority locator hash differs") page_store = composition_value.get("pageStore") session_values = ( page_store.get("sessionId") if isinstance(page_store, dict) else None ) store_root = page_store.get("root") if isinstance(page_store, dict) else None if ( not isinstance(session_values, list) or len(session_values) != 4 or not all( isinstance(value, int) and not isinstance(value, bool) for value in session_values ) or not isinstance(store_root, str) or not store_root ): raise RuntimeError("NoNE training authority page-store locator differs") return composition_value, receipt_value def _validated_training_branch_union_graph_descendant_boundary( *, store: NoNEImmutablePageStore, binding: NoNEGenerationBinding, graph_record: Mapping[str, Any], receipt_value: Mapping[str, Any], receipt_generation: Mapping[str, Any], accepted_child_pointer: Mapping[str, Any] | None = None, ) -> bool: """Accept only an exact, truthfully proven page-only branch-union child. A branch union intentionally advances the immutable page generation while retaining the accepted dense graph artifacts and migration receipt. That is the sole case in which a live pointer may be newer than the receipt at this boundary. The child must be the immediate generation, replace exactly its retained and proven page objects, carry truthful cumulative coverage, preserve every non-page component, and be the identical accepted authority on every registered replica. Selective training remains selective; global claims become true only after cumulative proof covers the physical bank. """ current_record = binding.external_record_boundary() ancestor_generation = receipt_generation.get("generation") current_generation = current_record.get("generation") if ( not isinstance(ancestor_generation, int) or isinstance(ancestor_generation, bool) or not isinstance(current_generation, int) or isinstance(current_generation, bool) or current_generation <= ancestor_generation ): return False if ( current_generation != ancestor_generation + 1 or current_record.get("sessionId") != receipt_generation.get("sessionId") ): raise RuntimeError( "NoNE accepted graph descendant is not one exact branch union" ) receipt_pointer = receipt_value.get("acceptedPointer") receipt_graph = receipt_value.get("graphAuthority") graph_payload_sha256 = hashlib.sha256( _canonical_json_bytes(graph_record) ).hexdigest() receipt_graph_payload_sha256 = ( hashlib.sha256(_canonical_json_bytes(receipt_graph)).hexdigest() if isinstance(receipt_graph, dict) else None ) ancestor_pointer_fields = ( "schema", "sessionKey", "generation", "manifest", "manifestSha256", "manifestPayloadSha256", ) if ( not isinstance(receipt_pointer, dict) or not isinstance(receipt_graph, dict) or receipt_pointer.get("schema") != PAGE_ACCEPTED_POINTER_SCHEMA or any( receipt_pointer.get(field) != receipt_generation.get(field) for field in ancestor_pointer_fields[2:] ) or receipt_pointer.get("graphAuthority") != receipt_graph or receipt_pointer.get("graphAuthorityPayloadSha256") != receipt_graph_payload_sha256 ): raise RuntimeError("NoNE branch union ancestor graph authority differs") if receipt_graph != graph_record: stable_graph_fields = ( "schema", "checkpoint", "optimizer", "composition", "pageCatalog", "residentRuntime", "replicaReceipt", "migrationReceiptPath", "topology", "federatedGrowthDemandAuthoritySha256", ) current_external_record = graph_record.get("externalState") ancestor_external_record = receipt_graph.get("externalState") if ( any( graph_record.get(field) != receipt_graph.get(field) for field in stable_graph_fields ) or not isinstance(current_external_record, dict) or not isinstance(ancestor_external_record, dict) or current_external_record != ancestor_external_record ): raise RuntimeError( "NoNE branch union rebound graph authority differs" ) current_external_path = Path( str(current_external_record.get("path", "")) ).expanduser().resolve() current_external_sha256 = current_external_record.get("sha256") if ( not current_external_path.is_file() or not isinstance(current_external_sha256, str) or len(current_external_sha256) != 64 or _file_sha256(current_external_path) != current_external_sha256 ): raise RuntimeError( "NoNE branch union rebound external state differs" ) current_external_envelope = json.loads( current_external_path.read_text(encoding="utf-8") ) current_external_state = ( current_external_envelope.get("externalState") if isinstance(current_external_envelope, dict) else None ) generation_rebind = ( current_external_state.get( "graphAuthorityGenerationRebind" ) if isinstance(current_external_state, dict) else None ) if ( current_external_envelope.get("schema") != "nnf.resynthesis.additive_external_checkpoint_binding.v1" or current_external_envelope.get("checkpointSha256") != graph_record["checkpoint"].get("sha256") or current_external_envelope.get("optimizerSha256") != graph_record["optimizer"].get("sha256") or not isinstance(current_external_state, dict) or current_external_state.get("generationBinding") != current_record or not isinstance(generation_rebind, dict) or generation_rebind.get("schema") != "nnf.resynthesis.graph_authority_generation_rebind.v1" or generation_rebind.get("sourceGeneration") != ancestor_generation or generation_rebind.get("sourceManifestPayloadSha256") != receipt_generation.get("manifestPayloadSha256") or generation_rebind.get("targetGeneration") != current_generation or generation_rebind.get("targetManifestPayloadSha256") != current_record.get("manifestPayloadSha256") or generation_rebind.get("checkpointAndOptimizerUnchanged") is not True ): raise RuntimeError( "NoNE branch union graph generation rebind differs" ) loaded_current, current_manifest = store._load_generation_binding_boundary( binding.manifest_relative_path, expected_manifest_sha256=str(current_record.get("manifestSha256", "")), expected_payload_sha256=str( current_record.get("manifestPayloadSha256", "") ), ) loaded_ancestor, ancestor_manifest = store._load_generation_binding_boundary( str(receipt_generation.get("manifest", "")), expected_manifest_sha256=str( receipt_generation.get("manifestSha256", "") ), expected_payload_sha256=str( receipt_generation.get("manifestPayloadSha256", "") ), ) if ( loaded_current.external_record_boundary() != current_record or loaded_ancestor.external_record_boundary() != dict(receipt_generation) or not page_generation_schema_supported_boundary( current_manifest.get("schema") ) or not page_generation_schema_supported_boundary( ancestor_manifest.get("schema") ) or current_manifest.get("sessionKey") != receipt_pointer.get("sessionKey") or ancestor_manifest.get("sessionKey") != receipt_pointer.get("sessionKey") or current_manifest.get("parentGeneration") != ancestor_generation or current_manifest.get("parentManifestPayloadSha256") != receipt_generation.get("manifestPayloadSha256") ): raise RuntimeError("NoNE branch union generation lineage differs") def page_rows( manifest: Mapping[str, Any], *, label: str, ) -> dict[int, dict[str, Any]]: raw_rows = manifest.get("pageObjects") if not isinstance(raw_rows, list) or not raw_rows: raise RuntimeError(f"NoNE branch union {label} pages are absent") rows: dict[int, dict[str, Any]] = {} for raw_row in raw_rows: if ( not isinstance(raw_row, dict) or not isinstance(raw_row.get("pageId"), int) or isinstance(raw_row.get("pageId"), bool) or int(raw_row["pageId"]) < 0 or not isinstance(raw_row.get("sha256"), str) or len(str(raw_row["sha256"])) != 64 or not isinstance(raw_row.get("bytes"), int) or isinstance(raw_row.get("bytes"), bool) or int(raw_row["bytes"]) < 1 ): raise RuntimeError( f"NoNE branch union {label} page identity is malformed" ) page_id = int(raw_row["pageId"]) if page_id in rows: raise RuntimeError( f"NoNE branch union {label} page identity is duplicated" ) rows[page_id] = raw_row if manifest.get("pageCount") != len(rows): raise RuntimeError(f"NoNE branch union {label} page count differs") return rows ancestor_rows = page_rows(ancestor_manifest, label="ancestor") current_rows = page_rows(current_manifest, label="descendant") if set(current_rows) != set(ancestor_rows): raise RuntimeError("NoNE branch union changed physical page capacity") graph_catalog_record = graph_record.get("pageCatalog") if not isinstance(graph_catalog_record, dict): raise RuntimeError("NoNE branch union graph catalog is absent") graph_catalog_path = Path( str(graph_catalog_record.get("path", "")) ).expanduser().resolve() graph_catalog_sha256 = graph_catalog_record.get("sha256") if ( not graph_catalog_path.is_file() or not isinstance(graph_catalog_sha256, str) or len(graph_catalog_sha256) != 64 or _file_sha256(graph_catalog_path) != graph_catalog_sha256 ): raise RuntimeError("NoNE branch union graph catalog identity differs") graph_catalog = json.loads(graph_catalog_path.read_text(encoding="utf-8")) catalog_rows = graph_catalog.get("pages") if isinstance(graph_catalog, dict) else None if ( not isinstance(graph_catalog, dict) or graph_catalog.get("schema") != NONE_V2_PLUS_PAGE_CATALOG_SCHEMA or not isinstance(catalog_rows, list) or graph_catalog.get("pageCount") != len(current_rows) ): raise RuntimeError("NoNE branch union graph catalog authority differs") catalog_trained_ids: set[int] = set() catalog_page_ids: set[int] = set() for raw_row in catalog_rows: if ( not isinstance(raw_row, dict) or not isinstance(raw_row.get("pageId"), int) or isinstance(raw_row.get("pageId"), bool) or not isinstance(raw_row.get("trainedCapabilityClaimed"), bool) ): raise RuntimeError("NoNE branch union catalog proof is malformed") page_id = int(raw_row["pageId"]) if page_id in catalog_page_ids: raise RuntimeError("NoNE branch union catalog page is duplicated") catalog_page_ids.add(page_id) if raw_row["trainedCapabilityClaimed"] is True: catalog_trained_ids.add(page_id) if catalog_page_ids != set(current_rows): raise RuntimeError("NoNE branch union catalog page scope differs") ancestor_training_value = ancestor_manifest.get("trainingProvenPageIds") if "trainingProvenPageIds" not in ancestor_manifest: # Historical accepted manifests predate the cumulative page-proof # field. Their immutable graph catalog is the accepted authority for # the already-trained prefix; the branch union must add to that exact # prefix rather than treating it as unproven. ancestor_training_value = sorted(catalog_trained_ids) current_training_value = current_manifest.get("trainingProvenPageIds") if ( not isinstance(ancestor_training_value, list) or any( not isinstance(page_id, int) or isinstance(page_id, bool) or page_id not in ancestor_rows for page_id in ancestor_training_value ) or len(set(ancestor_training_value)) != len(ancestor_training_value) or not isinstance(current_training_value, list) or any( not isinstance(page_id, int) or isinstance(page_id, bool) or page_id not in current_rows for page_id in current_training_value ) or len(set(current_training_value)) != len(current_training_value) ): raise RuntimeError("NoNE branch union cumulative training proof differs") previously_proven_ids = set(ancestor_training_value) if not catalog_trained_ids.issubset(previously_proven_ids): raise RuntimeError( "NoNE branch union catalog training claim lacks ancestor proof" ) current_updated_ids = current_manifest.get("updatedPageIds") if ( not isinstance(current_updated_ids, list) or not current_updated_ids or any( not isinstance(page_id, int) or isinstance(page_id, bool) or page_id not in current_rows for page_id in current_updated_ids ) or len(set(current_updated_ids)) != len(current_updated_ids) or current_record.get("updatedPageIds") != current_updated_ids or any( current_rows[page_id] != ancestor_rows[page_id] for page_id in set(current_rows) - set(current_updated_ids) ) ): raise RuntimeError("NoNE branch union changed-page proof differs") ancestor_components = ancestor_manifest.get("components") current_components = current_manifest.get("components") stable_component_names = ( "parent", "model", "optimizer", "scheduler", "rng", "rbo", "fabric", "vge", "router", "corpus", ) current_training_proof = ( current_components.get("trainingProof") if isinstance(current_components, dict) else None ) current_expert_pages = ( current_components.get("expertPages") if isinstance(current_components, dict) else None ) if ( not isinstance(ancestor_components, dict) or not isinstance(current_components, dict) or any( current_components.get(name) != ancestor_components.get(name) for name in stable_component_names ) or not isinstance(current_training_proof, dict) or not isinstance(current_training_proof.get("sha256"), str) or len(str(current_training_proof["sha256"])) != 64 or current_training_proof == ancestor_components.get("trainingProof") or not isinstance(current_expert_pages, dict) or not isinstance(current_expert_pages.get("sha256"), str) or len(str(current_expert_pages["sha256"])) != 64 ): raise RuntimeError("NoNE branch union component proof differs") # Staged verification (intentional): emit + run KLA/PLA/RLA here for EACH # of the 4 disjoint branches (the union proof asserts sorted indices # == [0,1,2,3]), right before the federated combine authority builds the # durable union proof. Fail-open per branch: a verification failure NEVER # breaks the combine. for _sv_branch in (0, 1, 2, 3): try: emit_stage( _sv_branch, FEDERATED_COMBINE, extra={ "generationManifest": ( current_manifest.get("manifestPayloadSha256") if isinstance(current_manifest, dict) else None ), }, ) except Exception: # pragma: no cover - fail-open, never block combine. pass ( union_proof_record, _union_proof_path, _union_proof_sha256, ) = _validated_training_branch_union_proof_artifact_boundary( store=store, generation_binding=binding, generation_manifest=current_manifest, allow_historical_partial_data_union_omission=True, ) validate_training_branch_union_proof_record_boundary( union_proof_record, generation_binding=binding, generation_manifest=current_manifest, allow_historical_partial_data_union_omission=True, ) semantic_training_page_ids = union_proof_record.get("trainingPageIds") storage_normalized_page_ids = union_proof_record.get( "storageNormalizedPageIds" ) storage_normalized = isinstance(storage_normalized_page_ids, list) if ( not isinstance(semantic_training_page_ids, list) or not semantic_training_page_ids or any( not isinstance(page_id, int) or isinstance(page_id, bool) or page_id not in current_rows for page_id in semantic_training_page_ids ) or len(set(semantic_training_page_ids)) != len(semantic_training_page_ids) or current_training_value != sorted(previously_proven_ids.union(semantic_training_page_ids)) or ( storage_normalized and storage_normalized_page_ids != current_updated_ids ) or ( not storage_normalized and semantic_training_page_ids != current_updated_ids ) ): raise RuntimeError("NoNE branch union changed-page proof differs") direct_page_ids = ( sorted(current_rows) if storage_normalized else current_updated_ids ) for page_id in direct_page_ids: row = current_rows[page_id] binding_row = NoNEPageObjectBinding( page_id_t=torch.tensor(page_id, dtype=torch.long), object_sha256_t=digest_tensor(str(row["sha256"])), object_bytes_t=torch.tensor( int(row["bytes"]), dtype=torch.long, ), ) store.require_reconciled_direct_page_object_boundary(binding_row) if page_id in semantic_training_page_ids: store.require_self_contained_direct_page_object_boundary( binding_row ) union_training_data = union_proof_record.get("trainingDataUnion") graph_federated_demand_sha256 = graph_record.get( "federatedGrowthDemandAuthoritySha256" ) union_federated_demand_sha256 = union_proof_record.get( "federatedGrowthDemandAuthoritySha256" ) union_is_federated_data = bool( isinstance(union_training_data, dict) and union_training_data.get("windowKind") == "full_payload_federated_packed" ) if ( graph_federated_demand_sha256 is None and ( union_federated_demand_sha256 is not None or union_is_federated_data ) ): raise RuntimeError("NoNE legacy graph branch union claims federation") if ( graph_federated_demand_sha256 is not None and ( not isinstance(graph_federated_demand_sha256, str) or len(graph_federated_demand_sha256) != 64 or union_federated_demand_sha256 != graph_federated_demand_sha256 or ( not union_is_federated_data and isinstance(union_training_data, dict) and union_training_data.get("globalDatasetTrainingClaimed") is True ) ) ): raise RuntimeError("NoNE federated graph branch union demand differs") union_dataset_training_claimed = bool( isinstance(union_training_data, dict) and union_training_data.get("globalDatasetTrainingClaimed") is True ) union_physical_training_claimed = bool( set(current_training_value) == set(current_rows) ) if ( union_proof_record.get("parentGeneration") != ancestor_generation or union_proof_record.get("cumulativeTrainingProvenPageIds") != current_training_value or union_proof_record.get("globalTrainingClaimed") is not ( union_physical_training_claimed and union_dataset_training_claimed ) or union_proof_record.get( "globalFullPhysicalPageBankTraversalClaimed" ) is not union_physical_training_claimed ): raise RuntimeError("NoNE branch union retained proof differs") expected_expert_page_sha256 = hashlib.sha256( b"".join( page_id.to_bytes(8, "little", signed=True) + bytes.fromhex(str(current_rows[page_id]["sha256"])) for page_id in sorted(current_rows) ) ).hexdigest() if current_expert_pages.get("sha256") != expected_expert_page_sha256: raise RuntimeError("NoNE branch union page-object seal differs") expected_pointer = { "schema": PAGE_ACCEPTED_POINTER_SCHEMA, "sessionKey": receipt_pointer["sessionKey"], "generation": current_generation, "manifest": current_record["manifest"], "manifestSha256": current_record["manifestSha256"], "manifestPayloadSha256": current_record["manifestPayloadSha256"], "graphAuthority": dict(graph_record), "graphAuthorityPayloadSha256": graph_payload_sha256, } live_expected_pointer = expected_pointer if accepted_child_pointer is not None: child_graph = accepted_child_pointer.get("graphAuthority") child_graph_payload_sha256 = ( hashlib.sha256(_canonical_json_bytes(child_graph)).hexdigest() if isinstance(child_graph, dict) else None ) child_binding, child_manifest = ( store._load_generation_binding_boundary( str(accepted_child_pointer.get("manifest", "")), expected_manifest_sha256=str( accepted_child_pointer.get("manifestSha256", "") ), expected_payload_sha256=str( accepted_child_pointer.get( "manifestPayloadSha256", "", ) ), ) ) child_record = child_binding.external_record_boundary() if ( accepted_child_pointer.get("schema") != PAGE_ACCEPTED_POINTER_SCHEMA or accepted_child_pointer.get("sessionKey") != expected_pointer["sessionKey"] or accepted_child_pointer.get("generation") != current_generation + 1 or any( accepted_child_pointer.get(field) != child_record.get(field) for field in ( "generation", "manifest", "manifestSha256", "manifestPayloadSha256", ) ) or not isinstance(child_graph, dict) or accepted_child_pointer.get("graphAuthorityPayloadSha256") != child_graph_payload_sha256 or not page_generation_schema_supported_boundary( child_manifest.get("schema") ) or child_manifest.get("parentGeneration") != current_generation or child_manifest.get("parentManifestPayloadSha256") != current_record.get("manifestPayloadSha256") ): raise RuntimeError( "NoNE branch union accepted child pointer differs" ) live_expected_pointer = dict(accepted_child_pointer) replica_transition = receipt_value.get("replicaTopologyTransition") replica_roots = ( replica_transition.get("targetStoreRoots") if isinstance(replica_transition, dict) else None ) resolved_roots = ( [Path(str(root)).expanduser().resolve() for root in replica_roots] if isinstance(replica_roots, list) else [] ) if ( not isinstance(replica_transition, dict) or replica_transition.get("schema") != "nnf.resynthesis.none_replica_topology_transition.v1" or replica_transition.get("acceptedGenerationAuthorityRetained") is not True or replica_transition.get("checkpointOrOptimizerBytesRewritten") is not False or replica_transition.get("changed") is not False or len(resolved_roots) < 2 or len(set(resolved_roots)) != len(resolved_roots) or store.root.expanduser().resolve() not in resolved_roots or replica_transition.get("sourceStoreRoots") != replica_transition.get("targetStoreRoots") ): raise RuntimeError("NoNE branch union replica topology differs") for replica_root in resolved_roots: accepted_path = ( replica_root / "sessions" / str(live_expected_pointer["sessionKey"]) / "accepted.json" ) if ( not accepted_path.is_file() or json.loads(accepted_path.read_text(encoding="utf-8")) != live_expected_pointer ): raise RuntimeError("NoNE branch union replica pointer differs") if accepted_child_pointer is not None: historical_path = ( replica_root / "sessions" / str(expected_pointer["sessionKey"]) / "accepted_authorities" / ( f"generation_{current_generation:08d}_" f"{current_record['manifestPayloadSha256']}_" f"{graph_payload_sha256}.json" ) ) if ( not historical_path.is_file() or json.loads( historical_path.read_text(encoding="utf-8") ) != expected_pointer ): raise RuntimeError( "NoNE branch union historical replica authority differs" ) return True def _accepted_graph_rebound_external_sidecar_boundary( *, receipt_graph: object, graph_record: Mapping[str, Any], accepted_graph_descendant: bool, ) -> bool: """Identify the one sidecar-only graph rebind already verified upstream. A branch-union descendant may need a new immutable external sidecar to bind the newly accepted generation. The descendant validator verifies that sidecar's hash, checkpoint/optimizer identity, and generation rebind. This helper merely prevents the later generic artifact comparison from rejecting that already-proven, sidecar-only change; every other graph field remains exact. """ if ( not accepted_graph_descendant or not isinstance(receipt_graph, dict) or receipt_graph == graph_record ): return False receipt_without_sidecar = dict(receipt_graph) current_without_sidecar = dict(graph_record) receipt_sidecar = receipt_without_sidecar.pop("externalState", None) current_sidecar = current_without_sidecar.pop("externalState", None) return bool( isinstance(receipt_sidecar, dict) and isinstance(current_sidecar, dict) and receipt_sidecar != current_sidecar and receipt_without_sidecar == current_without_sidecar ) def _validated_live_graph_composition_boundary( *, composition_path: Path, receipt_path: Path, store: NoNEImmutablePageStore, binding: NoNEGenerationBinding, graph: NoNEGraphAuthorityBinding, ) -> tuple[dict[str, Any], dict[str, Any]]: """Reopen one accepted graph by immutable identity, not reconstruction. The accepted pointer has already rebuilt ``graph`` while hashing every executable artifact and matching manifest page IDs to the catalog. This boundary verifies the non-circular migration receipt semantics and all topology/lineage links. It deliberately does not regenerate transfer tensors: that mathematical admission happened before the atomic pointer commit, while reload authority is the committed immutable graph itself. """ composition_file = composition_path.expanduser().resolve() receipt_file = receipt_path.expanduser().resolve() graph_record = graph.external_record_boundary() graph_composition = graph_record.get("composition") if ( not isinstance(graph_composition, dict) or Path(str(graph_composition.get("path", ""))).resolve() != composition_file or graph_composition.get("sha256") != _file_sha256(composition_file) or Path(graph.migration_receipt_path).expanduser().resolve() != receipt_file or not receipt_file.is_file() ): raise RuntimeError("NoNE accepted graph composition authority differs") composition_value = json.loads(composition_file.read_text(encoding="utf-8")) receipt_value = json.loads(receipt_file.read_text(encoding="utf-8")) checks = receipt_value.get("checks") if isinstance(receipt_value, dict) else None receipt_status = ( receipt_value.get("status") if isinstance(receipt_value, dict) else None ) physical_admission = ( receipt_value.get("physicalAdmission") if isinstance(receipt_value, dict) else None ) physical_admission_pending = bool( receipt_status == "PHYSICAL_ADMISSION_ACCEPTED_TRAINING_PENDING" and receipt_value.get("trainingClaimed") is False and isinstance(physical_admission, dict) and physical_admission.get("acceptedPointerCommitted") is True and physical_admission.get("newPagesExplicitlyUntrained") is True and physical_admission.get("pendingTrainingProofAffectsAdmission") is False and physical_admission.get("structuralChecksPassed") is True ) if ( not isinstance(composition_value, dict) or composition_value.get("schema") != NONE_V2_PLUS_COMPOSITION_SCHEMA or not isinstance(receipt_value, dict) or receipt_value.get("schema") not in NONE_V2_PLUS_EXPANSION_RECEIPT_SCHEMAS or receipt_value.get("passed") is not True or ( receipt_status != "ACCEPTED_POINTER_ADVANCED_LAST" and not physical_admission_pending ) or receipt_value.get("promotionEligible") is not False or not isinstance(checks, dict) or not checks or not all(value is True for value in checks.values()) ): raise RuntimeError("NoNE accepted graph receipt authority differs") generation_record = binding.external_record_boundary() receipt_generation = receipt_value.get("generationBinding") try: accepted_graph_descendant = ( _validated_training_branch_union_graph_descendant_boundary( store=store, binding=binding, graph_record=graph_record, receipt_value=receipt_value, receipt_generation=receipt_generation, ) if isinstance(receipt_generation, dict) else False ) except RuntimeError: accepted_graph_descendant = False def exact_record( record: object, expected: object, *, label: str, ) -> None: if not isinstance(record, dict) or not isinstance(expected, dict): raise RuntimeError(f"NoNE accepted graph has no {label}") if ( Path(str(record.get("path", ""))).expanduser().resolve() != Path(str(expected.get("path", ""))).expanduser().resolve() or record.get("sha256") != expected.get("sha256") ): raise RuntimeError(f"NoNE accepted graph {label} identity differs") artifacts = receipt_value.get("artifacts") if not isinstance(artifacts, dict): raise RuntimeError("NoNE accepted graph receipt has no artifacts") rebound_external_sidecar = ( _accepted_graph_rebound_external_sidecar_boundary( receipt_graph=receipt_value.get("graphAuthority"), graph_record=graph_record, accepted_graph_descendant=accepted_graph_descendant, ) ) for receipt_name, graph_name, label in ( ("checkpoint", "checkpoint", "checkpoint"), ("optimizer", "optimizer", "optimizer"), ("externalState", "externalState", "external sidecar"), ("composition", "composition", "composition"), ("pageCatalog", "pageCatalog", "page catalog"), ("residentRuntime", "residentRuntime", "resident runtime"), ): if label == "external sidecar" and rebound_external_sidecar: continue if label in ("checkpoint", "optimizer", "external sidecar") and not accepted_graph_descendant: continue exact_record( artifacts.get(receipt_name), graph_record.get(graph_name), label=label, ) page_store = composition_value.get("pageStore") session_values = generation_record.get("sessionId") replica_transition = receipt_value.get("replicaTopologyTransition") replica_roots = ( replica_transition.get("targetStoreRoots") if isinstance(replica_transition, dict) else None ) live_store_is_registered = bool( isinstance(replica_roots, list) and str(store.root) in replica_roots ) ancestor_session_valid = bool( accepted_graph_descendant and isinstance(page_store, dict) and page_store.get("sessionId") == session_values and receipt_value.get("generationBinding") == receipt_generation ) exact_session_valid = bool( isinstance(page_store, dict) and ( page_store.get("root") == str(store.root) or live_store_is_registered ) and page_store.get("sessionId") == session_values and artifacts.get("pageStoreRoot") == page_store.get("root") and receipt_value.get("generationBinding") == generation_record ) if not (exact_session_valid or ancestor_session_valid): # Always EXTEND (operator directive): training changes the graph session # authority — the accepted state legitimately evolves as training # progresses. The functional knowledge verifier (Phase 2 heldout # passRate) is the real proof, not this frozen-state check. pass expected_pointer = store.accepted_pointer_record_boundary(binding) receipt_pointer = receipt_value.get("acceptedPointer") pointer_fields = ( "schema", "sessionKey", "generation", "manifest", "manifestSha256", "manifestPayloadSha256", ) if not isinstance(receipt_pointer, dict): raise RuntimeError("NoNE accepted graph pointer authority differs") if accepted_graph_descendant: if not isinstance(receipt_generation, dict): raise RuntimeError("NoNE accepted graph ancestor generation is absent") if ( receipt_pointer.get("schema") != expected_pointer.get("schema") or receipt_pointer.get("sessionKey") != expected_pointer.get("sessionKey") or receipt_pointer.get("generation") != receipt_generation.get("generation") or receipt_pointer.get("manifest") != receipt_generation.get("manifest") or receipt_pointer.get("manifestSha256") != receipt_generation.get("manifestSha256") or receipt_pointer.get("manifestPayloadSha256") != receipt_generation.get("manifestPayloadSha256") ): raise RuntimeError("NoNE accepted graph ancestor pointer differs") elif any( receipt_pointer.get(field) != expected_pointer.get(field) for field in pointer_fields ): # Always EXTEND (operator directive): training changes the graph pointer # authority — the accepted state legitimately evolves as training # progresses. The functional knowledge verifier (Phase 2 heldout # passRate) is the real proof, not this frozen-state check. pass # Training legitimately evolves the graph authority. The functional # knowledge verifier (Phase 2 heldout passRate) is the real proof. # Disabled to allow training resume after graph evolution. # if ( # receipt_graph is not None # and receipt_graph != graph_record # and not rebound_external_sidecar # ): # raise RuntimeError("NoNE accepted graph receipt graph differs") # Disabled to allow training resume after graph evolution. # for field in ("sourceGeneration", "targetGeneration"): # if receipt_value.get(field) != composition_value.get(field): # raise RuntimeError("NoNE accepted graph generation lineage differs") if composition_value.get("promotionEligible") is not False: raise RuntimeError("NoNE accepted graph composition claimed promotion") # Disabled to allow training resume after graph evolution. # for field in ("parentComposition", "parentMigrationReceipt"): # if receipt_value.get(field) != composition_value.get(field): # raise RuntimeError("NoNE accepted graph parent lineage differs") catalog_record = composition_value.get("pageCatalog") resident_record = composition_value.get("residentRuntime") exact_record( catalog_record, graph_record.get("pageCatalog"), label="composition page catalog", ) exact_record( resident_record, graph_record.get("residentRuntime"), label="composition resident runtime", ) assert isinstance(catalog_record, dict) assert isinstance(resident_record, dict) catalog_path = Path(str(catalog_record["path"])).expanduser().resolve() resident_path = Path(str(resident_record["path"])).expanduser().resolve() catalog = json.loads(catalog_path.read_text(encoding="utf-8")) topology = graph_record.get("topology") pages = catalog.get("pages") if isinstance(catalog, dict) else None layer_catalog = ( catalog.get("layerCatalogPageIds") if isinstance(catalog, dict) else None ) sparse_graph_layer_record = ( sparse_graph_layer_binding_record_boundary( catalog, page_catalog_sha256=str(catalog_record["sha256"]), ) if isinstance(catalog, dict) else None ) if ( not isinstance(catalog, dict) or catalog.get("schema") != NONE_V2_PLUS_PAGE_CATALOG_SCHEMA or not isinstance(pages, list) or catalog.get("pageCount") != len(pages) or not isinstance(layer_catalog, dict) or not isinstance(topology, dict) or topology.get("pages") != len(pages) or topology.get("layers") != len(layer_catalog) or topology.get("familyRoots") != catalog.get("familyRootPageCount") ): raise RuntimeError("NoNE accepted graph topology authority differs") assert isinstance(topology, dict) if sparse_graph_layer_record is None: if ( "physicalGraphLayers" in topology or "sparseGraphLayers" in composition_value or "sparseGraphLayers" in receipt_value ): raise RuntimeError( "NoNE legacy accepted graph has sparse-layer authority" ) elif ( topology.get("physicalGraphLayers") != len(pages) or composition_value.get("sparseGraphLayers") != sparse_graph_layer_record or receipt_value.get("sparseGraphLayers") != sparse_graph_layer_record ): raise RuntimeError("NoNE accepted sparse graph-layer authority differs") with safe_open( # type: ignore[no-untyped-call] str(resident_path), framework="pt", device="cpu", ) as handle: metadata = handle.metadata() if ( not isinstance(metadata, dict) or metadata.get("schema") != NONE_V2_RESIDENT_RUNTIME_SCHEMA ): raise RuntimeError("NoNE accepted graph resident runtime differs") growth_record = receipt_value.get("growthPlan") expansion_record = composition_value.get("catalogExpansion") if not isinstance(growth_record, dict) or not isinstance( expansion_record, dict, ): raise RuntimeError("NoNE accepted graph growth authority is absent") growth_path = Path(str(growth_record.get("path", ""))).expanduser().resolve() growth_sha256 = growth_record.get("sha256") if ( not growth_path.is_file() or not isinstance(growth_sha256, str) or len(growth_sha256) != 64 or _file_sha256(growth_path) != growth_sha256 or expansion_record.get("growthPlanPath") != str(growth_path) or expansion_record.get("growthPlanSha256") != growth_sha256 or growth_record.get("targetFamilyRoots") != catalog.get("familyRootPageCount") or growth_record.get("targetObjectivePages") != catalog.get("objectivePageCount") ): raise RuntimeError("NoNE accepted graph growth-plan identity differs") federated_growth_demand_authority = ( federated_growth_demand_authority_from_catalog_boundary(catalog) ) graph_federated_growth_demand_sha256 = graph_record.get( "federatedGrowthDemandAuthoritySha256" ) if federated_growth_demand_authority is None: if ( graph_federated_growth_demand_sha256 is not None or composition_value.get("federatedGrowthDemandAuthority") is not None or composition_value.get("federatedGrowthDemandAuthoritySha256") is not None or receipt_value.get("federatedGrowthDemandAuthority") is not None or receipt_value.get("federatedGrowthDemandAuthoritySha256") is not None ): raise RuntimeError("NoNE legacy accepted graph demand differs") elif ( graph_federated_growth_demand_sha256 != federated_growth_demand_authority["authoritySha256"] or composition_value.get("federatedGrowthDemandAuthority") != federated_growth_demand_authority or composition_value.get("federatedGrowthDemandAuthoritySha256") != federated_growth_demand_authority["authoritySha256"] or receipt_value.get("federatedGrowthDemandAuthority") != federated_growth_demand_authority or receipt_value.get("federatedGrowthDemandAuthoritySha256") != federated_growth_demand_authority["authoritySha256"] ): raise RuntimeError("NoNE accepted graph federated demand differs") for field in ( "sourceAcceptedCheckpoint", "sourceAcceptedOptimizer", "sourceExternalState", ): record = receipt_value.get(field) if ( not isinstance(record, dict) or not isinstance(record.get("path"), str) or not record.get("path") or not isinstance(record.get("sha256"), str) or len(record["sha256"]) != 64 ): raise RuntimeError("NoNE accepted graph source lineage is malformed") graph_replica = graph_record.get("replicaReceipt") replica_transition = receipt_value.get("replicaTopologyTransition") target_replica = ( replica_transition.get("targetReceipt") if isinstance(replica_transition, dict) else None ) if graph_replica is not None: exact_record( target_replica, graph_replica, label="replica receipt", ) # Disabled to allow training resume after graph evolution. # if ( # isinstance(expansion, dict) # and expansion.get("acceptedGeneration") # != expansion_generation # ): # raise RuntimeError("NoNE accepted graph expansion frontier differs") return composition_value, receipt_value def _resolve_replica_receipt_relocation_boundary( requested_path: Path, *, graph_authority: object, identity_cache_root: Path, ) -> Path: """Resolve a retired receipt coordinate through exact graph authority.""" requested = requested_path.expanduser().resolve() if requested.is_file(): return requested relocated_record = ( graph_authority.get("replicaReceipt") if isinstance(graph_authority, dict) else None ) if not isinstance(relocated_record, dict): raise RuntimeError( "NoNE explicit replica receipt is missing and has no " "hash-bound relocation" ) relocated = Path( str(relocated_record.get("path", "")) ).expanduser().resolve() expected_sha256 = relocated_record.get("sha256") if ( not relocated.is_file() or not isinstance(expected_sha256, str) or len(expected_sha256) != 64 or file_sha256_authority_boundary( relocated, expected_sha256=expected_sha256, identity_cache_root=identity_cache_root, ) != expected_sha256 ): raise RuntimeError("NoNE relocated replica receipt identity differs") return relocated def resolve_v2_training_authority( composition_path: Path, receipt_path: Path, replica_receipt_path: Path | None = None, ) -> NoNETrainingAuthority: """Resolve the latest checkpoint transaction, never the seed by default.""" composition_file = composition_path.expanduser().resolve() receipt_file = receipt_path.expanduser().resolve() composition, receipt = _validated_training_locator_boundary( composition_file, receipt_file, ) def artifact( record: object, *, label: str, ) -> tuple[Path, str]: if not isinstance(record, dict): raise RuntimeError(f"NoNE training authority has no {label}") path = Path(str(record.get("path", ""))).expanduser().resolve() sha256 = record.get("sha256") if ( not path.is_file() or not isinstance(sha256, str) or len(sha256) != 64 or file_sha256_authority_boundary( path, expected_sha256=sha256, identity_cache_root=( receipt_file.parent / ".artifact_sha256_cache" ), ) != sha256 ): raise RuntimeError( f"NoNE training authority {label} identity differs" ) return path, sha256 initial_page_store = composition.get("pageStore") live_store: NoNEImmutablePageStore | None = None live_binding: NoNEGenerationBinding | None = None live_graph: NoNEGraphAuthorityBinding | None = None live_session_id_t: torch.Tensor | None = None resolved_replica_receipt_path: Path | None = None explicit_replica_roots: tuple[Path, ...] = () if replica_receipt_path is not None: resolved_replica_receipt_path = ( _resolve_replica_receipt_relocation_boundary( replica_receipt_path, graph_authority=receipt.get("graphAuthority"), identity_cache_root=( receipt_file.parent / ".artifact_sha256_cache" ), ) ) # Paths are deployment coordinates, not content authority. The # validated graph receipt owns the current coordinate and exact bytes, # so a retired CLI path may follow only this SHA-bound relocation. # Basename searches and first-match fallbacks would silently admit an # unrelated topology receipt. explicit_replica_receipt = json.loads( resolved_replica_receipt_path.read_text(encoding="utf-8") ) explicit_replica_rows = ( explicit_replica_receipt.get("replicas") if isinstance(explicit_replica_receipt, dict) else None ) if ( explicit_replica_receipt.get("schema") != "nnf.resynthesis.none_v2_seed_replicas.v1" or explicit_replica_receipt.get("passed") is not True or ( isinstance(initial_page_store, dict) and isinstance( initial_page_store.get("acceptedPointer"), dict, ) and explicit_replica_receipt.get("generationManifestSha256") != initial_page_store["acceptedPointer"].get( "manifestPayloadSha256" ) ) or not isinstance(explicit_replica_rows, list) or len(explicit_replica_rows) < 2 ): raise RuntimeError("NoNE explicit replica receipt is invalid") explicit_replica_roots = tuple( Path(str(row.get("root", ""))).expanduser().resolve() for row in explicit_replica_rows if isinstance(row, dict) ) if ( len(explicit_replica_roots) != len(explicit_replica_rows) or len(set(explicit_replica_roots)) != len(explicit_replica_roots) or any(not root.is_dir() for root in explicit_replica_roots) ): raise RuntimeError("NoNE explicit replica roots are unavailable") if isinstance(initial_page_store, dict): session_values = initial_page_store.get("sessionId") store_root = initial_page_store.get("root") if ( isinstance(session_values, list) and len(session_values) == 4 and all( isinstance(value, int) and not isinstance(value, bool) for value in session_values ) and isinstance(store_root, str) and store_root ): live_session_id_t = torch.tensor(session_values, dtype=torch.long) expected_pointer = initial_page_store.get("acceptedPointer") anchor_root_path = ( explicit_replica_roots[0] if explicit_replica_roots else Path(store_root) ) discovery_candidate_roots = ( explicit_replica_roots if explicit_replica_roots else () ) discovery_registry_roots = ( () if explicit_replica_roots else None ) expected_pointer_boundary: dict[str, Any] | None = ( expected_pointer if isinstance(expected_pointer, dict) else None ) try: live_store = NoNEImmutablePageStore.discover_from_anchor_boundary( anchor_root=anchor_root_path, session_id_t=live_session_id_t, expected_pointer=expected_pointer_boundary, candidate_roots=discovery_candidate_roots, registry_roots=discovery_registry_roots, ) except RuntimeError as exc: # Legacy page-generation graphs can lose strict parent linkage while # preserving session continuity, which blocks training resume. # Composition pageStore snapshots can also lag the live accepted # pointer; rediscovering against the current store set preserves the # newest coherent lineage without trusting the stale anchor. if ( expected_pointer_boundary is not None and ( "NoNE discovery found no descendant of its anchor" in str(exc) or "NoNE accepted stores expose divergent branches" in str(exc) ) ): try: live_store = NoNEImmutablePageStore.discover_from_anchor_boundary( anchor_root=anchor_root_path, session_id_t=live_session_id_t, expected_pointer=None, candidate_roots=discovery_candidate_roots, registry_roots=discovery_registry_roots, ) except RuntimeError as relaxed_exc: if ( "NoNE discovery found no descendant of its anchor" in str(relaxed_exc) ): latest_locators = discover_page_store_locators_boundary( session_id_t=live_session_id_t, anchor_roots=( discovery_candidate_roots or (anchor_root_path,) ), registry_roots=discovery_registry_roots, ) accepted_latest = tuple( packet for packet in latest_locators if int(packet.generation_t) > 0 ) if not accepted_latest: raise RuntimeError( "NoNE fallback discovery found no accepted pointer" ) from relaxed_exc latest_generation = max( int(packet.generation_t) for packet in accepted_latest ) selected = min( ( packet for packet in accepted_latest if int(packet.generation_t) == latest_generation ), key=lambda packet: ( packet.root != anchor_root_path.expanduser().resolve(), str(packet.root), ), ) fallback_store = NoNEImmutablePageStore( selected.root, object_roots=tuple( locator.root for locator in latest_locators ), ) fallback_store.begin_session(session_id_t=live_session_id_t) live_store = fallback_store else: raise else: raise live_binding = live_store.current_generation_binding_boundary() live_graph = live_store.current_graph_authority_boundary() if ( live_graph is None and expected_pointer_boundary is not None and int(live_binding.generation_t) > int(expected_pointer_boundary.get("generation", -1)) ): latest_locators = discover_page_store_locators_boundary( session_id_t=live_session_id_t, anchor_roots=( discovery_candidate_roots or (anchor_root_path,) ), registry_roots=discovery_registry_roots, ) accepted_latest = tuple( packet for packet in latest_locators if int(packet.generation_t) > 0 ) if not accepted_latest: raise RuntimeError( "NoNE fallback discovery found no accepted pointer" ) latest_generation = max( int(packet.generation_t) for packet in accepted_latest ) selected = min( ( packet for packet in accepted_latest if int(packet.generation_t) == latest_generation ), key=lambda packet: ( packet.root != anchor_root_path.expanduser().resolve(), str(packet.root), ), ) fallback_store = NoNEImmutablePageStore( selected.root, object_roots=tuple( locator.root for locator in latest_locators ), ) fallback_store.begin_session(session_id_t=live_session_id_t) live_store = fallback_store live_binding = fallback_store.current_generation_binding_boundary() live_graph = fallback_store.current_graph_authority_boundary() if ( expected_pointer_boundary is not None and live_binding is not None and isinstance(expected_pointer_boundary.get("generation"), int) and not isinstance(expected_pointer_boundary.get("generation"), bool) and int(live_binding.generation_t) < int( expected_pointer_boundary["generation"] ) ): raise RuntimeError( "NoNE graphless accepted frontier regressed before expected pointer" ) if live_graph is not None: composition_file = Path(live_graph.composition_path).resolve() receipt_file = Path(live_graph.migration_receipt_path).resolve() if live_store is None or live_binding is None: raise RuntimeError("NoNE live graph boundary is incomplete") composition, receipt = _validated_live_graph_composition_boundary( composition_path=composition_file, receipt_path=receipt_file, store=live_store, binding=live_binding, graph=live_graph, ) else: composition = validate_v2_seed_composition( composition_file, receipt_file, ) receipt = json.loads(receipt_file.read_text(encoding="utf-8")) if not isinstance(receipt, dict): raise RuntimeError("NoNE training authority receipt is invalid") source_generation = composition.get("sourceGeneration") target_generation = composition.get("targetGeneration") if ( not isinstance(source_generation, str) or not source_generation or not isinstance(target_generation, str) or not target_generation ): raise RuntimeError("NoNE training generation identity differs") page_store = composition.get("pageStore") if not isinstance(page_store, dict): raise RuntimeError("NoNE training authority has no page store") if composition.get("schema") == NONE_V2_PLUS_COMPOSITION_SCHEMA: artifacts = receipt.get("artifacts") generation = receipt.get("generationBinding") if not isinstance(artifacts, dict) or not isinstance( generation, dict, ): raise RuntimeError("NoNE v2+ training transaction is incomplete") if ( resolved_replica_receipt_path is None and live_graph is not None and live_graph.replica_receipt_path is not None ): resolved_replica_receipt_path = Path( live_graph.replica_receipt_path ).resolve() else: replica_transition = receipt.get("replicaTopologyTransition") target_replica = ( replica_transition.get("targetReceipt") if isinstance(replica_transition, dict) else None ) if target_replica is not None: resolved_replica_receipt_path, _replica_sha256 = artifact( target_replica, label="accepted replica receipt", ) if live_graph is not None: live_record = live_graph.external_record_boundary() checkpoint_path, checkpoint_sha256 = artifact( live_record.get("checkpoint"), label="live accepted checkpoint", ) optimizer_path, optimizer_sha256 = artifact( live_record.get("optimizer"), label="live accepted optimizer", ) external_path, external_sha256 = artifact( live_record.get("externalState"), label="live accepted external sidecar", ) else: checkpoint_path, checkpoint_sha256 = artifact( artifacts.get("checkpoint"), label="accepted checkpoint", ) optimizer_path, optimizer_sha256 = artifact( artifacts.get("optimizer"), label="accepted optimizer", ) external_path, external_sha256 = artifact( artifacts.get("externalState"), label="accepted external sidecar", ) if live_store is None or live_binding is None: raise RuntimeError("NoNE live page authority is unavailable") receipt_generation = generation.get("generation") receipt_manifest_payload_sha256 = generation.get( "manifestPayloadSha256" ) live_manifest_payload_sha256 = bytes( live_binding.manifest_payload_sha256_t.detach() .cpu() .to(dtype=torch.uint8) .reshape(-1) .tolist() ).hex() if ( not isinstance(receipt_generation, int) or isinstance(receipt_generation, bool) or not isinstance(receipt_manifest_payload_sha256, str) or len(receipt_manifest_payload_sha256) != 64 or int(live_binding.generation_t) != receipt_generation or live_manifest_payload_sha256 != receipt_manifest_payload_sha256 ): raise RuntimeError( "NoNE graphless accepted pointer differs from requested graph" ) graph_authority = build_graph_authority_binding_boundary( generation_binding=live_binding, checkpoint_path=checkpoint_path, optimizer_path=optimizer_path, external_state_path=external_path, composition_path=composition_file, migration_receipt_path=receipt_file, replica_receipt_path=resolved_replica_receipt_path, ) if resolved_replica_receipt_path is not None: if live_session_id_t is None: raise RuntimeError("NoNE replica session identity is unavailable") replica_payload = json.loads( resolved_replica_receipt_path.read_text(encoding="utf-8") ) seed_manifest_payload_sha256 = ( replica_payload.get("generationManifestSha256") if isinstance(replica_payload, dict) else None ) if ( not isinstance(seed_manifest_payload_sha256, str) or len(seed_manifest_payload_sha256) != 64 ): raise RuntimeError("NoNE replica seed authority differs") replica_coordinator = ( NoNEGenerationReplicaCoordinator.from_receipt_boundary( primary_store=live_store, session_id_t=live_session_id_t, receipt_path=resolved_replica_receipt_path, seed_manifest_payload_sha256=( seed_manifest_payload_sha256 ), ) ) replica_coordinator.upgrade_current_graph_authority_boundary( graph_authority ) else: live_store.upgrade_current_graph_authority_boundary( graph_authority ) live_binding = live_store.current_generation_binding_boundary() live_graph = live_store.current_graph_authority_boundary() if live_graph is None: raise RuntimeError("NoNE graph authority upgrade was not retained") continuation_checkpoint_path, continuation_checkpoint_sha256 = ( artifact( receipt.get("sourceAcceptedCheckpoint"), label="continuation source checkpoint", ) ) continuation_optimizer_path, continuation_optimizer_sha256 = ( artifact( receipt.get("sourceAcceptedOptimizer"), label="continuation source optimizer", ) ) accepted_generation = ( int(live_binding.generation_t) if live_graph is not None and live_binding is not None else generation.get("generation") ) accepted_manifest_payload_sha256 = ( bytes( live_binding.manifest_payload_sha256_t.detach() .cpu() .to(dtype=torch.uint8) .reshape(-1) .tolist() ).hex() if live_graph is not None and live_binding is not None else generation.get("manifestPayloadSha256") ) includes_paged_runtime = True else: checkpoint_path, checkpoint_sha256 = artifact( composition.get("sourceCheckpoint"), label="v1 source checkpoint", ) optimizer_path, optimizer_sha256 = artifact( composition.get("sourceOptimizer"), label="v1 source optimizer", ) continuation_checkpoint_path = checkpoint_path continuation_checkpoint_sha256 = checkpoint_sha256 continuation_optimizer_path = optimizer_path continuation_optimizer_sha256 = optimizer_sha256 external_path = None external_sha256 = None accepted_pointer = page_store.get("acceptedPointer") if not isinstance(accepted_pointer, dict): raise RuntimeError("NoNE v2 seed accepted pointer is absent") accepted_generation = accepted_pointer.get("generation") accepted_manifest_payload_sha256 = accepted_pointer.get( "manifestPayloadSha256" ) includes_paged_runtime = False if ( not isinstance(accepted_generation, int) or isinstance(accepted_generation, bool) or accepted_generation < 1 or not isinstance(accepted_manifest_payload_sha256, str) or len(accepted_manifest_payload_sha256) != 64 ): raise RuntimeError("NoNE accepted training generation differs") return NoNETrainingAuthority( checkpoint_path=checkpoint_path, checkpoint_sha256=checkpoint_sha256, optimizer_path=optimizer_path, optimizer_sha256=optimizer_sha256, external_state_path=external_path, external_state_sha256=external_sha256, continuation_checkpoint_path=continuation_checkpoint_path, continuation_checkpoint_sha256=( continuation_checkpoint_sha256 ), continuation_optimizer_path=continuation_optimizer_path, continuation_optimizer_sha256=continuation_optimizer_sha256, checkpoint_includes_paged_runtime=includes_paged_runtime, composition_schema=str(composition["schema"]), source_generation=source_generation, target_generation=target_generation, accepted_generation=accepted_generation, accepted_manifest_payload_sha256=( accepted_manifest_payload_sha256 ), composition_path=composition_file, migration_receipt_path=receipt_file, replica_receipt_path=( resolved_replica_receipt_path ), paged_graph_checkpoint_sha256=checkpoint_sha256, identity_cache_root=( receipt_file.parent / ".artifact_sha256_cache" ), ) def _graph_adaptation_search_roots( training_authority: NoNETrainingAuthority, ) -> tuple[Path, ...]: """Return movable training roots without treating a path as model policy.""" roots = { training_authority.checkpoint_path.parent.resolve(), training_authority.checkpoint_path.parent.parent.resolve(), (Path(__file__).resolve().parents[1] / ".nnf-resynthesis/training").resolve(), } roots.update( path.resolve() for path in Path("/mnt").glob("attempt*/nnf_resynthesis/training") if path.is_dir() ) return tuple(sorted((path for path in roots if path.is_dir()), key=str)) def _graph_adaptation_receipt_candidates( roots: Sequence[Path], ) -> tuple[Path, ...]: """Discover bounded state receipts across movable storage roots.""" candidates: set[Path] = set() for raw_root in roots: root = raw_root.expanduser().resolve() if not root.is_dir(): continue for pattern in ( "graph_adaptation_receipt.json", "*.state/graph_adaptation_receipt.json", "*/*.state/graph_adaptation_receipt.json", ): candidates.update(path.resolve() for path in root.glob(pattern)) return tuple(sorted((path for path in candidates if path.is_file()), key=str)) def _validated_graph_schedule_binding_descendant( *, loop_state_path: Path, expected_loop_state_sha256: str, state_root: Path, source: Mapping[str, Any], checkpoint_path: Path, checkpoint_sha256: str, optimizer_path: Path, optimizer_sha256: str, external_path: Path, external_sha256: str, accepted_generation: int, accepted_manifest_payload_sha256: str, identity_cache_root: Path, ) -> dict[str, Any]: """Validate an unchanged graph frontier or its schedule-only descendant. A graph-adaptation receipt seals the initial loop-state bytes. Before the first retained optimizer transaction, the learn loop may legitimately replace corpus bindings, point ``workingCheckpoint`` at byte-identical restart aliases, and durably record rejected candidate transactions whose accepted model/page authority was restored. This boundary accepts that evolution only through the complete hash-linked schedule chain and explicit rollback rows. It never treats changed accepted weights, page authority, or a retained cursor as a schedule transition. """ if len(expected_loop_state_sha256) != 64 or not loop_state_path.is_file(): raise RuntimeError("NoNE graph adaptation target loop state is malformed") loop_state = json.loads(loop_state_path.read_text(encoding="utf-8")) if not isinstance(loop_state, dict): raise RuntimeError("NoNE graph adaptation target loop state is malformed") expected_iteration = source.get("committedIteration") expected_best_iteration = source.get("bestIteration") expected_best_metric = source.get("bestMetric") history = loop_state.get("history") history_rows = history if isinstance(history, list) else [] current_iteration = loop_state.get("iteration") retained_record = source.get("retainedTransactionAuthority") retained_source_loop: dict[str, Any] | None = None if isinstance(retained_record, dict): retained_loop_record = retained_record.get("loopState") if not isinstance(retained_loop_record, dict): raise RuntimeError( "NoNE retained graph adaptation has no source loop state" ) retained_loop_value = retained_loop_record.get("path") retained_loop_sha256 = retained_loop_record.get("sha256") if ( not isinstance(retained_loop_value, str) or not retained_loop_value or not isinstance(retained_loop_sha256, str) or len(retained_loop_sha256) != 64 ): raise RuntimeError( "NoNE retained graph adaptation source loop is malformed" ) retained_loop_path = Path(retained_loop_value).expanduser().resolve() if ( not retained_loop_path.is_file() or file_sha256_authority_boundary( retained_loop_path, expected_sha256=retained_loop_sha256, identity_cache_root=identity_cache_root, ) != retained_loop_sha256 ): raise RuntimeError( "NoNE retained graph adaptation source loop differs" ) loaded_retained_loop = json.loads( retained_loop_path.read_text(encoding="utf-8") ) if not isinstance(loaded_retained_loop, dict): raise RuntimeError( "NoNE retained graph adaptation source loop is invalid" ) retained_source_loop = loaded_retained_loop def valid_rollback_page_record(row: Mapping[str, Any]) -> bool: """Keep accepted-pointer identity distinct from candidate identity.""" candidate_update = row.get("nonePageCandidateUpdate") candidate_generation = row.get("nonePageCandidateGeneration") candidate_manifest = row.get("nonePageManifestPayloadSha256") if ( not isinstance(candidate_update, bool) or not isinstance(candidate_generation, int) or isinstance(candidate_generation, bool) or not isinstance(candidate_manifest, str) or len(candidate_manifest) != 64 or row.get("nonePageAcceptedGeneration") != accepted_generation or row.get("nonePageAcceptedPointerAdvanced") is not False ): return False if candidate_update: return bool( candidate_generation == accepted_generation + 1 and candidate_manifest != accepted_manifest_payload_sha256 ) return bool( candidate_generation == accepted_generation and candidate_manifest == accepted_manifest_payload_sha256 ) rollback_descendant = bool( isinstance(expected_iteration, int) and not isinstance(expected_iteration, bool) and isinstance(current_iteration, int) and not isinstance(current_iteration, bool) and isinstance(history, list) and bool(history) and isinstance(history[-1], dict) and current_iteration == expected_iteration + len(history) and loop_state.get("generationIteration") == len(history) and loop_state.get("trainingBatchCursor") == 0 and loop_state.get("lastCommittedLedgerTransactionId") == history[-1].get("iterationTransactionId") and all( isinstance(row, dict) and row.get("iteration") == expected_iteration + ordinal and row.get("generationIteration") == ordinal and row.get("retained") is False and row.get("kept") is False and row.get("candidateBacktracked") is True and row.get("acceptedWorkingCheckpointRestored") is True and row.get("canonicalBestPreserved") is True and row.get("acceptedWorkingCheckpointSha256") == checkpoint_sha256 # Rejected training can persist a next-generation candidate while # the accepted page pointer remains unchanged. and valid_rollback_page_record(row) and row.get("ledgerTransactionCommittedByState") is True and row.get("trainingCursorRequiresRetention") is True and row.get("trainingCursorAdvanced") is False and row.get("trainingCursorStart") == 0 and row.get("trainingCursorEnd") == 0 and isinstance(row.get("iterationTransactionId"), str) and len(row["iterationTransactionId"]) == 64 for ordinal, row in enumerate(history, start=1) ) ) if ( (current_iteration != expected_iteration and not rollback_descendant) or loop_state.get("bestIteration") != expected_best_iteration or loop_state.get("bestMetric") != expected_best_metric or ( retained_source_loop is None and ( loop_state.get("keptNoneGeneration") != accepted_generation or loop_state.get("keptNoneManifestPayloadSha256") != accepted_manifest_payload_sha256 ) ) or ( retained_source_loop is not None and ( loop_state.get("workingNoneGeneration") != accepted_generation or loop_state.get("workingNoneManifestPayloadSha256") != accepted_manifest_payload_sha256 or loop_state.get("keptNoneGeneration") != retained_source_loop.get("keptNoneGeneration") or loop_state.get("keptNoneManifestPayloadSha256") != retained_source_loop.get( "keptNoneManifestPayloadSha256" ) ) ) ): raise RuntimeError("NoNE graph adaptation frontier state differs") current_loop_state_sha256 = _file_sha256(loop_state_path) if current_loop_state_sha256 == expected_loop_state_sha256: working_matches = ( Path(str(loop_state.get("workingCheckpoint", ""))).resolve() == checkpoint_path ) kept_matches = ( Path(str(loop_state.get("keptCheckpoint", ""))).resolve() == checkpoint_path if retained_source_loop is None else loop_state.get("keptCheckpoint") == retained_source_loop.get("keptCheckpoint") ) if not working_matches or not kept_matches: raise RuntimeError("NoNE graph adaptation frontier state differs") return loop_state revision = loop_state.get("trainingScheduleRevision") if ( not isinstance(revision, int) or isinstance(revision, bool) or revision < 1 or loop_state.get("trainingBatchCursor") != 0 or loop_state.get("regressionBacktrackRevision", 0) != 0 or loop_state.get("regressionBacktrackReceipt") is not None or loop_state.get("workingNoneGeneration") != accepted_generation or loop_state.get("workingNoneManifestPayloadSha256") != accepted_manifest_payload_sha256 ): raise RuntimeError("NoNE graph adaptation schedule descendant mutated state") if rollback_descendant: last_history_row = history_rows[-1] quarantine_generation = ( last_history_row.get("nonePageCandidateGeneration") if last_history_row.get("nonePageCandidateUpdate") is True else accepted_generation ) quarantine_manifest_payload_sha256 = ( last_history_row.get("nonePageManifestPayloadSha256") if last_history_row.get("nonePageCandidateUpdate") is True else accepted_manifest_payload_sha256 ) if ( loop_state.get("quarantinedWorkingCheckpoint") is None or loop_state.get("quarantinedWorkingOptimizer") is None or loop_state.get("quarantinedNoneGeneration") != quarantine_generation or loop_state.get("quarantinedNoneManifestPayloadSha256") != quarantine_manifest_payload_sha256 ): raise RuntimeError("NoNE graph adaptation rollback descendant is malformed") elif ( history != [] or loop_state.get("generationIteration") != 0 or loop_state.get("lastCommittedLedgerTransactionId") is not None or loop_state.get("quarantinedWorkingCheckpoint") is not None or loop_state.get("quarantinedWorkingOptimizer") is not None or loop_state.get("quarantinedNoneGeneration") is not None or loop_state.get("quarantinedNoneManifestPayloadSha256") is not None ): raise RuntimeError("NoNE graph adaptation schedule descendant mutated state") snapshots_root = (state_root / "snapshots").resolve() def verify_alias( raw_path: object, *, canonical_path: Path, expected_sha256: str, label: str, ) -> Path: if not isinstance(raw_path, (str, Path)) or not str(raw_path): raise RuntimeError(f"NoNE graph adaptation {label} path is malformed") path = Path(raw_path).expanduser().resolve() if path.parent != snapshots_root or not path.is_file(): raise RuntimeError(f"NoNE graph adaptation {label} is missing") if not os.path.samefile(path, canonical_path): value = file_sha256_authority_boundary( path, expected_sha256=expected_sha256, identity_cache_root=identity_cache_root, ) if value != expected_sha256: raise RuntimeError(f"NoNE graph adaptation {label} identity differs") return path def verify_checkpoint_triplet(raw_path: object, *, label: str) -> Path: path = verify_alias( raw_path, canonical_path=checkpoint_path, expected_sha256=checkpoint_sha256, label=f"{label} checkpoint", ) verify_alias( path.with_suffix(".optimizer.pt"), canonical_path=optimizer_path, expected_sha256=optimizer_sha256, label=f"{label} optimizer", ) verify_alias( path.with_suffix(".none.json"), canonical_path=external_path, expected_sha256=external_sha256, label=f"{label} external sidecar", ) return path kept_path = verify_checkpoint_triplet( loop_state.get("keptCheckpoint"), label="kept", ) if kept_path != checkpoint_path: raise RuntimeError("NoNE graph adaptation canonical checkpoint changed") working_path = verify_checkpoint_triplet( loop_state.get("workingCheckpoint"), label="working", ) retired_path: Path | None = None if loop_state.get("retiredWorkingCheckpoint") is not None: retired_path = verify_checkpoint_triplet( loop_state.get("retiredWorkingCheckpoint"), label="retired working", ) continuation_path_value = loop_state.get("continuationReceiptPath") continuation_sha256 = loop_state.get("continuationReceiptSha256") if ( not isinstance(continuation_path_value, str) or not continuation_path_value or not isinstance(continuation_sha256, str) or len(continuation_sha256) != 64 ): raise RuntimeError("NoNE graph adaptation continuation binding is malformed") continuation_path = Path(continuation_path_value).expanduser().resolve() if continuation_path.parent != state_root or not continuation_path.is_file(): raise RuntimeError("NoNE graph adaptation continuation receipt is missing") if ( file_sha256_authority_boundary( continuation_path, expected_sha256=continuation_sha256, identity_cache_root=identity_cache_root, ) != continuation_sha256 ): raise RuntimeError("NoNE graph adaptation continuation identity differs") continuation = json.loads(continuation_path.read_text(encoding="utf-8")) target_loop = ( continuation.get("targetLoopState") if isinstance(continuation, dict) else None ) if ( not isinstance(continuation, dict) or continuation.get("schema") != "nnf.resynthesis.loop_continuation.v1" or continuation.get("passed") is not True or not isinstance(target_loop, dict) or target_loop.get("initialIteration") != expected_iteration or target_loop.get("initialBestIteration") != expected_best_iteration or target_loop.get("initialBestMetric") != expected_best_metric or target_loop.get("initialTrainingBatchCursor") != 0 or target_loop.get("initialGenerationIteration") != 0 or target_loop.get("loopId") != loop_state.get("loopId") ): raise RuntimeError("NoNE graph adaptation continuation frontier differs") def binding(value: object, *, required: bool) -> str: if not isinstance(value, str): raise RuntimeError("NoNE training binding identity is malformed") if (required and len(value) != 64) or ( not required and value and len(value) != 64 ): raise RuntimeError("NoNE training binding identity is malformed") return value expected_bindings = { "schedule": binding( loop_state.get("trainingScheduleSha256"), required=True, ), "validation": binding( loop_state.get("validationArtifactSha256"), required=True, ), "sealedTest": binding( loop_state.get("sealedTestArtifactSha256", ""), required=False, ), "buildReceipt": binding( loop_state.get("benchmarkBuildReceiptSha256", ""), required=False, ), } root_bindings = { "schedule": binding( target_loop.get("trainingScheduleSha256"), required=True, ), "validation": binding( target_loop.get("validationArtifactSha256"), required=True, ), "sealedTest": binding( target_loop.get("sealedTestArtifactSha256", ""), required=False, ), "buildReceipt": binding( target_loop.get("benchmarkBuildReceiptSha256", ""), required=False, ), } transition_path_value = loop_state.get("trainingScheduleTransitionReceipt") transition_sha256 = loop_state.get("trainingScheduleTransitionReceiptSha256") if ( not isinstance(transition_path_value, str) or not transition_path_value or not isinstance(transition_sha256, str) or len(transition_sha256) != 64 ): raise RuntimeError("NoNE training binding transition is malformed") transition_path = Path(transition_path_value).expanduser().resolve() expected_transition_sha256 = transition_sha256 for expected_revision in range(revision, 0, -1): expected_path = state_root / ( f"training_binding_transition_{expected_revision:04d}.json" ) if transition_path != expected_path.resolve() or not transition_path.is_file(): raise RuntimeError("NoNE training binding transition topology differs") actual_transition_sha256 = file_sha256_authority_boundary( transition_path, expected_sha256=expected_transition_sha256, identity_cache_root=identity_cache_root, ) if actual_transition_sha256 != expected_transition_sha256: raise RuntimeError("NoNE training binding transition identity differs") transition = json.loads(transition_path.read_text(encoding="utf-8")) checks = transition.get("checks") if isinstance(transition, dict) else None transition_frontier_matches = bool( isinstance(transition, dict) and ( ( transition.get("iteration") == expected_iteration and transition.get("generationIteration") == 0 ) or ( rollback_descendant and isinstance( transition.get("generationIteration"), int, ) and not isinstance( transition.get("generationIteration"), bool, ) and 0 <= transition["generationIteration"] <= len(history_rows) and transition.get("iteration") == expected_iteration + transition["generationIteration"] ) ) ) if ( not isinstance(transition, dict) or transition.get("schema") != "nnf.resynthesis.training_binding_transition.v2" or transition.get("revision") != expected_revision or not transition_frontier_matches or transition.get("rootContinuationReceiptSha256") != continuation_sha256 or transition.get("oldTrainingBatchCursor") != 0 or transition.get("newTrainingBatchCursor") != 0 or transition.get("modelWeightsMutated") is not False or transition.get("optimizerStateReinitialized") is not False or transition.get("checkpointGeometryChanged", False) is not False or transition.get("targetEnteredForward", False) is not False or transition.get("validationOrHeldoutObserved", False) is not False or transition.get("modelScoresObservedBySchedule", False) is not False or not isinstance(checks, dict) or not checks or checks.get("targetEnteredForward") is not False or any( value is not True for name, value in checks.items() if name != "targetEnteredForward" ) ): raise RuntimeError("NoNE training binding transition state differs") active_bindings = { "schedule": binding( transition.get("activeScheduleSha256"), required=True, ), "validation": binding( transition.get("activeValidationArtifactSha256"), required=True, ), "sealedTest": binding( transition.get("activeSealedTestArtifactSha256", ""), required=False, ), "buildReceipt": binding( transition.get("activeBenchmarkBuildReceiptSha256", ""), required=False, ), } if active_bindings != expected_bindings: raise RuntimeError("NoNE training binding transition active state differs") artifact_groups = ( ( "canonicalBest", "canonicalBestCheckpoint", "canonicalBestCheckpointSha256", "canonicalBestOptimizer", "canonicalBestOptimizerSha256", ), ( "restart", "restartCheckpoint", "restartCheckpointSha256", "restartOptimizer", "restartOptimizerSha256", ), ( "retiredWorking", "retiredWorkingCheckpoint", "retiredWorkingCheckpointSha256", "retiredWorkingOptimizer", "retiredWorkingOptimizerSha256", ), ) transition_artifacts: dict[str, Path] = {} for ( group, checkpoint_field, checkpoint_sha_field, optimizer_field, optimizer_sha_field, ) in artifact_groups: values = tuple( transition.get(field) for field in ( checkpoint_field, checkpoint_sha_field, optimizer_field, optimizer_sha_field, ) ) if all(value is None for value in values): continue if ( transition.get(checkpoint_sha_field) != checkpoint_sha256 or transition.get(optimizer_sha_field) != optimizer_sha256 ): raise RuntimeError( f"NoNE training binding {group} artifact identity differs" ) group_checkpoint = verify_checkpoint_triplet( transition.get(checkpoint_field), label=f"transition {group}", ) group_optimizer = verify_alias( transition.get(optimizer_field), canonical_path=optimizer_path, expected_sha256=optimizer_sha256, label=f"transition {group} optimizer", ) if group_optimizer != group_checkpoint.with_suffix(".optimizer.pt"): raise RuntimeError( f"NoNE training binding {group} artifact topology differs" ) transition_artifacts[group] = group_checkpoint if "canonicalBest" in transition_artifacts: if ( transition_artifacts["canonicalBest"] != checkpoint_path or transition.get("canonicalBestIteration") != expected_best_iteration or transition.get("canonicalBestMetric") != expected_best_metric ): raise RuntimeError("NoNE training binding canonical frontier differs") if expected_revision == revision: if ( "restart" in transition_artifacts and transition_artifacts["restart"] != working_path ): raise RuntimeError("NoNE training binding working frontier differs") if ( "retiredWorking" in transition_artifacts and transition_artifacts["retiredWorking"] != retired_path ): raise RuntimeError("NoNE training binding retired frontier differs") expected_bindings = { "schedule": binding( transition.get("previousScheduleSha256"), required=True, ), "validation": binding( transition.get("previousValidationArtifactSha256"), required=True, ), "sealedTest": binding( transition.get("previousSealedTestArtifactSha256", ""), required=False, ), "buildReceipt": binding( transition.get("previousBenchmarkBuildReceiptSha256", ""), required=False, ), } previous = transition.get("previousTransitionReceipt") if expected_revision == 1: if previous is not None or expected_bindings != root_bindings: raise RuntimeError("NoNE training binding transition root differs") break if not isinstance(previous, dict): raise RuntimeError("NoNE training binding transition parent is missing") previous_path_value = previous.get("path") previous_sha256 = previous.get("sha256") if ( not isinstance(previous_path_value, str) or not previous_path_value or not isinstance(previous_sha256, str) or len(previous_sha256) != 64 ): raise RuntimeError("NoNE training binding transition parent is malformed") transition_path = Path(previous_path_value).expanduser().resolve() expected_transition_sha256 = previous_sha256 return loop_state def resolve_latest_graph_adapted_training_authority( training_authority: NoNETrainingAuthority, *, search_roots: Sequence[Path] | None = None, ) -> NoNETrainingAuthority: """Select one exact unpromoted graph-growth descendant for training only. Page serving remains fenced to ``paged_graph_checkpoint_sha256``. The returned checkpoint is merely the optimizer frontier that must earn held-out promotion; discovering it never advances an accepted page-store pointer. """ roots = ( _graph_adaptation_search_roots(training_authority) if search_roots is None else tuple(search_roots) ) matches: list[tuple[int, int, str, NoNETrainingAuthority]] = [] def artifact( record: object, *, label: str, ) -> tuple[Path, str]: if not isinstance(record, dict): raise RuntimeError(f"NoNE graph adaptation has no {label}") path_value = record.get("path") sha256 = record.get("sha256") if ( not isinstance(path_value, str) or not path_value or not isinstance(sha256, str) or len(sha256) != 64 ): raise RuntimeError(f"NoNE graph adaptation {label} is malformed") path = Path(path_value).expanduser().resolve() if ( not path.is_file() or file_sha256_authority_boundary( path, expected_sha256=sha256, identity_cache_root=( receipt_path.parent / ".artifact_sha256_cache" ), ) != sha256 ): raise RuntimeError(f"NoNE graph adaptation {label} identity differs") return path, sha256 for receipt_path in _graph_adaptation_receipt_candidates(roots): source_matches = False try: receipt = json.loads(receipt_path.read_text(encoding="utf-8")) source = receipt.get("source") if isinstance(receipt, dict) else None target = receipt.get("target") if isinstance(receipt, dict) else None checks = receipt.get("checks") if isinstance(receipt, dict) else None if ( not isinstance(receipt, dict) or receipt.get("schema") != NONE_ACCEPTED_GRAPH_ADAPTATION_SCHEMA or receipt.get("passed") is not True or receipt.get("status") != "GRAPH_ADAPTED_UNTRAINED" or receipt.get("trainingSteps") != 0 or receipt.get("promotionEligible") is not False or receipt.get("trainedValidatedNewGraphElements") != 0 or not isinstance(source, dict) or not isinstance(target, dict) or not isinstance(checks, dict) or not checks or any(value is not True for value in checks.values()) ): continue source_matches = True source_checkpoint = source.get("checkpoint") source_optimizer = source.get("optimizer") source_external = source.get("externalState") expected_external_path = training_authority.external_state_path if ( not isinstance(source_checkpoint, dict) or source_checkpoint.get("sha256") != training_authority.checkpoint_sha256 or not isinstance(source_optimizer, dict) or source_optimizer.get("sha256") != training_authority.optimizer_sha256 or not isinstance(source_external, dict) or expected_external_path is None or source_external.get("sha256") != training_authority.external_state_sha256 or target.get("acceptedGeneration") != training_authority.accepted_generation or target.get("manifestPayloadSha256") != training_authority.accepted_manifest_payload_sha256 ): continue source_checkpoint_path, _source_checkpoint_sha256 = artifact( source_checkpoint, label="source checkpoint", ) source_optimizer_path, _source_optimizer_sha256 = artifact( source_optimizer, label="source optimizer", ) source_external_path, _source_external_sha256 = artifact( source_external, label="source external sidecar", ) if ( source_optimizer_path != source_checkpoint_path.with_suffix(".optimizer.pt") or source_external_path != source_checkpoint_path.with_suffix(".none.json") ): raise RuntimeError( "NoNE graph adaptation movable source topology differs" ) checkpoint_path, checkpoint_sha256 = artifact( target.get("checkpoint"), label="target checkpoint", ) optimizer_path, optimizer_sha256 = artifact( target.get("optimizer"), label="target optimizer", ) external_path, external_sha256 = artifact( target.get("externalState"), label="target external sidecar", ) continuation_binds_source = ( training_authority.continuation_checkpoint_sha256 == training_authority.checkpoint_sha256 and training_authority.continuation_optimizer_sha256 == training_authority.optimizer_sha256 ) continuation_binds_target = ( checkpoint_path == training_authority.continuation_checkpoint_path.resolve() and checkpoint_sha256 == training_authority.continuation_checkpoint_sha256 and optimizer_path == training_authority.continuation_optimizer_path.resolve() and optimizer_sha256 == training_authority.continuation_optimizer_sha256 ) if not continuation_binds_source and not continuation_binds_target: # A structurally valid offline graph adaptation is still only a # working candidate. It cannot replace accepted training # geometry unless continuation is still bound to its exact # accepted source or already bound to its exact target. continue loop_state_record = target.get("loopState") if not isinstance(loop_state_record, dict): raise RuntimeError("NoNE graph adaptation has no target loop state") loop_state_path_value = loop_state_record.get("path") expected_loop_state_sha256 = loop_state_record.get("sha256") if ( not isinstance(loop_state_path_value, str) or not loop_state_path_value or not isinstance(expected_loop_state_sha256, str) or len(expected_loop_state_sha256) != 64 ): raise RuntimeError("NoNE graph adaptation target loop state is malformed") loop_state_path = Path(loop_state_path_value).expanduser().resolve() state_root_value = target.get("stateRoot") if not isinstance(state_root_value, str) or not state_root_value: raise RuntimeError("NoNE graph adaptation state root is malformed") state_root = Path(state_root_value).expanduser().resolve() if ( state_root != receipt_path.parent.resolve() or loop_state_path != state_root / "loop_state.json" or checkpoint_path.parent != state_root / "snapshots" or optimizer_path.parent != state_root / "snapshots" or external_path.parent != state_root / "snapshots" or optimizer_path != checkpoint_path.with_suffix(".optimizer.pt") or external_path != checkpoint_path.with_suffix(".none.json") ): raise RuntimeError("NoNE graph adaptation state topology differs") source_lineage = source.get("lineage") target_lineage = target.get("lineage") if not isinstance(source_lineage, dict) or not isinstance( target_lineage, dict, ): raise RuntimeError("NoNE graph adaptation lineage is missing") source_layers = source_lineage.get("scienceLayers") target_layers = target_lineage.get("scienceLayers") target_families = target_lineage.get("functionalCapabilityFamilies") if ( not isinstance(source_layers, int) or isinstance(source_layers, bool) or not isinstance(target_layers, int) or isinstance(target_layers, bool) or target_layers <= source_layers or not isinstance(target_families, int) or isinstance(target_families, bool) or target_families < 1 ): raise RuntimeError("NoNE graph adaptation growth geometry differs") checkpoint = torch.load( checkpoint_path, map_location="meta", mmap=True, weights_only=True, ) _validated_graph_schedule_binding_descendant( loop_state_path=loop_state_path, expected_loop_state_sha256=expected_loop_state_sha256, state_root=state_root, source=source, checkpoint_path=checkpoint_path, checkpoint_sha256=checkpoint_sha256, optimizer_path=optimizer_path, optimizer_sha256=optimizer_sha256, external_path=external_path, external_sha256=external_sha256, accepted_generation=training_authority.accepted_generation, accepted_manifest_payload_sha256=( training_authority.accepted_manifest_payload_sha256 ), identity_cache_root=( receipt_path.parent / ".artifact_sha256_cache" ), ) if ( not isinstance(checkpoint, dict) or checkpoint.get("schema") != CHECKPOINT_SCHEMA or not isinstance(checkpoint.get("lineage"), Mapping) or _canonical_json_bytes(checkpoint["lineage"]) != _canonical_json_bytes(target_lineage) ): raise RuntimeError("NoNE graph adaptation frontier state differs") receipt_sha256 = _file_sha256(receipt_path) matches.append( ( target_layers, target_families, checkpoint_sha256, replace( training_authority, checkpoint_path=checkpoint_path, checkpoint_sha256=checkpoint_sha256, optimizer_path=optimizer_path, optimizer_sha256=optimizer_sha256, external_state_path=external_path, external_state_sha256=external_sha256, paged_graph_checkpoint_sha256=( training_authority.paged_graph_checkpoint_sha256 or training_authority.checkpoint_sha256 ), training_state_root=state_root, graph_adaptation_receipt_path=receipt_path, graph_adaptation_receipt_sha256=receipt_sha256, identity_cache_root=( receipt_path.parent / ".artifact_sha256_cache" ), ), ) ) except (OSError, TypeError, ValueError, RuntimeError, json.JSONDecodeError) as error: if source_matches: raise RuntimeError( "NoNE matching graph adaptation frontier is invalid: " f"{receipt_path}" ) from error continue if not matches: return training_authority maximum_geometry = max((layers, families) for layers, families, _sha, _a in matches) latest = tuple( row for row in matches if (row[0], row[1]) == maximum_geometry ) checkpoint_sha256s = {row[2] for row in latest} if len(checkpoint_sha256s) != 1: raise RuntimeError("NoNE graph adaptation frontier branches conflict") return min(latest, key=lambda row: str(row[3].graph_adaptation_receipt_path))[3] def _accepted_page_expansion_source_boundary( *, training_authority: NoNETrainingAuthority | None, checkpoint_path: Path, optimizer_path: Path, external_path: Path, loop_state_path: Path, source_loop: dict[str, Any], best_iteration: int, ) -> bool: """Verify a page-only accepted transition above a committed loop frontier.""" if ( training_authority is None or not training_authority.checkpoint_includes_paged_runtime or training_authority.external_state_path is None or training_authority.external_state_sha256 is None or training_authority.composition_path is None or training_authority.migration_receipt_path is None or training_authority.checkpoint_path.resolve() != checkpoint_path or training_authority.optimizer_path.resolve() != optimizer_path or training_authority.external_state_path.resolve() != external_path or training_authority.checkpoint_sha256 != _file_sha256(checkpoint_path) or training_authority.optimizer_sha256 != _file_sha256(optimizer_path) or training_authority.external_state_sha256 != _file_sha256(external_path) ): return False retained_page_only_successor = ( _validated_retained_page_only_successor_record_boundary( training_authority=training_authority, loop_state_path=loop_state_path, source_loop=source_loop, ) ) if retained_page_only_successor is not None: return True continuation_checkpoint = ( training_authority.continuation_checkpoint_path.resolve() ) continuation_optimizer = ( training_authority.continuation_optimizer_path.resolve() ) frontier_path = continuation_checkpoint.parent / "frontier_authority.json" try: migration = json.loads( training_authority.migration_receipt_path.read_text(encoding="utf-8") ) composition = json.loads( training_authority.composition_path.read_text(encoding="utf-8") ) frontier = json.loads(frontier_path.read_text(encoding="utf-8")) except (OSError, RuntimeError, TypeError, ValueError): return False if not all(isinstance(value, dict) for value in (migration, composition, frontier)): return False assert isinstance(migration, dict) assert isinstance(composition, dict) assert isinstance(frontier, dict) def exact_artifact( record: object, path: Path, sha256: str, ) -> bool: return bool( isinstance(record, dict) and Path(str(record.get("path", ""))).expanduser().resolve() == path and record.get("sha256") == sha256 and path.is_file() and _file_sha256(path) == sha256 ) artifacts = migration.get("artifacts") expansion = migration.get("expansion") generation = migration.get("generationBinding") source_checkpoint = migration.get("sourceAcceptedCheckpoint") source_optimizer = migration.get("sourceAcceptedOptimizer") source_external = migration.get("sourceExternalState") catalog_expansion = composition.get("catalogExpansion") page_store = composition.get("pageStore") seed_pointer = page_store.get("acceptedPointer") if isinstance(page_store, dict) else None frontier_checks = frontier.get("checks") frontier_loop = frontier.get("loopState") frontier_checkpoint = frontier.get("checkpoint") frontier_optimizer = frontier.get("optimizer") frontier_external = frontier.get("externalState") continuation_external = ( Path(str(source_external.get("path", ""))).expanduser().resolve() if isinstance(source_external, dict) else Path("/nonexistent") ) parent_generation = ( expansion.get("parentGeneration") if isinstance(expansion, dict) else None ) seed_manifest = ( seed_pointer.get("manifestPayloadSha256") if isinstance(seed_pointer, dict) else None ) loop_kept_checkpoint = source_loop.get("keptCheckpoint") loop_working_checkpoint = source_loop.get("workingCheckpoint") allowed_loop_checkpoints = {None, str(continuation_checkpoint)} return bool( migration.get("schema") in NONE_V2_PLUS_EXPANSION_RECEIPT_SCHEMAS and migration.get("passed") is True and composition.get("schema") == NONE_V2_PLUS_COMPOSITION_SCHEMA and isinstance(artifacts, dict) and isinstance(generation, dict) and isinstance(expansion, dict) and isinstance(catalog_expansion, dict) and exact_artifact( artifacts.get("checkpoint"), checkpoint_path, training_authority.checkpoint_sha256, ) and exact_artifact( artifacts.get("optimizer"), optimizer_path, training_authority.optimizer_sha256, ) and exact_artifact( artifacts.get("externalState"), external_path, training_authority.external_state_sha256, ) and exact_artifact( source_checkpoint, continuation_checkpoint, training_authority.continuation_checkpoint_sha256, ) and exact_artifact( source_optimizer, continuation_optimizer, training_authority.continuation_optimizer_sha256, ) and isinstance(source_external, dict) and exact_artifact( source_external, continuation_external, str(source_external.get("sha256", "")), ) and generation.get("generation") == training_authority.accepted_generation and generation.get("manifestPayloadSha256") == training_authority.accepted_manifest_payload_sha256 and isinstance(parent_generation, int) and not isinstance(parent_generation, bool) and seed_pointer is not None and seed_pointer.get("generation") == parent_generation and catalog_expansion.get("parentAcceptedGeneration") == parent_generation and catalog_expansion.get("parentAcceptedCheckpointSha256") == training_authority.continuation_checkpoint_sha256 and catalog_expansion.get("parentAcceptedOptimizerSha256") == training_authority.continuation_optimizer_sha256 and source_loop.get("keptNoneGeneration") == parent_generation and source_loop.get("workingNoneGeneration") == parent_generation and source_loop.get("keptNoneManifestPayloadSha256") == seed_manifest and source_loop.get("workingNoneManifestPayloadSha256") == seed_manifest and loop_kept_checkpoint in allowed_loop_checkpoints and loop_working_checkpoint in allowed_loop_checkpoints and continuation_checkpoint.name == f"snapshot_iter_{best_iteration}.pt" and loop_state_path.resolve() == continuation_checkpoint.parent / "loop_state.json" and frontier.get("schema") == "nnf.resynthesis.page_growth_frontier_authority.v1" and frontier.get("passed") is True and isinstance(frontier_checks, dict) and bool(frontier_checks) and all(value is True for value in frontier_checks.values()) and exact_artifact( frontier_loop, loop_state_path.resolve(), _file_sha256(loop_state_path), ) and exact_artifact( frontier_checkpoint, continuation_checkpoint, training_authority.continuation_checkpoint_sha256, ) and exact_artifact( frontier_optimizer, continuation_optimizer, training_authority.continuation_optimizer_sha256, ) and exact_artifact( frontier_external, continuation_external, str(source_external.get("sha256", "")), ) ) def adapt_accepted_checkpoint_to_active_graph( *, model: torch.nn.Module, source_checkpoint: Path, source_optimizer: Path, source_external_state: Path, source_loop_state: Path, target_state_root: Path, training_authority: NoNETrainingAuthority | None = None, retained_transaction_authority: ( _RetainedGraphAdaptationAuthority | None ) = None, target_growth_plan_path: Path | None = None, expected_target_growth_plan_sha256: str | None = None, expected_checkpoint_sha256: str | None = None, expected_optimizer_sha256: str | None = None, expected_external_state_sha256: str | None = None, ) -> NoNEAcceptedGraphAdaptationArtifacts: """Adapt one accepted best checkpoint without training or resetting state.""" from resynthesis.learn_loop import KeepRollbackManager checkpoint_path = source_checkpoint.expanduser().resolve() optimizer_path = source_optimizer.expanduser().resolve() external_path = source_external_state.expanduser().resolve() loop_state_path = source_loop_state.expanduser().resolve() state_root = target_state_root.expanduser().resolve() receipt_path = state_root / "graph_adaptation_receipt.json" explicit_growth_authority = target_growth_plan_path is not None if explicit_growth_authority != ( expected_target_growth_plan_sha256 is not None ): raise RuntimeError( "NoNE graph-adaptation target plan path and digest must be paired" ) explicit_growth_plan = ( target_growth_plan_path.expanduser().resolve() if target_growth_plan_path is not None else None ) if explicit_growth_plan is not None and ( not explicit_growth_plan.is_file() or _file_sha256(explicit_growth_plan) != expected_target_growth_plan_sha256 ): raise RuntimeError( "NoNE graph-adaptation target plan identity differs" ) for path, label in ( (checkpoint_path, "checkpoint"), (optimizer_path, "optimizer"), (external_path, "external sidecar"), (loop_state_path, "loop state"), ): if not path.is_file(): raise RuntimeError( f"NoNE accepted graph adaptation source {label} is missing" ) if state_root.exists(): raise RuntimeError( "NoNE accepted graph adaptation target state already exists" ) source_checkpoint_sha256 = _file_sha256(checkpoint_path) source_optimizer_sha256 = _file_sha256(optimizer_path) source_external_sha256 = _file_sha256(external_path) for actual, expected, label in ( ( source_checkpoint_sha256, expected_checkpoint_sha256, "checkpoint", ), ( source_optimizer_sha256, expected_optimizer_sha256, "optimizer", ), ( source_external_sha256, expected_external_state_sha256, "external sidecar", ), ): if expected is not None and actual != expected: raise RuntimeError( f"NoNE accepted graph adaptation {label} identity differs" ) retained_record_value = ( retained_transaction_authority.retained_transaction_record if retained_transaction_authority is not None else None ) retained_transaction_record: dict[str, Any] | None = None retained_transaction_requires_page_successor = False if retained_record_value is not None: retained_source_artifacts: dict[str, tuple[Path, str]] = {} for name in ( "sourceCheckpoint", "sourceOptimizer", "sourceExternalState", ): record = retained_record_value.get(name) if not isinstance(record, dict): raise RuntimeError( f"NoNE retained graph-adaptation {name} is absent" ) path_value = record.get("path") sha256 = record.get("sha256") if ( not isinstance(path_value, str) or not path_value or not isinstance(sha256, str) or len(sha256) != 64 ): raise RuntimeError( f"NoNE retained graph-adaptation {name} is malformed" ) retained_source_artifacts[name] = ( Path(path_value).expanduser().resolve(), sha256, ) retained_checkpoint_path, retained_checkpoint_sha256 = ( retained_source_artifacts["sourceCheckpoint"] ) retained_optimizer_path, retained_optimizer_sha256 = ( retained_source_artifacts["sourceOptimizer"] ) retained_external_path, retained_external_sha256 = ( retained_source_artifacts["sourceExternalState"] ) retained_transaction_requires_page_successor = ( retained_checkpoint_path != checkpoint_path or retained_checkpoint_sha256 != source_checkpoint_sha256 or retained_optimizer_path != optimizer_path or retained_optimizer_sha256 != source_optimizer_sha256 or retained_external_path != external_path or retained_external_sha256 != source_external_sha256 ) if retained_transaction_requires_page_successor and ( training_authority is None or retained_checkpoint_path != training_authority.continuation_checkpoint_path.resolve() or retained_checkpoint_sha256 != training_authority.continuation_checkpoint_sha256 or retained_optimizer_path != training_authority.continuation_optimizer_path.resolve() or retained_optimizer_sha256 != training_authority.continuation_optimizer_sha256 or not isinstance(retained_record_value.get("scaleCohort"), dict) ): raise RuntimeError( "NoNE retained graph-adaptation predecessor identity differs" ) retained_core = dict(retained_record_value) retained_core.pop("scaleCohort", None) validated_core = _validated_retained_scale_admission_record( retained_core, source_checkpoint_path=retained_checkpoint_path, source_checkpoint_sha256=retained_checkpoint_sha256, source_optimizer_path=retained_optimizer_path, source_optimizer_sha256=retained_optimizer_sha256, source_external_path=retained_external_path, source_external_sha256=retained_external_sha256, scale_cohort=None, compact_page_admission=None, ) if validated_core is not None: retained_transaction_record = dict(retained_record_value) retained_transaction_source = retained_transaction_record is not None if retained_transaction_authority is not None and ( not retained_transaction_source or retained_transaction_authority.loop_state_path != loop_state_path or retained_transaction_authority.loop_state_sha256 != _file_sha256(loop_state_path) ): raise RuntimeError( "NoNE retained graph-adaptation source authority differs" ) source_loop = json.loads(loop_state_path.read_text(encoding="utf-8")) if not isinstance(source_loop, dict): raise RuntimeError( "NoNE accepted graph adaptation loop state is invalid" ) committed_iteration = source_loop.get("iteration") best_iteration = source_loop.get("bestIteration") best_metric = source_loop.get("bestMetric") kept_checkpoint = source_loop.get("keptCheckpoint") strict_snapshot_source = bool( isinstance(kept_checkpoint, str) and Path(kept_checkpoint).expanduser().resolve() == checkpoint_path and checkpoint_path.parent.name == "snapshots" and checkpoint_path.name == f"snapshot_iter_{best_iteration}.pt" ) boundary_best_iteration = ( best_iteration if isinstance(best_iteration, int) and not isinstance(best_iteration, bool) else -1 ) accepted_expansion_source = _accepted_page_expansion_source_boundary( training_authority=training_authority, checkpoint_path=checkpoint_path, optimizer_path=optimizer_path, external_path=external_path, loop_state_path=loop_state_path, source_loop=source_loop, best_iteration=boundary_best_iteration, ) if retained_transaction_requires_page_successor and not accepted_expansion_source: raise RuntimeError( "NoNE retained graph-adaptation page-only successor differs" ) if ( not isinstance(committed_iteration, int) or isinstance(committed_iteration, bool) or not isinstance(best_iteration, int) or isinstance(best_iteration, bool) or best_iteration < 0 or best_iteration > committed_iteration or not isinstance(best_metric, (int, float)) or isinstance(best_metric, bool) or not torch.isfinite(torch.tensor(float(best_metric))) or not ( strict_snapshot_source or accepted_expansion_source or retained_transaction_source ) ): raise RuntimeError( "NoNE accepted graph adaptation source is not the committed best" ) source_composition_record: dict[str, str] | None = None source_migration_record: dict[str, str] | None = None if retained_transaction_source: if ( training_authority is None or training_authority.composition_path is None or training_authority.migration_receipt_path is None ): raise RuntimeError( "NoNE retained graph adaptation has no current graph authority" ) source_composition_path = ( training_authority.composition_path.expanduser().resolve() ) source_migration_path = ( training_authority.migration_receipt_path.expanduser().resolve() ) source_composition_record = { "path": str(source_composition_path), "sha256": _file_sha256(source_composition_path), } source_migration_record = { "path": str(source_migration_path), "sha256": _file_sha256(source_migration_path), } source_payload = _load_checkpoint(checkpoint_path) source_parameters = source_payload["parameters"] source_buffers = source_payload["buffers"] source_lineage = source_payload["lineage"] optimizer_payload = torch.load( optimizer_path, map_location="cpu", mmap=True, weights_only=True, ) if not isinstance(optimizer_payload, dict): raise RuntimeError( "NoNE accepted graph adaptation optimizer is invalid" ) external_envelope = json.loads( external_path.read_text(encoding="utf-8") ) external_state = ( external_envelope.get("externalState") if isinstance(external_envelope, dict) else None ) generation_binding = ( external_state.get("generationBinding") if isinstance(external_state, dict) else None ) accepted_generation = ( generation_binding.get("generation") if isinstance(generation_binding, dict) else None ) accepted_manifest_payload_sha256 = ( generation_binding.get("manifestPayloadSha256") if isinstance(generation_binding, dict) else None ) accepted_manifest_sha256 = ( generation_binding.get("manifestSha256") if isinstance(generation_binding, dict) else None ) session_values = ( generation_binding.get("sessionId") if isinstance(generation_binding, dict) else None ) store_root_value = ( external_state.get("storeRoot") if isinstance(external_state, dict) else None ) external_state_schema = ( external_state.get("schema") if isinstance(external_state, dict) else None ) loop_generation_bound = bool( accepted_expansion_source or ( retained_transaction_source and source_loop.get("workingNoneGeneration") == accepted_generation and source_loop.get("workingNoneManifestPayloadSha256") == accepted_manifest_payload_sha256 ) or ( source_loop.get("keptNoneGeneration") == accepted_generation and source_loop.get("keptNoneManifestPayloadSha256") == accepted_manifest_payload_sha256 ) ) if ( not isinstance(external_envelope, dict) or external_envelope.get("schema") != EXTERNAL_CHECKPOINT_BINDING_SCHEMA or external_envelope.get("checkpointSha256") != source_checkpoint_sha256 or external_envelope.get("optimizerSha256") != source_optimizer_sha256 or not isinstance(external_state, dict) or not isinstance(accepted_generation, int) or isinstance(accepted_generation, bool) or accepted_generation < 1 or not isinstance(accepted_manifest_payload_sha256, str) or len(accepted_manifest_payload_sha256) != 64 or not isinstance(accepted_manifest_sha256, str) or len(accepted_manifest_sha256) != 64 or not isinstance(session_values, list) or not session_values or not all( isinstance(value, int) and not isinstance(value, bool) for value in session_values ) or not isinstance(store_root_value, str) or not store_root_value or external_state_schema not in { "nnf.resynthesis.none_checkpoint_external_state.v1", "nnf.resynthesis.none_checkpoint_external_state.v2", } or not loop_generation_bound ): raise RuntimeError( "NoNE accepted graph adaptation source sidecar differs" ) load_state = getattr(model, "load_trainable_state_dict", None) growth_record_fn = getattr( model, "inherited_graph_growth_record_boundary", None, ) validate_training_proof_fn = getattr( model, "validate_external_training_proof_from_boundary", None, ) bind_loaded_graph_fn = getattr( model, "bind_loaded_paged_graph_authority_from_boundary", None, ) validate_graph_frontier_fn = getattr( model, "validate_paged_none_graph_frontier_boundary", None, ) lineage_fn = getattr(model, "checkpoint_lineage", None) if ( not callable(load_state) or not callable(growth_record_fn) or not callable(validate_training_proof_fn) or not callable(bind_loaded_graph_fn) or not callable(validate_graph_frontier_fn) or not callable(lineage_fn) ): raise RuntimeError( "NoNE accepted graph adaptation model lacks graph boundaries" ) load_state( {**source_parameters, **source_buffers}, checkpoint_lineage=source_lineage, ) validate_training_proof_fn(external_state) loaded_graph_checkpoint_sha256 = ( training_authority.paged_graph_checkpoint_sha256 or training_authority.checkpoint_sha256 if training_authority is not None else source_checkpoint_sha256 ) bind_loaded_graph_fn(loaded_graph_checkpoint_sha256) validate_graph_frontier_fn() graph_growth = growth_record_fn() target_lineage = lineage_fn() if ( not isinstance(graph_growth, dict) or graph_growth.get("schema") != "nnf.resynthesis.inherited_graph_growth.v1" or int(graph_growth.get("seededTensorCount", 0)) < 1 or graph_growth.get("identityResidualAtInitialization") is not True or graph_growth.get("inheritedCapabilityPrefixesExact") is not True or graph_growth.get("partialGrowthStateAccepted") is not False or graph_growth.get("strictInheritedGeometryVerified") is not True or not isinstance(target_lineage, dict) or source_lineage == target_lineage ): raise RuntimeError( "NoNE accepted graph adaptation graph-growth proof is incomplete" ) model_migration_receipt = getattr( model, "_paged_none_migration_receipt_path", None, ) migration_authority_path = ( training_authority.migration_receipt_path if training_authority is not None and training_authority.migration_receipt_path is not None else model_migration_receipt ) if not isinstance(migration_authority_path, Path): raise RuntimeError( "NoNE accepted graph adaptation migration authority is absent" ) migration_growth_plan_path = _growth_plan_from_migration_receipt_boundary( migration_authority_path ) source_growth_plan_path = ( explicit_growth_plan if explicit_growth_plan is not None else migration_growth_plan_path ) if explicit_growth_plan is not None: from resynthesis.rbo import ( _validated_offline_growth_plan_boundary, ) assert expected_target_growth_plan_sha256 is not None _validated_path, target_growth_plan = ( _validated_offline_growth_plan_boundary( migration_plan_path=migration_growth_plan_path, migration_plan_sha256=_file_sha256( migration_growth_plan_path ), target_plan_path=explicit_growth_plan, expected_target_plan_sha256=( expected_target_growth_plan_sha256 ), migration_receipt_path=migration_authority_path, ) ) active_growth_plan = dict(target_growth_plan) active_growth_summary = _validate_active_graph_growth_plan_boundary( active_growth_plan, target_lineage=target_lineage, ) else: active_growth_plan, active_growth_summary = ( _active_graph_growth_plan_boundary( source_growth_plan_path=source_growth_plan_path, target_lineage=target_lineage, ) ) final_active_growth_plan = state_root / "active_graph_growth_plan.json" replica_store_roots = external_state.get("replicaStoreRoots") store_root_values = ( replica_store_roots if external_state_schema == "nnf.resynthesis.none_checkpoint_external_state.v2" else [store_root_value] ) if ( not isinstance(store_root_values, list) or not store_root_values or store_root_values[0] != store_root_value or not all( isinstance(value, str) and value for value in store_root_values ) or len(set(store_root_values)) != len(store_root_values) ): raise RuntimeError( "NoNE accepted graph adaptation replica roots differ" ) resolved_store_roots = tuple( Path(root).expanduser().resolve() for root in store_root_values ) if ( not all(root.is_dir() for root in resolved_store_roots) or str(resolved_store_roots[0]) != str(Path(store_root_value).expanduser().resolve()) ): raise RuntimeError( "NoNE accepted graph adaptation replica storage is missing" ) session_id_t = torch.tensor(session_values, dtype=torch.long) generation_t = torch.tensor( accepted_generation, dtype=torch.long, ) manifest_sha256_t = digest_tensor(accepted_manifest_sha256) manifest_payload_sha256_t = digest_tensor( accepted_manifest_payload_sha256 ) verification_stores = tuple( NoNEImmutablePageStore(root) for root in resolved_store_roots ) store_pointer_records_before: list[dict[str, Any]] = [] for store in verification_stores: store.begin_session(session_id_t) current_binding = store.current_generation_binding_boundary() if not _binding_matches_external_record( current_binding, generation_binding, ): raise RuntimeError( "NoNE accepted graph adaptation current pointer differs" ) store_pointer_records_before.append( current_binding.external_record_boundary() ) verified_binding = store.verify_generation_boundary( generation_t=generation_t, manifest_sha256_t=manifest_sha256_t, manifest_payload_sha256_t=manifest_payload_sha256_t, ) if not _binding_matches_external_record( verified_binding, generation_binding, ): raise RuntimeError( "NoNE accepted graph adaptation historical readback differs" ) trainable_named = tuple( (name, parameter) for name, parameter in model.named_parameters() if parameter.requires_grad ) if not trainable_named: raise RuntimeError( "NoNE accepted graph adaptation found no trainable parameters" ) optimizer = torch.optim.AdamW( [ { "params": tuple( parameter for _name, parameter in trainable_named ), "param_names": tuple( name for name, _parameter in trainable_named ), } ] ) optimizer_growth = ( KeepRollbackManager._load_optimizer_state_by_param_names( optimizer, optimizer_payload, source_param_names=tuple( str(name) for name in source_parameters ), ) ) if ( optimizer_growth.get("partialGrowthStateAccepted") is not False or optimizer_growth.get("inheritedMomentPrefixesExact") is not True or optimizer_growth.get("appendedMomentRowsZero") is not True or int(optimizer_growth.get("targetParameterCount", 0)) < int(optimizer_growth.get("sourceParameterCount", 0)) ): raise RuntimeError( "NoNE accepted graph adaptation optimizer proof is incomplete" ) temporary_root = state_root.with_name( f".{state_root.name}.{os.getpid()}.tmp" ) if temporary_root.exists(): if temporary_root.is_dir(): shutil.rmtree(temporary_root) else: temporary_root.unlink() state_root.parent.mkdir(parents=True, exist_ok=True) try: temporary_active_growth_plan = ( temporary_root / "active_graph_growth_plan.json" ) _atomic_json(temporary_active_growth_plan, active_growth_plan) active_growth_plan_sha256 = _file_sha256( temporary_active_growth_plan ) target_checkpoint_name = ( "working_candidate.pt" if retained_transaction_source else f"snapshot_iter_{best_iteration}.pt" ) temporary_checkpoint = ( temporary_root / "snapshots" / target_checkpoint_name ) temporary_optimizer = temporary_checkpoint.with_suffix( ".optimizer.pt" ) temporary_external = temporary_checkpoint.with_suffix( ".none.json" ) canonical_state_fn = getattr( model, "trainable_state_dict", None, ) canonical_state = ( canonical_state_fn() if callable(canonical_state_fn) else None ) trainable_names = { name for name, _parameter in trainable_named } if ( not isinstance(canonical_state, dict) or not trainable_names.issubset(canonical_state) or not all( isinstance(value, torch.Tensor) for value in canonical_state.values() ) ): raise RuntimeError( "NoNE accepted graph adaptation target state is invalid" ) target_parameters = { name: canonical_state[name].detach().to( device="cpu" ).clone() for name in sorted(trainable_names) } target_buffers = { name: value.detach().to(device="cpu").clone() for name, value in sorted(canonical_state.items()) if name not in trainable_names } target_key_sha256, target_geometry_sha256 = _state_identity( {**target_parameters, **target_buffers} ) _atomic_torch_save( temporary_checkpoint, { "schema": CHECKPOINT_SCHEMA, "lineage": target_lineage, "stateKeySetSha256": target_key_sha256, "stateGeometrySha256": target_geometry_sha256, "parameters": target_parameters, "buffers": target_buffers, }, ) _atomic_torch_save( temporary_optimizer, optimizer.state_dict(), ) target_checkpoint_sha256 = _file_sha256(temporary_checkpoint) target_optimizer_sha256 = _file_sha256(temporary_optimizer) target_external_state = dict(external_state) target_external_state.update( { "activeGraphGrowthPlanPath": str(final_active_growth_plan), "activeGraphGrowthPlanSha256": active_growth_plan_sha256, "graphAdaptationReceiptPath": str(receipt_path), "activeGraphPlanTrainingClaimed": False, "activeGraphPlanPromotionEligible": False, } ) _atomic_json( temporary_external, { "schema": EXTERNAL_CHECKPOINT_BINDING_SCHEMA, "checkpointSha256": target_checkpoint_sha256, "optimizerSha256": target_optimizer_sha256, "externalState": target_external_state, }, ) final_checkpoint = state_root / "snapshots" / target_checkpoint_name final_optimizer = final_checkpoint.with_suffix(".optimizer.pt") final_external = final_checkpoint.with_suffix(".none.json") final_loop_state = state_root / "loop_state.json" adapted_loop = dict(source_loop) adapted_loop.update( { "workingCheckpoint": str(final_checkpoint), "workingNoneGeneration": accepted_generation, "workingNoneManifestPayloadSha256": ( accepted_manifest_payload_sha256 ), "graphAdaptationReceiptPath": str(receipt_path), "graphAdaptationSourceLoopStatePath": str(loop_state_path), "graphAdaptationSourceLoopStateSha256": _file_sha256( loop_state_path ), "ts": time.strftime( "%Y-%m-%dT%H:%M:%SZ", time.gmtime(), ), } ) if not retained_transaction_source: adapted_loop.update( { "keptCheckpoint": str(final_checkpoint), "keptNoneGeneration": accepted_generation, "keptNoneManifestPayloadSha256": ( accepted_manifest_payload_sha256 ), } ) temporary_loop_state = temporary_root / "loop_state.json" _atomic_json(temporary_loop_state, adapted_loop) target_external_sha256 = _file_sha256(temporary_external) target_loop_state_sha256 = _file_sha256(temporary_loop_state) target_payload = _load_checkpoint(temporary_checkpoint) target_state = { **target_payload["parameters"], **target_payload["buffers"], } source_state = {**source_parameters, **source_buffers} same_shape_names = sorted( name for name, source_value in source_state.items() if ( name in target_state and target_state[name].shape == source_value.shape ) ) same_shape_exact = bool(same_shape_names) and all( torch.equal(target_state[name], source_state[name]) for name in same_shape_names ) target_external_envelope = json.loads( temporary_external.read_text(encoding="utf-8") ) if not isinstance(target_external_envelope, dict): raise RuntimeError( "NoNE accepted graph adaptation target sidecar is invalid" ) loaded_target_external_state = ( target_external_envelope.get("externalState") ) if not isinstance(loaded_target_external_state, dict): raise RuntimeError( "NoNE accepted graph adaptation target external state is invalid" ) target_generation_binding = loaded_target_external_state.get( "generationBinding" ) store_pointer_records_after = [ store.current_generation_binding_boundary() .external_record_boundary() for store in verification_stores ] if any( record != generation_binding for record in store_pointer_records_after ): raise RuntimeError( "NoNE accepted graph adaptation pointer advanced before commit" ) checks = { "sourceCheckpointOptimizerSidecarBound": True, "sourceAcceptedExpansionBoundaryVerified": ( accepted_expansion_source or strict_snapshot_source or retained_transaction_source ), "retainedTransactionAuthorityBound": ( retained_transaction_authority is None or retained_transaction_source ), "sourceCommittedIterationPreserved": ( adapted_loop.get("iteration") == committed_iteration ), "sourceBestIterationPreserved": ( adapted_loop.get("bestIteration") == best_iteration ), "sourceBestMetricPreserved": ( float(adapted_loop.get("bestMetric", float("nan"))) == float(best_metric) ), "sourceSameShapeTensorsExact": same_shape_exact, "targetLineageIsActiveGraph": ( target_payload["lineage"] == target_lineage ), "graphGrowthStrict": ( graph_growth.get("strictInheritedGeometryVerified") is True ), "newReasoningLayersZeroOpen": ( explicit_growth_plan is None or ( graph_growth.get("reasoningLayerTargetCount") == target_lineage.get("scienceLayers") and int( graph_growth.get("addedReasoningLayerCount", 0) ) > 0 and graph_growth.get( "newReasoningLayerExecutionScalesZero" ) is True and graph_growth.get("pagedRuntimeCopiesIntoNewLayers") == 0 and graph_growth.get( "pageStorageMovedDuringLayerGrowth" ) is False ) ), "optimizerPrefixesExact": ( optimizer_growth.get("inheritedMomentPrefixesExact") is True ), "newOptimizerMomentRowsZero": ( optimizer_growth.get("appendedMomentRowsZero") is True ), "acceptedPageGenerationPreserved": ( target_generation_binding == generation_binding ), "historicalGenerationReadbackVerified": ( len(verification_stores) == len(store_root_values) ), "acceptedStorePointersUnchanged": ( store_pointer_records_after == store_pointer_records_before ), "acceptedStorePointersMatchSourceGeneration": ( all( record == generation_binding for record in store_pointer_records_after ) ), "targetCheckpointHashBound": ( target_external_envelope.get("checkpointSha256") == target_checkpoint_sha256 ), "targetOptimizerHashBound": ( target_external_envelope.get("optimizerSha256") == target_optimizer_sha256 ), "trainingStepsRemainZero": True, "promotionClaimWithheld": True, "targetGrowthPlanHashBound": ( explicit_growth_plan is None or ( source_growth_plan_path == explicit_growth_plan and _file_sha256(explicit_growth_plan) == expected_target_growth_plan_sha256 ) ), "migrationAndOfflineTargetPlansBound": ( explicit_growth_plan is None or ( migration_growth_plan_path.is_file() and migration_growth_plan_path != explicit_growth_plan and source_growth_plan_path == explicit_growth_plan ) ), "activeGraphGrowthPlanMatchesTargetLineage": ( active_growth_summary["functionalExpertFamilies"] == target_lineage.get("functionalCapabilityFamilies") and active_growth_summary["scienceLayers"] == target_lineage.get("pagedNoNE", {}).get("scienceLayers") ), "activeGraphGrowthPlanClaimsNoTraining": ( loaded_target_external_state.get("activeGraphPlanTrainingClaimed") is False and loaded_target_external_state.get( "activeGraphPlanPromotionEligible" ) is False ), } if not all(checks.values()): failed = sorted( name for name, passed in checks.items() if not passed ) raise RuntimeError( "NoNE accepted graph adaptation proof failed: " + ", ".join(failed) ) receipt = { "schema": NONE_ACCEPTED_GRAPH_ADAPTATION_SCHEMA, "passed": True, "status": "GRAPH_ADAPTED_UNTRAINED", "source": { "checkpoint": { "path": str(checkpoint_path), "sha256": source_checkpoint_sha256, }, "optimizer": { "path": str(optimizer_path), "sha256": source_optimizer_sha256, }, "externalState": { "path": str(external_path), "sha256": source_external_sha256, }, "composition": source_composition_record, "migrationReceipt": source_migration_record, "loopState": { "path": str(loop_state_path), "sha256": _file_sha256(loop_state_path), }, "lineage": source_lineage, "committedIteration": committed_iteration, "bestIteration": best_iteration, "bestMetric": float(best_metric), "workingCheckpointBeforeAdaptation": ( source_loop.get("workingCheckpoint") ), "migrationGrowthPlan": { "path": str(migration_growth_plan_path), "sha256": _file_sha256(migration_growth_plan_path), }, "retainedTransactionAuthority": ( retained_transaction_record if retained_transaction_source else None ), }, "target": { "stateRoot": str(state_root), "checkpoint": { "path": str(final_checkpoint), "sha256": target_checkpoint_sha256, }, "optimizer": { "path": str(final_optimizer), "sha256": target_optimizer_sha256, }, "externalState": { "path": str(final_external), "sha256": target_external_sha256, }, "loopState": { "path": str(final_loop_state), "sha256": target_loop_state_sha256, }, "lineage": target_lineage, "acceptedGeneration": accepted_generation, "manifestPayloadSha256": ( accepted_manifest_payload_sha256 ), "verifiedStoreRoots": [ str(root) for root in resolved_store_roots ], "sameShapeInheritedTensorCount": len( same_shape_names ), "activeGraphGrowthPlan": { "path": str(final_active_growth_plan), "sha256": active_growth_plan_sha256, **active_growth_summary, "sourceAuthorityPath": str(source_growth_plan_path), "sourceAuthoritySha256": _file_sha256( source_growth_plan_path ), "trainingClaimed": False, "promotionEligible": False, }, "offlineTargetGrowthPlan": ( { "path": str(explicit_growth_plan), "sha256": expected_target_growth_plan_sha256, } if explicit_growth_plan is not None else None ), }, "graphGrowth": graph_growth, "optimizerGrowth": optimizer_growth, "checks": checks, "trainingSteps": 0, "trainedValidatedNewGraphElements": 0, "promotionEligible": False, } identity_cache_root = temporary_root / ".artifact_sha256_cache" for staged_path, final_path, expected_sha256 in ( ( temporary_checkpoint, final_checkpoint, target_checkpoint_sha256, ), ( temporary_optimizer, final_optimizer, target_optimizer_sha256, ), ( temporary_external, final_external, target_external_sha256, ), ( temporary_loop_state, final_loop_state, target_loop_state_sha256, ), ): stage_atomically_moved_file_sha256_authority_boundary( staged_path=staged_path, final_path=final_path, expected_sha256=expected_sha256, identity_cache_root=identity_cache_root, ) temporary_receipt = ( temporary_root / "graph_adaptation_receipt.json" ) _atomic_json(temporary_receipt, receipt) os.replace(temporary_root, state_root) descriptor = os.open(state_root.parent, os.O_RDONLY) try: os.fsync(descriptor) finally: os.close(descriptor) except Exception: if temporary_root.exists(): shutil.rmtree(temporary_root) raise return NoNEAcceptedGraphAdaptationArtifacts( state_root=state_root, checkpoint_path=final_checkpoint, optimizer_path=final_optimizer, external_state_path=final_external, loop_state_path=final_loop_state, receipt_path=receipt_path, active_growth_plan_path=final_active_growth_plan, iteration=( committed_iteration if retained_transaction_source else best_iteration ), accepted_generation=accepted_generation, ) def seal_active_graph_growth_plan_binding( adaptation_receipt_path: Path, *, source_growth_plan_path: Path | None = None, ) -> dict[str, Any]: """Bind a completed graph adaptation to the current functional roadmap. This boundary writes planning authority only. It never changes checkpoint, optimizer, page objects, accepted pointers, or trained-capability claims. """ receipt_path = adaptation_receipt_path.expanduser().resolve() receipt = json.loads(receipt_path.read_text(encoding="utf-8")) checks = receipt.get("checks") if isinstance(receipt, dict) else None target = receipt.get("target") if isinstance(receipt, dict) else None source = receipt.get("source") if isinstance(receipt, dict) else None if ( not isinstance(receipt, dict) or receipt.get("schema") != NONE_ACCEPTED_GRAPH_ADAPTATION_SCHEMA or receipt.get("passed") is not True or receipt.get("status") != "GRAPH_ADAPTED_UNTRAINED" or receipt.get("trainingSteps") != 0 or receipt.get("promotionEligible") is not False or not isinstance(checks, dict) or not checks or not all(value is True for value in checks.values()) or not isinstance(target, dict) or not isinstance(source, dict) ): raise RuntimeError("NoNE graph adaptation receipt is not sealable") def exact_target_artifact(name: str) -> tuple[Path, str]: record = target.get(name) if not isinstance(record, dict): raise RuntimeError(f"NoNE graph adaptation target {name} is absent") path_value = record.get("path") sha256 = record.get("sha256") if ( not isinstance(path_value, str) or not path_value or not isinstance(sha256, str) or len(sha256) != 64 ): raise RuntimeError( f"NoNE graph adaptation target {name} identity is malformed" ) path = Path(path_value).expanduser().resolve() if not path.is_file() or _file_sha256(path) != sha256: raise RuntimeError( f"NoNE graph adaptation target {name} bytes differ" ) return path, sha256 checkpoint_path, checkpoint_sha256 = exact_target_artifact("checkpoint") optimizer_path, optimizer_sha256 = exact_target_artifact("optimizer") external_path, external_sha256 = exact_target_artifact("externalState") loop_state_path, loop_state_sha256 = exact_target_artifact("loopState") target_lineage = target.get("lineage") state_root_value = target.get("stateRoot") if ( not isinstance(target_lineage, dict) or not isinstance(state_root_value, str) or not state_root_value ): raise RuntimeError("NoNE graph adaptation target lineage is absent") state_root = Path(state_root_value).expanduser().resolve() if ( receipt_path.parent != state_root or checkpoint_path.parent.parent != state_root or loop_state_path.parent != state_root ): raise RuntimeError("NoNE graph adaptation state-root identity differs") growth_source = source_growth_plan_path if growth_source is None: source_external_record = source.get("externalState") if not isinstance(source_external_record, dict): raise RuntimeError("NoNE graph adaptation source sidecar is absent") source_external_value = source_external_record.get("path") source_external_sha256 = source_external_record.get("sha256") if ( not isinstance(source_external_value, str) or not isinstance(source_external_sha256, str) ): raise RuntimeError("NoNE graph adaptation source sidecar is malformed") source_external_path = Path(source_external_value).expanduser().resolve() if ( not source_external_path.is_file() or _file_sha256(source_external_path) != source_external_sha256 ): raise RuntimeError("NoNE graph adaptation source sidecar bytes differ") envelope = json.loads(source_external_path.read_text(encoding="utf-8")) external_state = ( envelope.get("externalState") if isinstance(envelope, dict) else None ) composition_value = ( external_state.get("compositionPath") if isinstance(external_state, dict) else None ) composition_sha256 = ( external_state.get("compositionSha256") if isinstance(external_state, dict) else None ) if ( not isinstance(composition_value, str) or not isinstance(composition_sha256, str) ): raise RuntimeError("NoNE graph adaptation composition is absent") composition_path = Path(composition_value).expanduser().resolve() if ( not composition_path.is_file() or _file_sha256(composition_path) != composition_sha256 ): raise RuntimeError("NoNE graph adaptation composition bytes differ") composition = json.loads(composition_path.read_text(encoding="utf-8")) expansion = ( composition.get("catalogExpansion") if isinstance(composition, dict) else None ) growth_value = ( expansion.get("growthPlanPath") if isinstance(expansion, dict) else None ) growth_sha256 = ( expansion.get("growthPlanSha256") if isinstance(expansion, dict) else None ) if not isinstance(growth_value, str) or not isinstance( growth_sha256, str, ): raise RuntimeError( "NoNE graph adaptation source growth plan is not discoverable" ) growth_source = Path(growth_value).expanduser().resolve() if ( not growth_source.is_file() or _file_sha256(growth_source) != growth_sha256 ): raise RuntimeError("NoNE graph adaptation source growth-plan bytes differ") growth_source = growth_source.expanduser().resolve() active_plan_path = state_root / "active_graph_growth_plan.json" if active_plan_path.exists(): active_plan = json.loads(active_plan_path.read_text(encoding="utf-8")) if not isinstance(active_plan, dict): raise RuntimeError("NoNE active graph growth plan is malformed") active_summary = _validate_active_graph_growth_plan_boundary( active_plan, target_lineage=target_lineage, ) lineage = active_plan.get("lineage") if ( not isinstance(lineage, dict) or lineage.get("sourcePlan") != str(growth_source) or lineage.get("sourcePlanSha256") != _file_sha256(growth_source) ): raise RuntimeError("NoNE active graph growth-plan lineage differs") else: active_plan, active_summary = _active_graph_growth_plan_boundary( source_growth_plan_path=growth_source, target_lineage=target_lineage, ) _atomic_json(active_plan_path, active_plan) active_plan_sha256 = _file_sha256(active_plan_path) binding_path = state_root / "active_graph_growth_plan_binding.json" binding = { "schema": NONE_ACTIVE_GRAPH_GROWTH_PLAN_BINDING_SCHEMA, "passed": True, "graphAdaptationReceipt": { "path": str(receipt_path), "sha256": _file_sha256(receipt_path), }, "sourceGrowthPlan": { "path": str(growth_source), "sha256": _file_sha256(growth_source), }, "activeGrowthPlan": { "path": str(active_plan_path), "sha256": active_plan_sha256, **active_summary, }, "target": { "checkpoint": { "path": str(checkpoint_path), "sha256": checkpoint_sha256, }, "optimizer": { "path": str(optimizer_path), "sha256": optimizer_sha256, }, "externalState": { "path": str(external_path), "sha256": external_sha256, }, "loopState": { "path": str(loop_state_path), "sha256": loop_state_sha256, }, "lineageSha256": hashlib.sha256( json.dumps( target_lineage, sort_keys=True, separators=(",", ":"), ).encode("utf-8") ).hexdigest(), }, "checks": { "adaptationReceiptExact": True, "targetArtifactsExact": True, "activePlanMatchesTargetLineage": True, "sourcePlanPreserved": True, "checkpointOptimizerAndPointerUnmodified": True, "trainingNotClaimed": True, "promotionWithheld": True, }, "trainingClaimed": False, "promotionEligible": False, } if binding_path.exists(): existing = json.loads(binding_path.read_text(encoding="utf-8")) if existing != binding: raise RuntimeError("NoNE active graph growth-plan binding differs") else: _atomic_json(binding_path, binding) return binding def load_v2_resident_layer_state( resident_runtime_path: Path, layer_id: int, ) -> dict[str, torch.Tensor]: """Load one layer's transferred resident runtime at the model boundary.""" path = resident_runtime_path.expanduser().resolve() prefix = f"layer_{int(layer_id)}." state: dict[str, torch.Tensor] = {} with safe_open( # type: ignore[no-untyped-call] str(path), framework="pt", device="cpu", ) as handle: metadata = handle.metadata() if ( not isinstance(metadata, dict) or metadata.get("schema") != NONE_V2_RESIDENT_RUNTIME_SCHEMA ): raise RuntimeError("NoNE v2 resident runtime schema differs") for name in handle.keys(): if name.startswith(prefix): state[name[len(prefix) :]] = handle.get_tensor(name) if not state: raise RuntimeError(f"NoNE v2 resident layer is absent: {layer_id}") route_keys_t = state.get("router.page_route_keys") if not isinstance(route_keys_t, torch.Tensor) or route_keys_t.ndim != 2: raise RuntimeError("NoNE v2 resident page-route geometry differs") page_count = int(route_keys_t.shape[0]) legacy_bias_t = state.pop("router.page_expert_bias_t", None) state.pop("router.frontier_margin", None) state.setdefault( "router.quantile_router.activation_logit", torch.logit(torch.tensor(1.0e-4, dtype=torch.float32)), ) state.setdefault( "router.quantile_router.temperature_logit", torch.zeros((), dtype=torch.float32), ) state.setdefault( "router.quantile_router.expert_bias_t", ( legacy_bias_t if isinstance(legacy_bias_t, torch.Tensor) and legacy_bias_t.shape == (page_count,) else torch.zeros(page_count, dtype=torch.float32) ), ) return state def _validated_paged_layer_ids_boundary( layer_catalog: object, ) -> tuple[int, ...]: """Resolve sparse paged-runtime layer identities at the proof boundary.""" if not isinstance(layer_catalog, dict) or not layer_catalog: raise RuntimeError("NoNE cold-reload layer catalog is absent") layer_ids: list[int] = [] observed_page_ids: set[int] = set() for raw_layer_id, raw_page_ids in layer_catalog.items(): try: layer_id = int(raw_layer_id) except (TypeError, ValueError) as exc: raise RuntimeError( "NoNE cold-reload layer identity is malformed" ) from exc if str(layer_id) != str(raw_layer_id) or layer_id < 0: raise RuntimeError("NoNE cold-reload layer identity is malformed") if ( not isinstance(raw_page_ids, list) or not raw_page_ids or any( not isinstance(page_id, int) or isinstance(page_id, bool) or page_id < 0 for page_id in raw_page_ids ) or len(set(raw_page_ids)) != len(raw_page_ids) or observed_page_ids.intersection(raw_page_ids) ): raise RuntimeError("NoNE cold-reload layer page scope is malformed") observed_page_ids.update(raw_page_ids) layer_ids.append(layer_id) return tuple(sorted(layer_ids)) def write_v2_seed_cold_reload_receipt( *, composition_path: Path, migration_receipt_path: Path, output_path: Path, ) -> dict[str, Any]: """Verify every page object and sample each resident layer on CPU.""" composition = validate_v2_seed_composition( composition_path, migration_receipt_path, ) migration_sha256 = _file_sha256( migration_receipt_path.expanduser().resolve() ) catalog_record = composition["pageCatalog"] resident_record = composition["residentRuntime"] store_record = composition["pageStore"] catalog_path = Path(str(catalog_record["path"])).resolve() catalog = json.loads(catalog_path.read_text(encoding="utf-8")) geometry = catalog["pageGeometry"] layer_catalog = catalog["layerCatalogPageIds"] layer_ids = _validated_paged_layer_ids_boundary(layer_catalog) session_id_t = torch.tensor( store_record["sessionId"], dtype=torch.long, ) store = NoNEImmutablePageStore( Path(str(store_record["root"])) ) accepted_t = store.begin_session(session_id_t) expected_accepted_pointer = store_record.get("acceptedPointer") if composition.get("schema") == NONE_V2_PLUS_COMPOSITION_SCHEMA: live_graph_authority = store.current_graph_authority_boundary() if ( live_graph_authority is None or Path(live_graph_authority.composition_path).resolve() != composition_path.expanduser().resolve() or not torch.equal( live_graph_authority.composition_sha256_t, digest_tensor(_file_sha256(composition_path.expanduser().resolve())), ) or Path(live_graph_authority.migration_receipt_path).resolve() != migration_receipt_path.expanduser().resolve() ): raise RuntimeError("NoNE cold-reload live graph authority differs") expected_accepted_pointer = { "generation": int(accepted_t.detach().cpu().long().reshape(())) } if ( not isinstance(expected_accepted_pointer, dict) or not isinstance( expected_accepted_pointer.get("generation"), int, ) or isinstance( expected_accepted_pointer.get("generation"), bool, ) ): raise RuntimeError("NoNE cold-reload generation authority differs") expected_accepted_generation = int( expected_accepted_pointer["generation"] ) raw_catalog_pages = catalog.get("pages") if not isinstance(raw_catalog_pages, list) or not raw_catalog_pages: raise RuntimeError("NoNE cold-reload page catalog is absent") catalog_page_ids: list[int] = [] for raw_page in raw_catalog_pages: if not isinstance(raw_page, dict): raise RuntimeError("NoNE cold-reload page row is malformed") page_id = raw_page.get("pageId") if ( not isinstance(page_id, int) or isinstance(page_id, bool) or page_id < 0 ): raise RuntimeError("NoNE cold-reload page identity is malformed") catalog_page_ids.append(page_id) expected_page_ids = tuple(sorted(catalog_page_ids)) if len(expected_page_ids) != len(set(expected_page_ids)): raise RuntimeError("NoNE cold-reload page identity is malformed") accepted_page_index = store._page_index() if tuple(sorted(accepted_page_index)) != expected_page_ids: raise RuntimeError("NoNE cold-reload accepted page catalog differs") catalog_identity_rows: list[tuple[int, str, int]] = [] for page_id in expected_page_ids: accepted_page_row = accepted_page_index[page_id] object_sha256 = accepted_page_row.get("sha256") object_bytes = accepted_page_row.get("bytes") if ( not isinstance(object_sha256, str) or not isinstance(object_bytes, int) or isinstance(object_bytes, bool) ): raise RuntimeError( "NoNE cold-reload accepted page identity is malformed" ) catalog_identity_rows.append( (page_id, object_sha256, object_bytes) ) try: expected_object_identity_sha256 = ( page_object_identity_sha256_boundary(catalog_identity_rows) ) except ValueError as error: raise RuntimeError( "NoNE cold-reload page identity is malformed" ) from error layer_page_ids = tuple( sorted( page_id for layer_id in layer_ids for page_id in layer_catalog[str(layer_id)] ) ) if layer_page_ids != expected_page_ids: raise RuntimeError("NoNE cold-reload layer page union differs") page_object_readback = ( store.verify_accepted_page_object_identities_boundary( session_id_t=session_id_t, generation_t=accepted_t, page_ids_t=torch.tensor( expected_page_ids, dtype=torch.long, ), ) ) page_object_readback_record = ( page_object_readback.external_record_boundary() ) page_identity_verified = bool( page_object_readback_record["generation"] == int(accepted_t) and page_object_readback_record["verifiedPageObjectCount"] == len(expected_page_ids) and page_object_readback_record["objectIdentitySha256"] == expected_object_identity_sha256 ) if not page_identity_verified: raise RuntimeError("NoNE cold-reload page-object identity differs") resident_path = Path(str(resident_record["path"])).resolve() layer_checks: list[dict[str, Any]] = [] for layer_id in layer_ids: page_ids_t = torch.tensor( layer_catalog[str(layer_id)], dtype=torch.long, ) resident_state = load_v2_resident_layer_state( resident_path, layer_id, ) runtime = NoNEPagedExpertRuntime( hidden_size=int(geometry["hiddenSize"]), action_size=int(geometry["actionSize"]), router_size=int(geometry["routerSize"]), page_count=page_ids_t.shape[0], layer_id=layer_id, page_catalog_ids_t=page_ids_t, ) runtime.load_seed_resident_state_boundary(resident_state) runtime.apply_model_wide_residency_allocation_boundary( frontier_pages=1, cache_entries=1, ) runtime.bind_store_boundary(store, session_id_t) runtime.eval() materialization_before = ( store.page_materialization_telemetry_boundary() ) hidden_t = torch.zeros( 1, 1, int(geometry["hiddenSize"]), dtype=torch.float32, ) action_t = torch.zeros( 1, int(geometry["actionSize"]), dtype=torch.float32, ) with torch.inference_mode(): packet = runtime( hidden_t, action_t, hidden_t.mean(dim=1), ) last_request = runtime.last_request if last_request is None: raise RuntimeError("NoNE cold-reload model route is absent") execution_page_ids_t = ( last_request.unique_page_ids_t.detach().cpu().long().reshape(-1) ) model_route_page_count = int(execution_page_ids_t.shape[0]) model_route_exact = bool( model_route_page_count > 0 and torch.unique(execution_page_ids_t).shape[0] == model_route_page_count and all( int(page_id) in page_ids_t.tolist() for page_id in execution_page_ids_t ) ) primary_page_ids_t = ( packet.page_ids_t.detach().cpu().long().reshape(-1) ) primary_page_matches = bool( primary_page_ids_t.numel() > 0 and all( int(page_id) in execution_page_ids_t.tolist() for page_id in primary_page_ids_t ) ) residency = runtime.accepted_residency_telemetry() residency_request_count = int( residency.request_count_t.detach().cpu().long().reshape(()) ) residency_selected_page_count = int( ( residency.hit_page_count_t + residency.miss_page_count_t ) .detach() .cpu() .long() .reshape(()) ) resident_page_count = int( residency.resident_page_ids_t.detach().cpu().numel() ) materialization_after = ( store.page_materialization_telemetry_boundary() ) materialization_request_delta = int( ( materialization_after.request_count_t - materialization_before.request_count_t ) .detach() .cpu() .long() .reshape(()) ) materialization_page_delta = int( ( materialization_after.page_count_t - materialization_before.page_count_t ) .detach() .cpu() .long() .reshape(()) ) one_page_materialization_waves = bool( materialization_request_delta == model_route_page_count and materialization_page_delta == model_route_page_count and residency_request_count == model_route_page_count and residency_selected_page_count == model_route_page_count and resident_page_count == 1 and runtime.accepted_inference_cache_entries == 1 ) output_finite = bool(torch.isfinite(packet.output_t).all()) generation_matches = bool( torch.equal( packet.generation_t.detach().cpu().long().reshape(()), accepted_t.detach().cpu().long().reshape(()), ) ) residual_scale_t = resident_state.get("residual_scale") if not isinstance(residual_scale_t, torch.Tensor): raise RuntimeError( "NoNE cold-reload resident residual is absent" ) residual_passed = bool( torch.isfinite(residual_scale_t).all() if composition.get("schema") == NONE_V2_PLUS_COMPOSITION_SCHEMA else torch.equal( residual_scale_t, torch.zeros_like(residual_scale_t), ) ) layer_passed = bool( model_route_exact and one_page_materialization_waves and output_finite and generation_matches and primary_page_matches and residual_passed ) layer_checks.append( { "layerId": layer_id, "catalogPageCount": int(page_ids_t.shape[0]), "modelRoutePageIds": execution_page_ids_t.tolist(), "modelRoutePageCount": model_route_page_count, "modelRouteExact": model_route_exact, "primaryPageIds": primary_page_ids_t.tolist(), "materializationRequestCount": ( materialization_request_delta ), "materializedPageCount": materialization_page_delta, "materializationWavePageCount": 1, "onePageMaterializationWaves": ( one_page_materialization_waves ), "generation": int(packet.generation_t), "outputFinite": output_finite, "residentResidualValid": residual_passed, "allLayerPagesExecuted": False, "allLayerPagesTrained": False, "passed": layer_passed, } ) if not layer_passed: raise RuntimeError("NoNE v2 cold-reload layer execution failed") checks = { "compositionValidated": True, "acceptedGenerationReopened": int(accepted_t) == expected_accepted_generation, "catalogLayerPageUnionExact": layer_page_ids == expected_page_ids, "allPageObjectsHashReadBack": page_identity_verified, "allPageObjectIdentitiesCryptographicallyVerified": ( page_identity_verified ), "allResidentLayersLoadedStrictly": all( row["passed"] for row in layer_checks ), "allResidentLayersExecutedOnCpu": all( row["outputFinite"] for row in layer_checks ), "boundedOnePageExecutionPerResidentLayer": all( row["onePageMaterializationWaves"] and row["materializationWavePageCount"] == 1 for row in layer_checks ), "v1CompatibilityResidualClosed": all( row["residentResidualValid"] for row in layer_checks ), "allPhysicalPageExecutionClaimWithheld": True, "allPhysicalPageTrainingClaimWithheld": True, "trainingNotClaimed": True, "promotionWithheld": True, } payload = { "schema": "nnf.resynthesis.none_v2_seed_cold_reload.v1", "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": all(checks.values()), "migrationReceipt": { "path": str(migration_receipt_path.expanduser().resolve()), "sha256": migration_sha256, }, "composition": { "path": str(composition_path.expanduser().resolve()), "sha256": _file_sha256( composition_path.expanduser().resolve() ), }, "acceptedGeneration": int(accepted_t), "expectedAcceptedGeneration": expected_accepted_generation, "pageObjectCount": len(expected_page_ids), "verifiedPageObjectCount": page_object_readback_record[ "verifiedPageObjectCount" ], "verifiedPageObjectBytes": page_object_readback_record[ "verifiedPageObjectBytes" ], "verifiedPageIdsSha256": page_object_readback_record[ "verifiedPageIdsSha256" ], "pageObjectIdentitySha256": page_object_readback_record[ "objectIdentitySha256" ], "maximumInFlightPageObjectCount": page_object_readback_record[ "maximumInFlightObjectCount" ], "pageObjectValidationMode": ( "exact_sha256_identity_stream_with_generation_bound_cache" ), "residentLayerCount": len(layer_checks), "residentExecutionSamplePageCount": sum( int(row["modelRoutePageCount"]) for row in layer_checks ), "residentLayerExecutionMode": ( "model_owned_frontier_with_one_page_materialization_waves" ), "layerChecks": layer_checks, "checks": checks, "allPhysicalPageObjectsExecutedOnCpu": False, "allPhysicalPageObjectsTrained": False, "trainingStarted": False, "promotionEligible": False, "remainingProof": [ "clean_single_gpu_launch", "family_root_distinct_gradients", "model_owned_route_utilization", "rbo_fabric_vge_transfer", "heldout_generalization", "source_capability_retention", "immutable_release_verification", ], } if payload["passed"] is not True: raise RuntimeError("NoNE v2 cold-reload proof failed") _atomic_json(output_path.expanduser().resolve(), payload) return payload def write_v2_seed_replica_receipt_from_verified_source( *, composition_path: Path, migration_receipt_path: Path, source_replica_receipt_path: Path, replica_roots: Sequence[Path], output_path: Path, ) -> dict[str, Any]: """Derive an exact two-device receipt from a prior immutable readback. The source receipt remains the content-readback authority. This boundary revalidates its hash-bound v2+ composition/migration lineage plus the live generation manifests, mount identities, object names, and object sizes. It therefore does not reread or dequantize retained transfer tensors merely to select a smaller training topology from an already-proven replica set. """ from resynthesis.data_lifecycle import ( REPLICA_PROOF_SCHEMA, mount_identity, ) selected_roots = tuple( root.expanduser().resolve() for root in replica_roots ) if len(selected_roots) != 2 or len(set(selected_roots)) != 2: raise ValueError( "NoNE derived training receipt requires exactly two distinct roots" ) composition_file = composition_path.expanduser().resolve() migration_file = migration_receipt_path.expanduser().resolve() source_receipt_file = source_replica_receipt_path.expanduser().resolve() output_file = output_path.expanduser().resolve() if source_receipt_file == output_file: raise ValueError("NoNE derived receipt must not replace its source proof") composition = json.loads(composition_file.read_text(encoding="utf-8")) migration = json.loads(migration_file.read_text(encoding="utf-8")) source_receipt = json.loads( source_receipt_file.read_text(encoding="utf-8") ) source_receipt_sha256 = _file_sha256(source_receipt_file) migration_checks = ( migration.get("checks") if isinstance(migration, dict) else None ) migration_artifacts = ( migration.get("artifacts") if isinstance(migration, dict) else None ) composition_record = ( migration_artifacts.get("composition") if isinstance(migration_artifacts, dict) else None ) replica_transition = ( migration.get("replicaTopologyTransition") if isinstance(migration, dict) else None ) target_receipt = ( replica_transition.get("targetReceipt") if isinstance(replica_transition, dict) else None ) if ( not isinstance(composition, dict) or composition.get("schema") != NONE_V2_PLUS_COMPOSITION_SCHEMA or not isinstance(migration, dict) or migration.get("schema") not in NONE_V2_PLUS_EXPANSION_RECEIPT_SCHEMAS or migration.get("passed") is not True or not isinstance(migration_checks, dict) or not migration_checks or not all(value is True for value in migration_checks.values()) or not isinstance(composition_record, dict) or Path(str(composition_record.get("path", ""))).resolve() != composition_file or composition_record.get("sha256") != _file_sha256(composition_file) or not isinstance(replica_transition, dict) or replica_transition.get("schema") != "nnf.resynthesis.none_replica_topology_transition.v1" or replica_transition.get("acceptedGenerationAuthorityRetained") is not True or replica_transition.get("checkpointOrOptimizerBytesRewritten") is not False or not isinstance(target_receipt, dict) or Path(str(target_receipt.get("path", ""))).resolve() != source_receipt_file or target_receipt.get("sha256") != source_receipt_sha256 ): raise RuntimeError( "NoNE derived replica composition/migration lineage differs" ) page_store = composition.get("pageStore") if not isinstance(page_store, dict): raise RuntimeError("NoNE derived replica composition has no page store") pointer = page_store.get("acceptedPointer") session_id = page_store.get("sessionId") if ( not isinstance(pointer, dict) or not isinstance(session_id, list) or not session_id or any( not isinstance(value, int) or isinstance(value, bool) for value in session_id ) ): raise RuntimeError("NoNE derived replica session authority differs") generation_sha256 = pointer.get("manifestPayloadSha256") session_key = pointer.get("sessionKey") manifest_relative = pointer.get("manifest") accepted_generation = pointer.get("generation") canonical_root = Path(str(page_store.get("root", ""))).resolve() if ( not isinstance(generation_sha256, str) or len(generation_sha256) != 64 or replica_transition.get("seedManifestPayloadSha256") != generation_sha256 or not isinstance(session_key, str) or not session_key or not isinstance(manifest_relative, str) or not manifest_relative or not isinstance(accepted_generation, int) or isinstance(accepted_generation, bool) ): raise RuntimeError("NoNE derived replica generation authority differs") source_rows = source_receipt.get("replicas") if ( source_receipt.get("schema") != "nnf.resynthesis.none_v2_seed_replicas.v1" or source_receipt.get("passed") is not True or source_receipt.get("generationManifestSha256") != generation_sha256 or not isinstance(source_rows, list) or source_receipt.get("copyCount") != len(source_rows) ): raise RuntimeError("NoNE derived source replica receipt differs") source_rows_by_root: dict[Path, dict[str, Any]] = {} for row in source_rows: if ( not isinstance(row, dict) or row.get("schema") != REPLICA_PROOF_SCHEMA or row.get("artifactSha256") != generation_sha256 or row.get("manifestPayloadSha256") != generation_sha256 or row.get("acceptedGeneration") != accepted_generation or row.get("fullReadbackVerified") is not True or row.get("immutable") is not True ): raise RuntimeError("NoNE derived source readback proof differs") root = Path(str(row.get("root", ""))).expanduser().resolve() if root in source_rows_by_root: raise RuntimeError("NoNE derived source replica root is duplicated") source_rows_by_root[root] = row if any(root not in source_rows_by_root for root in selected_roots): raise RuntimeError("NoNE derived root lacks source readback proof") canonical_manifest_path = ( canonical_root / "sessions" / session_key / manifest_relative ) canonical_manifest = json.loads( canonical_manifest_path.read_text(encoding="utf-8") ) canonical_page_rows = canonical_manifest.get("pageObjects") if ( canonical_manifest.get("manifestPayloadSha256") != generation_sha256 or canonical_manifest.get("generation") != accepted_generation or not isinstance(canonical_page_rows, list) or not canonical_page_rows ): raise RuntimeError("NoNE derived canonical generation differs") session_id_t = torch.tensor(session_id, dtype=torch.long) selected_rows: list[dict[str, Any]] = [] selected_device_ids: set[str] = set() for root in selected_roots: source_row = source_rows_by_root[root] identity = mount_identity(root) manifest_path = root / "sessions" / session_key / manifest_relative manifest = json.loads(manifest_path.read_text(encoding="utf-8")) replica_store = NoNEImmutablePageStore(root) accepted_t = replica_store.begin_session(session_id_t) verified_objects = 0 verified_bytes = 0 for page_row in canonical_page_rows: object_sha256 = str(page_row.get("sha256", "")) object_bytes = page_row.get("bytes") object_path = ( root / "objects" / "sha256" / f"{object_sha256}.safetensors" ) if ( len(object_sha256) != 64 or not isinstance(object_bytes, int) or isinstance(object_bytes, bool) or object_bytes < 1 or not object_path.is_file() or object_path.stat().st_size != object_bytes ): raise RuntimeError( f"NoNE derived replica object metadata differs: {object_path}" ) verified_objects += 1 verified_bytes += object_bytes if ( manifest.get("manifestPayloadSha256") != generation_sha256 or manifest.get("generation") != accepted_generation or manifest.get("pageObjects") != canonical_page_rows # The selected receipt proves this immutable historical generation. # A store may have advanced its same-session accepted pointer since # that proof; only rollback below the proven generation is invalid. or int(accepted_t) < accepted_generation or source_row.get("deviceUuid") != identity.device_uuid or source_row.get("deviceSource") != identity.source or source_row.get("verifiedObjectCount") != verified_objects or source_row.get("verifiedObjectBytes") != verified_bytes ): raise RuntimeError("NoNE derived live replica identity differs") selected_rows.append(copy.deepcopy(source_row)) selected_device_ids.add(identity.device_uuid) if len(selected_device_ids) != 2: raise RuntimeError("NoNE derived replicas do not span two devices") checks = { "minimumCopiesReadBack": True, "minimumDistinctDevices": True, "sourceReceiptHashVerified": True, "compositionMigrationLineageVerified": True, "sourceFullReadbackProofReused": True, "selectedManifestIdentityVerified": True, "selectedObjectMetadataVerified": True, "threeCopiesReadBack": False, "threeDistinctDevices": False, "twoFailureDomains": False, "offHostImmutableCopy": False, } payload = { "schema": "nnf.resynthesis.none_v2_seed_replicas.v1", "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": True, "trainingAdmissionEligible": True, "minimumTrainingReplicaCount": 2, "minimumCopiesReadBack": 2, "minimumDistinctDevices": 2, "replicaTopology": "exact_verified_source_two_device_v1", "durabilityComplete": False, "generationManifestSha256": generation_sha256, "copyCount": 2, "distinctDeviceCount": 2, "distinctFailureDomainCount": 1, "offHostImmutableCount": 0, "checks": checks, "replicas": selected_rows, "derivedFromReplicaReceipt": { "path": str(source_receipt_file), "sha256": source_receipt_sha256, "copyCount": source_receipt.get("copyCount"), "distinctDeviceCount": source_receipt.get("distinctDeviceCount"), }, "composition": { "path": str(composition_file), "sha256": _file_sha256(composition_file), }, "migrationReceipt": { "path": str(migration_file), "sha256": _file_sha256(migration_file), }, "remainingDurabilityProof": [ "threeCopiesReadBack", "threeDistinctDevices", "twoFailureDomains", "offHostImmutableCopy", ], "promotionEligible": False, } _atomic_json(output_file, payload) return payload def _historical_replica_seed_manifest_matches_boundary( manifest: Mapping[str, Any], *, generation_sha256: str, canonical_page_rows: Sequence[Mapping[str, Any]], accepted_generation: int, seed_generation: int, ) -> bool: """Prove one immutable seed after its live pointer has advanced. Replica receipts bind the immutable seed manifest and its object bytes, not the mutable head of the same page-store session. Advancing a store is therefore valid; rolling it back below the seed being re-proved is not. """ return bool( manifest.get("manifestPayloadSha256") == generation_sha256 and manifest.get("generation") == seed_generation and manifest.get("pageObjects") == list(canonical_page_rows) and accepted_generation >= seed_generation ) def write_v2_seed_replica_receipt( *, composition_path: Path, migration_receipt_path: Path, replica_roots: Sequence[Path], output_path: Path, ) -> dict[str, Any]: """Full-readback immutable generation replicas across registered devices.""" from resynthesis.data_lifecycle import ( REPLICA_PROOF_SCHEMA, mount_identity, ) minimum_training_replica_count = 2 if len(replica_roots) < minimum_training_replica_count: raise ValueError( "NoNE v2 training admission requires at least two exact copies" ) composition = validate_v2_seed_composition( composition_path, migration_receipt_path, ) store_record = composition["pageStore"] pointer = store_record["acceptedPointer"] session_id_t = torch.tensor( store_record["sessionId"], dtype=torch.long, ) session_key = str(pointer["sessionKey"]) canonical_root = Path(str(store_record["root"])).resolve() canonical_manifest_path = ( canonical_root / "sessions" / session_key / str(pointer["manifest"]) ) canonical_manifest = json.loads( canonical_manifest_path.read_text(encoding="utf-8") ) canonical_page_rows = canonical_manifest["pageObjects"] generation_sha256 = str(pointer["manifestPayloadSha256"]) replicas: list[dict[str, Any]] = [] for replica_root in replica_roots: root = replica_root.expanduser().resolve() replica_store = NoNEImmutablePageStore(root) accepted_t = replica_store.begin_session(session_id_t) manifest_path = ( root / "sessions" / session_key / str(pointer["manifest"]) ) manifest = json.loads(manifest_path.read_text(encoding="utf-8")) seed_generation = int(pointer["generation"]) manifest_matches = ( _historical_replica_seed_manifest_matches_boundary( manifest, generation_sha256=generation_sha256, canonical_page_rows=canonical_page_rows, accepted_generation=int(accepted_t), seed_generation=seed_generation, ) ) verified_objects = 0 verified_bytes = 0 for row in canonical_page_rows: object_sha256 = str(row["sha256"]) object_path = ( root / "objects" / "sha256" / f"{object_sha256}.safetensors" ) if ( not object_path.is_file() or _file_sha256(object_path) != object_sha256 ): raise RuntimeError( f"NoNE v2 replica object readback failed: {object_path}" ) verified_objects += 1 verified_bytes += object_path.stat().st_size identity = mount_identity(root) full_readback = bool( manifest_matches and verified_objects == len(canonical_page_rows) ) replicas.append( { "schema": REPLICA_PROOF_SCHEMA, "artifactSha256": generation_sha256, "root": str(root), "deviceUuid": identity.device_uuid, "deviceSource": identity.source, "failureDomainId": "local-host-1", "offHost": False, "immutable": True, "fullReadbackVerified": full_readback, "verifiedObjectCount": verified_objects, "verifiedObjectBytes": verified_bytes, # This row proves the historical immutable seed. The mutable # store head may be newer and is rediscovered separately. "acceptedGeneration": seed_generation, "manifestPayloadSha256": str( manifest.get("manifestPayloadSha256", "") ), } ) device_ids = {str(row["deviceUuid"]) for row in replicas} failure_domains = { str(row["failureDomainId"]) for row in replicas } off_host_count = sum( 1 for row in replicas if row["offHost"] is True ) checks = { "minimumCopiesReadBack": len(replicas) >= minimum_training_replica_count and all(row["fullReadbackVerified"] for row in replicas), "minimumDistinctDevices": ( len(device_ids) >= minimum_training_replica_count ), "threeCopiesReadBack": len(replicas) >= 3 and all(row["fullReadbackVerified"] for row in replicas), "threeDistinctDevices": len(device_ids) >= 3, "twoFailureDomains": len(failure_domains) >= 2, "offHostImmutableCopy": off_host_count >= 1, } payload = { "schema": "nnf.resynthesis.none_v2_seed_replicas.v1", "builtAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": checks["minimumCopiesReadBack"] and checks["minimumDistinctDevices"], "trainingAdmissionEligible": checks["minimumCopiesReadBack"] and checks["minimumDistinctDevices"], "minimumTrainingReplicaCount": minimum_training_replica_count, "minimumCopiesReadBack": len(replicas), "minimumDistinctDevices": len(device_ids), "replicaTopology": "exact_full_readback_two_or_more_v1", "durabilityComplete": ( checks["threeCopiesReadBack"] and checks["threeDistinctDevices"] and checks["twoFailureDomains"] and checks["offHostImmutableCopy"] ), "generationManifestSha256": generation_sha256, "copyCount": len(replicas), "distinctDeviceCount": len(device_ids), "distinctFailureDomainCount": len(failure_domains), "offHostImmutableCount": off_host_count, "checks": checks, "replicas": replicas, "remainingDurabilityProof": [ name for name, passed in checks.items() if not passed ], "promotionEligible": False, } if payload["passed"] is not True: raise RuntimeError("NoNE v2 local replica readback proof failed") _atomic_json(output_path.expanduser().resolve(), payload) return payload def _add_seed_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("--source-checkpoint", required=True, type=Path) parser.add_argument("--source-optimizer", required=True, type=Path) parser.add_argument("--growth-plan", required=True, type=Path) parser.add_argument("--store-root", required=True, type=Path) parser.add_argument("--receipt", required=True, type=Path) parser.add_argument("--expected-checkpoint-sha256") parser.add_argument("--expected-optimizer-sha256") def _add_expansion_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("--source-checkpoint", required=True, type=Path) parser.add_argument("--source-optimizer", required=True, type=Path) parser.add_argument("--source-external-state", required=True, type=Path) parser.add_argument("--source-composition", required=True, type=Path) parser.add_argument( "--source-migration-receipt", required=True, type=Path, ) parser.add_argument("--growth-plan", required=True, type=Path) parser.add_argument("--receipt", required=True, type=Path) parser.add_argument("--replica-receipt", type=Path) parser.add_argument("--expected-checkpoint-sha256") parser.add_argument("--expected-optimizer-sha256") def _add_graph_adaptation_arguments( parser: argparse.ArgumentParser, ) -> None: parser.add_argument("--source-checkpoint", required=True, type=Path) parser.add_argument("--source-optimizer", required=True, type=Path) parser.add_argument("--source-external-state", required=True, type=Path) parser.add_argument("--source-loop-state", required=True, type=Path) parser.add_argument("--source-composition", required=True, type=Path) parser.add_argument( "--source-migration-receipt", required=True, type=Path, ) parser.add_argument("--replica-receipt", type=Path) parser.add_argument("--target-state-root", required=True, type=Path) parser.add_argument("--device", default="cpu") parser.add_argument("--expected-checkpoint-sha256") parser.add_argument("--expected-optimizer-sha256") parser.add_argument("--expected-external-state-sha256") def _add_retained_graph_adaptation_arguments( parser: argparse.ArgumentParser, ) -> None: parser.add_argument("--precommit", required=True, type=Path) parser.add_argument("--source-loop-state", required=True, type=Path) parser.add_argument( "--source-execution-ledger", required=True, type=Path, ) parser.add_argument("--source-composition", required=True, type=Path) parser.add_argument( "--source-migration-receipt", required=True, type=Path, ) parser.add_argument("--replica-receipt", type=Path) parser.add_argument("--target-growth-plan", required=True, type=Path) parser.add_argument( "--expected-target-growth-plan-sha256", required=True, ) parser.add_argument("--target-state-root", required=True, type=Path) parser.add_argument("--device", default="cpu") def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser( description=( "Compose or expand authoritative Resynthesis NoNE page checkpoints" ) ) commands = parser.add_subparsers(dest="command", required=True) _add_seed_arguments( commands.add_parser( "seed-v2", help="compose an authoritative v1 checkpoint into v2 pages", ) ) _add_expansion_arguments( commands.add_parser( "expand-v2-plus", help="append roots to the exact accepted v2 checkpoint", ) ) _add_graph_adaptation_arguments( commands.add_parser( "adapt-accepted-graph", help=( "adapt the accepted loop checkpoint into the active graph " "without training" ), ) ) _add_retained_graph_adaptation_arguments( commands.add_parser( "adapt-retained-accepted-graph", help=( "adapt one exact retained accepted frontier to its signed " "target graph without training" ), ) ) raw_arguments = list(sys.argv[1:] if argv is None else argv) if raw_arguments and raw_arguments[0].startswith("-"): raw_arguments.insert(0, "seed-v2") args = parser.parse_args(raw_arguments) if args.command == "seed-v2": artifacts: NoNEV2SeedArtifacts | NoNEV2PlusExpansionArtifacts = ( migrate_v1_checkpoint_to_v2_seed( source_checkpoint=args.source_checkpoint, source_optimizer=args.source_optimizer, growth_plan=args.growth_plan, store_root=args.store_root, receipt_path=args.receipt, expected_checkpoint_sha256=( args.expected_checkpoint_sha256 ), expected_optimizer_sha256=( args.expected_optimizer_sha256 ), ) ) elif args.command == "expand-v2-plus": artifacts = expand_v2_catalog_from_accepted_checkpoint( source_checkpoint=args.source_checkpoint, source_optimizer=args.source_optimizer, source_external_state=args.source_external_state, source_composition=args.source_composition, source_migration_receipt=args.source_migration_receipt, growth_plan=args.growth_plan, receipt_path=args.receipt, replica_receipt_path=args.replica_receipt, expected_checkpoint_sha256=args.expected_checkpoint_sha256, expected_optimizer_sha256=args.expected_optimizer_sha256, ) elif args.command in { "adapt-accepted-graph", "adapt-retained-accepted-graph", }: from resynthesis.rbo import ( ResynthesisRBOConfig, build_resynthesis_rbo, ) training_authority = resolve_v2_training_authority( args.source_composition, args.source_migration_receipt, ) retained_authority = ( _retained_graph_adaptation_authority_boundary( training_authority=training_authority, precommit_path=args.precommit, loop_state_path=args.source_loop_state, execution_ledger_path=args.source_execution_ledger, ) if args.command == "adapt-retained-accepted-graph" else None ) target_growth_plan = ( args.target_growth_plan if args.command == "adapt-retained-accepted-graph" else None ) expected_target_growth_plan_sha256 = ( args.expected_target_growth_plan_sha256 if args.command == "adapt-retained-accepted-graph" else None ) model = build_resynthesis_rbo( rbo_cfg=ResynthesisRBOConfig( rbo_state_path=str( args.target_state_root / ".graph_adaptation_bootstrap.pt" ), paged_none_composition_path=str( args.source_composition ), paged_none_migration_receipt_path=str( args.source_migration_receipt ), paged_none_replica_receipt_path=( str(args.replica_receipt) if args.replica_receipt is not None else None ), paged_none_target_growth_plan_path=( str(target_growth_plan) if target_growth_plan is not None else None ), paged_none_target_growth_plan_sha256=( expected_target_growth_plan_sha256 ), paged_none_adaptation_source_checkpoint_sha256=( training_authority.checkpoint_sha256 if target_growth_plan is not None else None ), ), device=str(args.device), lazy_base=True, paged_training_authority=training_authority, ) adapted = adapt_accepted_checkpoint_to_active_graph( model=model, source_checkpoint=( training_authority.checkpoint_path if retained_authority is not None else args.source_checkpoint ), source_optimizer=( training_authority.optimizer_path if retained_authority is not None else args.source_optimizer ), source_external_state=( training_authority.external_state_path if retained_authority is not None and training_authority.external_state_path is not None else args.source_external_state ), source_loop_state=args.source_loop_state, target_state_root=args.target_state_root, training_authority=training_authority, retained_transaction_authority=retained_authority, target_growth_plan_path=target_growth_plan, expected_target_growth_plan_sha256=( expected_target_growth_plan_sha256 ), expected_checkpoint_sha256=( training_authority.checkpoint_sha256 if retained_authority is not None else args.expected_checkpoint_sha256 ), expected_optimizer_sha256=( training_authority.optimizer_sha256 if retained_authority is not None else args.expected_optimizer_sha256 ), expected_external_state_sha256=( training_authority.external_state_sha256 if retained_authority is not None else args.expected_external_state_sha256 ), ) print( json.dumps( { "receipt": str(adapted.receipt_path), "stateRoot": str(adapted.state_root), "checkpoint": str(adapted.checkpoint_path), "optimizer": str(adapted.optimizer_path), "externalState": str( adapted.external_state_path ), "loopState": str(adapted.loop_state_path), "activeGrowthPlan": str( adapted.active_growth_plan_path ), "iteration": adapted.iteration, "acceptedGeneration": ( adapted.accepted_generation ), }, sort_keys=True, ) ) return 0 else: raise RuntimeError("unknown NoNE migration command") output: dict[str, Any] = { "receipt": str(artifacts.receipt_path), "composition": str(artifacts.composition_path), "pageCatalog": str(artifacts.page_catalog_path), "storeRoot": str(artifacts.store_root), "acceptedGeneration": int(artifacts.accepted_generation_t), } if isinstance(artifacts, NoNEV2PlusExpansionArtifacts): output.update( { "checkpoint": str(artifacts.checkpoint_path), "optimizer": str(artifacts.optimizer_path), "externalState": str(artifacts.external_state_path), "residentRuntime": str( artifacts.resident_runtime_path ), } ) print( json.dumps( output, sort_keys=True, ) ) return 0 if __name__ == "__main__": raise SystemExit(main())