#!/usr/bin/env python3 """Validate the public model-release candidate without the private source file.""" from __future__ import annotations import argparse import hashlib import json import math import re import subprocess from pathlib import Path from typing import Any from urllib.parse import urlsplit import torch from safetensors import safe_open MODEL_ID = "WeiboAI/VibeThinker-3B" MODEL_REVISION = "77bd2cced09193c8b9a59a32bd8577bbd1f3e01c" SOURCE_CHECKPOINT_SHA256 = ( "f36a99447623e0d777c70951a9148a7a52e42e0df82942e22ce6326f63d8d664" ) ARTIFACT_SHA256 = "089d776979408f23e5377539c15aa8025d171633718ccdce709bcd3372e7942c" ARTIFACT_SIZE_BYTES = 150_996_824 EVALUATION_LENS_SHA256 = ( "8f752032a26a5196c1cb447ef63f01e8a29820ff0c57178dd80c6e26d32b12a9" ) EVALUATION_ARTIFACT_SHA256 = ( "0cc184eb65d273ac8bfee5450a141c8cdf8dd6ec8caa68d7d47260ef621777d1" ) EVALUATION_ARTIFACT_SIZE_BYTES = 301_991_904 EVALUATION_FILE_SHA256 = ( "073f2886e370acec7dd1564f2d7e834b4253493e84b026ec0cc017a6069a5f78" ) SOURCE_LAYERS = tuple(range(0, 36, 2)) SELECTED_BAND = (24, 26, 28, 30, 32, 34) D_MODEL = 2048 TENSOR_NBYTES = D_MODEL * D_MODEL * 2 TENSOR_STORAGE_BYTES = len(SOURCE_LAYERS) * TENSOR_NBYTES EVALUATION_TENSOR_NBYTES = D_MODEL * D_MODEL * 4 EVALUATION_TENSOR_STORAGE_BYTES = len(SOURCE_LAYERS) * EVALUATION_TENSOR_NBYTES EXPECTED_KEYS = {f"J.{layer}" for layer in SOURCE_LAYERS} EXPECTED_TENSOR_RECORD_KEYS = { "dtype", "nbytes", "numel", "sha256_c_contiguous_little_endian_bytes", "shape", "source_layer", } EXPECTED_METADATA = { "artifact_kind": "jacobian_lens", "d_model": str(D_MODEL), "format": "pt", "model_id": MODEL_ID, "model_revision": MODEL_REVISION, "n_prompts": "1000", "schema_version": "1", "source_checkpoint_sha256": SOURCE_CHECKPOINT_SHA256, "source_layers": "[0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34]", "target_layer": "35", "tensor_dtype": "float16", "tensor_key_pattern": "J.{source_layer}", } EXPECTED_EVALUATION_METADATA = { "artifact_kind": "jacobian_lens_evaluation_fp32", "d_model": str(D_MODEL), "format": "pt", "model_id": MODEL_ID, "model_revision": MODEL_REVISION, "n_prompts": "1000", "schema_version": "1", "source_fit_checkpoint_sha256": ( "a1236cfe5d04601575b3de150ffe50e3a67e755ede1197ecf74c205a31bdc258" ), "source_fp32_checkpoint_sha256": EVALUATION_LENS_SHA256, "source_layers": "[0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34]", "target_layer": "35", "tensor_dtype": "float32", "tensor_key_pattern": "J.{source_layer}", } EXPECTED_CARD_FRONT_MATTER = """license: other license_name: qwen-research-license license_link: https://huggingface.co/JacobMolBio/vibethinker-3b-jlens-model/blob/main/LICENSES/QWEN-RESEARCH.txt thumbnail: https://huggingface.co/JacobMolBio/vibethinker-3b-jlens-model/resolve/main/assets/jlens-model-banner.png tags: - vibethinker-3b - jacobian-lens - mechanistic-interpretability - interpretability - llm-interpretability - activation-analysis - model-visualization - qwen - qwen2 - safetensors""" EXPECTED_RELEASE_FILES = { ".gitattributes", ".gitignore", "assets/jlens-model-banner.png", "assets/two-lens-files.png", "assets/two-lens-files.svg", "LICENSES/APACHE-2.0.txt", "LICENSES/QWEN-RESEARCH.txt", "LICENSES/VIBETHINKER-LICENSE-NOTE.txt", "NOTICE", "README.md", "SHA256SUMS", "THIRD_PARTY_NOTICES.md", "evaluation.json", "evaluation.safetensors", "evaluation_compatibility.json", "evaluation_provenance.json", "evaluation_tensor_manifest.json", "evaluation_validation.json", "lens_config.json", "model.safetensors", "provenance.json", "requirements.txt", "scripts/convert_checkpoint.py", "scripts/convert_evaluation_checkpoint.py", "scripts/validate_artifact.py", "tensor_manifest.json", "validation.json", } PINNED_PUBLIC_FILE_SHA256 = { ".gitattributes": "cd0273298656ca90cb8d08fd25b7601483f7e13417e3115ab809856c42e794f7", ".gitignore": "4c7486a4b7c5ad04e0e62225b9a55c9ae00b039168d34614d402ac1f73acc459", "assets/jlens-model-banner.png": "f27c4abb0c84481c2b0e67ed0c390716905bd1fe6fbcb09f2a65bbe15ecdc935", "LICENSES/APACHE-2.0.txt": "ec01a6d25ea6a6b50430eda7af9e23e1c510048502d0a3adb5b54559e687d4fa", "LICENSES/QWEN-RESEARCH.txt": "ef52482bb785733093dc9a2e8edd8e764c77d12d8e9d8f10a80c9b547d32d0f9", "LICENSES/VIBETHINKER-LICENSE-NOTE.txt": "a74ea19436fbab1210ff16aa9037337d95ca9f93675406da4383bd8de4f4d25b", "NOTICE": "35d9d57e97593a99aff4434875b43943d7627258495466bd5f8f81cdd8d40289", "THIRD_PARTY_NOTICES.md": "d68e3b08646b56bf8d8c80d1d1acd98edfb40e72cb287762a3150d9b9d75a18f", } PUBLICATION_ROWS = { "CODE_REPOSITORY_URL": "Source code and Pages source", "TRACE_REPOSITORY_URL": "Captured trace dataset", "PUBLIC_SITE_URL": "Static Pages site", "MODEL_REPOSITORY_ID": "Hugging Face model repository ID", } PUBLICATION_PLACEHOLDERS = {key: "{{" + key + "}}" for key in PUBLICATION_ROWS} CODE_REPOSITORY_NAME = "vibethinker-3b-jlens" TRACE_REPOSITORY_NAME = "vibethinker-3b-jlens-traces" MODEL_REPOSITORY_NAME = "vibethinker-3b-jlens-model" REPOSITORY_COMPONENT = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*") SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") PUBLICATION_SENTINELS = { "", "example", "local", "none", "null", "org", "organization", "placeholder", "repo", "repository", "tbd", "todo", "user", "username", } FORBIDDEN_PUBLIC_PATTERNS = { "email address": re.compile( r"\b[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+" + chr(64) + r"[A-Za-z0-9](?:[A-Za-z0-9.-]{0,61}[A-Za-z0-9])?" + r"\.[A-Za-z]{2,}\b" ), "AWS access key": re.compile(r"\b" + "AK" + r"IA[0-9A-Z]{16}\b"), "AWS temporary access key": re.compile(r"\b" + "AS" + r"IA[0-9A-Z]{16}\b"), "GitHub access token": re.compile(r"\b" + "gh" + r"[pousr]_[A-Za-z0-9]{20,}\b"), "GitHub fine-grained token": re.compile( r"\b" + "github" + r"_pat_[A-Za-z0-9_]{20,}\b" ), "GitLab access token": re.compile(r"\b" + "gl" + r"pat-[A-Za-z0-9_-]{20,}\b"), "Hugging Face access token": re.compile(r"\b" + "hf" + r"_[A-Za-z0-9]{20,}\b"), "OpenAI-style access token": re.compile(r"\b" + "sk" + r"-[A-Za-z0-9_-]{20,}\b"), "Google API key": re.compile(r"\b" + "AI" + r"za[0-9A-Za-z_-]{30,}\b"), "Slack access token": re.compile(r"\b" + "xo" + r"[abprs]-[A-Za-z0-9-]{20,}\b"), "private key": re.compile( "-----BEGIN " + r"(?:DSA |EC |OPENSSH |RSA )?PRIVATE KEY-----" ), "bearer credential": re.compile( r"\b" + "Bearer" + r"\s+[A-Za-z0-9._~+/=-]{20,}", re.IGNORECASE ), "local file URI": re.compile(r"\b" + "file:" + r"//", re.IGNORECASE), "macOS user path": re.compile(r"(? None: if not condition: raise ValueError(message) def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def tensor_sha256(tensor: torch.Tensor) -> str: contiguous = tensor.detach().cpu().contiguous() if tensor.dtype == torch.float16: array = contiguous.view(torch.int16).numpy() payload = array.astype(" dict[str, Any]: value = json.loads(path.read_text(encoding="utf-8")) require(isinstance(value, dict), f"{path.name} must contain a JSON object") return value def normalized_words(value: str) -> str: return " ".join(value.split()) def extract_card_front_matter(readme: str) -> str: require(readme.startswith("---\n"), "README model-card front matter is missing") closing = readme.find("\n---\n", 4) require(closing != -1, "README model-card front matter is not closed") return readme[4:closing] def validate_release_file_set(root: Path) -> None: actual_files: set[str] = set() for path in root.rglob("*"): relative = path.relative_to(root) if relative.parts and relative.parts[0] == ".git": continue require(not path.is_symlink(), f"release tree contains a symlink: {relative}") if path.is_file(): actual_files.add(relative.as_posix()) require( actual_files == EXPECTED_RELEASE_FILES, "release file set mismatch: " f"missing={sorted(EXPECTED_RELEASE_FILES - actual_files)}, " f"extra={sorted(actual_files - EXPECTED_RELEASE_FILES)}", ) def validate_pinned_public_files(root: Path) -> None: for relative_path, expected_sha256 in PINNED_PUBLIC_FILE_SHA256.items(): require( sha256_file(root / relative_path) == expected_sha256, f"pinned public file hash mismatch: {relative_path}", ) def validate_public_text(root: Path) -> None: binary_files = { "assets/jlens-model-banner.png", "assets/two-lens-files.png", "evaluation.safetensors", "model.safetensors", } for relative_path in sorted(EXPECTED_RELEASE_FILES - binary_files): text = (root / relative_path).read_text(encoding="utf-8") for label, pattern in FORBIDDEN_PUBLIC_PATTERNS.items(): require( pattern.search(text) is None, f"{relative_path} contains a forbidden {label}", ) for label, literal in FORBIDDEN_PUBLIC_LITERALS.items(): require( literal not in text, f"{relative_path} contains a forbidden {label}", ) def parse_publication_rows(readme: str) -> tuple[dict[str, str], list[str]]: values: dict[str, str] = {} for key, label in PUBLICATION_ROWS.items(): pattern = re.compile( rf"^\| {re.escape(label)} \| (?:`([^`\r\n]+)`|\[([^\]\r\n]+)\]\([^\)\r\n]+\)) \|$", re.MULTILINE, ) matches = pattern.findall(readme) require(len(matches) == 1, f"README publication row mismatch: {key}") values[key] = matches[0][0] or matches[0][1] placeholders_in_readme = set(re.findall(r"\{\{([A-Z0-9_]+)\}\}", readme)) unresolved = sorted( key for key, value in values.items() if value == PUBLICATION_PLACEHOLDERS[key] ) require( placeholders_in_readme == set(unresolved), "README publication placeholders do not match the publication table", ) require( len(unresolved) in {0, len(PUBLICATION_ROWS)}, "publication fields must be fully unresolved or fully resolved", ) require( f'export JLENS_CODE_REPO_URL="{values["CODE_REPOSITORY_URL"]}"' in readme, "source URL example differs from the publication table", ) require( f'export JLENS_MODEL_REPO_ID="{values["MODEL_REPOSITORY_ID"]}"' in readme, "model repository example differs from the publication table", ) return values, unresolved def require_public_component(value: str, label: str) -> None: require(value == value.strip(), f"{label} has surrounding whitespace") require(value.casefold() not in PUBLICATION_SENTINELS, f"{label} is a placeholder") require("{{" not in value and "}}" not in value, f"{label} is unresolved") require( not any(character.isspace() for character in value), f"{label} contains whitespace", ) require("\\" not in value, f"{label} contains a local path separator") def parse_https_url(value: str, label: str) -> Any: require_public_component(value, label) parsed = urlsplit(value) require(parsed.scheme == "https", f"{label} must use HTTPS") require(parsed.hostname is not None, f"{label} has no hostname") require( parsed.username is None and parsed.password is None, f"{label} contains credentials", ) try: port = parsed.port except ValueError as error: raise ValueError(f"{label} has an invalid port") from error require(port is None, f"{label} must not specify a port") require(not parsed.query, f"{label} must not contain a query") require(not parsed.fragment, f"{label} must not contain a fragment") require("%" not in parsed.path, f"{label} must not contain encoded path fragments") require("//" not in parsed.path, f"{label} contains an empty path fragment") require(";" not in parsed.path, f"{label} contains a parameter fragment") return parsed def validate_publication_values(values: dict[str, str]) -> None: code = parse_https_url(values["CODE_REPOSITORY_URL"], "source repository URL") trace = parse_https_url(values["TRACE_REPOSITORY_URL"], "trace repository URL") site = parse_https_url(values["PUBLIC_SITE_URL"], "Pages site URL") model_id = values["MODEL_REPOSITORY_ID"] require_public_component(model_id, "model repository ID") code_parts = [part for part in code.path.split("/") if part] require( code.hostname.casefold() == "github.com", "source repository must use github.com", ) require( len(code_parts) == 2, "source repository URL must contain namespace/repository" ) require( code.path == "/" + "/".join(code_parts), "source repository URL is not canonical", ) require( all(REPOSITORY_COMPONENT.fullmatch(part) for part in code_parts), "source repository URL contains an invalid path fragment", ) require( code_parts[0].casefold() not in PUBLICATION_SENTINELS, "source namespace is a placeholder", ) require(code_parts[1] == CODE_REPOSITORY_NAME, "source repository name mismatch") trace_parts = [part for part in trace.path.split("/") if part] require( trace.hostname.casefold() == "huggingface.co", "trace repository must use huggingface.co", ) require( len(trace_parts) == 3 and trace_parts[0] == "datasets", "trace repository URL must use /datasets/namespace/repository", ) require( trace.path == "/" + "/".join(trace_parts), "trace repository URL is not canonical", ) require( all(REPOSITORY_COMPONENT.fullmatch(part) for part in trace_parts[1:]), "trace repository URL contains an invalid path fragment", ) require( trace_parts[1].casefold() not in PUBLICATION_SENTINELS, "trace namespace is a placeholder", ) require(trace_parts[2] == TRACE_REPOSITORY_NAME, "trace repository name mismatch") model_parts = model_id.split("/") require(len(model_parts) == 2, "model repository ID must use namespace/repository") require( all(REPOSITORY_COMPONENT.fullmatch(part) for part in model_parts), "model repository ID contains an invalid fragment", ) require( model_parts[0].casefold() not in PUBLICATION_SENTINELS, "model namespace is a placeholder", ) require(model_parts[1] == MODEL_REPOSITORY_NAME, "model repository name mismatch") require( model_parts[0].casefold() == trace_parts[1].casefold(), "model and trace repositories must use the same Hugging Face namespace", ) require( site.hostname.casefold() == f"{code_parts[0].casefold()}.github.io", "Pages hostname must match the source repository namespace", ) require( site.path.rstrip("/") == f"/{CODE_REPOSITORY_NAME}", "Pages path must match the source repository name", ) require( site.path in {f"/{CODE_REPOSITORY_NAME}", f"/{CODE_REPOSITORY_NAME}/"}, "Pages URL is not canonical", ) def validate_model_card(readme: str) -> None: require( extract_card_front_matter(readme) == EXPECTED_CARD_FRONT_MATTER, "README model-card front matter mismatch", ) readme_words = normalized_words(readme) for required_text in ( MODEL_ID, MODEL_REVISION, ARTIFACT_SHA256, EVALUATION_LENS_SHA256, EVALUATION_ARTIFACT_SHA256, "LICENSES/VIBETHINKER-LICENSE-NOTE.txt", "The static site cannot analyze a new prompt.", "50,050 readout rows", ): require( required_text in readme_words, f"README missing required binding: {required_text}", ) def validate_licenses_and_notices(root: Path) -> None: notice = (root / "NOTICE").read_text(encoding="utf-8") third_party = (root / "THIRD_PARTY_NOTICES.md").read_text(encoding="utf-8") note = (root / "LICENSES/VIBETHINKER-LICENSE-NOTE.txt").read_text(encoding="utf-8") qwen_license = (root / "LICENSES/QWEN-RESEARCH.txt").read_text(encoding="utf-8") required_qwen_notice = ( "Qwen is licensed under the Qwen RESEARCH LICENSE AGREEMENT, " "Copyright (c) Alibaba Cloud. All Rights Reserved." ) require( required_qwen_notice in normalized_words(notice), "NOTICE is missing the required Qwen attribution", ) require( required_qwen_notice in normalized_words(qwen_license), "Qwen license is missing its attribution clause", ) require( "Built with Qwen." in notice, "NOTICE is missing the Qwen product attribution" ) require( "LICENSES/VIBETHINKER-LICENSE-NOTE.txt" in notice and "LICENSES/VIBETHINKER-LICENSE-NOTE.txt" in third_party, "VibeThinker metadata note is not linked from the notices", ) require( "license: mit" in normalized_words(note), "VibeThinker metadata declaration missing", ) require(MODEL_REVISION in note, "VibeThinker metadata note revision mismatch") require( "It is not an upstream license text" in normalized_words(note), "VibeThinker metadata note scope missing", ) require( not (root / "LICENSES" / ("VIBETHINKER-" + "MIT.txt")).exists(), "removed VibeThinker license file is present", ) def validate_metadata_records( config: dict[str, Any], provenance: dict[str, Any], frozen_validation: dict[str, Any], evaluation: dict[str, Any], evaluation_path: Path, ) -> None: require(config.get("schema_version") == 1, "config schema mismatch") require(config.get("artifact_kind") == "jacobian_lens", "config kind mismatch") require(config.get("d_model") == D_MODEL, "config width mismatch") require(config.get("n_prompts") == 1000, "config prompt count mismatch") require( config.get("source_layers") == list(SOURCE_LAYERS), "config source layers mismatch", ) require(config.get("target_layer") == 35, "config target layer mismatch") require(config.get("tensor_dtype") == "float16", "config tensor dtype mismatch") require( config.get("tensor_key_pattern") == "J.{source_layer}", "config key pattern mismatch", ) require( config.get("model") == { "architecture": "Qwen2ForCausalLM", "id": MODEL_ID, "n_layers": 36, "revision": MODEL_REVISION, }, "config model binding mismatch", ) require( config.get("source_checkpoint", {}).get("sha256") == SOURCE_CHECKPOINT_SHA256, "config source checkpoint mismatch", ) require( config.get("artifact") == { "filename": "model.safetensors", "format": "safetensors", "sha256": ARTIFACT_SHA256, "size_bytes": ARTIFACT_SIZE_BYTES, }, "config artifact binding mismatch", ) require( config.get("evaluation_artifact") == { "filename": "evaluation.safetensors", "format": "safetensors", "sha256": EVALUATION_ARTIFACT_SHA256, "size_bytes": EVALUATION_ARTIFACT_SIZE_BYTES, "source_checkpoint_sha256": EVALUATION_LENS_SHA256, "tensor_conversion": "lossless_fp32_reserialization", "tensor_dtype": "float32", }, "config evaluation artifact binding mismatch", ) require(provenance.get("schema_version") == 1, "provenance schema mismatch") require( provenance.get("model", {}).get("id") == MODEL_ID, "provenance model ID mismatch", ) require( provenance.get("model", {}).get("revision") == MODEL_REVISION, "provenance model revision mismatch", ) require( provenance.get("artifact", {}).get("sha256") == ARTIFACT_SHA256, "provenance artifact hash mismatch", ) require( provenance.get("artifact", {}).get("size_bytes") == ARTIFACT_SIZE_BYTES, "provenance artifact size mismatch", ) require( provenance.get("artifact", {}).get("tensor_conversion") == "lossless_fp16_reserialization", "provenance conversion boundary mismatch", ) require( provenance.get("source_checkpoint", {}).get("sha256") == SOURCE_CHECKPOINT_SHA256, "provenance source checkpoint mismatch", ) require(frozen_validation.get("schema_version") == 1, "validation schema mismatch") require( frozen_validation.get("artifact") == "model.safetensors", "validation filename mismatch", ) require( frozen_validation.get("artifact_sha256") == ARTIFACT_SHA256, "validation artifact hash mismatch", ) require( frozen_validation.get("artifact_size_bytes") == ARTIFACT_SIZE_BYTES, "validation artifact size mismatch", ) require(frozen_validation.get("ok") is True, "frozen validation is not successful") require( frozen_validation.get("tensor_values_changed") == 0, "frozen validation records changed values", ) require( frozen_validation.get("exact_tensor_matches") == len(SOURCE_LAYERS), "frozen validation tensor count mismatch", ) require( frozen_validation.get("expected_tensor_matches") == len(SOURCE_LAYERS), "frozen validation expected tensor count mismatch", ) require( frozen_validation.get("source_checkpoint_sha256") == SOURCE_CHECKPOINT_SHA256, "frozen validation source checkpoint mismatch", ) checks = frozen_validation.get("checks") require( isinstance(checks, dict) and checks and all(value is True for value in checks.values()), "frozen validation contains a failed or malformed check", ) require( sha256_file(evaluation_path) == EVALUATION_FILE_SHA256, "evaluation file hash mismatch", ) require( evaluation.get("artifact_kind") == "frozen_vibethinker_v1_readout_reference", "evaluation artifact kind mismatch", ) require( evaluation.get("classification") == "validated_readout_only", "evaluation classification mismatch", ) require(evaluation.get("model") == MODEL_ID, "evaluation model ID mismatch") require( evaluation.get("model_revision") == MODEL_REVISION, "evaluation model revision mismatch", ) require( evaluation.get("lens_sha256") == EVALUATION_LENS_SHA256, "evaluation lens hash mismatch", ) require( evaluation.get("lens_variant") == "fp32_evaluation_safetensors_included", "evaluation lens variant mismatch", ) require( evaluation.get("released_lens_artifact") == { "filename": "evaluation.safetensors", "format": "safetensors", "sha256": EVALUATION_ARTIFACT_SHA256, "size_bytes": EVALUATION_ARTIFACT_SIZE_BYTES, "source_checkpoint_sha256": EVALUATION_LENS_SHA256, "tensor_conversion": "lossless_fp32_reserialization", }, "evaluation released artifact binding mismatch", ) require( evaluation.get("claim_boundary") == EXPECTED_EVALUATION_CLAIM_BOUNDARY, "evaluation claim boundary mismatch", ) require( evaluation.get("note") == EXPECTED_EVALUATION_NOTE, "evaluation FP16/FP32 note mismatch", ) require( evaluation.get("selected_band") == list(SELECTED_BAND), "evaluation selected band mismatch", ) require( evaluation.get("coverage_scope") == "frozen_readout_source_run_not_the_100_prompt_ui_test_pack", "evaluation coverage scope mismatch", ) require( evaluation.get("task_definition") == EXPECTED_EVALUATION_TASK, "evaluation task definition mismatch", ) evidence = evaluation.get("evidence", {}) require( evidence.get("validated_readout_signal") is True, "evaluation readout signal boundary mismatch", ) require( evidence.get("token_specific_vs_shuffled_target") is True, "evaluation token control boundary mismatch", ) require( evidence.get("layer_mapping_specific_vs_shuffled_jacobian") is True, "evaluation layer control boundary mismatch", ) require( evidence.get("incremental_over_ordinary_logit_lens") is False, "evaluation logit-lens boundary mismatch", ) require( evidence.get("paired_bootstrap") == { "confidence": 0.95, "input": "paired_item_metric_differences", "interval": "percentile", "metrics": ["pass@10", "mean_reciprocal_rank"], "samples": 2000, "scope": { "kind": "selected_band", "source_layers": list(SELECTED_BAND), }, "split": "test", }, "evaluation paired-bootstrap method mismatch", ) require( evidence.get("paired_bootstrap_decisions") == { "incremental_over_ordinary_logit_lens": { "comparison": "jlens_band_minus_logit_lens_band", "criterion": "at_least_one_lower_bound_greater_than_zero", "met": False, }, "layer_mapping_specific_vs_shuffled_jacobian": { "comparison": "jlens_band_minus_shuffled_layer_band", "criterion": "both_lower_bounds_greater_than_zero", "met": True, }, "token_specific_vs_shuffled_target": { "comparison": "jlens_band_minus_shuffled_token_band", "criterion": "both_lower_bounds_greater_than_zero", "met": True, }, }, "evaluation paired-bootstrap decisions mismatch", ) require( evidence.get("specificity_scope") == {"kind": "selected_band", "source_layers": list(SELECTED_BAND)}, "evaluation specificity scope mismatch", ) availability = evaluation.get("source_artifact_availability", {}) require( availability == { "fp32_compatibility_check_output_included": True, "fp32_compatibility_check_output_path": ("evaluation_compatibility.json"), "fp32_derivation_record_included": True, "fp32_derivation_record_path": "evaluation_provenance.json", "fp32_evaluation_lens_included": True, "fp32_evaluation_lens_path": "evaluation.safetensors", "item_level_evaluation_rows_included": False, "item_level_evaluation_rows_location": ( "companion_trace_repository:data/evaluation-results/" "readout-trials.jsonl" ), "paired_bootstrap_interval_bounds_included": False, "paired_bootstrap_interval_bounds_location": ( "companion_trace_repository:data/evaluation-results/" "readout-bootstrap-intervals.json" ), "source_evaluation_bundle_included": False, "source_hashes_included": True, "source_hashes_location": ( "evaluation_provenance.json_and_companion_trace_repository:" "data/readout-reference.json" ), "release_content": ( "fp16_trace_lens_fp32_evaluation_lens_aggregate_reference_" "provenance_and_compatibility" ), }, "evaluation source-artifact boundary mismatch", ) require( EVALUATION_LENS_SHA256 != ARTIFACT_SHA256, "FP32 and FP16 artifact hashes were conflated", ) def validate_tensor_manifest(manifest: dict[str, Any]) -> None: require( set(manifest) == { "artifact", "artifact_sha256", "artifact_size_bytes", "schema_version", "tensor_count", "tensor_storage_bytes", "tensors", }, "tensor manifest top-level fields mismatch", ) require( manifest.get("artifact") == "model.safetensors", "tensor manifest filename mismatch", ) require( manifest.get("artifact_sha256") == ARTIFACT_SHA256, "tensor manifest artifact hash mismatch", ) require( manifest.get("artifact_size_bytes") == ARTIFACT_SIZE_BYTES, "tensor manifest artifact size mismatch", ) require(manifest.get("schema_version") == 1, "tensor manifest schema mismatch") require( manifest.get("tensor_count") == len(SOURCE_LAYERS), "tensor manifest count mismatch", ) require( manifest.get("tensor_storage_bytes") == TENSOR_STORAGE_BYTES, "tensor manifest storage total mismatch", ) tensors = manifest.get("tensors") require(isinstance(tensors, dict), "tensor manifest tensors must be an object") require(set(tensors) == EXPECTED_KEYS, "tensor manifest key set mismatch") recorded_storage = 0 for layer in SOURCE_LAYERS: key = f"J.{layer}" record = tensors[key] require(isinstance(record, dict), f"{key} descriptor must be an object") require( set(record) == EXPECTED_TENSOR_RECORD_KEYS, f"{key} descriptor fields mismatch", ) require(record.get("source_layer") == layer, f"{key} source layer mismatch") require(record.get("dtype") == "float16", f"{key} descriptor dtype mismatch") require( record.get("shape") == [D_MODEL, D_MODEL], f"{key} descriptor shape mismatch", ) require( record.get("numel") == D_MODEL * D_MODEL, f"{key} descriptor element count mismatch", ) require( record.get("nbytes") == TENSOR_NBYTES, f"{key} descriptor byte count mismatch", ) tensor_hash = record.get("sha256_c_contiguous_little_endian_bytes") require( isinstance(tensor_hash, str) and SHA256_PATTERN.fullmatch(tensor_hash), f"{key} descriptor hash format mismatch", ) recorded_storage += record["nbytes"] require( recorded_storage == TENSOR_STORAGE_BYTES, "tensor descriptor byte total mismatch", ) def validate_evaluation_artifact( root: Path, manifest: dict[str, Any], provenance: dict[str, Any], frozen_validation: dict[str, Any], compatibility: dict[str, Any], ) -> int: artifact_path = root / "evaluation.safetensors" require( set(manifest) == { "artifact", "artifact_sha256", "artifact_size_bytes", "schema_version", "source_checkpoint_sha256", "tensor_count", "tensor_storage_bytes", "tensors", }, "evaluation tensor manifest top-level fields mismatch", ) require( manifest.get("artifact") == "evaluation.safetensors", "evaluation tensor manifest filename mismatch", ) require( manifest.get("artifact_sha256") == EVALUATION_ARTIFACT_SHA256, "evaluation tensor manifest artifact hash mismatch", ) require( manifest.get("artifact_size_bytes") == EVALUATION_ARTIFACT_SIZE_BYTES, "evaluation tensor manifest artifact size mismatch", ) require( manifest.get("source_checkpoint_sha256") == EVALUATION_LENS_SHA256, "evaluation tensor manifest source hash mismatch", ) require( manifest.get("tensor_count") == len(SOURCE_LAYERS), "evaluation tensor manifest count mismatch", ) require( manifest.get("tensor_storage_bytes") == EVALUATION_TENSOR_STORAGE_BYTES, "evaluation tensor manifest storage total mismatch", ) tensors = manifest.get("tensors") require(isinstance(tensors, dict), "evaluation manifest tensors must be an object") require(set(tensors) == EXPECTED_KEYS, "evaluation manifest key set mismatch") for layer in SOURCE_LAYERS: key = f"J.{layer}" record = tensors[key] require( set(record) == EXPECTED_TENSOR_RECORD_KEYS, f"evaluation {key} descriptor fields mismatch", ) require(record.get("source_layer") == layer, f"evaluation {key} layer mismatch") require(record.get("dtype") == "float32", f"evaluation {key} dtype mismatch") require( record.get("shape") == [D_MODEL, D_MODEL], f"evaluation {key} shape mismatch", ) require( record.get("numel") == D_MODEL * D_MODEL, f"evaluation {key} element count mismatch", ) require( record.get("nbytes") == EVALUATION_TENSOR_NBYTES, f"evaluation {key} byte count mismatch", ) require( isinstance(record.get("sha256_c_contiguous_little_endian_bytes"), str) and SHA256_PATTERN.fullmatch( record["sha256_c_contiguous_little_endian_bytes"] ), f"evaluation {key} tensor hash format mismatch", ) require( provenance.get("artifact_kind") == "jacobian_lens_evaluation_fp32_provenance", "evaluation provenance kind mismatch", ) require( provenance.get("artifact") == { "filename": "evaluation.safetensors", "format": "safetensors", "sha256": EVALUATION_ARTIFACT_SHA256, "size_bytes": EVALUATION_ARTIFACT_SIZE_BYTES, "tensor_conversion": "lossless_fp32_reserialization", }, "evaluation provenance artifact binding mismatch", ) require( provenance.get("source_checkpoint") == {"format": "pytorch", "sha256": EVALUATION_LENS_SHA256}, "evaluation provenance source binding mismatch", ) derivation = provenance.get("derivation", {}) require( derivation.get("formula") == "jacobian_sum[layer] / n_done", "evaluation derivation formula mismatch", ) require(derivation.get("n_done") == 1000, "evaluation derivation count mismatch") require( derivation.get("source_fit_checkpoint_sha256") == EXPECTED_EVALUATION_METADATA["source_fit_checkpoint_sha256"], "evaluation fit checkpoint binding mismatch", ) require( provenance.get("model", {}).get("id") == MODEL_ID and provenance.get("model", {}).get("revision") == MODEL_REVISION, "evaluation provenance model binding mismatch", ) require( provenance.get("lens", {}).get("dtype") == "float32" and provenance.get("lens", {}).get("source_layers") == list(SOURCE_LAYERS) and provenance.get("lens", {}).get("target_layer") == 35 and provenance.get("lens", {}).get("n_prompts") == 1000, "evaluation provenance lens metadata mismatch", ) runtime = provenance.get("software", {}).get("conversion_runtime", {}) require( runtime.get("safetensors") == "0.8.0" and runtime.get("torch") == "2.13.0", "evaluation conversion runtime mismatch", ) require( frozen_validation.get("artifact") == "evaluation.safetensors", "evaluation validation filename mismatch", ) require( frozen_validation.get("artifact_sha256") == EVALUATION_ARTIFACT_SHA256, "evaluation validation artifact hash mismatch", ) require( frozen_validation.get("artifact_size_bytes") == EVALUATION_ARTIFACT_SIZE_BYTES, "evaluation validation artifact size mismatch", ) require( frozen_validation.get("source_checkpoint_sha256") == EVALUATION_LENS_SHA256, "evaluation validation source hash mismatch", ) require(frozen_validation.get("ok") is True, "evaluation validation failed") require( frozen_validation.get("tensor_values_changed") == 0, "evaluation validation records changed values", ) require( frozen_validation.get("exact_tensor_matches") == len(SOURCE_LAYERS), "evaluation validation tensor count mismatch", ) checks = frozen_validation.get("checks") require( isinstance(checks, dict) and checks and all(value is True for value in checks.values()), "evaluation validation contains a failed check", ) require( compatibility.get("artifact_kind") == "fp32_to_fp16_lens_compatibility", "evaluation compatibility kind mismatch", ) require( compatibility.get("fp32_artifact") == "evaluation.safetensors" and compatibility.get("fp32_artifact_sha256") == EVALUATION_ARTIFACT_SHA256 and compatibility.get("fp32_source_checkpoint_sha256") == EVALUATION_LENS_SHA256, "evaluation compatibility FP32 binding mismatch", ) require( compatibility.get("fp16_artifact") == "model.safetensors" and compatibility.get("fp16_artifact_sha256") == ARTIFACT_SHA256, "evaluation compatibility FP16 binding mismatch", ) require( compatibility.get("all_layer_casts_match_exactly") is True, "evaluation compatibility records a cast mismatch", ) require( compatibility.get("max_absolute_error") == 0.00048828125, "evaluation compatibility maximum error mismatch", ) require( math.isclose( compatibility.get("relative_frobenius_error", math.inf), 0.00020901276774552773, rel_tol=1e-12, abs_tol=0.0, ), "evaluation compatibility relative error mismatch", ) per_layer = compatibility.get("per_layer") require( isinstance(per_layer, dict) and set(per_layer) == {str(layer) for layer in SOURCE_LAYERS}, "evaluation compatibility layer set mismatch", ) require( all( record.get("cast_matches_model_safetensors_exactly") is True for record in per_layer.values() ), "evaluation compatibility layer cast mismatch", ) require( sha256_file(artifact_path) == EVALUATION_ARTIFACT_SHA256, "evaluation artifact SHA-256 mismatch", ) require( artifact_path.stat().st_size == EVALUATION_ARTIFACT_SIZE_BYTES, "evaluation artifact size mismatch", ) checked = 0 with ( safe_open(artifact_path, framework="pt", device="cpu") as artifact, safe_open( root / "model.safetensors", framework="pt", device="cpu" ) as fp16_artifact, ): require( (artifact.metadata() or {}) == EXPECTED_EVALUATION_METADATA, "evaluation Safetensors metadata mismatch", ) require(set(artifact.keys()) == EXPECTED_KEYS, "evaluation tensor key mismatch") for layer in SOURCE_LAYERS: key = f"J.{layer}" tensor = artifact.get_tensor(key) record = tensors[key] require(tensor.dtype == torch.float32, f"evaluation {key} dtype mismatch") require( tuple(tensor.shape) == (D_MODEL, D_MODEL), f"evaluation {key} shape mismatch", ) require(tensor.is_contiguous(), f"evaluation {key} is not contiguous") require( bool(torch.isfinite(tensor).all()), f"evaluation {key} contains non-finite values", ) require( tensor_sha256(tensor) == record["sha256_c_contiguous_little_endian_bytes"], f"evaluation {key} tensor hash mismatch", ) require( torch.equal( tensor.to(torch.float16).view(torch.int16), fp16_artifact.get_tensor(key).view(torch.int16), ), f"evaluation {key} FP16 cast mismatch", ) checked += 1 return checked def run_git(root: Path, *arguments: str) -> subprocess.CompletedProcess[bytes]: return subprocess.run( ["git", "-C", str(root), *arguments], check=False, capture_output=True, ) def validate_git_lfs( root: Path, *, require_evaluation_head: bool = False ) -> tuple[bool, bool]: repository_check = run_git(root, "rev-parse", "--is-inside-work-tree") if repository_check.returncode != 0: return False, False model_pointer = ( "version https://git-lfs.github.com/spec/v1\n" f"oid sha256:{ARTIFACT_SHA256}\n" f"size {ARTIFACT_SIZE_BYTES}\n" ).encode("ascii") for object_name in ("HEAD:model.safetensors", ":model.safetensors"): pointer = run_git(root, "show", object_name) require(pointer.returncode == 0, f"cannot read Git LFS pointer: {object_name}") require( pointer.stdout == model_pointer, f"Git LFS pointer mismatch: {object_name}" ) evaluation_pointer = ( "version https://git-lfs.github.com/spec/v1\n" f"oid sha256:{EVALUATION_ARTIFACT_SHA256}\n" f"size {EVALUATION_ARTIFACT_SIZE_BYTES}\n" ).encode("ascii") staged_evaluation = run_git(root, "show", ":evaluation.safetensors") require( staged_evaluation.returncode == 0, "cannot read staged Git LFS pointer: evaluation.safetensors", ) require( staged_evaluation.stdout == evaluation_pointer, "staged Git LFS pointer mismatch: evaluation.safetensors", ) head_evaluation = run_git(root, "show", "HEAD:evaluation.safetensors") if require_evaluation_head: require( head_evaluation.returncode == 0, "publication check requires evaluation.safetensors in HEAD", ) if head_evaluation.returncode == 0: require( head_evaluation.stdout == evaluation_pointer, "Git LFS pointer mismatch: HEAD:evaluation.safetensors", ) fsck = run_git(root, "lfs", "fsck", "HEAD") require(fsck.returncode == 0, "git lfs fsck failed") return True, True def validate(root: Path, *, require_publication_fields: bool = False) -> dict[str, Any]: artifact_path = root / "model.safetensors" readme = (root / "README.md").read_text(encoding="utf-8") config = read_json(root / "lens_config.json") manifest = read_json(root / "tensor_manifest.json") provenance = read_json(root / "provenance.json") frozen_validation = read_json(root / "validation.json") evaluation_path = root / "evaluation.json" evaluation = read_json(evaluation_path) evaluation_manifest = read_json(root / "evaluation_tensor_manifest.json") evaluation_provenance = read_json(root / "evaluation_provenance.json") evaluation_validation = read_json(root / "evaluation_validation.json") evaluation_compatibility = read_json(root / "evaluation_compatibility.json") validate_release_file_set(root) validate_pinned_public_files(root) validate_public_text(root) require( (root / "requirements.txt").read_text(encoding="utf-8") == "safetensors==0.8.0\ntorch==2.13.0\n", "requirements do not match the evaluation conversion runtime", ) validate_model_card(readme) validate_licenses_and_notices(root) validate_tensor_manifest(manifest) evaluation_tensors_checked = validate_evaluation_artifact( root, evaluation_manifest, evaluation_provenance, evaluation_validation, evaluation_compatibility, ) validate_metadata_records( config, provenance, frozen_validation, evaluation, evaluation_path, ) publication_values, unresolved_publication_fields = parse_publication_rows(readme) if require_publication_fields: require( not unresolved_publication_fields, "publication fields remain unresolved: " + ", ".join(unresolved_publication_fields), ) if not unresolved_publication_fields: validate_publication_values(publication_values) expected_checksum_file = ( f"{ARTIFACT_SHA256} model.safetensors\n" f"{EVALUATION_ARTIFACT_SHA256} evaluation.safetensors\n" ) require( (root / "SHA256SUMS").read_text(encoding="ascii") == expected_checksum_file, "SHA256SUMS content mismatch", ) artifact_checksum = sha256_file(artifact_path) require(artifact_checksum == ARTIFACT_SHA256, "artifact SHA-256 mismatch") require( artifact_path.stat().st_size == ARTIFACT_SIZE_BYTES, "artifact size mismatch" ) checked = 0 with safe_open(artifact_path, framework="pt", device="cpu") as artifact: require( (artifact.metadata() or {}) == EXPECTED_METADATA, "safetensors metadata mismatch", ) require( set(artifact.keys()) == EXPECTED_KEYS, "safetensors tensor key mismatch" ) for layer in SOURCE_LAYERS: key = f"J.{layer}" tensor = artifact.get_tensor(key) record = manifest["tensors"][key] require(tensor.dtype == torch.float16, f"{key} dtype mismatch") require(tuple(tensor.shape) == (D_MODEL, D_MODEL), f"{key} shape mismatch") require(tensor.is_contiguous(), f"{key} is not contiguous") require( bool(torch.isfinite(tensor).all()), f"{key} contains non-finite values" ) require(tensor.numel() == record["numel"], f"{key} element count mismatch") require( tensor.numel() * tensor.element_size() == record["nbytes"], f"{key} byte count mismatch", ) require( tensor_sha256(tensor) == record["sha256_c_contiguous_little_endian_bytes"], f"{key} tensor hash mismatch", ) checked += 1 lfs_pointer_exact, lfs_fsck_ok = validate_git_lfs( root, require_evaluation_head=require_publication_fields, ) if require_publication_fields: require( lfs_pointer_exact and lfs_fsck_ok, "publication check requires Git LFS validation", ) return { "artifact": artifact_path.name, "artifact_kind": EXPECTED_METADATA["artifact_kind"], "artifact_sha256": artifact_checksum, "artifact_size_bytes": artifact_path.stat().st_size, "evaluation_artifact_separation": True, "evaluation_artifact_sha256": EVALUATION_ARTIFACT_SHA256, "evaluation_artifact_size_bytes": EVALUATION_ARTIFACT_SIZE_BYTES, "evaluation_lens_sha256": EVALUATION_LENS_SHA256, "evaluation_tensor_count": evaluation_tensors_checked, "fp32_to_fp16_cast_exact": True, "license_notice_hashes_exact": True, "lfs_fsck_ok": lfs_fsck_ok, "lfs_pointer_exact": lfs_pointer_exact, "metadata_exact": True, "model_binding_exact": True, "ok": ( checked == len(EXPECTED_KEYS) and evaluation_tensors_checked == len(EXPECTED_KEYS) ), "privacy_text_scan": True, "publication_fields_resolved": not unresolved_publication_fields, "release_file_set_exact": True, "tensor_count": checked, "tensor_descriptors_exact": True, "tensor_hashes_exact": True, "unresolved_publication_fields": unresolved_publication_fields, } def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--root", type=Path, default=Path(__file__).resolve().parents[1], help="Artifact repository root", ) parser.add_argument( "--publication", action="store_true", help="Require resolved and valid companion publication fields", ) args = parser.parse_args() try: result = validate( args.root.resolve(), require_publication_fields=args.publication, ) except ValueError as error: parser.error(str(error)) print( json.dumps( result, indent=2, sort_keys=True, ) ) if __name__ == "__main__": main()