| |
| """Validate the public model-release candidate without the private source file.""" |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import hashlib |
| import json |
| import math |
| import re |
| import subprocess |
| from pathlib import Path |
| from typing import Any |
| from urllib.parse import urlsplit |
|
|
| import torch |
| from safetensors import safe_open |
|
|
| MODEL_ID = "WeiboAI/VibeThinker-3B" |
| MODEL_REVISION = "77bd2cced09193c8b9a59a32bd8577bbd1f3e01c" |
| SOURCE_CHECKPOINT_SHA256 = ( |
| "f36a99447623e0d777c70951a9148a7a52e42e0df82942e22ce6326f63d8d664" |
| ) |
| ARTIFACT_SHA256 = "089d776979408f23e5377539c15aa8025d171633718ccdce709bcd3372e7942c" |
| ARTIFACT_SIZE_BYTES = 150_996_824 |
| EVALUATION_LENS_SHA256 = ( |
| "8f752032a26a5196c1cb447ef63f01e8a29820ff0c57178dd80c6e26d32b12a9" |
| ) |
| EVALUATION_ARTIFACT_SHA256 = ( |
| "0cc184eb65d273ac8bfee5450a141c8cdf8dd6ec8caa68d7d47260ef621777d1" |
| ) |
| EVALUATION_ARTIFACT_SIZE_BYTES = 301_991_904 |
| EVALUATION_FILE_SHA256 = ( |
| "073f2886e370acec7dd1564f2d7e834b4253493e84b026ec0cc017a6069a5f78" |
| ) |
| SOURCE_LAYERS = tuple(range(0, 36, 2)) |
| SELECTED_BAND = (24, 26, 28, 30, 32, 34) |
| D_MODEL = 2048 |
| TENSOR_NBYTES = D_MODEL * D_MODEL * 2 |
| TENSOR_STORAGE_BYTES = len(SOURCE_LAYERS) * TENSOR_NBYTES |
| EVALUATION_TENSOR_NBYTES = D_MODEL * D_MODEL * 4 |
| EVALUATION_TENSOR_STORAGE_BYTES = len(SOURCE_LAYERS) * EVALUATION_TENSOR_NBYTES |
| EXPECTED_KEYS = {f"J.{layer}" for layer in SOURCE_LAYERS} |
| EXPECTED_TENSOR_RECORD_KEYS = { |
| "dtype", |
| "nbytes", |
| "numel", |
| "sha256_c_contiguous_little_endian_bytes", |
| "shape", |
| "source_layer", |
| } |
| EXPECTED_METADATA = { |
| "artifact_kind": "jacobian_lens", |
| "d_model": str(D_MODEL), |
| "format": "pt", |
| "model_id": MODEL_ID, |
| "model_revision": MODEL_REVISION, |
| "n_prompts": "1000", |
| "schema_version": "1", |
| "source_checkpoint_sha256": SOURCE_CHECKPOINT_SHA256, |
| "source_layers": "[0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34]", |
| "target_layer": "35", |
| "tensor_dtype": "float16", |
| "tensor_key_pattern": "J.{source_layer}", |
| } |
| EXPECTED_EVALUATION_METADATA = { |
| "artifact_kind": "jacobian_lens_evaluation_fp32", |
| "d_model": str(D_MODEL), |
| "format": "pt", |
| "model_id": MODEL_ID, |
| "model_revision": MODEL_REVISION, |
| "n_prompts": "1000", |
| "schema_version": "1", |
| "source_fit_checkpoint_sha256": ( |
| "a1236cfe5d04601575b3de150ffe50e3a67e755ede1197ecf74c205a31bdc258" |
| ), |
| "source_fp32_checkpoint_sha256": EVALUATION_LENS_SHA256, |
| "source_layers": "[0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34]", |
| "target_layer": "35", |
| "tensor_dtype": "float32", |
| "tensor_key_pattern": "J.{source_layer}", |
| } |
| EXPECTED_CARD_FRONT_MATTER = """license: other |
| license_name: qwen-research-license |
| license_link: https://huggingface.co/JacobMolBio/vibethinker-3b-jlens-model/blob/main/LICENSES/QWEN-RESEARCH.txt |
| thumbnail: https://huggingface.co/JacobMolBio/vibethinker-3b-jlens-model/resolve/main/assets/jlens-model-banner.png |
| tags: |
| - vibethinker-3b |
| - jacobian-lens |
| - mechanistic-interpretability |
| - interpretability |
| - llm-interpretability |
| - activation-analysis |
| - model-visualization |
| - qwen |
| - qwen2 |
| - safetensors""" |
| EXPECTED_RELEASE_FILES = { |
| ".gitattributes", |
| ".gitignore", |
| "assets/jlens-model-banner.png", |
| "assets/two-lens-files.png", |
| "assets/two-lens-files.svg", |
| "LICENSES/APACHE-2.0.txt", |
| "LICENSES/QWEN-RESEARCH.txt", |
| "LICENSES/VIBETHINKER-LICENSE-NOTE.txt", |
| "NOTICE", |
| "README.md", |
| "SHA256SUMS", |
| "THIRD_PARTY_NOTICES.md", |
| "evaluation.json", |
| "evaluation.safetensors", |
| "evaluation_compatibility.json", |
| "evaluation_provenance.json", |
| "evaluation_tensor_manifest.json", |
| "evaluation_validation.json", |
| "lens_config.json", |
| "model.safetensors", |
| "provenance.json", |
| "requirements.txt", |
| "scripts/convert_checkpoint.py", |
| "scripts/convert_evaluation_checkpoint.py", |
| "scripts/validate_artifact.py", |
| "tensor_manifest.json", |
| "validation.json", |
| } |
| PINNED_PUBLIC_FILE_SHA256 = { |
| ".gitattributes": "cd0273298656ca90cb8d08fd25b7601483f7e13417e3115ab809856c42e794f7", |
| ".gitignore": "4c7486a4b7c5ad04e0e62225b9a55c9ae00b039168d34614d402ac1f73acc459", |
| "assets/jlens-model-banner.png": "f27c4abb0c84481c2b0e67ed0c390716905bd1fe6fbcb09f2a65bbe15ecdc935", |
| "LICENSES/APACHE-2.0.txt": "ec01a6d25ea6a6b50430eda7af9e23e1c510048502d0a3adb5b54559e687d4fa", |
| "LICENSES/QWEN-RESEARCH.txt": "ef52482bb785733093dc9a2e8edd8e764c77d12d8e9d8f10a80c9b547d32d0f9", |
| "LICENSES/VIBETHINKER-LICENSE-NOTE.txt": "a74ea19436fbab1210ff16aa9037337d95ca9f93675406da4383bd8de4f4d25b", |
| "NOTICE": "35d9d57e97593a99aff4434875b43943d7627258495466bd5f8f81cdd8d40289", |
| "THIRD_PARTY_NOTICES.md": "d68e3b08646b56bf8d8c80d1d1acd98edfb40e72cb287762a3150d9b9d75a18f", |
| } |
| PUBLICATION_ROWS = { |
| "CODE_REPOSITORY_URL": "Source code and Pages source", |
| "TRACE_REPOSITORY_URL": "Captured trace dataset", |
| "PUBLIC_SITE_URL": "Static Pages site", |
| "MODEL_REPOSITORY_ID": "Hugging Face model repository ID", |
| } |
| PUBLICATION_PLACEHOLDERS = {key: "{{" + key + "}}" for key in PUBLICATION_ROWS} |
| CODE_REPOSITORY_NAME = "vibethinker-3b-jlens" |
| TRACE_REPOSITORY_NAME = "vibethinker-3b-jlens-traces" |
| MODEL_REPOSITORY_NAME = "vibethinker-3b-jlens-model" |
| REPOSITORY_COMPONENT = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*") |
| SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") |
| PUBLICATION_SENTINELS = { |
| "", |
| "example", |
| "local", |
| "none", |
| "null", |
| "org", |
| "organization", |
| "placeholder", |
| "repo", |
| "repository", |
| "tbd", |
| "todo", |
| "user", |
| "username", |
| } |
| FORBIDDEN_PUBLIC_PATTERNS = { |
| "email address": re.compile( |
| r"\b[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+" |
| + chr(64) |
| + r"[A-Za-z0-9](?:[A-Za-z0-9.-]{0,61}[A-Za-z0-9])?" |
| + r"\.[A-Za-z]{2,}\b" |
| ), |
| "AWS access key": re.compile(r"\b" + "AK" + r"IA[0-9A-Z]{16}\b"), |
| "AWS temporary access key": re.compile(r"\b" + "AS" + r"IA[0-9A-Z]{16}\b"), |
| "GitHub access token": re.compile(r"\b" + "gh" + r"[pousr]_[A-Za-z0-9]{20,}\b"), |
| "GitHub fine-grained token": re.compile( |
| r"\b" + "github" + r"_pat_[A-Za-z0-9_]{20,}\b" |
| ), |
| "GitLab access token": re.compile(r"\b" + "gl" + r"pat-[A-Za-z0-9_-]{20,}\b"), |
| "Hugging Face access token": re.compile(r"\b" + "hf" + r"_[A-Za-z0-9]{20,}\b"), |
| "OpenAI-style access token": re.compile(r"\b" + "sk" + r"-[A-Za-z0-9_-]{20,}\b"), |
| "Google API key": re.compile(r"\b" + "AI" + r"za[0-9A-Za-z_-]{30,}\b"), |
| "Slack access token": re.compile(r"\b" + "xo" + r"[abprs]-[A-Za-z0-9-]{20,}\b"), |
| "private key": re.compile( |
| "-----BEGIN " + r"(?:DSA |EC |OPENSSH |RSA )?PRIVATE KEY-----" |
| ), |
| "bearer credential": re.compile( |
| r"\b" + "Bearer" + r"\s+[A-Za-z0-9._~+/=-]{20,}", re.IGNORECASE |
| ), |
| "local file URI": re.compile(r"\b" + "file:" + r"//", re.IGNORECASE), |
| "macOS user path": re.compile(r"(?<![A-Za-z0-9:])/" + r"Users/[^/\s]+/"), |
| "mounted volume path": re.compile(r"(?<![A-Za-z0-9:])/" + r"Volumes/[^/\s]+/"), |
| "Unix home path": re.compile(r"(?<![A-Za-z0-9:])/" + r"home/[^/\s]+/"), |
| "root home path": re.compile(r"(?<![A-Za-z0-9:])/" + r"root(?:/|\b)"), |
| "temporary path": re.compile( |
| r"(?<![A-Za-z0-9:])/" + r"(?:tmp|private/tmp|var/folders)/" |
| ), |
| "workspace path": re.compile(r"(?<![A-Za-z0-9:])/" + r"workspaces?/[^\s]+"), |
| "mounted data path": re.compile(r"(?<![A-Za-z0-9:])/" + r"mnt/[^\s]+"), |
| "Windows user path": re.compile(r"[A-Za-z]:\\" + r"Users\\[^\\\s]+\\"), |
| "loopback hostname": re.compile(r"\b" + "local" + r"host\b", re.IGNORECASE), |
| "loopback IPv4 address": re.compile(r"\b127(?:\.[0-9]{1,3}){3}\b"), |
| "unspecified IPv4 address": re.compile(r"\b0\.0\.0\.0\b"), |
| } |
| FORBIDDEN_PUBLIC_LITERALS = { |
| "local account name": "jacob" + "vogan", |
| "local account alias": "jaco" + "vogan", |
| "invented VibeThinker copyright": "Copyright (c) 2025 " + "WeiboAI", |
| "removed VibeThinker license filename": "VIBETHINKER-" + "MIT.txt", |
| } |
| EXPECTED_EVALUATION_CLAIM_BOUNDARY = ( |
| "This reference records the V1 readout results. It excludes causal-assay " |
| "results and does not establish free-generation steering or a global " |
| "workspace." |
| ) |
| EXPECTED_EVALUATION_NOTE = ( |
| "The recorded readout metrics bind to evaluation.safetensors, the FP32 " |
| "evaluation lens in this repository. They do not evaluate " |
| "model.safetensors, the FP16 lens used for the captured traces." |
| ) |
| EXPECTED_EVALUATION_TASK = { |
| "aggregate_mean_reciprocal_rank": ( |
| "mean_across_eligible_target_terms_of_reciprocal_best_rank" |
| ), |
| "eligible_target": ("at_least_one_candidate_form_tokenizes_to_exactly_one_token"), |
| "final_model": ( |
| "rank_target_terms_in_the_model_next_token_logits_at_the_score_position" |
| ), |
| "item": "one_prompt_with_one_or_more_target_terms", |
| "layer_scope_reduction": ("best_target_rank_across_layers_in_the_reported_scope"), |
| "no_eligible_target_item": ( |
| "none_of_the_item_target_terms_has_an_eligible_single_token_form" |
| ), |
| "paired_bootstrap_mean_reciprocal_rank": ( |
| "within_item_mean_of_reciprocal_best_rank_across_eligible_target_terms" |
| ), |
| "pass_at_k": ( |
| "mean_across_items_of_the_fraction_of_item_target_terms_with_best_rank_at_most_k" |
| ), |
| "score_position": { |
| "default": "final_prompt_token", |
| "poetry": "last_newline_token", |
| }, |
| "split": { |
| "dev_fraction": 0.3, |
| "method": "sha256_stable_split", |
| "seed": "vibethinker-jlens-v1", |
| }, |
| "suites": [ |
| "association", |
| "multihop", |
| "multilingual", |
| "order-ops", |
| "poetry", |
| "typo", |
| ], |
| "target_candidate_forms": [ |
| "original_lowercase_and_capitalized_forms_with_and_without_leading_space", |
| "order_ops_also_adds_configured_operation_synonyms_and_number_word_digit_forms", |
| ], |
| "target_rank": ("best_one_based_vocabulary_rank_among_eligible_target_token_ids"), |
| } |
|
|
|
|
| def require(condition: bool, message: str) -> None: |
| if not condition: |
| raise ValueError(message) |
|
|
|
|
| def sha256_file(path: Path) -> str: |
| digest = hashlib.sha256() |
| with path.open("rb") as handle: |
| for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): |
| digest.update(chunk) |
| return digest.hexdigest() |
|
|
|
|
| def tensor_sha256(tensor: torch.Tensor) -> str: |
| contiguous = tensor.detach().cpu().contiguous() |
| if tensor.dtype == torch.float16: |
| array = contiguous.view(torch.int16).numpy() |
| payload = array.astype("<i2", copy=False).tobytes(order="C") |
| elif tensor.dtype == torch.float32: |
| array = contiguous.view(torch.int32).numpy() |
| payload = array.astype("<i4", copy=False).tobytes(order="C") |
| else: |
| raise ValueError(f"unsupported tensor dtype for hashing: {tensor.dtype}") |
| return hashlib.sha256(payload).hexdigest() |
|
|
|
|
| def read_json(path: Path) -> dict[str, Any]: |
| value = json.loads(path.read_text(encoding="utf-8")) |
| require(isinstance(value, dict), f"{path.name} must contain a JSON object") |
| return value |
|
|
|
|
| def normalized_words(value: str) -> str: |
| return " ".join(value.split()) |
|
|
|
|
| def extract_card_front_matter(readme: str) -> str: |
| require(readme.startswith("---\n"), "README model-card front matter is missing") |
| closing = readme.find("\n---\n", 4) |
| require(closing != -1, "README model-card front matter is not closed") |
| return readme[4:closing] |
|
|
|
|
| def validate_release_file_set(root: Path) -> None: |
| actual_files: set[str] = set() |
| for path in root.rglob("*"): |
| relative = path.relative_to(root) |
| if relative.parts and relative.parts[0] == ".git": |
| continue |
| require(not path.is_symlink(), f"release tree contains a symlink: {relative}") |
| if path.is_file(): |
| actual_files.add(relative.as_posix()) |
| require( |
| actual_files == EXPECTED_RELEASE_FILES, |
| "release file set mismatch: " |
| f"missing={sorted(EXPECTED_RELEASE_FILES - actual_files)}, " |
| f"extra={sorted(actual_files - EXPECTED_RELEASE_FILES)}", |
| ) |
|
|
|
|
| def validate_pinned_public_files(root: Path) -> None: |
| for relative_path, expected_sha256 in PINNED_PUBLIC_FILE_SHA256.items(): |
| require( |
| sha256_file(root / relative_path) == expected_sha256, |
| f"pinned public file hash mismatch: {relative_path}", |
| ) |
|
|
|
|
| def validate_public_text(root: Path) -> None: |
| binary_files = { |
| "assets/jlens-model-banner.png", |
| "assets/two-lens-files.png", |
| "evaluation.safetensors", |
| "model.safetensors", |
| } |
| for relative_path in sorted(EXPECTED_RELEASE_FILES - binary_files): |
| text = (root / relative_path).read_text(encoding="utf-8") |
| for label, pattern in FORBIDDEN_PUBLIC_PATTERNS.items(): |
| require( |
| pattern.search(text) is None, |
| f"{relative_path} contains a forbidden {label}", |
| ) |
| for label, literal in FORBIDDEN_PUBLIC_LITERALS.items(): |
| require( |
| literal not in text, |
| f"{relative_path} contains a forbidden {label}", |
| ) |
|
|
|
|
| def parse_publication_rows(readme: str) -> tuple[dict[str, str], list[str]]: |
| values: dict[str, str] = {} |
| for key, label in PUBLICATION_ROWS.items(): |
| pattern = re.compile( |
| rf"^\| {re.escape(label)} \| (?:`([^`\r\n]+)`|\[([^\]\r\n]+)\]\([^\)\r\n]+\)) \|$", |
| re.MULTILINE, |
| ) |
| matches = pattern.findall(readme) |
| require(len(matches) == 1, f"README publication row mismatch: {key}") |
| values[key] = matches[0][0] or matches[0][1] |
|
|
| placeholders_in_readme = set(re.findall(r"\{\{([A-Z0-9_]+)\}\}", readme)) |
| unresolved = sorted( |
| key for key, value in values.items() if value == PUBLICATION_PLACEHOLDERS[key] |
| ) |
| require( |
| placeholders_in_readme == set(unresolved), |
| "README publication placeholders do not match the publication table", |
| ) |
| require( |
| len(unresolved) in {0, len(PUBLICATION_ROWS)}, |
| "publication fields must be fully unresolved or fully resolved", |
| ) |
| require( |
| f'export JLENS_CODE_REPO_URL="{values["CODE_REPOSITORY_URL"]}"' in readme, |
| "source URL example differs from the publication table", |
| ) |
| require( |
| f'export JLENS_MODEL_REPO_ID="{values["MODEL_REPOSITORY_ID"]}"' in readme, |
| "model repository example differs from the publication table", |
| ) |
| return values, unresolved |
|
|
|
|
| def require_public_component(value: str, label: str) -> None: |
| require(value == value.strip(), f"{label} has surrounding whitespace") |
| require(value.casefold() not in PUBLICATION_SENTINELS, f"{label} is a placeholder") |
| require("{{" not in value and "}}" not in value, f"{label} is unresolved") |
| require( |
| not any(character.isspace() for character in value), |
| f"{label} contains whitespace", |
| ) |
| require("\\" not in value, f"{label} contains a local path separator") |
|
|
|
|
| def parse_https_url(value: str, label: str) -> Any: |
| require_public_component(value, label) |
| parsed = urlsplit(value) |
| require(parsed.scheme == "https", f"{label} must use HTTPS") |
| require(parsed.hostname is not None, f"{label} has no hostname") |
| require( |
| parsed.username is None and parsed.password is None, |
| f"{label} contains credentials", |
| ) |
| try: |
| port = parsed.port |
| except ValueError as error: |
| raise ValueError(f"{label} has an invalid port") from error |
| require(port is None, f"{label} must not specify a port") |
| require(not parsed.query, f"{label} must not contain a query") |
| require(not parsed.fragment, f"{label} must not contain a fragment") |
| require("%" not in parsed.path, f"{label} must not contain encoded path fragments") |
| require("//" not in parsed.path, f"{label} contains an empty path fragment") |
| require(";" not in parsed.path, f"{label} contains a parameter fragment") |
| return parsed |
|
|
|
|
| def validate_publication_values(values: dict[str, str]) -> None: |
| code = parse_https_url(values["CODE_REPOSITORY_URL"], "source repository URL") |
| trace = parse_https_url(values["TRACE_REPOSITORY_URL"], "trace repository URL") |
| site = parse_https_url(values["PUBLIC_SITE_URL"], "Pages site URL") |
| model_id = values["MODEL_REPOSITORY_ID"] |
| require_public_component(model_id, "model repository ID") |
|
|
| code_parts = [part for part in code.path.split("/") if part] |
| require( |
| code.hostname.casefold() == "github.com", |
| "source repository must use github.com", |
| ) |
| require( |
| len(code_parts) == 2, "source repository URL must contain namespace/repository" |
| ) |
| require( |
| code.path == "/" + "/".join(code_parts), |
| "source repository URL is not canonical", |
| ) |
| require( |
| all(REPOSITORY_COMPONENT.fullmatch(part) for part in code_parts), |
| "source repository URL contains an invalid path fragment", |
| ) |
| require( |
| code_parts[0].casefold() not in PUBLICATION_SENTINELS, |
| "source namespace is a placeholder", |
| ) |
| require(code_parts[1] == CODE_REPOSITORY_NAME, "source repository name mismatch") |
|
|
| trace_parts = [part for part in trace.path.split("/") if part] |
| require( |
| trace.hostname.casefold() == "huggingface.co", |
| "trace repository must use huggingface.co", |
| ) |
| require( |
| len(trace_parts) == 3 and trace_parts[0] == "datasets", |
| "trace repository URL must use /datasets/namespace/repository", |
| ) |
| require( |
| trace.path == "/" + "/".join(trace_parts), |
| "trace repository URL is not canonical", |
| ) |
| require( |
| all(REPOSITORY_COMPONENT.fullmatch(part) for part in trace_parts[1:]), |
| "trace repository URL contains an invalid path fragment", |
| ) |
| require( |
| trace_parts[1].casefold() not in PUBLICATION_SENTINELS, |
| "trace namespace is a placeholder", |
| ) |
| require(trace_parts[2] == TRACE_REPOSITORY_NAME, "trace repository name mismatch") |
|
|
| model_parts = model_id.split("/") |
| require(len(model_parts) == 2, "model repository ID must use namespace/repository") |
| require( |
| all(REPOSITORY_COMPONENT.fullmatch(part) for part in model_parts), |
| "model repository ID contains an invalid fragment", |
| ) |
| require( |
| model_parts[0].casefold() not in PUBLICATION_SENTINELS, |
| "model namespace is a placeholder", |
| ) |
| require(model_parts[1] == MODEL_REPOSITORY_NAME, "model repository name mismatch") |
| require( |
| model_parts[0].casefold() == trace_parts[1].casefold(), |
| "model and trace repositories must use the same Hugging Face namespace", |
| ) |
|
|
| require( |
| site.hostname.casefold() == f"{code_parts[0].casefold()}.github.io", |
| "Pages hostname must match the source repository namespace", |
| ) |
| require( |
| site.path.rstrip("/") == f"/{CODE_REPOSITORY_NAME}", |
| "Pages path must match the source repository name", |
| ) |
| require( |
| site.path in {f"/{CODE_REPOSITORY_NAME}", f"/{CODE_REPOSITORY_NAME}/"}, |
| "Pages URL is not canonical", |
| ) |
|
|
|
|
| def validate_model_card(readme: str) -> None: |
| require( |
| extract_card_front_matter(readme) == EXPECTED_CARD_FRONT_MATTER, |
| "README model-card front matter mismatch", |
| ) |
| readme_words = normalized_words(readme) |
| for required_text in ( |
| MODEL_ID, |
| MODEL_REVISION, |
| ARTIFACT_SHA256, |
| EVALUATION_LENS_SHA256, |
| EVALUATION_ARTIFACT_SHA256, |
| "LICENSES/VIBETHINKER-LICENSE-NOTE.txt", |
| "The static site cannot analyze a new prompt.", |
| "50,050 readout rows", |
| ): |
| require( |
| required_text in readme_words, |
| f"README missing required binding: {required_text}", |
| ) |
|
|
|
|
| def validate_licenses_and_notices(root: Path) -> None: |
| notice = (root / "NOTICE").read_text(encoding="utf-8") |
| third_party = (root / "THIRD_PARTY_NOTICES.md").read_text(encoding="utf-8") |
| note = (root / "LICENSES/VIBETHINKER-LICENSE-NOTE.txt").read_text(encoding="utf-8") |
| qwen_license = (root / "LICENSES/QWEN-RESEARCH.txt").read_text(encoding="utf-8") |
| required_qwen_notice = ( |
| "Qwen is licensed under the Qwen RESEARCH LICENSE AGREEMENT, " |
| "Copyright (c) Alibaba Cloud. All Rights Reserved." |
| ) |
| require( |
| required_qwen_notice in normalized_words(notice), |
| "NOTICE is missing the required Qwen attribution", |
| ) |
| require( |
| required_qwen_notice in normalized_words(qwen_license), |
| "Qwen license is missing its attribution clause", |
| ) |
| require( |
| "Built with Qwen." in notice, "NOTICE is missing the Qwen product attribution" |
| ) |
| require( |
| "LICENSES/VIBETHINKER-LICENSE-NOTE.txt" in notice |
| and "LICENSES/VIBETHINKER-LICENSE-NOTE.txt" in third_party, |
| "VibeThinker metadata note is not linked from the notices", |
| ) |
| require( |
| "license: mit" in normalized_words(note), |
| "VibeThinker metadata declaration missing", |
| ) |
| require(MODEL_REVISION in note, "VibeThinker metadata note revision mismatch") |
| require( |
| "It is not an upstream license text" in normalized_words(note), |
| "VibeThinker metadata note scope missing", |
| ) |
| require( |
| not (root / "LICENSES" / ("VIBETHINKER-" + "MIT.txt")).exists(), |
| "removed VibeThinker license file is present", |
| ) |
|
|
|
|
| def validate_metadata_records( |
| config: dict[str, Any], |
| provenance: dict[str, Any], |
| frozen_validation: dict[str, Any], |
| evaluation: dict[str, Any], |
| evaluation_path: Path, |
| ) -> None: |
| require(config.get("schema_version") == 1, "config schema mismatch") |
| require(config.get("artifact_kind") == "jacobian_lens", "config kind mismatch") |
| require(config.get("d_model") == D_MODEL, "config width mismatch") |
| require(config.get("n_prompts") == 1000, "config prompt count mismatch") |
| require( |
| config.get("source_layers") == list(SOURCE_LAYERS), |
| "config source layers mismatch", |
| ) |
| require(config.get("target_layer") == 35, "config target layer mismatch") |
| require(config.get("tensor_dtype") == "float16", "config tensor dtype mismatch") |
| require( |
| config.get("tensor_key_pattern") == "J.{source_layer}", |
| "config key pattern mismatch", |
| ) |
| require( |
| config.get("model") |
| == { |
| "architecture": "Qwen2ForCausalLM", |
| "id": MODEL_ID, |
| "n_layers": 36, |
| "revision": MODEL_REVISION, |
| }, |
| "config model binding mismatch", |
| ) |
| require( |
| config.get("source_checkpoint", {}).get("sha256") == SOURCE_CHECKPOINT_SHA256, |
| "config source checkpoint mismatch", |
| ) |
| require( |
| config.get("artifact") |
| == { |
| "filename": "model.safetensors", |
| "format": "safetensors", |
| "sha256": ARTIFACT_SHA256, |
| "size_bytes": ARTIFACT_SIZE_BYTES, |
| }, |
| "config artifact binding mismatch", |
| ) |
| require( |
| config.get("evaluation_artifact") |
| == { |
| "filename": "evaluation.safetensors", |
| "format": "safetensors", |
| "sha256": EVALUATION_ARTIFACT_SHA256, |
| "size_bytes": EVALUATION_ARTIFACT_SIZE_BYTES, |
| "source_checkpoint_sha256": EVALUATION_LENS_SHA256, |
| "tensor_conversion": "lossless_fp32_reserialization", |
| "tensor_dtype": "float32", |
| }, |
| "config evaluation artifact binding mismatch", |
| ) |
|
|
| require(provenance.get("schema_version") == 1, "provenance schema mismatch") |
| require( |
| provenance.get("model", {}).get("id") == MODEL_ID, |
| "provenance model ID mismatch", |
| ) |
| require( |
| provenance.get("model", {}).get("revision") == MODEL_REVISION, |
| "provenance model revision mismatch", |
| ) |
| require( |
| provenance.get("artifact", {}).get("sha256") == ARTIFACT_SHA256, |
| "provenance artifact hash mismatch", |
| ) |
| require( |
| provenance.get("artifact", {}).get("size_bytes") == ARTIFACT_SIZE_BYTES, |
| "provenance artifact size mismatch", |
| ) |
| require( |
| provenance.get("artifact", {}).get("tensor_conversion") |
| == "lossless_fp16_reserialization", |
| "provenance conversion boundary mismatch", |
| ) |
| require( |
| provenance.get("source_checkpoint", {}).get("sha256") |
| == SOURCE_CHECKPOINT_SHA256, |
| "provenance source checkpoint mismatch", |
| ) |
|
|
| require(frozen_validation.get("schema_version") == 1, "validation schema mismatch") |
| require( |
| frozen_validation.get("artifact") == "model.safetensors", |
| "validation filename mismatch", |
| ) |
| require( |
| frozen_validation.get("artifact_sha256") == ARTIFACT_SHA256, |
| "validation artifact hash mismatch", |
| ) |
| require( |
| frozen_validation.get("artifact_size_bytes") == ARTIFACT_SIZE_BYTES, |
| "validation artifact size mismatch", |
| ) |
| require(frozen_validation.get("ok") is True, "frozen validation is not successful") |
| require( |
| frozen_validation.get("tensor_values_changed") == 0, |
| "frozen validation records changed values", |
| ) |
| require( |
| frozen_validation.get("exact_tensor_matches") == len(SOURCE_LAYERS), |
| "frozen validation tensor count mismatch", |
| ) |
| require( |
| frozen_validation.get("expected_tensor_matches") == len(SOURCE_LAYERS), |
| "frozen validation expected tensor count mismatch", |
| ) |
| require( |
| frozen_validation.get("source_checkpoint_sha256") == SOURCE_CHECKPOINT_SHA256, |
| "frozen validation source checkpoint mismatch", |
| ) |
| checks = frozen_validation.get("checks") |
| require( |
| isinstance(checks, dict) |
| and checks |
| and all(value is True for value in checks.values()), |
| "frozen validation contains a failed or malformed check", |
| ) |
|
|
| require( |
| sha256_file(evaluation_path) == EVALUATION_FILE_SHA256, |
| "evaluation file hash mismatch", |
| ) |
| require( |
| evaluation.get("artifact_kind") == "frozen_vibethinker_v1_readout_reference", |
| "evaluation artifact kind mismatch", |
| ) |
| require( |
| evaluation.get("classification") == "validated_readout_only", |
| "evaluation classification mismatch", |
| ) |
| require(evaluation.get("model") == MODEL_ID, "evaluation model ID mismatch") |
| require( |
| evaluation.get("model_revision") == MODEL_REVISION, |
| "evaluation model revision mismatch", |
| ) |
| require( |
| evaluation.get("lens_sha256") == EVALUATION_LENS_SHA256, |
| "evaluation lens hash mismatch", |
| ) |
| require( |
| evaluation.get("lens_variant") == "fp32_evaluation_safetensors_included", |
| "evaluation lens variant mismatch", |
| ) |
| require( |
| evaluation.get("released_lens_artifact") |
| == { |
| "filename": "evaluation.safetensors", |
| "format": "safetensors", |
| "sha256": EVALUATION_ARTIFACT_SHA256, |
| "size_bytes": EVALUATION_ARTIFACT_SIZE_BYTES, |
| "source_checkpoint_sha256": EVALUATION_LENS_SHA256, |
| "tensor_conversion": "lossless_fp32_reserialization", |
| }, |
| "evaluation released artifact binding mismatch", |
| ) |
| require( |
| evaluation.get("claim_boundary") == EXPECTED_EVALUATION_CLAIM_BOUNDARY, |
| "evaluation claim boundary mismatch", |
| ) |
| require( |
| evaluation.get("note") == EXPECTED_EVALUATION_NOTE, |
| "evaluation FP16/FP32 note mismatch", |
| ) |
| require( |
| evaluation.get("selected_band") == list(SELECTED_BAND), |
| "evaluation selected band mismatch", |
| ) |
| require( |
| evaluation.get("coverage_scope") |
| == "frozen_readout_source_run_not_the_100_prompt_ui_test_pack", |
| "evaluation coverage scope mismatch", |
| ) |
| require( |
| evaluation.get("task_definition") == EXPECTED_EVALUATION_TASK, |
| "evaluation task definition mismatch", |
| ) |
| evidence = evaluation.get("evidence", {}) |
| require( |
| evidence.get("validated_readout_signal") is True, |
| "evaluation readout signal boundary mismatch", |
| ) |
| require( |
| evidence.get("token_specific_vs_shuffled_target") is True, |
| "evaluation token control boundary mismatch", |
| ) |
| require( |
| evidence.get("layer_mapping_specific_vs_shuffled_jacobian") is True, |
| "evaluation layer control boundary mismatch", |
| ) |
| require( |
| evidence.get("incremental_over_ordinary_logit_lens") is False, |
| "evaluation logit-lens boundary mismatch", |
| ) |
| require( |
| evidence.get("paired_bootstrap") |
| == { |
| "confidence": 0.95, |
| "input": "paired_item_metric_differences", |
| "interval": "percentile", |
| "metrics": ["pass@10", "mean_reciprocal_rank"], |
| "samples": 2000, |
| "scope": { |
| "kind": "selected_band", |
| "source_layers": list(SELECTED_BAND), |
| }, |
| "split": "test", |
| }, |
| "evaluation paired-bootstrap method mismatch", |
| ) |
| require( |
| evidence.get("paired_bootstrap_decisions") |
| == { |
| "incremental_over_ordinary_logit_lens": { |
| "comparison": "jlens_band_minus_logit_lens_band", |
| "criterion": "at_least_one_lower_bound_greater_than_zero", |
| "met": False, |
| }, |
| "layer_mapping_specific_vs_shuffled_jacobian": { |
| "comparison": "jlens_band_minus_shuffled_layer_band", |
| "criterion": "both_lower_bounds_greater_than_zero", |
| "met": True, |
| }, |
| "token_specific_vs_shuffled_target": { |
| "comparison": "jlens_band_minus_shuffled_token_band", |
| "criterion": "both_lower_bounds_greater_than_zero", |
| "met": True, |
| }, |
| }, |
| "evaluation paired-bootstrap decisions mismatch", |
| ) |
| require( |
| evidence.get("specificity_scope") |
| == {"kind": "selected_band", "source_layers": list(SELECTED_BAND)}, |
| "evaluation specificity scope mismatch", |
| ) |
| availability = evaluation.get("source_artifact_availability", {}) |
| require( |
| availability |
| == { |
| "fp32_compatibility_check_output_included": True, |
| "fp32_compatibility_check_output_path": ("evaluation_compatibility.json"), |
| "fp32_derivation_record_included": True, |
| "fp32_derivation_record_path": "evaluation_provenance.json", |
| "fp32_evaluation_lens_included": True, |
| "fp32_evaluation_lens_path": "evaluation.safetensors", |
| "item_level_evaluation_rows_included": False, |
| "item_level_evaluation_rows_location": ( |
| "companion_trace_repository:data/evaluation-results/" |
| "readout-trials.jsonl" |
| ), |
| "paired_bootstrap_interval_bounds_included": False, |
| "paired_bootstrap_interval_bounds_location": ( |
| "companion_trace_repository:data/evaluation-results/" |
| "readout-bootstrap-intervals.json" |
| ), |
| "source_evaluation_bundle_included": False, |
| "source_hashes_included": True, |
| "source_hashes_location": ( |
| "evaluation_provenance.json_and_companion_trace_repository:" |
| "data/readout-reference.json" |
| ), |
| "release_content": ( |
| "fp16_trace_lens_fp32_evaluation_lens_aggregate_reference_" |
| "provenance_and_compatibility" |
| ), |
| }, |
| "evaluation source-artifact boundary mismatch", |
| ) |
| require( |
| EVALUATION_LENS_SHA256 != ARTIFACT_SHA256, |
| "FP32 and FP16 artifact hashes were conflated", |
| ) |
|
|
|
|
| def validate_tensor_manifest(manifest: dict[str, Any]) -> None: |
| require( |
| set(manifest) |
| == { |
| "artifact", |
| "artifact_sha256", |
| "artifact_size_bytes", |
| "schema_version", |
| "tensor_count", |
| "tensor_storage_bytes", |
| "tensors", |
| }, |
| "tensor manifest top-level fields mismatch", |
| ) |
| require( |
| manifest.get("artifact") == "model.safetensors", |
| "tensor manifest filename mismatch", |
| ) |
| require( |
| manifest.get("artifact_sha256") == ARTIFACT_SHA256, |
| "tensor manifest artifact hash mismatch", |
| ) |
| require( |
| manifest.get("artifact_size_bytes") == ARTIFACT_SIZE_BYTES, |
| "tensor manifest artifact size mismatch", |
| ) |
| require(manifest.get("schema_version") == 1, "tensor manifest schema mismatch") |
| require( |
| manifest.get("tensor_count") == len(SOURCE_LAYERS), |
| "tensor manifest count mismatch", |
| ) |
| require( |
| manifest.get("tensor_storage_bytes") == TENSOR_STORAGE_BYTES, |
| "tensor manifest storage total mismatch", |
| ) |
| tensors = manifest.get("tensors") |
| require(isinstance(tensors, dict), "tensor manifest tensors must be an object") |
| require(set(tensors) == EXPECTED_KEYS, "tensor manifest key set mismatch") |
|
|
| recorded_storage = 0 |
| for layer in SOURCE_LAYERS: |
| key = f"J.{layer}" |
| record = tensors[key] |
| require(isinstance(record, dict), f"{key} descriptor must be an object") |
| require( |
| set(record) == EXPECTED_TENSOR_RECORD_KEYS, |
| f"{key} descriptor fields mismatch", |
| ) |
| require(record.get("source_layer") == layer, f"{key} source layer mismatch") |
| require(record.get("dtype") == "float16", f"{key} descriptor dtype mismatch") |
| require( |
| record.get("shape") == [D_MODEL, D_MODEL], |
| f"{key} descriptor shape mismatch", |
| ) |
| require( |
| record.get("numel") == D_MODEL * D_MODEL, |
| f"{key} descriptor element count mismatch", |
| ) |
| require( |
| record.get("nbytes") == TENSOR_NBYTES, |
| f"{key} descriptor byte count mismatch", |
| ) |
| tensor_hash = record.get("sha256_c_contiguous_little_endian_bytes") |
| require( |
| isinstance(tensor_hash, str) and SHA256_PATTERN.fullmatch(tensor_hash), |
| f"{key} descriptor hash format mismatch", |
| ) |
| recorded_storage += record["nbytes"] |
| require( |
| recorded_storage == TENSOR_STORAGE_BYTES, |
| "tensor descriptor byte total mismatch", |
| ) |
|
|
|
|
| def validate_evaluation_artifact( |
| root: Path, |
| manifest: dict[str, Any], |
| provenance: dict[str, Any], |
| frozen_validation: dict[str, Any], |
| compatibility: dict[str, Any], |
| ) -> int: |
| artifact_path = root / "evaluation.safetensors" |
| require( |
| set(manifest) |
| == { |
| "artifact", |
| "artifact_sha256", |
| "artifact_size_bytes", |
| "schema_version", |
| "source_checkpoint_sha256", |
| "tensor_count", |
| "tensor_storage_bytes", |
| "tensors", |
| }, |
| "evaluation tensor manifest top-level fields mismatch", |
| ) |
| require( |
| manifest.get("artifact") == "evaluation.safetensors", |
| "evaluation tensor manifest filename mismatch", |
| ) |
| require( |
| manifest.get("artifact_sha256") == EVALUATION_ARTIFACT_SHA256, |
| "evaluation tensor manifest artifact hash mismatch", |
| ) |
| require( |
| manifest.get("artifact_size_bytes") == EVALUATION_ARTIFACT_SIZE_BYTES, |
| "evaluation tensor manifest artifact size mismatch", |
| ) |
| require( |
| manifest.get("source_checkpoint_sha256") == EVALUATION_LENS_SHA256, |
| "evaluation tensor manifest source hash mismatch", |
| ) |
| require( |
| manifest.get("tensor_count") == len(SOURCE_LAYERS), |
| "evaluation tensor manifest count mismatch", |
| ) |
| require( |
| manifest.get("tensor_storage_bytes") == EVALUATION_TENSOR_STORAGE_BYTES, |
| "evaluation tensor manifest storage total mismatch", |
| ) |
| tensors = manifest.get("tensors") |
| require(isinstance(tensors, dict), "evaluation manifest tensors must be an object") |
| require(set(tensors) == EXPECTED_KEYS, "evaluation manifest key set mismatch") |
| for layer in SOURCE_LAYERS: |
| key = f"J.{layer}" |
| record = tensors[key] |
| require( |
| set(record) == EXPECTED_TENSOR_RECORD_KEYS, |
| f"evaluation {key} descriptor fields mismatch", |
| ) |
| require(record.get("source_layer") == layer, f"evaluation {key} layer mismatch") |
| require(record.get("dtype") == "float32", f"evaluation {key} dtype mismatch") |
| require( |
| record.get("shape") == [D_MODEL, D_MODEL], |
| f"evaluation {key} shape mismatch", |
| ) |
| require( |
| record.get("numel") == D_MODEL * D_MODEL, |
| f"evaluation {key} element count mismatch", |
| ) |
| require( |
| record.get("nbytes") == EVALUATION_TENSOR_NBYTES, |
| f"evaluation {key} byte count mismatch", |
| ) |
| require( |
| isinstance(record.get("sha256_c_contiguous_little_endian_bytes"), str) |
| and SHA256_PATTERN.fullmatch( |
| record["sha256_c_contiguous_little_endian_bytes"] |
| ), |
| f"evaluation {key} tensor hash format mismatch", |
| ) |
|
|
| require( |
| provenance.get("artifact_kind") == "jacobian_lens_evaluation_fp32_provenance", |
| "evaluation provenance kind mismatch", |
| ) |
| require( |
| provenance.get("artifact") |
| == { |
| "filename": "evaluation.safetensors", |
| "format": "safetensors", |
| "sha256": EVALUATION_ARTIFACT_SHA256, |
| "size_bytes": EVALUATION_ARTIFACT_SIZE_BYTES, |
| "tensor_conversion": "lossless_fp32_reserialization", |
| }, |
| "evaluation provenance artifact binding mismatch", |
| ) |
| require( |
| provenance.get("source_checkpoint") |
| == {"format": "pytorch", "sha256": EVALUATION_LENS_SHA256}, |
| "evaluation provenance source binding mismatch", |
| ) |
| derivation = provenance.get("derivation", {}) |
| require( |
| derivation.get("formula") == "jacobian_sum[layer] / n_done", |
| "evaluation derivation formula mismatch", |
| ) |
| require(derivation.get("n_done") == 1000, "evaluation derivation count mismatch") |
| require( |
| derivation.get("source_fit_checkpoint_sha256") |
| == EXPECTED_EVALUATION_METADATA["source_fit_checkpoint_sha256"], |
| "evaluation fit checkpoint binding mismatch", |
| ) |
| require( |
| provenance.get("model", {}).get("id") == MODEL_ID |
| and provenance.get("model", {}).get("revision") == MODEL_REVISION, |
| "evaluation provenance model binding mismatch", |
| ) |
| require( |
| provenance.get("lens", {}).get("dtype") == "float32" |
| and provenance.get("lens", {}).get("source_layers") == list(SOURCE_LAYERS) |
| and provenance.get("lens", {}).get("target_layer") == 35 |
| and provenance.get("lens", {}).get("n_prompts") == 1000, |
| "evaluation provenance lens metadata mismatch", |
| ) |
| runtime = provenance.get("software", {}).get("conversion_runtime", {}) |
| require( |
| runtime.get("safetensors") == "0.8.0" and runtime.get("torch") == "2.13.0", |
| "evaluation conversion runtime mismatch", |
| ) |
|
|
| require( |
| frozen_validation.get("artifact") == "evaluation.safetensors", |
| "evaluation validation filename mismatch", |
| ) |
| require( |
| frozen_validation.get("artifact_sha256") == EVALUATION_ARTIFACT_SHA256, |
| "evaluation validation artifact hash mismatch", |
| ) |
| require( |
| frozen_validation.get("artifact_size_bytes") == EVALUATION_ARTIFACT_SIZE_BYTES, |
| "evaluation validation artifact size mismatch", |
| ) |
| require( |
| frozen_validation.get("source_checkpoint_sha256") == EVALUATION_LENS_SHA256, |
| "evaluation validation source hash mismatch", |
| ) |
| require(frozen_validation.get("ok") is True, "evaluation validation failed") |
| require( |
| frozen_validation.get("tensor_values_changed") == 0, |
| "evaluation validation records changed values", |
| ) |
| require( |
| frozen_validation.get("exact_tensor_matches") == len(SOURCE_LAYERS), |
| "evaluation validation tensor count mismatch", |
| ) |
| checks = frozen_validation.get("checks") |
| require( |
| isinstance(checks, dict) |
| and checks |
| and all(value is True for value in checks.values()), |
| "evaluation validation contains a failed check", |
| ) |
|
|
| require( |
| compatibility.get("artifact_kind") == "fp32_to_fp16_lens_compatibility", |
| "evaluation compatibility kind mismatch", |
| ) |
| require( |
| compatibility.get("fp32_artifact") == "evaluation.safetensors" |
| and compatibility.get("fp32_artifact_sha256") == EVALUATION_ARTIFACT_SHA256 |
| and compatibility.get("fp32_source_checkpoint_sha256") |
| == EVALUATION_LENS_SHA256, |
| "evaluation compatibility FP32 binding mismatch", |
| ) |
| require( |
| compatibility.get("fp16_artifact") == "model.safetensors" |
| and compatibility.get("fp16_artifact_sha256") == ARTIFACT_SHA256, |
| "evaluation compatibility FP16 binding mismatch", |
| ) |
| require( |
| compatibility.get("all_layer_casts_match_exactly") is True, |
| "evaluation compatibility records a cast mismatch", |
| ) |
| require( |
| compatibility.get("max_absolute_error") == 0.00048828125, |
| "evaluation compatibility maximum error mismatch", |
| ) |
| require( |
| math.isclose( |
| compatibility.get("relative_frobenius_error", math.inf), |
| 0.00020901276774552773, |
| rel_tol=1e-12, |
| abs_tol=0.0, |
| ), |
| "evaluation compatibility relative error mismatch", |
| ) |
| per_layer = compatibility.get("per_layer") |
| require( |
| isinstance(per_layer, dict) |
| and set(per_layer) == {str(layer) for layer in SOURCE_LAYERS}, |
| "evaluation compatibility layer set mismatch", |
| ) |
| require( |
| all( |
| record.get("cast_matches_model_safetensors_exactly") is True |
| for record in per_layer.values() |
| ), |
| "evaluation compatibility layer cast mismatch", |
| ) |
|
|
| require( |
| sha256_file(artifact_path) == EVALUATION_ARTIFACT_SHA256, |
| "evaluation artifact SHA-256 mismatch", |
| ) |
| require( |
| artifact_path.stat().st_size == EVALUATION_ARTIFACT_SIZE_BYTES, |
| "evaluation artifact size mismatch", |
| ) |
| checked = 0 |
| with ( |
| safe_open(artifact_path, framework="pt", device="cpu") as artifact, |
| safe_open( |
| root / "model.safetensors", framework="pt", device="cpu" |
| ) as fp16_artifact, |
| ): |
| require( |
| (artifact.metadata() or {}) == EXPECTED_EVALUATION_METADATA, |
| "evaluation Safetensors metadata mismatch", |
| ) |
| require(set(artifact.keys()) == EXPECTED_KEYS, "evaluation tensor key mismatch") |
| for layer in SOURCE_LAYERS: |
| key = f"J.{layer}" |
| tensor = artifact.get_tensor(key) |
| record = tensors[key] |
| require(tensor.dtype == torch.float32, f"evaluation {key} dtype mismatch") |
| require( |
| tuple(tensor.shape) == (D_MODEL, D_MODEL), |
| f"evaluation {key} shape mismatch", |
| ) |
| require(tensor.is_contiguous(), f"evaluation {key} is not contiguous") |
| require( |
| bool(torch.isfinite(tensor).all()), |
| f"evaluation {key} contains non-finite values", |
| ) |
| require( |
| tensor_sha256(tensor) |
| == record["sha256_c_contiguous_little_endian_bytes"], |
| f"evaluation {key} tensor hash mismatch", |
| ) |
| require( |
| torch.equal( |
| tensor.to(torch.float16).view(torch.int16), |
| fp16_artifact.get_tensor(key).view(torch.int16), |
| ), |
| f"evaluation {key} FP16 cast mismatch", |
| ) |
| checked += 1 |
| return checked |
|
|
|
|
| def run_git(root: Path, *arguments: str) -> subprocess.CompletedProcess[bytes]: |
| return subprocess.run( |
| ["git", "-C", str(root), *arguments], |
| check=False, |
| capture_output=True, |
| ) |
|
|
|
|
| def validate_git_lfs( |
| root: Path, *, require_evaluation_head: bool = False |
| ) -> tuple[bool, bool]: |
| repository_check = run_git(root, "rev-parse", "--is-inside-work-tree") |
| if repository_check.returncode != 0: |
| return False, False |
|
|
| model_pointer = ( |
| "version https://git-lfs.github.com/spec/v1\n" |
| f"oid sha256:{ARTIFACT_SHA256}\n" |
| f"size {ARTIFACT_SIZE_BYTES}\n" |
| ).encode("ascii") |
| for object_name in ("HEAD:model.safetensors", ":model.safetensors"): |
| pointer = run_git(root, "show", object_name) |
| require(pointer.returncode == 0, f"cannot read Git LFS pointer: {object_name}") |
| require( |
| pointer.stdout == model_pointer, f"Git LFS pointer mismatch: {object_name}" |
| ) |
|
|
| evaluation_pointer = ( |
| "version https://git-lfs.github.com/spec/v1\n" |
| f"oid sha256:{EVALUATION_ARTIFACT_SHA256}\n" |
| f"size {EVALUATION_ARTIFACT_SIZE_BYTES}\n" |
| ).encode("ascii") |
| staged_evaluation = run_git(root, "show", ":evaluation.safetensors") |
| require( |
| staged_evaluation.returncode == 0, |
| "cannot read staged Git LFS pointer: evaluation.safetensors", |
| ) |
| require( |
| staged_evaluation.stdout == evaluation_pointer, |
| "staged Git LFS pointer mismatch: evaluation.safetensors", |
| ) |
| head_evaluation = run_git(root, "show", "HEAD:evaluation.safetensors") |
| if require_evaluation_head: |
| require( |
| head_evaluation.returncode == 0, |
| "publication check requires evaluation.safetensors in HEAD", |
| ) |
| if head_evaluation.returncode == 0: |
| require( |
| head_evaluation.stdout == evaluation_pointer, |
| "Git LFS pointer mismatch: HEAD:evaluation.safetensors", |
| ) |
|
|
| fsck = run_git(root, "lfs", "fsck", "HEAD") |
| require(fsck.returncode == 0, "git lfs fsck failed") |
| return True, True |
|
|
|
|
| def validate(root: Path, *, require_publication_fields: bool = False) -> dict[str, Any]: |
| artifact_path = root / "model.safetensors" |
| readme = (root / "README.md").read_text(encoding="utf-8") |
| config = read_json(root / "lens_config.json") |
| manifest = read_json(root / "tensor_manifest.json") |
| provenance = read_json(root / "provenance.json") |
| frozen_validation = read_json(root / "validation.json") |
| evaluation_path = root / "evaluation.json" |
| evaluation = read_json(evaluation_path) |
| evaluation_manifest = read_json(root / "evaluation_tensor_manifest.json") |
| evaluation_provenance = read_json(root / "evaluation_provenance.json") |
| evaluation_validation = read_json(root / "evaluation_validation.json") |
| evaluation_compatibility = read_json(root / "evaluation_compatibility.json") |
|
|
| validate_release_file_set(root) |
| validate_pinned_public_files(root) |
| validate_public_text(root) |
| require( |
| (root / "requirements.txt").read_text(encoding="utf-8") |
| == "safetensors==0.8.0\ntorch==2.13.0\n", |
| "requirements do not match the evaluation conversion runtime", |
| ) |
| validate_model_card(readme) |
| validate_licenses_and_notices(root) |
| validate_tensor_manifest(manifest) |
| evaluation_tensors_checked = validate_evaluation_artifact( |
| root, |
| evaluation_manifest, |
| evaluation_provenance, |
| evaluation_validation, |
| evaluation_compatibility, |
| ) |
| validate_metadata_records( |
| config, |
| provenance, |
| frozen_validation, |
| evaluation, |
| evaluation_path, |
| ) |
|
|
| publication_values, unresolved_publication_fields = parse_publication_rows(readme) |
| if require_publication_fields: |
| require( |
| not unresolved_publication_fields, |
| "publication fields remain unresolved: " |
| + ", ".join(unresolved_publication_fields), |
| ) |
| if not unresolved_publication_fields: |
| validate_publication_values(publication_values) |
|
|
| expected_checksum_file = ( |
| f"{ARTIFACT_SHA256} model.safetensors\n" |
| f"{EVALUATION_ARTIFACT_SHA256} evaluation.safetensors\n" |
| ) |
| require( |
| (root / "SHA256SUMS").read_text(encoding="ascii") == expected_checksum_file, |
| "SHA256SUMS content mismatch", |
| ) |
| artifact_checksum = sha256_file(artifact_path) |
| require(artifact_checksum == ARTIFACT_SHA256, "artifact SHA-256 mismatch") |
| require( |
| artifact_path.stat().st_size == ARTIFACT_SIZE_BYTES, "artifact size mismatch" |
| ) |
|
|
| checked = 0 |
| with safe_open(artifact_path, framework="pt", device="cpu") as artifact: |
| require( |
| (artifact.metadata() or {}) == EXPECTED_METADATA, |
| "safetensors metadata mismatch", |
| ) |
| require( |
| set(artifact.keys()) == EXPECTED_KEYS, "safetensors tensor key mismatch" |
| ) |
| for layer in SOURCE_LAYERS: |
| key = f"J.{layer}" |
| tensor = artifact.get_tensor(key) |
| record = manifest["tensors"][key] |
| require(tensor.dtype == torch.float16, f"{key} dtype mismatch") |
| require(tuple(tensor.shape) == (D_MODEL, D_MODEL), f"{key} shape mismatch") |
| require(tensor.is_contiguous(), f"{key} is not contiguous") |
| require( |
| bool(torch.isfinite(tensor).all()), f"{key} contains non-finite values" |
| ) |
| require(tensor.numel() == record["numel"], f"{key} element count mismatch") |
| require( |
| tensor.numel() * tensor.element_size() == record["nbytes"], |
| f"{key} byte count mismatch", |
| ) |
| require( |
| tensor_sha256(tensor) |
| == record["sha256_c_contiguous_little_endian_bytes"], |
| f"{key} tensor hash mismatch", |
| ) |
| checked += 1 |
|
|
| lfs_pointer_exact, lfs_fsck_ok = validate_git_lfs( |
| root, |
| require_evaluation_head=require_publication_fields, |
| ) |
| if require_publication_fields: |
| require( |
| lfs_pointer_exact and lfs_fsck_ok, |
| "publication check requires Git LFS validation", |
| ) |
|
|
| return { |
| "artifact": artifact_path.name, |
| "artifact_kind": EXPECTED_METADATA["artifact_kind"], |
| "artifact_sha256": artifact_checksum, |
| "artifact_size_bytes": artifact_path.stat().st_size, |
| "evaluation_artifact_separation": True, |
| "evaluation_artifact_sha256": EVALUATION_ARTIFACT_SHA256, |
| "evaluation_artifact_size_bytes": EVALUATION_ARTIFACT_SIZE_BYTES, |
| "evaluation_lens_sha256": EVALUATION_LENS_SHA256, |
| "evaluation_tensor_count": evaluation_tensors_checked, |
| "fp32_to_fp16_cast_exact": True, |
| "license_notice_hashes_exact": True, |
| "lfs_fsck_ok": lfs_fsck_ok, |
| "lfs_pointer_exact": lfs_pointer_exact, |
| "metadata_exact": True, |
| "model_binding_exact": True, |
| "ok": ( |
| checked == len(EXPECTED_KEYS) |
| and evaluation_tensors_checked == len(EXPECTED_KEYS) |
| ), |
| "privacy_text_scan": True, |
| "publication_fields_resolved": not unresolved_publication_fields, |
| "release_file_set_exact": True, |
| "tensor_count": checked, |
| "tensor_descriptors_exact": True, |
| "tensor_hashes_exact": True, |
| "unresolved_publication_fields": unresolved_publication_fields, |
| } |
|
|
|
|
| def main() -> None: |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument( |
| "--root", |
| type=Path, |
| default=Path(__file__).resolve().parents[1], |
| help="Artifact repository root", |
| ) |
| parser.add_argument( |
| "--publication", |
| action="store_true", |
| help="Require resolved and valid companion publication fields", |
| ) |
| args = parser.parse_args() |
| try: |
| result = validate( |
| args.root.resolve(), |
| require_publication_fields=args.publication, |
| ) |
| except ValueError as error: |
| parser.error(str(error)) |
| print( |
| json.dumps( |
| result, |
| indent=2, |
| sort_keys=True, |
| ) |
| ) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
