"""CPU checks for the Hugging Face package metadata.""" import json import os import sys import tempfile from types import SimpleNamespace sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from scripts.hf_metadata import ( # noqa: E402 ADDITIONAL_SPECIAL_TOKENS, _interval, _metric, _verdict, development_evaluation_section, file_sha256, generation_config, llama_config, mtp_config, require_unchanged_files, render_evaluation_section, render_model_card, special_tokens_map, tokenizer_config, validate_external_verification_receipt, validate_export_checkpoint, verify_export_manifest, write_json_atomic, write_export_manifest, ) def main(): args = SimpleNamespace( vocab_size=32768, dim=768, n_layers=12, n_heads=12, n_kv_heads=4, head_dim=64, ffn_hidden=2048, max_seq_len=2048, rope_theta=100000.0, norm_eps=1e-5, tie_embeddings=True, mtp_layers=1, mtp_depth=2, ) config = llama_config(args) assert config["architectures"] == ["LlamaForCausalLM"] assert config["vocab_size"] == 32768 assert config["eos_token_id"] == 0 assert config["pad_token_id"] == 1 assert config["bos_token_id"] is None tokenizer = tokenizer_config(args) assert tokenizer["tokenizer_class"] == "PreTrainedTokenizerFast" assert tokenizer["model_max_length"] == args.max_seq_len assert tokenizer["eos_token"] == "<|endoftext|>" assert tokenizer["pad_token"] == "<|pad|>" assert tokenizer["additional_special_tokens"] == ADDITIONAL_SPECIAL_TOKENS special = special_tokens_map() assert special["additional_special_tokens"] == ADDITIONAL_SPECIAL_TOKENS generation = generation_config() assert generation["eos_token_id"] == config["eos_token_id"] assert generation["pad_token_id"] == config["pad_token_id"] mtp = mtp_config(args, {"step": 123}) assert mtp["trained_steps"] == 123 assert mtp["recursive"] is True assert "does not guarantee close distributions" in mtp["note"] validation_report = { "publication_ready": True, "summary": { "main_loss": {"mean": 4.2, "ci95": [4.1, 4.3]}, "main_perplexity": 66.69, "mtp_loss": [ {"mean": 4.4, "ci95": [4.3, 4.5]}, {"mean": 4.6, "ci95": [4.5, 4.7]}, ], }, } acceptance_report = { "schema_version": 1, "documents": 200, "trained_primary_endpoint": { "ratio": 1.1, "ci95": [1.02, 1.18], "verdict": "POSITIVE", }, "trained_minus_untrained_ratio": { "difference": 0.08, "ci95": [0.01, 0.15], "verdict": "CLEARS_CONTROL", }, "combined_interpretation": "POSITIVE_AND_CLEARS_CONTROL", } format_ablation_report = { "schema_version": 1, "publication_ready": True, "documents": 200, "baseline_revision_evidence": { "limitation": ( "Repository revisions were captured after the run 1 corpus " "build, so source-stream drift cannot be ruled out." ), }, "runtime_code_evidence": { "limitation": ( "Run 1 source hashes were not recorded, so exact source-state " "equivalence cannot be proven from its checkpoint." ), }, "primary_endpoint": { "difference_in_differences": 0.06, "ci95": [0.01, 0.11], "verdict": "POSITIVE", }, "secondary_endpoint": { "difference_in_differences": 0.03, "ci95": [-0.01, 0.07], "verdict": "NULL: the interval includes zero", }, } rollout_report = { "schema_version": 2, "instrument_version": 3, "publication_ready": True, "quality_gate": { "reference": "branch_local_full_sequence_replay", "rule": ( "every_emitted_token_equals_reference_argmax_or_is_a_" "certified_bf16_near_tie_on_the_same_realized_prefix" ), "near_tie_max_ulps": 8, "scored_policy_documents": 600, "scored_tokens": 38400, "exact_argmax_tokens": 38399, "certified_near_tie_tokens": 1, "failed_tokens": 0, "branch_replay_passes": 600, "cross_policy_trajectory_matches": 100, "passed": True, }, "primary_endpoint": { "comparison": "selected_adaptive_minus_selected_fixed", "metric": "accepted_drafts_per_verification", "difference": 0.2, "ci95": [0.1, 0.3], "verdict": "POSITIVE", "fixed_policy": "fixed_d2", "adaptive_policy": "adaptive_h0.5", "documents": 60, }, "secondary_target_forward_endpoint": { "comparison": "selected_adaptive_minus_selected_fixed", "metric": "output_tokens_per_target_forward", "adaptive_policy": "adaptive_h0.5", "fixed_policy": "fixed_d2", "difference": 0.04, "ci95": [0.01, 0.07], "documents": 60, "verdict": "POSITIVE", }, "secondary_draft_issued_proxy_endpoint": { "comparison": "selected_adaptive_minus_selected_fixed", "metric": "drafts_issued_per_output_token", "adaptive_policy": "adaptive_h0.5", "fixed_policy": "fixed_d2", "difference": -0.03, "ci95": [-0.05, -0.01], "documents": 60, "verdict": "NEGATIVE", }, "secondary_draft_work_endpoint": { "comparison": "selected_adaptive_minus_selected_fixed", "metric": "draft_recursions_per_output_token", "adaptive_policy": "adaptive_h0.5", "fixed_policy": "fixed_d2", "difference": -0.02, "ci95": [-0.04, 0.00], "documents": 60, "verdict": "NULL: the interval includes 0", }, "endpoint_scope_note": ( "The primary endpoint measures accepted drafts per verification. " "It does not establish verification-width cost or deployment " "latency. Draft recursions per output token is the registered " "drafter-work companion; issued drafts per output token is retained " "only as an issuance proxy. Target forwards exclude the added " "post-hoc branch-replay forward and independent verification pass." ), } evaluation = render_evaluation_section( validation_report, acceptance_report, format_ablation_report, rollout_report, ) assert "Final validation main NLL | 4.2000" in evaluation assert "POSITIVE_AND_CLEARS_CONTROL" in evaluation assert "FIM-training effect" in evaluation assert "source-stream drift cannot be ruled out" in evaluation assert "exact source-state equivalence cannot be proven" in evaluation assert "adaptive_h0.5 versus fixed_d2" in evaluation assert ( "Exact argmax on 38399 of 38400 emitted tokens; " "1 token certified as a bfloat16 near-tie within 8 ulps" ) in evaluation assert "Branch-local greedy replay" in evaluation assert "Cross-policy output identity" in evaluation assert "drafts issued per output token" in evaluation assert "draft recursions per output token" in evaluation assert "issuance proxy" in evaluation assert "verification-width cost" in evaluation assert "deployment latency" in evaluation assert "post-hoc branch-replay forward" in evaluation assert "not a signed external or trusted-execution witness" in evaluation assert "Greedy-output equivalence" not in evaluation all_exact = render_evaluation_section( validation_report, acceptance_report, format_ablation_report, { **rollout_report, "quality_gate": { **rollout_report["quality_gate"], "exact_argmax_tokens": 38400, "certified_near_tie_tokens": 0, }, }, ) assert "Exact argmax for all 38400 emitted tokens" in all_exact def _expect_evaluation_reject( bad_validation=None, bad_acceptance=None, bad_format_ablation=None, bad_rollout=None, expected_substring=None, ): try: render_evaluation_section( bad_validation if bad_validation is not None else validation_report, bad_acceptance if bad_acceptance is not None else acceptance_report, bad_format_ablation if bad_format_ablation is not None else format_ablation_report, bad_rollout if bad_rollout is not None else rollout_report, ) except ValueError as exc: assert expected_substring in str(exc), ( f"expected {expected_substring!r} in {exc}" ) else: raise AssertionError( f"an evaluation section that should have failed on " f"{expected_substring!r} was accepted" ) _expect_evaluation_reject( bad_validation={**validation_report, "publication_ready": False}, expected_substring="validation report is not publication-ready", ) _expect_evaluation_reject( bad_rollout={**rollout_report, "publication_ready": False}, expected_substring="rollout report is not publication-ready", ) _expect_evaluation_reject( bad_acceptance={**acceptance_report, "schema_version": 2}, expected_substring="acceptance comparison schema is not 1", ) _expect_evaluation_reject( bad_format_ablation={**format_ablation_report, "schema_version": 2}, expected_substring="format-ablation comparison schema is not 1", ) _expect_evaluation_reject( bad_format_ablation={ **format_ablation_report, "publication_ready": False, }, expected_substring="format-ablation comparison is not publication-ready", ) _expect_evaluation_reject( bad_validation={ **validation_report, "summary": { **validation_report["summary"], "mtp_loss": [{"mean": 4.4, "ci95": [4.3, 4.5]}], }, }, expected_substring="does not have two MTP losses", ) _expect_evaluation_reject( bad_acceptance={**acceptance_report, "documents": 1}, expected_substring="acceptance document count is invalid", ) _expect_evaluation_reject( bad_format_ablation={**format_ablation_report, "documents": 1}, expected_substring="format-ablation document count is invalid", ) _expect_evaluation_reject( bad_rollout={ **rollout_report, "quality_gate": { **rollout_report["quality_gate"], "passed": False, }, }, expected_substring="rollout branch-local replay did not pass", ) _expect_evaluation_reject( bad_rollout={ **rollout_report, "quality_gate": { **rollout_report["quality_gate"], "certified_near_tie_tokens": -1, }, }, expected_substring="invalid certified near tie tokens count", ) _expect_evaluation_reject( bad_rollout={ **rollout_report, "schema_version": 1, "instrument_version": 2, "quality_gate": { "reference": "greedy_ar", "rule": "exact_token_match_or_certified_near_tie", "near_tie_max_ulps": 8, "exact_ar_matches": 510, "certified_divergences": 90, "passed": True, }, }, expected_substring="requires rollout schema 2 instrument 3", ) _expect_evaluation_reject( bad_rollout={ **rollout_report, "quality_gate": { **rollout_report["quality_gate"], "exact_argmax_tokens": 38398, }, }, expected_substring="rollout branch-local replay did not pass", ) _expect_evaluation_reject( bad_rollout={ **rollout_report, "quality_gate": { **rollout_report["quality_gate"], "branch_replay_passes": 599, }, }, expected_substring="rollout branch-local replay did not pass", ) _expect_evaluation_reject( bad_rollout={ **rollout_report, "quality_gate": { **rollout_report["quality_gate"], "cross_policy_trajectory_matches": 99, }, }, expected_substring="rollout cross-policy output identity did not pass", ) _expect_evaluation_reject( bad_rollout={ **rollout_report, "primary_endpoint": { **rollout_report["primary_endpoint"], "documents": 1, }, }, expected_substring="rollout test document count is invalid", ) try: _metric(float("nan"), "a metric") except ValueError as exc: assert "is not finite" in str(exc) else: raise AssertionError("a NaN metric value was accepted") try: _interval([1.0], "an interval") except ValueError as exc: assert "is not a two-element interval" in str(exc) else: raise AssertionError("a one-element interval was accepted") try: _interval([2.0, 1.0], "an interval") except ValueError as exc: assert "is reversed" in str(exc) else: raise AssertionError("a reversed interval was accepted") try: _verdict("line one\nline two", "a verdict") except ValueError as exc: assert "not a safe single-line verdict" in str(exc) else: raise AssertionError("a multi-line verdict was accepted") try: _verdict("has | a pipe", "a verdict") except ValueError as exc: assert "not a safe single-line verdict" in str(exc) else: raise AssertionError("a verdict containing a table pipe was accepted") development = development_evaluation_section(100, 123) assert "step 100 of 123" in development assert "no final registered evaluation claims" in development with tempfile.NamedTemporaryFile("w", encoding="utf-8") as f: f.write("load {{REPO_ID}} here\n{{FINAL_EVALUATION}}\n") f.flush() rendered = render_model_card(f.name, "owner/model", evaluation) assert rendered.startswith("load owner/model here\n") assert "Final validation main NLL" in rendered assert "{{FINAL_EVALUATION}}" not in rendered try: render_model_card(f.name, "missing-slash", evaluation) except ValueError: pass else: raise AssertionError("invalid repository ID was accepted") with tempfile.NamedTemporaryFile("w", encoding="utf-8") as f: f.write("no repo id placeholder here\n{{FINAL_EVALUATION}}\n") f.flush() try: render_model_card(f.name, "owner/model", evaluation) except ValueError as exc: assert "must contain the placeholder {{REPO_ID}}" in str(exc) else: raise AssertionError( "a template missing the repo-id placeholder was accepted" ) with tempfile.NamedTemporaryFile("w", encoding="utf-8") as f: f.write("{{REPO_ID}}\nno evaluation placeholder here\n") f.flush() try: render_model_card(f.name, "owner/model", evaluation) except ValueError as exc: assert "must contain the placeholder {{FINAL_EVALUATION}}" in str(exc) else: raise AssertionError( "a template missing the evaluation placeholder was accepted" ) with tempfile.NamedTemporaryFile("w", encoding="utf-8") as f: f.write("{{REPO_ID}}\n{{FINAL_EVALUATION}}\n") f.flush() try: render_model_card(f.name, "owner/model", " ") except ValueError as exc: assert "evaluation text is empty" in str(exc) else: raise AssertionError("blank evaluation text was accepted") # A leaked literal placeholder inside the evaluation text survives # rendering unresolved: REPO_ID substitution runs on the raw template # before the evaluation text is spliced in, so a placeholder-looking # string arriving *inside* evaluation_markdown is never touched. leaking_evaluation = "See {{REPO_ID}} for details." try: render_model_card(f.name, "owner/model", leaking_evaluation) except ValueError as exc: assert "unresolved placeholder" in str(exc) else: raise AssertionError( "a literal placeholder leaking through evaluation text was " "accepted" ) with tempfile.NamedTemporaryFile("w+b") as f: f.write(b"frozen") f.flush() evidence = { "holdout": { "path": f.name, "sha256": file_sha256(f.name), } } require_unchanged_files(evidence, "test evaluation") f.write(b"mutation") f.flush() try: require_unchanged_files(evidence, "test evaluation") except RuntimeError as exc: assert "holdout changed during test evaluation" in str(exc) else: raise AssertionError("an input mutation passed snapshot validation") final_meta = { "step": 123, "config": { "max_steps": 123, "run_name": "wisp-test", }, "model_args": {"dim": 768}, "optimizer_state_included": True, } assert validate_export_checkpoint(final_meta)["complete"] is True unattested_final = { **final_meta, "optimizer_state_included": False, } try: validate_export_checkpoint(unattested_final) except ValueError as exc: assert "optimizer state was saved" in str(exc) else: raise AssertionError("an unattested final checkpoint was releasable") snapshot_meta = { **final_meta, "step": 100, } try: validate_export_checkpoint(snapshot_meta) except ValueError as exc: assert "not final step" in str(exc) else: raise AssertionError("an incomplete checkpoint was releasable by default") assert validate_export_checkpoint( snapshot_meta, allow_incomplete=True )["complete"] is False control_meta = { **final_meta, "config": { **final_meta["config"], "initialization_only": True, }, } try: validate_export_checkpoint(control_meta) except ValueError as exc: assert "initialization-only control" in str(exc) else: raise AssertionError("an untrained control was releasable") required = [ "LICENSE", "README.md", "config.json", "generation_config.json", "model.safetensors", "mtp.safetensors", "mtp_config.json", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json", ] with tempfile.TemporaryDirectory() as out_dir: for name in required: with open(os.path.join(out_dir, name), "wb") as f: content = ( b"owner/model" if name == "README.md" else name.encode() ) f.write(content) write_export_manifest( out_dir, final_meta, 100, 20, "owner/model", { "meta_sha256": "a" * 64, "master_sha256": "b" * 64, "optimizer_sha256": "c" * 64, }, True, { "validation": {"sha256": "d" * 64}, "acceptance_comparison": {"sha256": "e" * 64}, "format_ablation": {"sha256": "1" * 64}, "rollout": {"sha256": "f" * 64}, }, "0" * 64, ) with open(os.path.join(out_dir, "export_manifest.json")) as f: manifest = json.load(f) assert manifest["schema_version"] == 3 assert manifest["repo_id"] == "owner/model" assert manifest["release_complete"] is True assert manifest["evaluation_sources"]["validation"]["sha256"] == ( "d" * 64 ) assert manifest["model_card_template_sha256"] == "0" * 64 assert manifest["source_checkpoint"]["step"] == 123 assert manifest["source_checkpoint"]["meta_sha256"] == "a" * 64 assert manifest["source_checkpoint"]["master_sha256"] == "b" * 64 assert manifest["source_checkpoint"]["optimizer_sha256"] == "c" * 64 assert manifest["trunk_parameters"] == 100 assert manifest["mtp_parameters_excluding_shared_embedding_and_head"] == 20 assert sorted(manifest["files"]) == required assert all(len(item["sha256"]) == 64 for item in manifest["files"].values()) verify_export_manifest(out_dir) manifest_path = os.path.join(out_dir, "export_manifest.json") invalid_manifest = json.loads(json.dumps(manifest)) invalid_manifest["evaluation_sources"] = None with open(manifest_path, "w", encoding="utf-8") as f: json.dump(invalid_manifest, f, indent=2, sort_keys=True) f.write("\n") try: verify_export_manifest(out_dir) except ValueError as exc: assert "evaluation sources" in str(exc) else: raise AssertionError( "a final export without evaluation sources passed verification" ) with open(manifest_path, "w", encoding="utf-8") as f: json.dump(manifest, f, indent=2, sort_keys=True) f.write("\n") manifest_sha256 = file_sha256( manifest_path ) receipt = { "schema_version": 1, "created_at": "2026-07-25T23:00:00+00:00", "status": "verified", "passed": True, "package": { "export_dir": os.path.abspath(out_dir), "repo_id": "owner/model", "export_manifest_sha256": manifest_sha256, "source_checkpoint": manifest["source_checkpoint"], }, "checkpoint": { "path": os.path.abspath("/tmp/checkpoint"), "meta_sha256": "a" * 64, "master_sha256": "b" * 64, "optimizer_sha256": "c" * 64, }, "probe": { "seed": 0, "length": 24, "token_ids_sha256": "d" * 64, }, "tokenizer": { "special_token_ids": list(range(7)), "eos_token_id": 0, "pad_token_id": 1, "byte_roundtrip_exact": True, "probe_sha256": "e" * 64, }, "logits": { "shape": [24, 32768], "max_abs_delta": 0.02, "logit_scale": 10.0, "relative_max_abs_delta": 0.002, "relative_delta_threshold": 0.01, "argmax_agreement": 1.0, }, "toolchain": { "python": "3.12.0", "platform": "macOS-26-arm64", "mlx": "0.29.0", "torch": "2.9.0", "transformers": "4.57.0", }, } validate_external_verification_receipt( receipt, manifest, manifest_sha256 ) receipt_path = os.path.join(out_dir, "external-receipt.json") write_json_atomic(receipt_path, receipt) try: write_json_atomic(receipt_path, receipt) except FileExistsError as exc: assert "refusing to replace JSON artifact" in str(exc) else: raise AssertionError("an external receipt was silently overwritten") # The TOCTOU guard (path appears between staging and rename) is # distinct from the refuse-to-overwrite guard just exercised above -- # neutralizing either one alone still raises FileExistsError from the # other, so a bare `except FileExistsError` can't tell them apart. # Simulated by making os.path.lexists lie: absent on the first check, # present on the second, since there's no real concurrency here. real_lexists = os.path.lexists call_count = [0] racing_path = os.path.join(out_dir, "racing.json") def racing_lexists(path): call_count[0] += 1 if call_count[0] == 1: return False if path == os.path.abspath(racing_path): return True return real_lexists(path) try: os.path.lexists = racing_lexists try: write_json_atomic(racing_path, {"k": "v"}) except FileExistsError as exc: assert "JSON target appeared during staging" in str(exc) else: raise AssertionError( "a JSON target appearing mid-staging was not caught" ) finally: os.path.lexists = real_lexists assert not real_lexists(racing_path), ( "the racing rename should never have actually happened" ) os.unlink(receipt_path) mutated_receipt = json.loads(json.dumps(receipt)) mutated_receipt["checkpoint"]["master_sha256"] = "f" * 64 try: validate_external_verification_receipt( mutated_receipt, manifest, manifest_sha256 ) except ValueError as exc: assert "checkpoint hashes" in str(exc) else: raise AssertionError("a receipt for different weights was accepted") mutated_receipt = json.loads(json.dumps(receipt)) mutated_receipt["logits"]["argmax_agreement"] = 0.99 try: validate_external_verification_receipt( mutated_receipt, manifest, manifest_sha256 ) except ValueError as exc: assert "argmax agreement" in str(exc) else: raise AssertionError("a nonexact argmax comparison was accepted") mutated_receipt = json.loads(json.dumps(receipt)) mutated_receipt["logits"]["relative_max_abs_delta"] = -1.0 try: validate_external_verification_receipt( mutated_receipt, manifest, manifest_sha256 ) except ValueError as exc: assert "relative logit delta" in str(exc) else: raise AssertionError("a negative relative delta was accepted") config_path = os.path.join(out_dir, "config.json") with open(config_path, "ab") as f: f.write(b"mutation") try: verify_export_manifest(out_dir) except ValueError as exc: assert "does not match" in str(exc) else: raise AssertionError("a mutated export payload passed verification") with tempfile.TemporaryDirectory() as out_dir: for name in required: with open(os.path.join(out_dir, name), "wb") as f: content = ( b"owner/model" if name == "README.md" else name.encode() ) f.write(content) write_export_manifest( out_dir, final_meta, 100, 20, "owner/model", { "meta_sha256": "a" * 64, "master_sha256": "b" * 64, "optimizer_sha256": "c" * 64, }, True, { "validation": {"sha256": "d" * 64}, "acceptance_comparison": {"sha256": "e" * 64}, "format_ablation": {"sha256": "1" * 64}, "rollout": {"sha256": "f" * 64}, }, "0" * 64, ) os.makedirs(os.path.join(out_dir, "unexpected")) try: verify_export_manifest(out_dir) except ValueError as exc: assert "unexpected or missing" in str(exc) else: raise AssertionError("an extra export directory passed verification") with tempfile.TemporaryDirectory() as out_dir: for name in required[:-1]: with open(os.path.join(out_dir, name), "wb") as f: f.write(name.encode()) try: write_export_manifest( out_dir, final_meta, 100, 20, "owner/model", { "meta_sha256": "a" * 64, "master_sha256": "b" * 64, "optimizer_sha256": "c" * 64, }, True, { "validation": {"sha256": "d" * 64}, "acceptance_comparison": {"sha256": "e" * 64}, "format_ablation": {"sha256": "1" * 64}, "rollout": {"sha256": "f" * 64}, }, "0" * 64, ) except FileNotFoundError as exc: assert "release artifact is missing" in str(exc) else: raise AssertionError( "an export directory missing a required file was accepted" ) def build_manifest_dict(): return { "schema_version": 3, "repo_id": "owner/model", "release_complete": True, "evaluation_sources": { "validation": {"sha256": "d" * 64}, "acceptance_comparison": {"sha256": "e" * 64}, "format_ablation": {"sha256": "1" * 64}, "rollout": {"sha256": "f" * 64}, }, "model_card_template_sha256": "0" * 64, "source_checkpoint": { "step": 123, "meta_sha256": "a" * 64, "master_sha256": "b" * 64, "optimizer_sha256": "c" * 64, }, "trunk_parameters": 100, "mtp_parameters_excluding_shared_embedding_and_head": 20, "files": {}, } def build_export_dir(mutate=None): out_dir = tempfile.mkdtemp() for name in required: path = os.path.join(out_dir, name) content = b"owner/model" if name == "README.md" else name.encode() with open(path, "wb") as f: f.write(content) manifest = build_manifest_dict() manifest["files"] = { name: { "bytes": os.path.getsize(os.path.join(out_dir, name)), "sha256": file_sha256(os.path.join(out_dir, name)), } for name in required } if mutate is not None: mutate(manifest) with open( os.path.join(out_dir, "export_manifest.json"), "w", encoding="utf-8" ) as f: json.dump(manifest, f, indent=2, sort_keys=True) f.write("\n") return out_dir def _expect_manifest_reject(mutate, expected_substring): out_dir = build_export_dir(mutate) try: verify_export_manifest(out_dir) except ValueError as exc: assert expected_substring in str(exc), ( f"expected {expected_substring!r} in {exc}" ) else: raise AssertionError( f"a manifest that should have failed on " f"{expected_substring!r} passed verification" ) _expect_manifest_reject( lambda m: m.__setitem__("schema_version", 2), "must use schema_version 3", ) _expect_manifest_reject( lambda m: m.__setitem__("release_complete", "yes"), "release_complete is not boolean", ) _expect_manifest_reject( lambda m: m.__setitem__("model_card_template_sha256", "not-hex"), "model card template hash is not a SHA-256", ) _expect_manifest_reject( lambda m: ( m.__setitem__("release_complete", False), m.__setitem__( "evaluation_sources", {"validation": {"sha256": "d" * 64}} ), ), "development export must not declare final evaluation sources", ) _expect_manifest_reject( lambda m: m.__setitem__("files", "not-a-dict"), "export manifest has no files object", ) _expect_manifest_reject( lambda m: m["files"].pop("LICENSE"), "export manifest payload differs from required files", ) _expect_manifest_reject( lambda m: m.__setitem__("source_checkpoint", "not-a-dict"), "export manifest has no source_checkpoint", ) _expect_manifest_reject( lambda m: m["source_checkpoint"].__setitem__("step", -1), "source_checkpoint.step is not a positive integer", ) _expect_manifest_reject( lambda m: m["source_checkpoint"].__setitem__("meta_sha256", "not-hex"), "source_checkpoint.meta_sha256 is not a SHA-256", ) # README.md must actually contain the manifest's repo_id string. The # README's hash has to match what the manifest recorded (a hash mismatch # would trip the "does not match export manifest" check first) but its # *content* must lack the repo_id, so build a directory whose README # never contained it in the first place. out_dir = tempfile.mkdtemp() for name in required: path = os.path.join(out_dir, name) content = ( b"a different repo entirely, no repo id here" if name == "README.md" else name.encode() ) with open(path, "wb") as f: f.write(content) manifest = build_manifest_dict() manifest["files"] = { name: { "bytes": os.path.getsize(os.path.join(out_dir, name)), "sha256": file_sha256(os.path.join(out_dir, name)), } for name in required } with open( os.path.join(out_dir, "export_manifest.json"), "w", encoding="utf-8" ) as f: json.dump(manifest, f, indent=2, sort_keys=True) f.write("\n") try: verify_export_manifest(out_dir) except ValueError as exc: assert "does not match export manifest repo_id" in str(exc) else: raise AssertionError( "a rendered model card missing the repo id was accepted" ) print("Hugging Face export metadata: PASS") if __name__ == "__main__": main()