evidence/source/e3_rollout_v3/scripts/test_export_metadata.py

"""CPU checks for the Hugging Face package metadata."""

import json
import os
import sys
import tempfile
from types import SimpleNamespace

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from scripts.hf_metadata import (  # noqa: E402
    ADDITIONAL_SPECIAL_TOKENS,
    _interval,
    _metric,
    _verdict,
    development_evaluation_section,
    file_sha256,
    generation_config,
    llama_config,
    mtp_config,
    require_unchanged_files,
    render_evaluation_section,
    render_model_card,
    special_tokens_map,
    tokenizer_config,
    validate_external_verification_receipt,
    validate_export_checkpoint,
    verify_export_manifest,
    write_json_atomic,
    write_export_manifest,
)


def main():
    args = SimpleNamespace(
        vocab_size=32768,
        dim=768,
        n_layers=12,
        n_heads=12,
        n_kv_heads=4,
        head_dim=64,
        ffn_hidden=2048,
        max_seq_len=2048,
        rope_theta=100000.0,
        norm_eps=1e-5,
        tie_embeddings=True,
        mtp_layers=1,
        mtp_depth=2,
    )
    config = llama_config(args)
    assert config["architectures"] == ["LlamaForCausalLM"]
    assert config["vocab_size"] == 32768
    assert config["eos_token_id"] == 0
    assert config["pad_token_id"] == 1
    assert config["bos_token_id"] is None

    tokenizer = tokenizer_config(args)
    assert tokenizer["tokenizer_class"] == "PreTrainedTokenizerFast"
    assert tokenizer["model_max_length"] == args.max_seq_len
    assert tokenizer["eos_token"] == "<|endoftext|>"
    assert tokenizer["pad_token"] == "<|pad|>"
    assert tokenizer["additional_special_tokens"] == ADDITIONAL_SPECIAL_TOKENS

    special = special_tokens_map()
    assert special["additional_special_tokens"] == ADDITIONAL_SPECIAL_TOKENS
    generation = generation_config()
    assert generation["eos_token_id"] == config["eos_token_id"]
    assert generation["pad_token_id"] == config["pad_token_id"]

    mtp = mtp_config(args, {"step": 123})
    assert mtp["trained_steps"] == 123
    assert mtp["recursive"] is True
    assert "does not guarantee close distributions" in mtp["note"]

    validation_report = {
        "publication_ready": True,
        "summary": {
            "main_loss": {"mean": 4.2, "ci95": [4.1, 4.3]},
            "main_perplexity": 66.69,
            "mtp_loss": [
                {"mean": 4.4, "ci95": [4.3, 4.5]},
                {"mean": 4.6, "ci95": [4.5, 4.7]},
            ],
        },
    }
    acceptance_report = {
        "schema_version": 1,
        "documents": 200,
        "trained_primary_endpoint": {
            "ratio": 1.1,
            "ci95": [1.02, 1.18],
            "verdict": "POSITIVE",
        },
        "trained_minus_untrained_ratio": {
            "difference": 0.08,
            "ci95": [0.01, 0.15],
            "verdict": "CLEARS_CONTROL",
        },
        "combined_interpretation": "POSITIVE_AND_CLEARS_CONTROL",
    }
    format_ablation_report = {
        "schema_version": 1,
        "publication_ready": True,
        "documents": 200,
        "baseline_revision_evidence": {
            "limitation": (
                "Repository revisions were captured after the run 1 corpus "
                "build, so source-stream drift cannot be ruled out."
            ),
        },
        "runtime_code_evidence": {
            "limitation": (
                "Run 1 source hashes were not recorded, so exact source-state "
                "equivalence cannot be proven from its checkpoint."
            ),
        },
        "primary_endpoint": {
            "difference_in_differences": 0.06,
            "ci95": [0.01, 0.11],
            "verdict": "POSITIVE",
        },
        "secondary_endpoint": {
            "difference_in_differences": 0.03,
            "ci95": [-0.01, 0.07],
            "verdict": "NULL: the interval includes zero",
        },
    }
    rollout_report = {
        "schema_version": 2,
        "instrument_version": 3,
        "publication_ready": True,
        "quality_gate": {
            "reference": "branch_local_full_sequence_replay",
            "rule": (
                "every_emitted_token_equals_reference_argmax_or_is_a_"
                "certified_bf16_near_tie_on_the_same_realized_prefix"
            ),
            "near_tie_max_ulps": 8,
            "scored_policy_documents": 600,
            "scored_tokens": 38400,
            "exact_argmax_tokens": 38399,
            "certified_near_tie_tokens": 1,
            "failed_tokens": 0,
            "branch_replay_passes": 600,
            "cross_policy_trajectory_matches": 100,
            "passed": True,
        },
        "primary_endpoint": {
            "comparison": "selected_adaptive_minus_selected_fixed",
            "metric": "accepted_drafts_per_verification",
            "difference": 0.2,
            "ci95": [0.1, 0.3],
            "verdict": "POSITIVE",
            "fixed_policy": "fixed_d2",
            "adaptive_policy": "adaptive_h0.5",
            "documents": 60,
        },
        "secondary_target_forward_endpoint": {
            "comparison": "selected_adaptive_minus_selected_fixed",
            "metric": "output_tokens_per_target_forward",
            "adaptive_policy": "adaptive_h0.5",
            "fixed_policy": "fixed_d2",
            "difference": 0.04,
            "ci95": [0.01, 0.07],
            "documents": 60,
            "verdict": "POSITIVE",
        },
        "secondary_draft_issued_proxy_endpoint": {
            "comparison": "selected_adaptive_minus_selected_fixed",
            "metric": "drafts_issued_per_output_token",
            "adaptive_policy": "adaptive_h0.5",
            "fixed_policy": "fixed_d2",
            "difference": -0.03,
            "ci95": [-0.05, -0.01],
            "documents": 60,
            "verdict": "NEGATIVE",
        },
        "secondary_draft_work_endpoint": {
            "comparison": "selected_adaptive_minus_selected_fixed",
            "metric": "draft_recursions_per_output_token",
            "adaptive_policy": "adaptive_h0.5",
            "fixed_policy": "fixed_d2",
            "difference": -0.02,
            "ci95": [-0.04, 0.00],
            "documents": 60,
            "verdict": "NULL: the interval includes 0",
        },
        "endpoint_scope_note": (
            "The primary endpoint measures accepted drafts per verification. "
            "It does not establish verification-width cost or deployment "
            "latency. Draft recursions per output token is the registered "
            "drafter-work companion; issued drafts per output token is retained "
            "only as an issuance proxy. Target forwards exclude the added "
            "post-hoc branch-replay forward and independent verification pass."
        ),
    }
    evaluation = render_evaluation_section(
        validation_report,
        acceptance_report,
        format_ablation_report,
        rollout_report,
    )
    assert "Final validation main NLL | 4.2000" in evaluation
    assert "POSITIVE_AND_CLEARS_CONTROL" in evaluation
    assert "FIM-training effect" in evaluation
    assert "source-stream drift cannot be ruled out" in evaluation
    assert "exact source-state equivalence cannot be proven" in evaluation
    assert "adaptive_h0.5 versus fixed_d2" in evaluation
    assert (
        "Exact argmax on 38399 of 38400 emitted tokens; "
        "1 token certified as a bfloat16 near-tie within 8 ulps"
    ) in evaluation
    assert "Branch-local greedy replay" in evaluation
    assert "Cross-policy output identity" in evaluation
    assert "drafts issued per output token" in evaluation
    assert "draft recursions per output token" in evaluation
    assert "issuance proxy" in evaluation
    assert "verification-width cost" in evaluation
    assert "deployment latency" in evaluation
    assert "post-hoc branch-replay forward" in evaluation
    assert "not a signed external or trusted-execution witness" in evaluation
    assert "Greedy-output equivalence" not in evaluation

    all_exact = render_evaluation_section(
        validation_report,
        acceptance_report,
        format_ablation_report,
        {
            **rollout_report,
            "quality_gate": {
                **rollout_report["quality_gate"],
                "exact_argmax_tokens": 38400,
                "certified_near_tie_tokens": 0,
            },
        },
    )
    assert "Exact argmax for all 38400 emitted tokens" in all_exact

    def _expect_evaluation_reject(
        bad_validation=None,
        bad_acceptance=None,
        bad_format_ablation=None,
        bad_rollout=None,
        expected_substring=None,
    ):
        try:
            render_evaluation_section(
                bad_validation if bad_validation is not None else validation_report,
                bad_acceptance if bad_acceptance is not None else acceptance_report,
                bad_format_ablation
                if bad_format_ablation is not None
                else format_ablation_report,
                bad_rollout if bad_rollout is not None else rollout_report,
            )
        except ValueError as exc:
            assert expected_substring in str(exc), (
                f"expected {expected_substring!r} in {exc}"
            )
        else:
            raise AssertionError(
                f"an evaluation section that should have failed on "
                f"{expected_substring!r} was accepted"
            )

    _expect_evaluation_reject(
        bad_validation={**validation_report, "publication_ready": False},
        expected_substring="validation report is not publication-ready",
    )
    _expect_evaluation_reject(
        bad_rollout={**rollout_report, "publication_ready": False},
        expected_substring="rollout report is not publication-ready",
    )
    _expect_evaluation_reject(
        bad_acceptance={**acceptance_report, "schema_version": 2},
        expected_substring="acceptance comparison schema is not 1",
    )
    _expect_evaluation_reject(
        bad_format_ablation={**format_ablation_report, "schema_version": 2},
        expected_substring="format-ablation comparison schema is not 1",
    )
    _expect_evaluation_reject(
        bad_format_ablation={
            **format_ablation_report,
            "publication_ready": False,
        },
        expected_substring="format-ablation comparison is not publication-ready",
    )
    _expect_evaluation_reject(
        bad_validation={
            **validation_report,
            "summary": {
                **validation_report["summary"],
                "mtp_loss": [{"mean": 4.4, "ci95": [4.3, 4.5]}],
            },
        },
        expected_substring="does not have two MTP losses",
    )
    _expect_evaluation_reject(
        bad_acceptance={**acceptance_report, "documents": 1},
        expected_substring="acceptance document count is invalid",
    )
    _expect_evaluation_reject(
        bad_format_ablation={**format_ablation_report, "documents": 1},
        expected_substring="format-ablation document count is invalid",
    )
    _expect_evaluation_reject(
        bad_rollout={
            **rollout_report,
            "quality_gate": {
                **rollout_report["quality_gate"],
                "passed": False,
            },
        },
        expected_substring="rollout branch-local replay did not pass",
    )
    _expect_evaluation_reject(
        bad_rollout={
            **rollout_report,
            "quality_gate": {
                **rollout_report["quality_gate"],
                "certified_near_tie_tokens": -1,
            },
        },
        expected_substring="invalid certified near tie tokens count",
    )
    _expect_evaluation_reject(
        bad_rollout={
            **rollout_report,
            "schema_version": 1,
            "instrument_version": 2,
            "quality_gate": {
                "reference": "greedy_ar",
                "rule": "exact_token_match_or_certified_near_tie",
                "near_tie_max_ulps": 8,
                "exact_ar_matches": 510,
                "certified_divergences": 90,
                "passed": True,
            },
        },
        expected_substring="requires rollout schema 2 instrument 3",
    )
    _expect_evaluation_reject(
        bad_rollout={
            **rollout_report,
            "quality_gate": {
                **rollout_report["quality_gate"],
                "exact_argmax_tokens": 38398,
            },
        },
        expected_substring="rollout branch-local replay did not pass",
    )
    _expect_evaluation_reject(
        bad_rollout={
            **rollout_report,
            "quality_gate": {
                **rollout_report["quality_gate"],
                "branch_replay_passes": 599,
            },
        },
        expected_substring="rollout branch-local replay did not pass",
    )
    _expect_evaluation_reject(
        bad_rollout={
            **rollout_report,
            "quality_gate": {
                **rollout_report["quality_gate"],
                "cross_policy_trajectory_matches": 99,
            },
        },
        expected_substring="rollout cross-policy output identity did not pass",
    )
    _expect_evaluation_reject(
        bad_rollout={
            **rollout_report,
            "primary_endpoint": {
                **rollout_report["primary_endpoint"],
                "documents": 1,
            },
        },
        expected_substring="rollout test document count is invalid",
    )

    try:
        _metric(float("nan"), "a metric")
    except ValueError as exc:
        assert "is not finite" in str(exc)
    else:
        raise AssertionError("a NaN metric value was accepted")

    try:
        _interval([1.0], "an interval")
    except ValueError as exc:
        assert "is not a two-element interval" in str(exc)
    else:
        raise AssertionError("a one-element interval was accepted")

    try:
        _interval([2.0, 1.0], "an interval")
    except ValueError as exc:
        assert "is reversed" in str(exc)
    else:
        raise AssertionError("a reversed interval was accepted")

    try:
        _verdict("line one\nline two", "a verdict")
    except ValueError as exc:
        assert "not a safe single-line verdict" in str(exc)
    else:
        raise AssertionError("a multi-line verdict was accepted")

    try:
        _verdict("has | a pipe", "a verdict")
    except ValueError as exc:
        assert "not a safe single-line verdict" in str(exc)
    else:
        raise AssertionError("a verdict containing a table pipe was accepted")

    development = development_evaluation_section(100, 123)
    assert "step 100 of 123" in development
    assert "no final registered evaluation claims" in development

    with tempfile.NamedTemporaryFile("w", encoding="utf-8") as f:
        f.write("load {{REPO_ID}} here\n{{FINAL_EVALUATION}}\n")
        f.flush()
        rendered = render_model_card(f.name, "owner/model", evaluation)
        assert rendered.startswith("load owner/model here\n")
        assert "Final validation main NLL" in rendered
        assert "{{FINAL_EVALUATION}}" not in rendered
        try:
            render_model_card(f.name, "missing-slash", evaluation)
        except ValueError:
            pass
        else:
            raise AssertionError("invalid repository ID was accepted")

    with tempfile.NamedTemporaryFile("w", encoding="utf-8") as f:
        f.write("no repo id placeholder here\n{{FINAL_EVALUATION}}\n")
        f.flush()
        try:
            render_model_card(f.name, "owner/model", evaluation)
        except ValueError as exc:
            assert "must contain the placeholder {{REPO_ID}}" in str(exc)
        else:
            raise AssertionError(
                "a template missing the repo-id placeholder was accepted"
            )

    with tempfile.NamedTemporaryFile("w", encoding="utf-8") as f:
        f.write("{{REPO_ID}}\nno evaluation placeholder here\n")
        f.flush()
        try:
            render_model_card(f.name, "owner/model", evaluation)
        except ValueError as exc:
            assert "must contain the placeholder {{FINAL_EVALUATION}}" in str(exc)
        else:
            raise AssertionError(
                "a template missing the evaluation placeholder was accepted"
            )

    with tempfile.NamedTemporaryFile("w", encoding="utf-8") as f:
        f.write("{{REPO_ID}}\n{{FINAL_EVALUATION}}\n")
        f.flush()
        try:
            render_model_card(f.name, "owner/model", "   ")
        except ValueError as exc:
            assert "evaluation text is empty" in str(exc)
        else:
            raise AssertionError("blank evaluation text was accepted")

        # A leaked literal placeholder inside the evaluation text survives
        # rendering unresolved: REPO_ID substitution runs on the raw template
        # before the evaluation text is spliced in, so a placeholder-looking
        # string arriving *inside* evaluation_markdown is never touched.
        leaking_evaluation = "See {{REPO_ID}} for details."
        try:
            render_model_card(f.name, "owner/model", leaking_evaluation)
        except ValueError as exc:
            assert "unresolved placeholder" in str(exc)
        else:
            raise AssertionError(
                "a literal placeholder leaking through evaluation text was "
                "accepted"
            )

    with tempfile.NamedTemporaryFile("w+b") as f:
        f.write(b"frozen")
        f.flush()
        evidence = {
            "holdout": {
                "path": f.name,
                "sha256": file_sha256(f.name),
            }
        }
        require_unchanged_files(evidence, "test evaluation")
        f.write(b"mutation")
        f.flush()
        try:
            require_unchanged_files(evidence, "test evaluation")
        except RuntimeError as exc:
            assert "holdout changed during test evaluation" in str(exc)
        else:
            raise AssertionError("an input mutation passed snapshot validation")

    final_meta = {
        "step": 123,
        "config": {
            "max_steps": 123,
            "run_name": "wisp-test",
        },
        "model_args": {"dim": 768},
        "optimizer_state_included": True,
    }
    assert validate_export_checkpoint(final_meta)["complete"] is True
    unattested_final = {
        **final_meta,
        "optimizer_state_included": False,
    }
    try:
        validate_export_checkpoint(unattested_final)
    except ValueError as exc:
        assert "optimizer state was saved" in str(exc)
    else:
        raise AssertionError("an unattested final checkpoint was releasable")
    snapshot_meta = {
        **final_meta,
        "step": 100,
    }
    try:
        validate_export_checkpoint(snapshot_meta)
    except ValueError as exc:
        assert "not final step" in str(exc)
    else:
        raise AssertionError("an incomplete checkpoint was releasable by default")
    assert validate_export_checkpoint(
        snapshot_meta, allow_incomplete=True
    )["complete"] is False
    control_meta = {
        **final_meta,
        "config": {
            **final_meta["config"],
            "initialization_only": True,
        },
    }
    try:
        validate_export_checkpoint(control_meta)
    except ValueError as exc:
        assert "initialization-only control" in str(exc)
    else:
        raise AssertionError("an untrained control was releasable")

    required = [
        "LICENSE",
        "README.md",
        "config.json",
        "generation_config.json",
        "model.safetensors",
        "mtp.safetensors",
        "mtp_config.json",
        "special_tokens_map.json",
        "tokenizer.json",
        "tokenizer_config.json",
    ]
    with tempfile.TemporaryDirectory() as out_dir:
        for name in required:
            with open(os.path.join(out_dir, name), "wb") as f:
                content = (
                    b"owner/model" if name == "README.md" else name.encode()
                )
                f.write(content)
        write_export_manifest(
            out_dir,
            final_meta,
            100,
            20,
            "owner/model",
            {
                "meta_sha256": "a" * 64,
                "master_sha256": "b" * 64,
                "optimizer_sha256": "c" * 64,
            },
            True,
            {
                "validation": {"sha256": "d" * 64},
                "acceptance_comparison": {"sha256": "e" * 64},
                "format_ablation": {"sha256": "1" * 64},
                "rollout": {"sha256": "f" * 64},
            },
            "0" * 64,
        )
        with open(os.path.join(out_dir, "export_manifest.json")) as f:
            manifest = json.load(f)
        assert manifest["schema_version"] == 3
        assert manifest["repo_id"] == "owner/model"
        assert manifest["release_complete"] is True
        assert manifest["evaluation_sources"]["validation"]["sha256"] == (
            "d" * 64
        )
        assert manifest["model_card_template_sha256"] == "0" * 64
        assert manifest["source_checkpoint"]["step"] == 123
        assert manifest["source_checkpoint"]["meta_sha256"] == "a" * 64
        assert manifest["source_checkpoint"]["master_sha256"] == "b" * 64
        assert manifest["source_checkpoint"]["optimizer_sha256"] == "c" * 64
        assert manifest["trunk_parameters"] == 100
        assert manifest["mtp_parameters_excluding_shared_embedding_and_head"] == 20
        assert sorted(manifest["files"]) == required
        assert all(len(item["sha256"]) == 64 for item in manifest["files"].values())
        verify_export_manifest(out_dir)

        manifest_path = os.path.join(out_dir, "export_manifest.json")
        invalid_manifest = json.loads(json.dumps(manifest))
        invalid_manifest["evaluation_sources"] = None
        with open(manifest_path, "w", encoding="utf-8") as f:
            json.dump(invalid_manifest, f, indent=2, sort_keys=True)
            f.write("\n")
        try:
            verify_export_manifest(out_dir)
        except ValueError as exc:
            assert "evaluation sources" in str(exc)
        else:
            raise AssertionError(
                "a final export without evaluation sources passed verification"
            )
        with open(manifest_path, "w", encoding="utf-8") as f:
            json.dump(manifest, f, indent=2, sort_keys=True)
            f.write("\n")

        manifest_sha256 = file_sha256(
            manifest_path
        )
        receipt = {
            "schema_version": 1,
            "created_at": "2026-07-25T23:00:00+00:00",
            "status": "verified",
            "passed": True,
            "package": {
                "export_dir": os.path.abspath(out_dir),
                "repo_id": "owner/model",
                "export_manifest_sha256": manifest_sha256,
                "source_checkpoint": manifest["source_checkpoint"],
            },
            "checkpoint": {
                "path": os.path.abspath("/tmp/checkpoint"),
                "meta_sha256": "a" * 64,
                "master_sha256": "b" * 64,
                "optimizer_sha256": "c" * 64,
            },
            "probe": {
                "seed": 0,
                "length": 24,
                "token_ids_sha256": "d" * 64,
            },
            "tokenizer": {
                "special_token_ids": list(range(7)),
                "eos_token_id": 0,
                "pad_token_id": 1,
                "byte_roundtrip_exact": True,
                "probe_sha256": "e" * 64,
            },
            "logits": {
                "shape": [24, 32768],
                "max_abs_delta": 0.02,
                "logit_scale": 10.0,
                "relative_max_abs_delta": 0.002,
                "relative_delta_threshold": 0.01,
                "argmax_agreement": 1.0,
            },
            "toolchain": {
                "python": "3.12.0",
                "platform": "macOS-26-arm64",
                "mlx": "0.29.0",
                "torch": "2.9.0",
                "transformers": "4.57.0",
            },
        }
        validate_external_verification_receipt(
            receipt, manifest, manifest_sha256
        )
        receipt_path = os.path.join(out_dir, "external-receipt.json")
        write_json_atomic(receipt_path, receipt)
        try:
            write_json_atomic(receipt_path, receipt)
        except FileExistsError as exc:
            assert "refusing to replace JSON artifact" in str(exc)
        else:
            raise AssertionError("an external receipt was silently overwritten")

        # The TOCTOU guard (path appears between staging and rename) is
        # distinct from the refuse-to-overwrite guard just exercised above --
        # neutralizing either one alone still raises FileExistsError from the
        # other, so a bare `except FileExistsError` can't tell them apart.
        # Simulated by making os.path.lexists lie: absent on the first check,
        # present on the second, since there's no real concurrency here.
        real_lexists = os.path.lexists
        call_count = [0]
        racing_path = os.path.join(out_dir, "racing.json")

        def racing_lexists(path):
            call_count[0] += 1
            if call_count[0] == 1:
                return False
            if path == os.path.abspath(racing_path):
                return True
            return real_lexists(path)

        try:
            os.path.lexists = racing_lexists
            try:
                write_json_atomic(racing_path, {"k": "v"})
            except FileExistsError as exc:
                assert "JSON target appeared during staging" in str(exc)
            else:
                raise AssertionError(
                    "a JSON target appearing mid-staging was not caught"
                )
        finally:
            os.path.lexists = real_lexists
        assert not real_lexists(racing_path), (
            "the racing rename should never have actually happened"
        )

        os.unlink(receipt_path)

        mutated_receipt = json.loads(json.dumps(receipt))
        mutated_receipt["checkpoint"]["master_sha256"] = "f" * 64
        try:
            validate_external_verification_receipt(
                mutated_receipt, manifest, manifest_sha256
            )
        except ValueError as exc:
            assert "checkpoint hashes" in str(exc)
        else:
            raise AssertionError("a receipt for different weights was accepted")

        mutated_receipt = json.loads(json.dumps(receipt))
        mutated_receipt["logits"]["argmax_agreement"] = 0.99
        try:
            validate_external_verification_receipt(
                mutated_receipt, manifest, manifest_sha256
            )
        except ValueError as exc:
            assert "argmax agreement" in str(exc)
        else:
            raise AssertionError("a nonexact argmax comparison was accepted")

        mutated_receipt = json.loads(json.dumps(receipt))
        mutated_receipt["logits"]["relative_max_abs_delta"] = -1.0
        try:
            validate_external_verification_receipt(
                mutated_receipt, manifest, manifest_sha256
            )
        except ValueError as exc:
            assert "relative logit delta" in str(exc)
        else:
            raise AssertionError("a negative relative delta was accepted")

        config_path = os.path.join(out_dir, "config.json")
        with open(config_path, "ab") as f:
            f.write(b"mutation")
        try:
            verify_export_manifest(out_dir)
        except ValueError as exc:
            assert "does not match" in str(exc)
        else:
            raise AssertionError("a mutated export payload passed verification")

    with tempfile.TemporaryDirectory() as out_dir:
        for name in required:
            with open(os.path.join(out_dir, name), "wb") as f:
                content = (
                    b"owner/model" if name == "README.md" else name.encode()
                )
                f.write(content)
        write_export_manifest(
            out_dir,
            final_meta,
            100,
            20,
            "owner/model",
            {
                "meta_sha256": "a" * 64,
                "master_sha256": "b" * 64,
                "optimizer_sha256": "c" * 64,
            },
            True,
            {
                "validation": {"sha256": "d" * 64},
                "acceptance_comparison": {"sha256": "e" * 64},
                "format_ablation": {"sha256": "1" * 64},
                "rollout": {"sha256": "f" * 64},
            },
            "0" * 64,
        )
        os.makedirs(os.path.join(out_dir, "unexpected"))
        try:
            verify_export_manifest(out_dir)
        except ValueError as exc:
            assert "unexpected or missing" in str(exc)
        else:
            raise AssertionError("an extra export directory passed verification")

    with tempfile.TemporaryDirectory() as out_dir:
        for name in required[:-1]:
            with open(os.path.join(out_dir, name), "wb") as f:
                f.write(name.encode())
        try:
            write_export_manifest(
                out_dir,
                final_meta,
                100,
                20,
                "owner/model",
                {
                    "meta_sha256": "a" * 64,
                    "master_sha256": "b" * 64,
                    "optimizer_sha256": "c" * 64,
                },
                True,
                {
                    "validation": {"sha256": "d" * 64},
                    "acceptance_comparison": {"sha256": "e" * 64},
                    "format_ablation": {"sha256": "1" * 64},
                    "rollout": {"sha256": "f" * 64},
                },
                "0" * 64,
            )
        except FileNotFoundError as exc:
            assert "release artifact is missing" in str(exc)
        else:
            raise AssertionError(
                "an export directory missing a required file was accepted"
            )

    def build_manifest_dict():
        return {
            "schema_version": 3,
            "repo_id": "owner/model",
            "release_complete": True,
            "evaluation_sources": {
                "validation": {"sha256": "d" * 64},
                "acceptance_comparison": {"sha256": "e" * 64},
                "format_ablation": {"sha256": "1" * 64},
                "rollout": {"sha256": "f" * 64},
            },
            "model_card_template_sha256": "0" * 64,
            "source_checkpoint": {
                "step": 123,
                "meta_sha256": "a" * 64,
                "master_sha256": "b" * 64,
                "optimizer_sha256": "c" * 64,
            },
            "trunk_parameters": 100,
            "mtp_parameters_excluding_shared_embedding_and_head": 20,
            "files": {},
        }

    def build_export_dir(mutate=None):
        out_dir = tempfile.mkdtemp()
        for name in required:
            path = os.path.join(out_dir, name)
            content = b"owner/model" if name == "README.md" else name.encode()
            with open(path, "wb") as f:
                f.write(content)
        manifest = build_manifest_dict()
        manifest["files"] = {
            name: {
                "bytes": os.path.getsize(os.path.join(out_dir, name)),
                "sha256": file_sha256(os.path.join(out_dir, name)),
            }
            for name in required
        }
        if mutate is not None:
            mutate(manifest)
        with open(
            os.path.join(out_dir, "export_manifest.json"), "w", encoding="utf-8"
        ) as f:
            json.dump(manifest, f, indent=2, sort_keys=True)
            f.write("\n")
        return out_dir

    def _expect_manifest_reject(mutate, expected_substring):
        out_dir = build_export_dir(mutate)
        try:
            verify_export_manifest(out_dir)
        except ValueError as exc:
            assert expected_substring in str(exc), (
                f"expected {expected_substring!r} in {exc}"
            )
        else:
            raise AssertionError(
                f"a manifest that should have failed on "
                f"{expected_substring!r} passed verification"
            )

    _expect_manifest_reject(
        lambda m: m.__setitem__("schema_version", 2),
        "must use schema_version 3",
    )
    _expect_manifest_reject(
        lambda m: m.__setitem__("release_complete", "yes"),
        "release_complete is not boolean",
    )
    _expect_manifest_reject(
        lambda m: m.__setitem__("model_card_template_sha256", "not-hex"),
        "model card template hash is not a SHA-256",
    )
    _expect_manifest_reject(
        lambda m: (
            m.__setitem__("release_complete", False),
            m.__setitem__(
                "evaluation_sources", {"validation": {"sha256": "d" * 64}}
            ),
        ),
        "development export must not declare final evaluation sources",
    )
    _expect_manifest_reject(
        lambda m: m.__setitem__("files", "not-a-dict"),
        "export manifest has no files object",
    )
    _expect_manifest_reject(
        lambda m: m["files"].pop("LICENSE"),
        "export manifest payload differs from required files",
    )
    _expect_manifest_reject(
        lambda m: m.__setitem__("source_checkpoint", "not-a-dict"),
        "export manifest has no source_checkpoint",
    )
    _expect_manifest_reject(
        lambda m: m["source_checkpoint"].__setitem__("step", -1),
        "source_checkpoint.step is not a positive integer",
    )
    _expect_manifest_reject(
        lambda m: m["source_checkpoint"].__setitem__("meta_sha256", "not-hex"),
        "source_checkpoint.meta_sha256 is not a SHA-256",
    )

    # README.md must actually contain the manifest's repo_id string. The
    # README's hash has to match what the manifest recorded (a hash mismatch
    # would trip the "does not match export manifest" check first) but its
    # *content* must lack the repo_id, so build a directory whose README
    # never contained it in the first place.
    out_dir = tempfile.mkdtemp()
    for name in required:
        path = os.path.join(out_dir, name)
        content = (
            b"a different repo entirely, no repo id here"
            if name == "README.md"
            else name.encode()
        )
        with open(path, "wb") as f:
            f.write(content)
    manifest = build_manifest_dict()
    manifest["files"] = {
        name: {
            "bytes": os.path.getsize(os.path.join(out_dir, name)),
            "sha256": file_sha256(os.path.join(out_dir, name)),
        }
        for name in required
    }
    with open(
        os.path.join(out_dir, "export_manifest.json"), "w", encoding="utf-8"
    ) as f:
        json.dump(manifest, f, indent=2, sort_keys=True)
        f.write("\n")
    try:
        verify_export_manifest(out_dir)
    except ValueError as exc:
        assert "does not match export manifest repo_id" in str(exc)
    else:
        raise AssertionError(
            "a rendered model card missing the repo id was accepted"
        )

    print("Hugging Face export metadata: PASS")


if __name__ == "__main__":
    main()