evidence/source/e3_rollout_v3/scripts/hf_metadata.py

"""Pure Python metadata helpers for the Wisp Hugging Face package."""

import hashlib
import json
import math
import os
import tempfile

EOS_TOKEN = "<|endoftext|>"
PAD_TOKEN = "<|pad|>"
ADDITIONAL_SPECIAL_TOKENS = [
    "<|fim_prefix|>",
    "<|fim_middle|>",
    "<|fim_suffix|>",
    "<|repo_name|>",
    "<|file_sep|>",
]
REPO_ID_PLACEHOLDER = "{{REPO_ID}}"
EVALUATION_PLACEHOLDER = "{{FINAL_EVALUATION}}"
EVALUATION_SOURCE_KEYS = (
    "validation",
    "acceptance_comparison",
    "format_ablation",
    "rollout",
)
E3_ROLLOUT_SCHEMA_VERSION = 2
E3_ROLLOUT_INSTRUMENT_VERSION = 3
E3_REPLAY_REFERENCE = "branch_local_full_sequence_replay"
E3_REPLAY_RULE = (
    "every_emitted_token_equals_reference_argmax_or_is_a_certified_"
    "bf16_near_tie_on_the_same_realized_prefix"
)
E3_NEAR_TIE_MAX_ULPS = 8
E3_SCORED_POLICY_DOCUMENTS = 600
E3_SCORED_TOKENS = 38400
E3_CROSS_POLICY_DOCUMENTS = 100
E3_TEST_DOCUMENTS = 60
E3_ENDPOINT_SCOPE_NOTE = (
    "The primary endpoint measures accepted drafts per verification. "
    "It does not establish verification-width cost or deployment latency. "
    "Draft recursions per output token is the registered drafter-work "
    "companion; issued drafts per output token is retained only as an issuance "
    "proxy. Target forwards exclude the added post-hoc branch-replay forward "
    "and independent verification pass."
)
E3_ATTESTATION_PROVENANCE_SCOPE = (
    "Unsigned local attestation bound to a pushed pre-execution receipt commit "
    "and the registered source, inputs, checkpoint, and argv; it is not a "
    "signed external or trusted-execution witness."
)
RELEASE_FILES = [
    "LICENSE",
    "README.md",
    "config.json",
    "generation_config.json",
    "model.safetensors",
    "mtp.safetensors",
    "mtp_config.json",
    "special_tokens_map.json",
    "tokenizer.json",
    "tokenizer_config.json",
]


def llama_config(args):
    return {
        "architectures": ["LlamaForCausalLM"],
        "model_type": "llama",
        "hidden_size": args.dim,
        "intermediate_size": args.ffn_hidden,
        "num_hidden_layers": args.n_layers,
        "num_attention_heads": args.n_heads,
        "num_key_value_heads": args.n_kv_heads,
        "head_dim": args.head_dim,
        "max_position_embeddings": args.max_seq_len,
        "rms_norm_eps": args.norm_eps,
        "rope_theta": args.rope_theta,
        "vocab_size": args.vocab_size,
        "tie_word_embeddings": bool(args.tie_embeddings),
        "hidden_act": "silu",
        "attention_bias": False,
        "mlp_bias": False,
        "torch_dtype": "bfloat16",
        "bos_token_id": None,
        "eos_token_id": 0,
        "pad_token_id": 1,
    }


def tokenizer_config(args):
    return {
        "tokenizer_class": "PreTrainedTokenizerFast",
        "model_max_length": args.max_seq_len,
        "clean_up_tokenization_spaces": False,
        "bos_token": None,
        "eos_token": EOS_TOKEN,
        "pad_token": PAD_TOKEN,
        "unk_token": None,
        "additional_special_tokens": ADDITIONAL_SPECIAL_TOKENS,
    }


def special_tokens_map():
    return {
        "eos_token": EOS_TOKEN,
        "pad_token": PAD_TOKEN,
        "additional_special_tokens": ADDITIONAL_SPECIAL_TOKENS,
    }


def generation_config():
    return {
        "_from_model_config": True,
        "bos_token_id": None,
        "eos_token_id": 0,
        "pad_token_id": 1,
    }


def mtp_config(args, meta):
    return {
        "mtp_layers": args.mtp_layers,
        "mtp_depth_trained": args.mtp_depth,
        "shared_lm_head": True,
        "recursive": True,
        "note": (
            "One shared MTP module applied recursively, Qwen3-Next style. It "
            "consumes the trunk hidden state at position i and the embedding of "
            "the token at i+k, and predicts the token at i+k+1. The LM head is "
            "shared with the trunk, which ties both computations to one output "
            "projection but does not guarantee close distributions. The module "
            "contains a transformer block whose attention was trained under a "
            "causal mask over the whole window: at inference it must be given "
            "the sequence, not a single position."
        ),
        "trained_steps": meta.get("step"),
    }


def write_json(path, value):
    with open(path, "w", encoding="utf-8") as f:
        json.dump(value, f, indent=2, sort_keys=True)
        f.write("\n")


def write_json_atomic(path, value):
    path = os.path.abspath(path)
    if os.path.lexists(path):
        raise FileExistsError(f"refusing to replace JSON artifact: {path}")
    parent = os.path.dirname(path)
    os.makedirs(parent, exist_ok=True)
    rendered = json.dumps(value, indent=2, sort_keys=True, allow_nan=False)
    temporary = None
    try:
        with tempfile.NamedTemporaryFile(
            "w",
            encoding="utf-8",
            prefix=f".{os.path.basename(path)}.",
            suffix=".tmp",
            dir=parent,
            delete=False,
        ) as f:
            temporary = f.name
            f.write(rendered)
            f.write("\n")
            f.flush()
            os.fsync(f.fileno())
        if os.path.lexists(path):
            raise FileExistsError(
                f"JSON target appeared during staging: {path}"
            )
        os.rename(temporary, path)
        temporary = None
    finally:
        if temporary is not None and os.path.isfile(temporary):
            os.unlink(temporary)
    return path


def validate_repo_id(repo_id):
    if (
        not isinstance(repo_id, str)
        or repo_id != repo_id.strip()
        or repo_id.count("/") != 1
        or any(not part for part in repo_id.split("/"))
        or any(character.isspace() for character in repo_id)
    ):
        raise ValueError("--repo-id must have the form namespace/model")
    return repo_id


def render_model_card(path, repo_id, evaluation_markdown):
    validate_repo_id(repo_id)
    with open(path, encoding="utf-8") as f:
        card = f.read()
    if REPO_ID_PLACEHOLDER not in card:
        raise ValueError(
            f"model card must contain the placeholder {REPO_ID_PLACEHOLDER}"
        )
    if EVALUATION_PLACEHOLDER not in card:
        raise ValueError(
            f"model card must contain the placeholder {EVALUATION_PLACEHOLDER}"
        )
    if (
        not isinstance(evaluation_markdown, str)
        or not evaluation_markdown.strip()
    ):
        raise ValueError("model card evaluation text is empty")
    rendered = card.replace(REPO_ID_PLACEHOLDER, repo_id).replace(
        EVALUATION_PLACEHOLDER,
        evaluation_markdown.strip(),
    )
    if (
        REPO_ID_PLACEHOLDER in rendered
        or EVALUATION_PLACEHOLDER in rendered
    ):
        raise ValueError("unresolved placeholder in rendered model card")
    return rendered


def _metric(value, label):
    if (
        not isinstance(value, (int, float))
        or isinstance(value, bool)
        or not math.isfinite(value)
    ):
        raise ValueError(f"{label} is not finite")
    return float(value)


def _interval(value, label):
    if not isinstance(value, list) or len(value) != 2:
        raise ValueError(f"{label} is not a two-element interval")
    lo = _metric(value[0], f"{label} lower")
    hi = _metric(value[1], f"{label} upper")
    if lo > hi:
        raise ValueError(f"{label} is reversed")
    return lo, hi


def _verdict(value, label):
    if (
        not isinstance(value, str)
        or not value.strip()
        or "\n" in value
        or "|" in value
    ):
        raise ValueError(f"{label} is not a safe single-line verdict")
    return value


def render_evaluation_section(
    validation,
    acceptance,
    format_ablation,
    rollout,
):
    """Render final model-card numbers from structured registered reports."""
    if validation.get("publication_ready") is not True:
        raise ValueError("validation report is not publication-ready")
    if rollout.get("publication_ready") is not True:
        raise ValueError("rollout report is not publication-ready")
    if (
        rollout.get("schema_version") != E3_ROLLOUT_SCHEMA_VERSION
        or rollout.get("instrument_version") != E3_ROLLOUT_INSTRUMENT_VERSION
    ):
        raise ValueError(
            "final metadata requires rollout schema 2 instrument 3; "
            "historical rollout instruments are not publication evidence"
        )
    if acceptance.get("schema_version") != 1:
        raise ValueError("acceptance comparison schema is not 1")
    if format_ablation.get("schema_version") != 1:
        raise ValueError("format-ablation comparison schema is not 1")
    if format_ablation.get("publication_ready") is not True:
        raise ValueError("format-ablation comparison is not publication-ready")

    summary = validation.get("summary", {})
    main = summary.get("main_loss", {})
    main_mean = _metric(main.get("mean"), "validation main loss")
    main_lo, main_hi = _interval(
        main.get("ci95"), "validation main loss interval"
    )
    perplexity = _metric(
        summary.get("main_perplexity"), "validation perplexity"
    )
    mtp = summary.get("mtp_loss")
    if not isinstance(mtp, list) or len(mtp) != 2:
        raise ValueError("validation report does not have two MTP losses")
    mtp_values = []
    for index, row in enumerate(mtp, 1):
        mean = _metric(row.get("mean"), f"MTP depth {index} loss")
        lo, hi = _interval(
            row.get("ci95"), f"MTP depth {index} loss interval"
        )
        mtp_values.append((mean, lo, hi))

    primary = acceptance.get("trained_primary_endpoint", {})
    primary_ratio = _metric(primary.get("ratio"), "acceptance primary ratio")
    primary_lo, primary_hi = _interval(
        primary.get("ci95"), "acceptance primary interval"
    )
    primary_verdict = _verdict(
        primary.get("verdict"), "acceptance primary verdict"
    )
    adjusted = acceptance.get("trained_minus_untrained_ratio", {})
    adjusted_difference = _metric(
        adjusted.get("difference"), "acceptance control-adjusted difference"
    )
    adjusted_lo, adjusted_hi = _interval(
        adjusted.get("ci95"), "acceptance control-adjusted interval"
    )
    adjusted_verdict = _verdict(
        adjusted.get("verdict"), "acceptance control verdict"
    )
    combined = _verdict(
        acceptance.get("combined_interpretation"),
        "acceptance combined interpretation",
    )
    acceptance_documents = acceptance.get("documents")
    if (
        not isinstance(acceptance_documents, int)
        or isinstance(acceptance_documents, bool)
        or acceptance_documents < 2
    ):
        raise ValueError("acceptance document count is invalid")

    format_documents = format_ablation.get("documents")
    if (
        not isinstance(format_documents, int)
        or isinstance(format_documents, bool)
        or format_documents < 2
    ):
        raise ValueError("format-ablation document count is invalid")
    format_primary = format_ablation.get("primary_endpoint", {})
    format_primary_difference = _metric(
        format_primary.get("difference_in_differences"),
        "format-ablation primary difference",
    )
    format_primary_lo, format_primary_hi = _interval(
        format_primary.get("ci95"),
        "format-ablation primary interval",
    )
    format_primary_verdict = _verdict(
        format_primary.get("verdict"),
        "format-ablation primary verdict",
    )
    format_secondary = format_ablation.get("secondary_endpoint", {})
    format_secondary_difference = _metric(
        format_secondary.get("difference_in_differences"),
        "format-ablation secondary difference",
    )
    format_secondary_lo, format_secondary_hi = _interval(
        format_secondary.get("ci95"),
        "format-ablation secondary interval",
    )
    format_secondary_verdict = _verdict(
        format_secondary.get("verdict"),
        "format-ablation secondary verdict",
    )
    format_limitation = _verdict(
        format_ablation.get("baseline_revision_evidence", {}).get(
            "limitation"
        ),
        "format-ablation source limitation",
    )
    format_runtime_limitation = _verdict(
        format_ablation.get("runtime_code_evidence", {}).get("limitation"),
        "format-ablation runtime limitation",
    )

    quality = rollout.get("quality_gate", {})

    def quality_count(name):
        value = quality.get(name)
        if (
            not isinstance(value, int)
            or isinstance(value, bool)
            or value < 0
        ):
            raise ValueError(
                "rollout branch-local replay has an invalid "
                f"{name.replace('_', ' ')} count"
            )
        return value

    scored_policy_documents = quality_count("scored_policy_documents")
    scored_tokens = quality_count("scored_tokens")
    exact_argmax_tokens = quality_count("exact_argmax_tokens")
    certified_near_tie_tokens = quality_count(
        "certified_near_tie_tokens"
    )
    failed_tokens = quality_count("failed_tokens")
    branch_replay_passes = quality_count("branch_replay_passes")
    cross_policy_matches = quality_count(
        "cross_policy_trajectory_matches"
    )
    if (
        quality.get("reference") != E3_REPLAY_REFERENCE
        or quality.get("rule") != E3_REPLAY_RULE
        or quality.get("near_tie_max_ulps") != E3_NEAR_TIE_MAX_ULPS
        or quality.get("passed") is not True
        or scored_policy_documents != E3_SCORED_POLICY_DOCUMENTS
        or scored_tokens != E3_SCORED_TOKENS
        or exact_argmax_tokens + certified_near_tie_tokens
        + failed_tokens != scored_tokens
        or failed_tokens != 0
        or branch_replay_passes != scored_policy_documents
    ):
        raise ValueError("rollout branch-local replay did not pass")
    if cross_policy_matches != E3_CROSS_POLICY_DOCUMENTS:
        raise ValueError("rollout cross-policy output identity did not pass")
    if certified_near_tie_tokens == 0:
        replay_cell = (
            "| Branch-local greedy replay | "
            f"Exact argmax for all {scored_tokens} emitted tokens across "
            f"{scored_policy_documents} scored policy-document rollouts |"
        )
    else:
        near_tie_claim = (
            "1 token certified as a bfloat16 near-tie"
            if certified_near_tie_tokens == 1
            else (
                f"{certified_near_tie_tokens} tokens certified as "
                "bfloat16 near-ties"
            )
        )
        replay_cell = (
            "| Branch-local greedy replay | "
            f"Exact argmax on {exact_argmax_tokens} of {scored_tokens} "
            f"emitted tokens; {near_tie_claim} within "
            f"{E3_NEAR_TIE_MAX_ULPS} ulps on "
            "their realized branches |"
        )
    identity_cell = (
        "| Cross-policy output identity | "
        f"Identical realized output branches for all {cross_policy_matches} "
        "calibration/test documents across compared policies |"
    )
    rollout_primary = rollout.get("primary_endpoint", {})
    if (
        rollout_primary.get("comparison")
        != "selected_adaptive_minus_selected_fixed"
        or rollout_primary.get("metric")
        != "accepted_drafts_per_verification"
    ):
        raise ValueError("rollout primary endpoint differs from registration")
    rollout_difference = _metric(
        rollout_primary.get("difference"), "rollout primary difference"
    )
    rollout_lo, rollout_hi = _interval(
        rollout_primary.get("ci95"), "rollout primary interval"
    )
    rollout_verdict = _verdict(
        rollout_primary.get("verdict"), "rollout primary verdict"
    )
    fixed_policy = _verdict(
        rollout_primary.get("fixed_policy"), "rollout fixed policy"
    )
    adaptive_policy = _verdict(
        rollout_primary.get("adaptive_policy"), "rollout adaptive policy"
    )
    rollout_documents = rollout_primary.get("documents")
    if (
        not isinstance(rollout_documents, int)
        or isinstance(rollout_documents, bool)
        or rollout_documents != E3_TEST_DOCUMENTS
    ):
        raise ValueError("rollout test document count is invalid")

    def companion_endpoint(key, expected_metric, label):
        endpoint = rollout.get(key, {})
        if (
            endpoint.get("comparison")
            != "selected_adaptive_minus_selected_fixed"
            or endpoint.get("metric") != expected_metric
            or endpoint.get("adaptive_policy") != adaptive_policy
            or endpoint.get("fixed_policy") != fixed_policy
            or endpoint.get("documents") != rollout_documents
        ):
            raise ValueError(f"{label} differs from registration")
        difference = _metric(
            endpoint.get("difference"), f"{label} difference"
        )
        lo, hi = _interval(endpoint.get("ci95"), f"{label} interval")
        verdict = _verdict(endpoint.get("verdict"), f"{label} verdict")
        return difference, lo, hi, verdict

    target_forward = companion_endpoint(
        "secondary_target_forward_endpoint",
        "output_tokens_per_target_forward",
        "rollout target-forward companion endpoint",
    )
    draft_issued_proxy = companion_endpoint(
        "secondary_draft_issued_proxy_endpoint",
        "drafts_issued_per_output_token",
        "rollout draft-issuance proxy endpoint",
    )
    draft_work = companion_endpoint(
        "secondary_draft_work_endpoint",
        "draft_recursions_per_output_token",
        "rollout draft-work companion endpoint",
    )
    if rollout.get("endpoint_scope_note") != E3_ENDPOINT_SCOPE_NOTE:
        raise ValueError("rollout endpoint scope disclosure differs from code")

    return "\n".join([
        "### Registered final results",
        "",
        "| Measurement | Result |",
        "|---|---|",
        (
            "| Final validation main NLL | "
            f"{main_mean:.4f} [{main_lo:.4f}, {main_hi:.4f}], "
            f"perplexity {perplexity:.2f} |"
        ),
        (
            "| Validation MTP depth 1 NLL | "
            f"{mtp_values[0][0]:.4f} "
            f"[{mtp_values[0][1]:.4f}, {mtp_values[0][2]:.4f}] |"
        ),
        (
            "| Validation MTP depth 2 NLL | "
            f"{mtp_values[1][0]:.4f} "
            f"[{mtp_values[1][1]:.4f}, {mtp_values[1][2]:.4f}] |"
        ),
        (
            "| FIM / shuffled-suffix acceptance, depth 2 | "
            f"{primary_ratio:.4f} [{primary_lo:.4f}, {primary_hi:.4f}], "
            f"{primary_verdict}, {acceptance_documents} documents |"
        ),
        (
            "| Trained minus initialized acceptance-ratio lift | "
            f"{adjusted_difference:+.4f} "
            f"[{adjusted_lo:+.4f}, {adjusted_hi:+.4f}], "
            f"{adjusted_verdict} |"
        ),
        f"| Acceptance interpretation | {combined} |",
        (
            "| FIM-training effect on shuffled-FIM minus L2R acceptance | "
            f"{format_primary_difference:+.4f} "
            f"[{format_primary_lo:+.4f}, {format_primary_hi:+.4f}], "
            f"{format_primary_verdict}, {format_documents} documents |"
        ),
        (
            "| FIM-training effect on true-suffix minus shuffled-suffix "
            "acceptance | "
            f"{format_secondary_difference:+.4f} "
            f"[{format_secondary_lo:+.4f}, {format_secondary_hi:+.4f}], "
            f"{format_secondary_verdict} |"
        ),
        (
            "| Adaptive minus fixed accepted drafts per verification | "
            f"{rollout_difference:+.4f} "
            f"[{rollout_lo:+.4f}, {rollout_hi:+.4f}], "
            f"{rollout_verdict}, {rollout_documents} test documents |"
        ),
        (
            "| Adaptive minus fixed output tokens per target forward | "
            f"{target_forward[0]:+.4f} "
            f"[{target_forward[1]:+.4f}, {target_forward[2]:+.4f}], "
            f"{target_forward[3]}, {rollout_documents} test documents |"
        ),
        (
            "| Adaptive minus fixed drafts issued per output token | "
            f"{draft_issued_proxy[0]:+.4f} "
            f"[{draft_issued_proxy[1]:+.4f}, "
            f"{draft_issued_proxy[2]:+.4f}], "
            f"{draft_issued_proxy[3]}, {rollout_documents} test documents "
            "(issuance proxy) |"
        ),
        (
            "| Adaptive minus fixed draft recursions per output token | "
            f"{draft_work[0]:+.4f} "
            f"[{draft_work[1]:+.4f}, {draft_work[2]:+.4f}], "
            f"{draft_work[3]}, {rollout_documents} test documents |"
        ),
        f"| Rollout policies selected on calibration | {adaptive_policy} versus {fixed_policy} |",
        replay_cell,
        identity_cell,
        "",
        (
            "Validation intervals measure Monte Carlo uncertainty from the frozen "
            "random-window sampler. Acceptance and rollout intervals resample "
            "paired target documents. They do not measure training-run or model "
            "uncertainty. Null and negative outcomes are retained rather than "
            "filtered from the release."
        ),
        "",
        f"Rollout endpoint scope: {E3_ENDPOINT_SCOPE_NOTE}",
        "",
        (
            "Independent replay provenance scope: "
            f"{E3_ATTESTATION_PROVENANCE_SCOPE}"
        ),
        "",
        f"Format-ablation provenance limitation: {format_limitation}",
        "",
        f"Format-ablation runtime limitation: {format_runtime_limitation}",
    ])


def development_evaluation_section(step, max_steps):
    return (
        "### Development snapshot\n\n"
        f"This package is an incomplete checkpoint at step {step} of "
        f"{max_steps}. It has no final registered evaluation claims and must "
        "not be published as the Wisp release."
    )


def validate_export_checkpoint(meta, allow_incomplete=False):
    """Reject controls, malformed metadata, and accidental snapshot releases."""
    problems = []
    step = meta.get("step")
    config = meta.get("config")
    model_args = meta.get("model_args")
    if not isinstance(step, int) or isinstance(step, bool) or step < 1:
        problems.append(f"checkpoint step must be a positive integer, got {step!r}")
    if not isinstance(config, dict):
        problems.append("checkpoint config is missing")
        config = {}
    if not isinstance(model_args, dict) or not model_args:
        problems.append("checkpoint model_args are missing")
    max_steps = config.get("max_steps")
    if not isinstance(max_steps, int) or isinstance(max_steps, bool) or max_steps < 1:
        problems.append(
            f"checkpoint config max_steps must be a positive integer, got {max_steps!r}"
        )
    elif isinstance(step, int) and step > max_steps:
        problems.append(f"checkpoint step {step} exceeds max_steps {max_steps}")
    elif isinstance(step, int) and step != max_steps and not allow_incomplete:
        problems.append(
            f"checkpoint step {step} is not final step {max_steps}; "
            "use --allow-incomplete only for a development export"
        )
    if config.get("initialization_only") is True:
        problems.append("an initialization-only control cannot be exported")
    if not config.get("run_name"):
        problems.append("checkpoint config run_name is missing")
    if step == max_steps and meta.get("optimizer_state_included") is not True:
        problems.append(
            "final checkpoint does not attest that optimizer state was saved"
        )
    if problems:
        raise ValueError("checkpoint is not releasable:\n- " + "\n- ".join(problems))
    return {
        "step": step,
        "max_steps": max_steps,
        "run_name": config["run_name"],
        "complete": step == max_steps,
    }


def file_sha256(path):
    digest = hashlib.sha256()
    with open(path, "rb") as f:
        for block in iter(lambda: f.read(1024 * 1024), b""):
            digest.update(block)
    return digest.hexdigest()


def require_unchanged_files(artifacts, activity):
    """Fail if any named file differs from its start-of-activity snapshot."""
    for label, evidence in artifacts.items():
        path = evidence.get("path")
        expected = evidence.get("sha256")
        if (
            not isinstance(path, str)
            or not isinstance(expected, str)
            or file_sha256(path) != expected
        ):
            raise RuntimeError(f"{label} changed during {activity}")


def write_export_manifest(
    out_dir,
    meta,
    n_trunk,
    n_mtp,
    repo_id,
    checkpoint_hashes,
    release_complete,
    evaluation_sources,
    model_card_template_sha256,
):
    files = {}
    for name in RELEASE_FILES:
        path = os.path.join(out_dir, name)
        if not os.path.isfile(path):
            raise FileNotFoundError(f"release artifact is missing {name}")
        files[name] = {
            "bytes": os.path.getsize(path),
            "sha256": file_sha256(path),
        }
    manifest = {
        "schema_version": 3,
        "repo_id": repo_id,
        "release_complete": release_complete,
        "evaluation_sources": evaluation_sources,
        "model_card_template_sha256": model_card_template_sha256,
        "source_checkpoint": {
            "step": meta.get("step"),
            "meta_sha256": checkpoint_hashes["meta_sha256"],
            "master_sha256": checkpoint_hashes["master_sha256"],
            "optimizer_sha256": checkpoint_hashes["optimizer_sha256"],
        },
        "trunk_parameters": n_trunk,
        "mtp_parameters_excluding_shared_embedding_and_head": n_mtp,
        "files": files,
    }
    write_json(os.path.join(out_dir, "export_manifest.json"), manifest)


def verify_export_manifest(out_dir):
    """Verify an export is exactly the payload recorded by its manifest."""
    manifest_path = os.path.join(out_dir, "export_manifest.json")
    with open(manifest_path, encoding="utf-8") as f:
        manifest = json.load(f)
    if manifest.get("schema_version") != 3:
        raise ValueError("export manifest must use schema_version 3")
    release_complete = manifest.get("release_complete")
    if not isinstance(release_complete, bool):
        raise ValueError("export manifest release_complete is not boolean")
    template_digest = manifest.get("model_card_template_sha256")
    if not _is_sha256(template_digest):
        raise ValueError("model card template hash is not a SHA-256")
    evaluation_sources = manifest.get("evaluation_sources")
    if release_complete:
        if (
            not isinstance(evaluation_sources, dict)
            or sorted(evaluation_sources) != sorted(EVALUATION_SOURCE_KEYS)
        ):
            raise ValueError(
                "final export does not declare exact evaluation sources"
            )
        for key in EVALUATION_SOURCE_KEYS:
            if not _is_sha256(evaluation_sources.get(key, {}).get("sha256")):
                raise ValueError(
                    f"evaluation source {key} hash is not a SHA-256"
                )
    elif evaluation_sources is not None:
        raise ValueError(
            "development export must not declare final evaluation sources"
        )
    declared = manifest.get("files")
    if not isinstance(declared, dict):
        raise ValueError("export manifest has no files object")
    if sorted(declared) != RELEASE_FILES:
        raise ValueError(
            f"export manifest payload differs from required files: {sorted(declared)}"
        )
    actual_names = sorted(os.listdir(out_dir))
    expected_names = sorted([*RELEASE_FILES, "export_manifest.json"])
    if actual_names != expected_names:
        raise ValueError(
            f"export directory contains unexpected or missing files: {actual_names}"
        )
    for name in RELEASE_FILES:
        path = os.path.join(out_dir, name)
        actual = {
            "bytes": os.path.getsize(path),
            "sha256": file_sha256(path),
        }
        if declared[name] != actual:
            raise ValueError(
                f"release artifact {name} does not match export manifest"
            )
    checkpoint = manifest.get("source_checkpoint")
    if not isinstance(checkpoint, dict):
        raise ValueError("export manifest has no source_checkpoint")
    step = checkpoint.get("step")
    if not isinstance(step, int) or isinstance(step, bool) or step < 1:
        raise ValueError("source_checkpoint.step is not a positive integer")
    for key in ("meta_sha256", "master_sha256", "optimizer_sha256"):
        digest = checkpoint.get(key)
        valid_digest = (
            isinstance(digest, str)
            and len(digest) == 64
            and all(
                character in "0123456789abcdef"
                for character in digest.lower()
            )
        )
        if not valid_digest:
            raise ValueError(f"source_checkpoint.{key} is not a SHA-256")
    repo_id = validate_repo_id(manifest.get("repo_id"))
    with open(os.path.join(out_dir, "README.md"), encoding="utf-8") as f:
        card = f.read()
    if (
        REPO_ID_PLACEHOLDER in card
        or EVALUATION_PLACEHOLDER in card
        or repo_id not in card
    ):
        raise ValueError(
            "rendered model card does not match export manifest repo_id"
        )
    return manifest


def validate_external_verification_receipt(receipt, manifest, manifest_sha256):
    """Verify a structured Transformers comparison receipt fails closed."""
    problems = []
    if receipt.get("schema_version") != 1:
        problems.append("schema_version is not 1")
    if receipt.get("status") != "verified" or receipt.get("passed") is not True:
        problems.append("receipt is not a verified pass")
    package = receipt.get("package", {})
    if package.get("export_manifest_sha256") != manifest_sha256:
        problems.append("export manifest hash does not match")
    if package.get("repo_id") != manifest.get("repo_id"):
        problems.append("repository ID does not match export manifest")
    if package.get("source_checkpoint") != manifest.get("source_checkpoint"):
        problems.append("source checkpoint does not match export manifest")
    export_dir = package.get("export_dir")
    if not isinstance(export_dir, str) or not os.path.isabs(export_dir):
        problems.append("export directory is not an absolute path")
    checkpoint = receipt.get("checkpoint", {})
    source_checkpoint = manifest.get("source_checkpoint", {})
    checkpoint_hashes = {
        key: checkpoint.get(key)
        for key in ("meta_sha256", "master_sha256", "optimizer_sha256")
    }
    expected_hashes = {
        key: source_checkpoint.get(key)
        for key in ("meta_sha256", "master_sha256", "optimizer_sha256")
    }
    if checkpoint_hashes != expected_hashes:
        problems.append("verified checkpoint hashes do not match package source")
    checkpoint_path = checkpoint.get("path")
    if not isinstance(checkpoint_path, str) or not os.path.isabs(checkpoint_path):
        problems.append("checkpoint path is not absolute")
    probe = receipt.get("probe", {})
    probe_length = probe.get("length")
    probe_digest = probe.get("token_ids_sha256")
    if (
        probe.get("seed") != 0
        or not isinstance(probe_length, int)
        or isinstance(probe_length, bool)
        or probe_length < 1
        or not _is_sha256(probe_digest)
    ):
        problems.append("deterministic token probe is malformed")
    tokenizer = receipt.get("tokenizer", {})
    if (
        tokenizer.get("special_token_ids") != list(range(7))
        or tokenizer.get("eos_token_id") != 0
        or tokenizer.get("pad_token_id") != 1
        or tokenizer.get("byte_roundtrip_exact") is not True
        or not _is_sha256(tokenizer.get("probe_sha256"))
    ):
        problems.append("tokenizer verification is incomplete")
    logits = receipt.get("logits", {})
    shape = logits.get("shape")
    max_delta = logits.get("max_abs_delta")
    scale = logits.get("logit_scale")
    relative = logits.get("relative_max_abs_delta")
    threshold = logits.get("relative_delta_threshold")
    agreement = logits.get("argmax_agreement")
    if (
        not isinstance(shape, list)
        or len(shape) != 2
        or any(
            not isinstance(item, int) or isinstance(item, bool) or item < 1
            for item in shape
        )
        or shape[0] != probe_length
    ):
        problems.append("logit shape does not match deterministic probe")
    if (
        not _is_nonnegative_finite(max_delta)
        or not _is_nonnegative_finite(scale)
        or not _is_nonnegative_finite(relative)
        or threshold != 1e-2
        or relative >= threshold
    ):
        problems.append("relative logit delta did not pass its threshold")
    expected_relative = (
        max_delta / max(scale, 1e-9)
        if _is_nonnegative_finite(max_delta) and _is_nonnegative_finite(scale)
        else None
    )
    if (
        expected_relative is None
        or not _is_nonnegative_finite(relative)
        or not math.isclose(relative, expected_relative, rel_tol=1e-12, abs_tol=0.0)
    ):
        problems.append("relative logit delta is inconsistent with raw metrics")
    if agreement != 1.0:
        problems.append("argmax agreement is not exact")
    toolchain = receipt.get("toolchain", {})
    for key in ("python", "platform", "mlx", "torch", "transformers"):
        if not isinstance(toolchain.get(key), str) or not toolchain[key].strip():
            problems.append(f"toolchain {key} is missing")
    if problems:
        raise ValueError(
            "external verification receipt is invalid:\n- "
            + "\n- ".join(problems)
        )
    return receipt


def _is_sha256(value):
    return (
        isinstance(value, str)
        and len(value) == 64
        and all(character in "0123456789abcdef" for character in value.lower())
    )


def _is_nonnegative_finite(value):
    return (
        isinstance(value, (int, float))
        and not isinstance(value, bool)
        and math.isfinite(value)
        and value >= 0
    )