scripts/validate_model_card.py

#!/usr/bin/env python3
"""Validate the model card before publishing.

Checks the YAML front matter, verifies that every number in a results table is
backed by a file under evaluation/results or benchmarks/results, and refuses
placeholder values.  The point is to make it hard to reintroduce the original
release's defect: benchmark numbers measured on a different checkpoint.

    python scripts/validate_model_card.py --readme README.md
"""

from __future__ import annotations

import argparse
import json
import re
import sys
from pathlib import Path
from typing import Any

REQUIRED_METADATA = ["license", "language", "library_name", "pipeline_tag", "base_model", "tags"]

REQUIRED_SECTIONS = [
    "Model summary",
    "Relationship to the base model",
    "Intended uses",
    "Out-of-scope uses",
    "Architecture",
    "Training",
    "Evaluation",
    "Benchmark results",
    "Hardware requirements",
    "Quantization",
    "Limitations",
    "Reproducibility",
    "Citation",
    "License",
]

PLACEHOLDERS = [
    "TODO",
    "FIXME",
    "XXX",
    "[More Information Needed]",
    "lorem ipsum",
    "TBD",
]

# Anything that looks like a leaked local path.
PATH_PATTERNS = [
    re.compile(r"/mnt/[a-z]/"),
    re.compile(r"/home/[A-Za-z0-9_.-]+/"),
    re.compile(r"[Cc]:[\\/]+Users[\\/]+"),
]

SECRET_PATTERNS = [
    re.compile(r"\bhf_[A-Za-z0-9]{34,}\b"),
    re.compile(r"\bsk-[A-Za-z0-9]{20,}\b"),
    re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
]

ALLOWED_STATUSES = {
    "not run",
    "unavailable",
    "could not verify",
    "not tested",
    "n/a",
    "—",
    "-",
}


def parse_front_matter(text: str) -> tuple[dict[str, Any] | None, str]:
    if not text.startswith("---"):
        return None, text
    end = text.find("\n---", 3)
    if end == -1:
        return None, text
    block = text[3:end]
    body = text[end + 4 :]
    try:
        import yaml

        return yaml.safe_load(block), body
    except ImportError:
        # Fall back to a shallow key scan so the check still works without pyyaml.
        keys = dict(re.findall(r"^([a-z_]+):", block, re.MULTILINE))
        return {k: True for k in keys}, body


def collect_measured_numbers(results_dirs: list[Path]) -> set[str]:
    """Every percentage and ratio that appears in a committed result file."""
    found: set[str] = set()
    for directory in results_dirs:
        for path in directory.rglob("*.json"):
            try:
                data = json.loads(path.read_text(encoding="utf-8"))
            except (json.JSONDecodeError, OSError):
                continue
            _walk(data, found)
    return found


def _walk(node: Any, found: set[str]) -> None:
    if isinstance(node, dict):
        for key, value in node.items():
            if key in ("accuracy", "score") and isinstance(value, (int, float)):
                found.add(f"{value * 100:.1f}")
                found.add(f"{value * 100:.0f}")
            if key in ("passed", "total") and isinstance(value, int):
                found.add(str(value))
            _walk(value, found)
    elif isinstance(node, list):
        for item in node:
            _walk(item, found)


def check_tables(body: str, measured: set[str]) -> list[str]:
    """Every percentage in a results table must be measured or an allowed status."""
    problems: list[str] = []
    in_results = False
    for line in body.splitlines():
        lowered = line.lower()
        if lowered.startswith("#"):
            in_results = "benchmark" in lowered or "result" in lowered
            continue
        if not in_results or not line.strip().startswith("|"):
            continue
        cells = [c.strip() for c in line.strip().strip("|").split("|")]
        for cell in cells:
            stripped = cell.replace("**", "").strip()
            if stripped.lower() in ALLOWED_STATUSES or not stripped:
                continue
            match = re.fullmatch(r"([+-]?\d+(?:\.\d+)?)\s*%", stripped)
            if not match:
                continue
            value = match.group(1).lstrip("+-")
            if value.startswith("0.") or value in ("0",):
                continue
            normalised = {f"{float(value):.1f}", f"{float(value):.0f}"}
            if not (normalised & measured):
                problems.append(
                    f"  {stripped} appears in a results table but is not in any "
                    f"committed result file"
                )
    return problems


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--readme", type=Path, default=Path("README.md"))
    parser.add_argument(
        "--results",
        type=Path,
        nargs="*",
        default=[Path("evaluation/results"), Path("benchmarks/results")],
    )
    parser.add_argument(
        "--strict-numbers",
        action="store_true",
        help="Fail on unbacked table numbers instead of warning.",
    )
    args = parser.parse_args()

    if not args.readme.is_file():
        sys.exit(f"Model card not found: {args.readme}")

    text = args.readme.read_text(encoding="utf-8")
    metadata, body = parse_front_matter(text)

    errors: list[str] = []
    warnings: list[str] = []

    if metadata is None:
        errors.append("No YAML front matter found.")
    else:
        for key in REQUIRED_METADATA:
            if key not in metadata:
                errors.append(f"Front matter is missing required key: {key}")
        license_is_apache = str(metadata.get("license", "")).lower() == "apache-2.0"
        if license_is_apache and not Path("NOTICE").is_file():
            errors.append(
                "License is apache-2.0 but no NOTICE file exists. Upstream MIT "
                "attribution must be preserved."
            )

    for section in REQUIRED_SECTIONS:
        if section.lower() not in body.lower():
            warnings.append(f"No section mentioning: {section}")

    for placeholder in PLACEHOLDERS:
        if placeholder.lower() in body.lower():
            errors.append(f"Placeholder text present: {placeholder!r}")

    for pattern in PATH_PATTERNS:
        for hit in set(pattern.findall(text)):
            errors.append(f"Local path leaked in model card: {hit!r}")

    for pattern in SECRET_PATTERNS:
        if pattern.search(text):
            errors.append("Possible secret detected in model card.")

    existing = [d for d in args.results if d.is_dir()]
    if existing:
        measured = collect_measured_numbers(existing)
        table_problems = check_tables(body, measured)
        (errors if args.strict_numbers else warnings).extend(table_problems)
    else:
        warnings.append("No result directories found; skipped number verification.")

    for link in re.findall(r"\]\((?!https?://|#)([^)]+)\)", body):
        target = (args.readme.parent / link.split("#")[0]).resolve()
        if not target.exists():
            errors.append(f"Broken relative link: {link}")

    for warning in warnings:
        print(f"WARN  {warning}")
    for error in errors:
        print(f"ERROR {error}")

    print(f"\n{len(errors)} error(s), {len(warnings)} warning(s) in {args.readme}")
    sys.exit(1 if errors else 0)


if __name__ == "__main__":
    main()