#!/usr/bin/env python3 """Validate the model card before publishing. Checks the YAML front matter, verifies that every number in a results table is backed by a file under evaluation/results or benchmarks/results, and refuses placeholder values. The point is to make it hard to reintroduce the original release's defect: benchmark numbers measured on a different checkpoint. python scripts/validate_model_card.py --readme README.md """ from __future__ import annotations import argparse import json import re import sys from pathlib import Path from typing import Any REQUIRED_METADATA = ["license", "language", "library_name", "pipeline_tag", "base_model", "tags"] REQUIRED_SECTIONS = [ "Model summary", "Relationship to the base model", "Intended uses", "Out-of-scope uses", "Architecture", "Training", "Evaluation", "Benchmark results", "Hardware requirements", "Quantization", "Limitations", "Reproducibility", "Citation", "License", ] PLACEHOLDERS = [ "TODO", "FIXME", "XXX", "[More Information Needed]", "lorem ipsum", "TBD", ] # Anything that looks like a leaked local path. PATH_PATTERNS = [ re.compile(r"/mnt/[a-z]/"), re.compile(r"/home/[A-Za-z0-9_.-]+/"), re.compile(r"[Cc]:[\\/]+Users[\\/]+"), ] SECRET_PATTERNS = [ re.compile(r"\bhf_[A-Za-z0-9]{34,}\b"), re.compile(r"\bsk-[A-Za-z0-9]{20,}\b"), re.compile(r"\bAKIA[0-9A-Z]{16}\b"), ] ALLOWED_STATUSES = { "not run", "unavailable", "could not verify", "not tested", "n/a", "—", "-", } def parse_front_matter(text: str) -> tuple[dict[str, Any] | None, str]: if not text.startswith("---"): return None, text end = text.find("\n---", 3) if end == -1: return None, text block = text[3:end] body = text[end + 4 :] try: import yaml return yaml.safe_load(block), body except ImportError: # Fall back to a shallow key scan so the check still works without pyyaml. keys = dict(re.findall(r"^([a-z_]+):", block, re.MULTILINE)) return {k: True for k in keys}, body def collect_measured_numbers(results_dirs: list[Path]) -> set[str]: """Every percentage and ratio that appears in a committed result file.""" found: set[str] = set() for directory in results_dirs: for path in directory.rglob("*.json"): try: data = json.loads(path.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError): continue _walk(data, found) return found def _walk(node: Any, found: set[str]) -> None: if isinstance(node, dict): for key, value in node.items(): if key in ("accuracy", "score") and isinstance(value, (int, float)): found.add(f"{value * 100:.1f}") found.add(f"{value * 100:.0f}") if key in ("passed", "total") and isinstance(value, int): found.add(str(value)) _walk(value, found) elif isinstance(node, list): for item in node: _walk(item, found) def check_tables(body: str, measured: set[str]) -> list[str]: """Every percentage in a results table must be measured or an allowed status.""" problems: list[str] = [] in_results = False for line in body.splitlines(): lowered = line.lower() if lowered.startswith("#"): in_results = "benchmark" in lowered or "result" in lowered continue if not in_results or not line.strip().startswith("|"): continue cells = [c.strip() for c in line.strip().strip("|").split("|")] for cell in cells: stripped = cell.replace("**", "").strip() if stripped.lower() in ALLOWED_STATUSES or not stripped: continue match = re.fullmatch(r"([+-]?\d+(?:\.\d+)?)\s*%", stripped) if not match: continue value = match.group(1).lstrip("+-") if value.startswith("0.") or value in ("0",): continue normalised = {f"{float(value):.1f}", f"{float(value):.0f}"} if not (normalised & measured): problems.append( f" {stripped} appears in a results table but is not in any " f"committed result file" ) return problems def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--readme", type=Path, default=Path("README.md")) parser.add_argument( "--results", type=Path, nargs="*", default=[Path("evaluation/results"), Path("benchmarks/results")], ) parser.add_argument( "--strict-numbers", action="store_true", help="Fail on unbacked table numbers instead of warning.", ) args = parser.parse_args() if not args.readme.is_file(): sys.exit(f"Model card not found: {args.readme}") text = args.readme.read_text(encoding="utf-8") metadata, body = parse_front_matter(text) errors: list[str] = [] warnings: list[str] = [] if metadata is None: errors.append("No YAML front matter found.") else: for key in REQUIRED_METADATA: if key not in metadata: errors.append(f"Front matter is missing required key: {key}") license_is_apache = str(metadata.get("license", "")).lower() == "apache-2.0" if license_is_apache and not Path("NOTICE").is_file(): errors.append( "License is apache-2.0 but no NOTICE file exists. Upstream MIT " "attribution must be preserved." ) for section in REQUIRED_SECTIONS: if section.lower() not in body.lower(): warnings.append(f"No section mentioning: {section}") for placeholder in PLACEHOLDERS: if placeholder.lower() in body.lower(): errors.append(f"Placeholder text present: {placeholder!r}") for pattern in PATH_PATTERNS: for hit in set(pattern.findall(text)): errors.append(f"Local path leaked in model card: {hit!r}") for pattern in SECRET_PATTERNS: if pattern.search(text): errors.append("Possible secret detected in model card.") existing = [d for d in args.results if d.is_dir()] if existing: measured = collect_measured_numbers(existing) table_problems = check_tables(body, measured) (errors if args.strict_numbers else warnings).extend(table_problems) else: warnings.append("No result directories found; skipped number verification.") for link in re.findall(r"\]\((?!https?://|#)([^)]+)\)", body): target = (args.readme.parent / link.split("#")[0]).resolve() if not target.exists(): errors.append(f"Broken relative link: {link}") for warning in warnings: print(f"WARN {warning}") for error in errors: print(f"ERROR {error}") print(f"\n{len(errors)} error(s), {len(warnings)} warning(s) in {args.readme}") sys.exit(1 if errors else 0) if __name__ == "__main__": main()