#!/usr/bin/env python from __future__ import annotations import argparse import json import subprocess import sys from datetime import datetime, timezone from pathlib import Path from typing import Any NON_MARKDOWN_BACKFILLS: tuple[str, ...] = ( "config.yaml", "train.log", "eval.log", "metrics_by_task.json", "metrics_by_seed.json", ) def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description=( "Backfill transparent non-Markdown run-contract artifacts for runs " "referenced by the CTT paper audit. Existing files are preserved." ) ) parser.add_argument("--repo-root", type=Path, default=Path.cwd()) parser.add_argument( "--audit", type=Path, default=Path("runs/paper_ctt_audit/audit.json"), help="Audit JSON produced by scripts/audit_ctt_paper_artifacts.py.", ) parser.add_argument( "--dry-run", action="store_true", help="Print planned writes without creating files.", ) args = parser.parse_args(argv) repo_root = args.repo_root.resolve() audit_path = _resolve(repo_root, args.audit) if not audit_path.exists(): raise SystemExit(f"audit not found: {audit_path}") audit = json.loads(audit_path.read_text()) timestamp = datetime.now(timezone.utc).isoformat() git_hash = _git_hash(repo_root) writes: list[dict[str, str]] = [] for row in audit.get("run_artifacts", []): run_dir = repo_root / str(row.get("run_dir", "")) if not run_dir.exists() or not run_dir.is_dir(): continue missing = set(row.get("missing_advisor_contract", [])) for filename in NON_MARKDOWN_BACKFILLS: if filename in missing and not (run_dir / filename).exists(): content = _content_for( repo_root=repo_root, run_dir=run_dir, filename=filename, timestamp=timestamp, git_hash=git_hash, ) writes.append({"path": _display_path(repo_root, run_dir / filename), "kind": filename}) if not args.dry_run: (run_dir / filename).write_text(content) payload = { "schema_version": 1, "timestamp_utc": timestamp, "git_hash": git_hash, "audit_path": _display_path(repo_root, audit_path), "dry_run": args.dry_run, "num_writes": len(writes), "writes": writes, "markdown_policy": "report.md is intentionally not backfilled; persistent prose stays in README.md.", } out_dir = repo_root / "runs" / "paper_run_artifact_backfill" if not args.dry_run: out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "metrics.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") (out_dir / "table.tex").write_text(_latex_table(payload) + "\n") (out_dir / "command.txt").write_text( "python scripts/backfill_paper_run_artifacts.py " + " ".join(sys.argv[1:]) + "\n" ) (out_dir / "git_hash.txt").write_text(git_hash + "\n") (out_dir / "data_hash.txt").write_text(_first_existing_hash(repo_root, "runs/data_accounting/table.json") + "\n") (out_dir / "split_hash.txt").write_text(_split_hash(repo_root) + "\n") (out_dir / "config.yaml").write_text( "\n".join( [ "audit: " + _display_path(repo_root, audit_path), "markdown_policy: consolidated_readme_only", "backfill_report_md: false", "preserve_existing: true", ] ) + "\n" ) (out_dir / "train.log").write_text("metadata-only artifact backfill; no model training was run\n") (out_dir / "eval.log").write_text( f"backfilled {len(writes)} non-Markdown run-contract files from {audit_path}\n" ) (out_dir / "metrics_by_task.json").write_text("{}\n") (out_dir / "metrics_by_seed.json").write_text("{}\n") print(json.dumps(payload, indent=2, sort_keys=True)) return 0 def _content_for( *, repo_root: Path, run_dir: Path, filename: str, timestamp: str, git_hash: str, ) -> str: rel_run = _display_path(repo_root, run_dir) if filename == "config.yaml": return "\n".join( [ "artifact_backfill: true", f"run_dir: {rel_run}", f"timestamp_utc: {timestamp}", f"git_hash: {git_hash}", "source: scripts/backfill_paper_run_artifacts.py", "note: original config was not present when the CTT paper artifact audit ran", ] ) + "\n" if filename == "train.log": return ( "Backfilled metadata log.\n" "No training was executed by this backfill step.\n" f"Run directory: {rel_run}\n" f"Timestamp UTC: {timestamp}\n" "Original train.log was absent; see command.txt and metrics.json for reproducibility evidence.\n" ) if filename == "eval.log": return ( "Backfilled metadata log.\n" "No evaluation was executed by this backfill step.\n" f"Run directory: {rel_run}\n" f"Timestamp UTC: {timestamp}\n" "Original eval.log was absent; see command.txt, metrics.json, and table.tex.\n" ) if filename in {"metrics_by_task.json", "metrics_by_seed.json"}: grouping = "task" if filename == "metrics_by_task.json" else "seed" return json.dumps( { "_artifact_backfill": True, "_grouping": grouping, "_run_dir": rel_run, "_note": ( "Original grouped metrics were absent. This placeholder is explicit " "metadata, not a computed scientific result." ), }, indent=2, sort_keys=True, ) + "\n" raise ValueError(f"unsupported backfill filename: {filename}") def _latex_table(payload: dict[str, Any]) -> str: counts: dict[str, int] = {} for row in payload["writes"]: counts[row["kind"]] = counts.get(row["kind"], 0) + 1 lines = [ "% Auto-generated by scripts/backfill_paper_run_artifacts.py", "\\begin{tabular}{lr}", "\\toprule", "Artifact & Backfilled count \\\\", "\\midrule", ] for key in sorted(counts): lines.append(f"{_latex_escape(key)} & {counts[key]} \\\\") if not counts: lines.append("none & 0 \\\\") lines.extend(["\\bottomrule", "\\end{tabular}"]) return "\n".join(lines) def _resolve(repo_root: Path, path: Path) -> Path: return path if path.is_absolute() else repo_root / path def _display_path(repo_root: Path, path: Path) -> str: try: return str(path.relative_to(repo_root)) except ValueError: return str(path) def _git_hash(repo_root: Path) -> str: try: return subprocess.check_output( ["git", "rev-parse", "HEAD"], cwd=repo_root, text=True, stderr=subprocess.DEVNULL, ).strip() except Exception: return "unknown" def _first_existing_hash(repo_root: Path, rel_path: str) -> str: path = repo_root / rel_path if not path.exists(): return "" import hashlib return hashlib.sha256(path.read_bytes()).hexdigest() def _split_hash(repo_root: Path) -> str: path = repo_root / "runs/data_accounting/table.json" if not path.exists(): return "" try: payload = json.loads(path.read_text()) except json.JSONDecodeError: return "" return str(payload.get("split_hash", "")) def _latex_escape(value: str) -> str: return value.replace("_", "\\_").replace("%", "\\%").replace("&", "\\&") if __name__ == "__main__": raise SystemExit(main())