workspace/tests/test_backfill_paper_run_artifacts.py

from __future__ import annotations

import json
from pathlib import Path

from scripts import backfill_paper_run_artifacts


def test_backfill_writes_non_markdown_contract_only(tmp_path: Path) -> None:
    run_dir = tmp_path / "runs" / "example"
    audit_dir = tmp_path / "runs" / "paper_ctt_audit"
    run_dir.mkdir(parents=True)
    audit_dir.mkdir(parents=True)
    (run_dir / "config.yaml").write_text("keep: original\n")
    (audit_dir / "audit.json").write_text(
        json.dumps(
            {
                "run_artifacts": [
                    {
                        "run_dir": "runs/example",
                        "missing_advisor_contract": [
                            "config.yaml",
                            "train.log",
                            "eval.log",
                            "metrics_by_task.json",
                            "metrics_by_seed.json",
                            "report.md",
                        ],
                    }
                ]
            }
        )
    )

    assert (
        backfill_paper_run_artifacts.main(
            [
                "--repo-root",
                str(tmp_path),
                "--audit",
                "runs/paper_ctt_audit/audit.json",
            ]
        )
        == 0
    )

    assert (run_dir / "config.yaml").read_text() == "keep: original\n"
    assert (run_dir / "train.log").exists()
    assert (run_dir / "eval.log").exists()
    assert json.loads((run_dir / "metrics_by_task.json").read_text())["_artifact_backfill"]
    assert json.loads((run_dir / "metrics_by_seed.json").read_text())["_artifact_backfill"]
    assert not (run_dir / "report.md").exists()

    backfill_metrics = json.loads(
        (tmp_path / "runs" / "paper_run_artifact_backfill" / "metrics.json").read_text()
    )
    assert backfill_metrics["num_writes"] == 4
    assert "report.md" in backfill_metrics["markdown_policy"]