File size: 1,917 Bytes
dfcaf7a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | from __future__ import annotations
import json
from pathlib import Path
from scripts import backfill_paper_run_artifacts
def test_backfill_writes_non_markdown_contract_only(tmp_path: Path) -> None:
run_dir = tmp_path / "runs" / "example"
audit_dir = tmp_path / "runs" / "paper_ctt_audit"
run_dir.mkdir(parents=True)
audit_dir.mkdir(parents=True)
(run_dir / "config.yaml").write_text("keep: original\n")
(audit_dir / "audit.json").write_text(
json.dumps(
{
"run_artifacts": [
{
"run_dir": "runs/example",
"missing_advisor_contract": [
"config.yaml",
"train.log",
"eval.log",
"metrics_by_task.json",
"metrics_by_seed.json",
"report.md",
],
}
]
}
)
)
assert (
backfill_paper_run_artifacts.main(
[
"--repo-root",
str(tmp_path),
"--audit",
"runs/paper_ctt_audit/audit.json",
]
)
== 0
)
assert (run_dir / "config.yaml").read_text() == "keep: original\n"
assert (run_dir / "train.log").exists()
assert (run_dir / "eval.log").exists()
assert json.loads((run_dir / "metrics_by_task.json").read_text())["_artifact_backfill"]
assert json.loads((run_dir / "metrics_by_seed.json").read_text())["_artifact_backfill"]
assert not (run_dir / "report.md").exists()
backfill_metrics = json.loads(
(tmp_path / "runs" / "paper_run_artifact_backfill" / "metrics.json").read_text()
)
assert backfill_metrics["num_writes"] == 4
assert "report.md" in backfill_metrics["markdown_policy"]
|