auto-sync 2026-07-04T06:38:25Z workspace (part 4)

workspace/scripts/audit_ctt_paper_artifacts.py

@@ -108,13 +108,15 @@ ADVISOR_RUN_FILES: tuple[str, ...] = (
     "eval.log",
     "metrics_by_task.json",
     "metrics_by_seed.json",
-    "report.md",
 )
 
 
 INPUT_RE = re.compile(r"\\input\{([^}]+)\}")
 
 
+ALLOWED_MARKDOWN_FILES: tuple[str, ...] = ("README.md",)
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(
         description=(
@@ -143,7 +145,15 @@ def main(argv: list[str] | None = None) -> int:
     paper_inputs = _paper_input_checks(repo_root, paper_path, paper_text)
     required_paths = [] if args.skip_implementation_checks else _required_path_checks(repo_root)
     run_artifacts = _run_artifact_checks(repo_root, paper_inputs["run_dirs"])
-    summary = _summary(forbidden, phrase_checks, paper_inputs, required_paths, run_artifacts)
+    markdown_policy = _markdown_policy_checks(repo_root)
+    summary = _summary(
+        forbidden,
+        phrase_checks,
+        paper_inputs,
+        required_paths,
+        run_artifacts,
+        markdown_policy,
+    )
 
     payload = {
         "schema_version": 1,
@@ -151,9 +161,10 @@ def main(argv: list[str] | None = None) -> int:
             "markdown_policy": "consolidated_readme_only",
             "markdown_note": (
                 "Advisor checklist asks for report.md files, but the current workspace "
-                "policy keeps README.md as the only Markdown document. Missing report.md "
-                "is recorded as a warning, not regenerated here."
+                "policy keeps README.md as the only Markdown document. This audit now "
+                "fails on additional Markdown files instead of regenerating report.md."
             ),
+            "allowed_markdown_files": list(ALLOWED_MARKDOWN_FILES),
         },
         "paper": str(paper_path.relative_to(repo_root)),
         "summary": summary,
@@ -162,6 +173,7 @@ def main(argv: list[str] | None = None) -> int:
         "paper_inputs": paper_inputs,
         "required_paths": required_paths,
         "run_artifacts": run_artifacts,
+        "markdown_policy": markdown_policy,
     }
 
     out_dir = _resolve(repo_root, args.out_dir)
@@ -175,6 +187,8 @@ def main(argv: list[str] | None = None) -> int:
                 "paper: " + str(paper_path.relative_to(repo_root)),
                 "markdown_policy: consolidated_readme_only",
                 "strict_report_md: false",
+                "allowed_markdown_files:",
+                *[f"  - {name}" for name in ALLOWED_MARKDOWN_FILES],
             ]
         )
         + "\n"
@@ -293,22 +307,52 @@ def _run_artifact_checks(repo_root: Path, run_dirs: list[str]) -> list[dict[str,
                 "status": status,
                 "missing_required": missing_required,
                 "missing_advisor_contract": missing_advisor,
-                "markdown_report_policy": (
-                    "warning_only_consolidated_readme"
-                    if "report.md" in missing_advisor
-                    else "present"
-                ),
+                "markdown_report_policy": "consolidated_readme_only",
             }
         )
     return rows
 
 
+def _markdown_policy_checks(repo_root: Path) -> dict[str, Any]:
+    ignored_parts = {".git", ".venv"}
+    rows = []
+    for path in sorted(repo_root.rglob("*.md")):
+        try:
+            rel = path.relative_to(repo_root)
+        except ValueError:
+            continue
+        if any(part in ignored_parts for part in rel.parts):
+            continue
+        rel_text = rel.as_posix()
+        rows.append(
+            {
+                "path": rel_text,
+                "status": "pass" if rel_text in ALLOWED_MARKDOWN_FILES else "fail",
+            }
+        )
+    unexpected = [row["path"] for row in rows if row["status"] != "pass"]
+    missing_allowed = [
+        name for name in ALLOWED_MARKDOWN_FILES if not (repo_root / name).exists()
+    ]
+    return {
+        "policy": "consolidated_readme_only",
+        "allowed": list(ALLOWED_MARKDOWN_FILES),
+        "rows": rows,
+        "unexpected_markdown": unexpected,
+        "missing_allowed": missing_allowed,
+        "num_markdown_files": len(rows),
+        "num_unexpected_markdown": len(unexpected),
+        "status": "pass" if not unexpected and not missing_allowed else "fail",
+    }
+
+
 def _summary(
     forbidden: list[dict[str, Any]],
     phrase_checks: list[dict[str, Any]],
     paper_inputs: dict[str, Any],
     required_paths: list[dict[str, Any]],
     run_artifacts: list[dict[str, Any]],
+    markdown_policy: dict[str, Any],
 ) -> dict[str, Any]:
     failures = (
         len(forbidden)
@@ -316,6 +360,7 @@ def _summary(
         + sum(1 for item in phrase_checks if item["status"] == "fail")
         + sum(1 for item in required_paths if item["status"] == "fail")
         + sum(1 for item in run_artifacts if item["status"] == "fail")
+        + (0 if markdown_policy["status"] == "pass" else 1)
     )
     warnings = sum(len(item.get("missing_advisor_contract", [])) for item in run_artifacts)
     return {
@@ -323,6 +368,7 @@ def _summary(
         "num_failures": failures,
         "num_warnings": warnings,
         "num_forbidden_matches": len(forbidden),
+        "num_unexpected_markdown": markdown_policy["num_unexpected_markdown"],
         "num_paper_inputs": paper_inputs["num_inputs"],
         "num_run_dirs_in_paper": len(paper_inputs["run_dirs"]),
     }
@@ -340,7 +386,8 @@ def _latex_table(payload: dict[str, Any]) -> str:
         f"Paper inputs & {summary['num_paper_inputs']} & {_latex_status(payload['paper_inputs']['num_missing'] == 0)} \\\\",
         f"Run dirs in paper & {summary['num_run_dirs_in_paper']} & {_latex_status(all(row['status'] == 'pass' for row in payload['run_artifacts']))} \\\\",
         f"Implementation paths & {len(payload['required_paths'])} & {_latex_status(all(row['status'] == 'pass' for row in payload['required_paths']))} \\\\",
-        f"Advisor-contract warnings & {summary['num_warnings']} & warn \\\\",
+        f"Unexpected Markdown files & {summary['num_unexpected_markdown']} & {_latex_status(summary['num_unexpected_markdown'] == 0)} \\\\",
+        f"Advisor-contract warnings & {summary['num_warnings']} & {'pass' if summary['num_warnings'] == 0 else 'warn'} \\\\",
         "\\bottomrule",
         "\\end{tabular}",
     ]