"""Contract tests for strict runbook consistency validation.""" from __future__ import annotations import hashlib import importlib.util import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] SPEC = importlib.util.spec_from_file_location( "validate_runbook_consistency", ROOT / "scripts" / "validate_runbook_consistency.py" ) assert SPEC and SPEC.loader validator = importlib.util.module_from_spec(SPEC) sys.modules[SPEC.name] = validator SPEC.loader.exec_module(validator) def _task( task_id: str, *, checked: bool, checkpoint: str, dependency: str = "None", complete_lists: bool = True, evidence: str = "Files `a.py` · Tests `pytest` · Result `PASS` · Commit `abc` · Limitations `None` · Completed `2026-07-23`", ) -> str: mark = "x" if checked else " " item = "x" if complete_lists else " " return f""" ### [{mark}] {task_id} — Task {task_id} **Type:** ENGINEERING TASK **Dependencies:** {dependency} **Checkpoint:** `{checkpoint}` · Session `test` · Verification `PASS` **Exact next action:** Continue. #### Implementation checklist - [{item}] implementation #### Verification checklist - [{item}] verification #### Acceptance criteria - [{item}] acceptance **Completion evidence:** {evidence} --- """ def _runbook( *, first: str | None = None, second: str | None = None, current_task: str = "P0-T02", current_status: str = "IN_PROGRESS", last_done: str | None = "P0-T01", dashboard_done: int = 1, dashboard_total: int = 2, dashboard_status: str = "IN_PROGRESS", dashboard_task: str = "P0-T02", dashboard_phase: str = "Phase 0", handoff_task: str = "P0-T02", handoff_phase: str = "Phase 0", handoff_status: str = "IN_PROGRESS", ) -> str: first = first or _task("P0-T01", checked=True, checkpoint="DONE_VERIFIED") second = second or _task( "P0-T02", checked=False, checkpoint="IN_PROGRESS", dependency="P0-T01", complete_lists=False, evidence="Files `Pending` · Tests `Pending` · Result `Pending`", ) last_value = "null" if last_done is None else f'"{last_done}"' return f"""--- runbook_version: "test" project_name: "test" current_track: "A_SOFTWARE_REMEDIATION" current_phase: "Phase 0" current_task: "{current_task}" current_task_status: "{current_status}" last_done_verified_task: {last_value} current_release_mode: "PAPER_ONLY" paper_only_required: true testnet_execution_approved: false live_shadow_approved: false limited_live_approved: false autonomous_execution_approved: false open_p0_gaps: 0 open_p1_gaps: 0 operator_actions_open: 0 --- # 4. Progress Dashboard | Phase | Name | Status | Verified / Total | Open blockers | Open P0 gaps | Last updated | |---:|---|---|---:|---:|---:|---| | 0 | Test | {dashboard_status} | {dashboard_done} / {dashboard_total} | 0 | 0 | now | ## Current operational state - **Active phase:** {dashboard_phase} - **Active task:** `{dashboard_task}` — test --- # Phase 0 — Test {first} {second} # Gap Register No registered gaps. # Session Log ## Current Session Handoff - **Active phase:** {handoff_phase} - **Active task:** {handoff_task} - **Task status at end:** {handoff_status} - **Exact next file and symbol:** `a.py:test` - **Exact next command:** `pytest` - **Exact expected result:** tests pass - **Tracker save verified:** YES --- # Final Rule Paper only. """ def _errors(text: str, tmp_path: Path | None = None, evidence: Path | None = None) -> list[str]: result = validator.validate_text( text, root=tmp_path or ROOT, evidence_path=evidence, check_evidence_hashes=evidence is not None, ) return result.errors def test_valid_open_active_task_is_consistent(): assert _errors(_runbook()) == [] def test_checked_task_with_unchecked_internal_item_fails(): bad = _task("P0-T01", checked=True, checkpoint="DONE_VERIFIED", complete_lists=False) assert any("unfinished internal checklist" in e for e in _errors(_runbook(first=bad))) def test_checked_task_with_todo_checkpoint_fails(): bad = _task("P0-T01", checked=True, checkpoint="TODO") errors = _errors(_runbook(first=bad, last_done=None)) assert any("incomplete checkpoint TODO" in e for e in errors) def test_checked_task_with_pending_evidence_fails(): bad = _task("P0-T01", checked=True, checkpoint="DONE_VERIFIED", evidence="Files `Pending`") assert any("pending or missing completion evidence" in e for e in _errors(_runbook(first=bad))) def test_checked_task_with_unmet_dependency_fails(): first = _task("P0-T01", checked=False, checkpoint="TODO", complete_lists=False, evidence="Pending") second = _task("P0-T02", checked=True, checkpoint="DONE_VERIFIED", dependency="P0-T01") errors = _errors(_runbook( first=first, second=second, current_task="P0-T01", current_status="TODO", last_done="P0-T02", dashboard_done=1, dashboard_task="P0-T01", handoff_task="P0-T01", handoff_status="TODO", )) assert any("unmet dependency P0-T01" in e for e in errors) def test_yaml_dashboard_and_handoff_disagreement_fails(): errors = _errors(_runbook(dashboard_task="P0-T01", handoff_phase="Phase 1")) assert any("Dashboard active task" in e for e in errors) assert any("Session Handoff active phase" in e for e in errors) def test_progress_counter_mismatch_fails(): assert any("counter 0/2 does not match actual 1/2" in e for e in _errors(_runbook(dashboard_done=0))) def test_active_task_already_completed_fails(): second = _task("P0-T02", checked=True, checkpoint="DONE_VERIFIED", dependency="P0-T01") errors = _errors(_runbook( second=second, current_status="DONE_VERIFIED", last_done="P0-T02", dashboard_done=2, dashboard_status="DONE_VERIFIED", handoff_status="DONE_VERIFIED", )) assert any("active task P0-T02 is already completed" in e for e in errors) def test_completed_task_before_later_dependency_fails(): first = _task("P0-T01", checked=True, checkpoint="DONE_VERIFIED", dependency="P0-T02") second = _task("P0-T02", checked=True, checkpoint="DONE_VERIFIED") errors = _errors(_runbook( first=first, second=second, current_task="OP-P0-01", current_status="TODO", last_done="P0-T02", dashboard_done=2, dashboard_total=2, dashboard_status="DONE_VERIFIED", dashboard_task="OP-P0-01", handoff_task="OP-P0-01", handoff_status="TODO", )) assert any("completed before later-listed dependency P0-T02" in e for e in errors) def test_evidence_hash_mismatch_fails(tmp_path: Path): target = tmp_path / "artifact.txt" target.write_text("actual", encoding="utf-8") evidence = tmp_path / "evidence.md" wrong = hashlib.sha256(b"different").hexdigest() evidence.write_text(f"- `artifact.txt`: `{wrong}`\n", encoding="utf-8") assert any("evidence hash mismatch" in e for e in _errors(_runbook(), tmp_path, evidence)) def test_checked_task_with_unmet_phase_dependency_fails(): first = _task("P0-T01", checked=False, checkpoint="TODO", complete_lists=False, evidence="Pending") second = _task("P0-T02", checked=True, checkpoint="DONE_VERIFIED", dependency="Phase 0") errors = _errors(_runbook( first=first, second=second, current_task="P0-T01", current_status="TODO", last_done="P0-T02", dashboard_done=1, dashboard_task="P0-T01", handoff_task="P0-T01", handoff_status="TODO", )) assert any("unmet phase dependency Phase 0" in error for error in errors) def test_checked_task_requires_approved_release_gate(): first = _task("P0-T01", checked=True, checkpoint="DONE_VERIFIED") second = _task("P0-T02", checked=True, checkpoint="DONE_VERIFIED", dependency="Release Gate B") text = _runbook( first=first, second=second, current_task="OP-P0-01", current_status="TODO", last_done="P0-T02", dashboard_done=2, dashboard_status="DONE_VERIFIED", dashboard_task="OP-P0-01", handoff_task="OP-P0-01", handoff_status="TODO", ).replace( "# Final Rule", "## Gate B — Testnet Read-Only\n\n**Status:** `NOT_REVIEWED`\n\n# Final Rule", ) errors = _errors(text) assert any("requires Gate B APPROVED" in error for error in errors) def test_blocked_open_task_requires_concrete_evidence(): second = _task( "P0-T02", checked=False, checkpoint="BLOCKED", dependency="P0-T01", complete_lists=False, evidence="Files `Pending` · Tests `NOT_RUN`", ) errors = _errors(_runbook(second=second, current_status="BLOCKED", handoff_status="BLOCKED")) assert any("requires concrete evidence" in error for error in errors) def _provisional_task(task_id: str = "P0-T01", *, desktop_item: str = "Run Docker container scan on Desktop") -> str: return f""" ### [~] {task_id} — Task {task_id} **Type:** ENGINEERING TASK **Dependencies:** None **Checkpoint:** `WEB_IMPLEMENTED_PENDING_DESKTOP_VERIFICATION` · Session `test` · Verification `PARTIAL_PASS` **Exact next action:** Run Docker verification on Desktop. #### Implementation checklist - [x] implementation #### Verification checklist - [ ] {desktop_item} #### Acceptance criteria - [x] acceptance **Completion evidence:** Files `a.py` · Tests `pytest` · Result `PARTIAL_PASS` · Commit `workspace` · Limitations `Docker Desktop verification required` · Completed `not final` --- """ def test_desktop_pending_task_allows_next_active_implementation() -> None: text = _runbook( first=_provisional_task(), current_task="P0-T02", current_status="IN_PROGRESS", last_done=None, dashboard_done=0, dashboard_task="P0-T02", handoff_task="P0-T02", handoff_status="IN_PROGRESS", ) assert _errors(text) == [] def test_desktop_pending_task_rejects_non_desktop_unfinished_verification() -> None: text = _runbook( first=_provisional_task(desktop_item="Run ordinary unit tests"), current_task="P0-T02", current_status="IN_PROGRESS", last_done=None, dashboard_done=0, dashboard_task="P0-T02", handoff_task="P0-T02", handoff_status="IN_PROGRESS", ) assert any("non-Desktop verification" in error for error in _errors(text)) def test_checked_task_may_depend_on_desktop_pending_implementation() -> None: second = _task("P0-T02", checked=True, checkpoint="DONE_VERIFIED", dependency="P0-T01") text = _runbook( first=_provisional_task(), second=second, current_task="P0-T01", current_status="WEB_IMPLEMENTED_PENDING_DESKTOP_VERIFICATION", last_done="P0-T02", dashboard_done=1, dashboard_status="IN_PROGRESS", dashboard_task="P0-T01", handoff_task="P0-T01", handoff_status="WEB_IMPLEMENTED_PENDING_DESKTOP_VERIFICATION", ) assert _errors(text) == []