hermes_overlay/tests/test_runbook_validator.py

"""Contract tests for strict runbook consistency validation."""
from __future__ import annotations

import hashlib
import importlib.util
import sys
from pathlib import Path


ROOT = Path(__file__).resolve().parents[2]
SPEC = importlib.util.spec_from_file_location(
    "validate_runbook_consistency", ROOT / "scripts" / "validate_runbook_consistency.py"
)
assert SPEC and SPEC.loader
validator = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = validator
SPEC.loader.exec_module(validator)


def _task(
    task_id: str,
    *,
    checked: bool,
    checkpoint: str,
    dependency: str = "None",
    complete_lists: bool = True,
    evidence: str = "Files `a.py` · Tests `pytest` · Result `PASS` · Commit `abc` · Limitations `None` · Completed `2026-07-23`",
) -> str:
    mark = "x" if checked else " "
    item = "x" if complete_lists else " "
    return f"""
### [{mark}] {task_id} — Task {task_id}

**Type:** ENGINEERING TASK
**Dependencies:** {dependency}

<!-- TASK_CHECKPOINT_START:{task_id} -->
**Checkpoint:** `{checkpoint}` · Session `test` · Verification `PASS`
**Exact next action:** Continue.
<!-- TASK_CHECKPOINT_END:{task_id} -->

#### Implementation checklist

- [{item}] implementation

#### Verification checklist

- [{item}] verification

#### Acceptance criteria

- [{item}] acceptance

**Completion evidence:** {evidence}

---
"""


def _runbook(
    *,
    first: str | None = None,
    second: str | None = None,
    current_task: str = "P0-T02",
    current_status: str = "IN_PROGRESS",
    last_done: str | None = "P0-T01",
    dashboard_done: int = 1,
    dashboard_total: int = 2,
    dashboard_status: str = "IN_PROGRESS",
    dashboard_task: str = "P0-T02",
    dashboard_phase: str = "Phase 0",
    handoff_task: str = "P0-T02",
    handoff_phase: str = "Phase 0",
    handoff_status: str = "IN_PROGRESS",
) -> str:
    first = first or _task("P0-T01", checked=True, checkpoint="DONE_VERIFIED")
    second = second or _task(
        "P0-T02",
        checked=False,
        checkpoint="IN_PROGRESS",
        dependency="P0-T01",
        complete_lists=False,
        evidence="Files `Pending` · Tests `Pending` · Result `Pending`",
    )
    last_value = "null" if last_done is None else f'"{last_done}"'
    return f"""---
runbook_version: "test"
project_name: "test"
current_track: "A_SOFTWARE_REMEDIATION"
current_phase: "Phase 0"
current_task: "{current_task}"
current_task_status: "{current_status}"
last_done_verified_task: {last_value}
current_release_mode: "PAPER_ONLY"
paper_only_required: true
testnet_execution_approved: false
live_shadow_approved: false
limited_live_approved: false
autonomous_execution_approved: false
open_p0_gaps: 0
open_p1_gaps: 0
operator_actions_open: 0
---

# 4. Progress Dashboard

| Phase | Name | Status | Verified / Total | Open blockers | Open P0 gaps | Last updated |
|---:|---|---|---:|---:|---:|---|
| 0 | Test | {dashboard_status} | {dashboard_done} / {dashboard_total} | 0 | 0 | now |

## Current operational state

- **Active phase:** {dashboard_phase}
- **Active task:** `{dashboard_task}` — test

---

# Phase 0 — Test

{first}
{second}

# Gap Register

No registered gaps.

# Session Log

## Current Session Handoff

- **Active phase:** {handoff_phase}
- **Active task:** {handoff_task}
- **Task status at end:** {handoff_status}
- **Exact next file and symbol:** `a.py:test`
- **Exact next command:** `pytest`
- **Exact expected result:** tests pass
- **Tracker save verified:** YES

---

# Final Rule
Paper only.
"""


def _errors(text: str, tmp_path: Path | None = None, evidence: Path | None = None) -> list[str]:
    result = validator.validate_text(
        text,
        root=tmp_path or ROOT,
        evidence_path=evidence,
        check_evidence_hashes=evidence is not None,
    )
    return result.errors


def test_valid_open_active_task_is_consistent():
    assert _errors(_runbook()) == []


def test_checked_task_with_unchecked_internal_item_fails():
    bad = _task("P0-T01", checked=True, checkpoint="DONE_VERIFIED", complete_lists=False)
    assert any("unfinished internal checklist" in e for e in _errors(_runbook(first=bad)))


def test_checked_task_with_todo_checkpoint_fails():
    bad = _task("P0-T01", checked=True, checkpoint="TODO")
    errors = _errors(_runbook(first=bad, last_done=None))
    assert any("incomplete checkpoint TODO" in e for e in errors)


def test_checked_task_with_pending_evidence_fails():
    bad = _task("P0-T01", checked=True, checkpoint="DONE_VERIFIED", evidence="Files `Pending`")
    assert any("pending or missing completion evidence" in e for e in _errors(_runbook(first=bad)))


def test_checked_task_with_unmet_dependency_fails():
    first = _task("P0-T01", checked=False, checkpoint="TODO", complete_lists=False, evidence="Pending")
    second = _task("P0-T02", checked=True, checkpoint="DONE_VERIFIED", dependency="P0-T01")
    errors = _errors(_runbook(
        first=first,
        second=second,
        current_task="P0-T01",
        current_status="TODO",
        last_done="P0-T02",
        dashboard_done=1,
        dashboard_task="P0-T01",
        handoff_task="P0-T01",
        handoff_status="TODO",
    ))
    assert any("unmet dependency P0-T01" in e for e in errors)


def test_yaml_dashboard_and_handoff_disagreement_fails():
    errors = _errors(_runbook(dashboard_task="P0-T01", handoff_phase="Phase 1"))
    assert any("Dashboard active task" in e for e in errors)
    assert any("Session Handoff active phase" in e for e in errors)


def test_progress_counter_mismatch_fails():
    assert any("counter 0/2 does not match actual 1/2" in e for e in _errors(_runbook(dashboard_done=0)))


def test_active_task_already_completed_fails():
    second = _task("P0-T02", checked=True, checkpoint="DONE_VERIFIED", dependency="P0-T01")
    errors = _errors(_runbook(
        second=second,
        current_status="DONE_VERIFIED",
        last_done="P0-T02",
        dashboard_done=2,
        dashboard_status="DONE_VERIFIED",
        handoff_status="DONE_VERIFIED",
    ))
    assert any("active task P0-T02 is already completed" in e for e in errors)


def test_completed_task_before_later_dependency_fails():
    first = _task("P0-T01", checked=True, checkpoint="DONE_VERIFIED", dependency="P0-T02")
    second = _task("P0-T02", checked=True, checkpoint="DONE_VERIFIED")
    errors = _errors(_runbook(
        first=first,
        second=second,
        current_task="OP-P0-01",
        current_status="TODO",
        last_done="P0-T02",
        dashboard_done=2,
        dashboard_total=2,
        dashboard_status="DONE_VERIFIED",
        dashboard_task="OP-P0-01",
        handoff_task="OP-P0-01",
        handoff_status="TODO",
    ))
    assert any("completed before later-listed dependency P0-T02" in e for e in errors)


def test_evidence_hash_mismatch_fails(tmp_path: Path):
    target = tmp_path / "artifact.txt"
    target.write_text("actual", encoding="utf-8")
    evidence = tmp_path / "evidence.md"
    wrong = hashlib.sha256(b"different").hexdigest()
    evidence.write_text(f"- `artifact.txt`: `{wrong}`\n", encoding="utf-8")
    assert any("evidence hash mismatch" in e for e in _errors(_runbook(), tmp_path, evidence))


def test_checked_task_with_unmet_phase_dependency_fails():
    first = _task("P0-T01", checked=False, checkpoint="TODO", complete_lists=False, evidence="Pending")
    second = _task("P0-T02", checked=True, checkpoint="DONE_VERIFIED", dependency="Phase 0")
    errors = _errors(_runbook(
        first=first,
        second=second,
        current_task="P0-T01",
        current_status="TODO",
        last_done="P0-T02",
        dashboard_done=1,
        dashboard_task="P0-T01",
        handoff_task="P0-T01",
        handoff_status="TODO",
    ))
    assert any("unmet phase dependency Phase 0" in error for error in errors)


def test_checked_task_requires_approved_release_gate():
    first = _task("P0-T01", checked=True, checkpoint="DONE_VERIFIED")
    second = _task("P0-T02", checked=True, checkpoint="DONE_VERIFIED", dependency="Release Gate B")
    text = _runbook(
        first=first,
        second=second,
        current_task="OP-P0-01",
        current_status="TODO",
        last_done="P0-T02",
        dashboard_done=2,
        dashboard_status="DONE_VERIFIED",
        dashboard_task="OP-P0-01",
        handoff_task="OP-P0-01",
        handoff_status="TODO",
    ).replace(
        "# Final Rule",
        "## Gate B — Testnet Read-Only\n\n**Status:** `NOT_REVIEWED`\n\n# Final Rule",
    )
    errors = _errors(text)
    assert any("requires Gate B APPROVED" in error for error in errors)


def test_blocked_open_task_requires_concrete_evidence():
    second = _task(
        "P0-T02",
        checked=False,
        checkpoint="BLOCKED",
        dependency="P0-T01",
        complete_lists=False,
        evidence="Files `Pending` · Tests `NOT_RUN`",
    )
    errors = _errors(_runbook(second=second, current_status="BLOCKED", handoff_status="BLOCKED"))
    assert any("requires concrete evidence" in error for error in errors)


def _provisional_task(task_id: str = "P0-T01", *, desktop_item: str = "Run Docker container scan on Desktop") -> str:
    return f"""
### [~] {task_id} — Task {task_id}

**Type:** ENGINEERING TASK
**Dependencies:** None

<!-- TASK_CHECKPOINT_START:{task_id} -->
**Checkpoint:** `WEB_IMPLEMENTED_PENDING_DESKTOP_VERIFICATION` · Session `test` · Verification `PARTIAL_PASS`
**Exact next action:** Run Docker verification on Desktop.
<!-- TASK_CHECKPOINT_END:{task_id} -->

#### Implementation checklist

- [x] implementation

#### Verification checklist

- [ ] {desktop_item}

#### Acceptance criteria

- [x] acceptance

**Completion evidence:** Files `a.py` · Tests `pytest` · Result `PARTIAL_PASS` · Commit `workspace` · Limitations `Docker Desktop verification required` · Completed `not final`

---
"""


def test_desktop_pending_task_allows_next_active_implementation() -> None:
    text = _runbook(
        first=_provisional_task(),
        current_task="P0-T02",
        current_status="IN_PROGRESS",
        last_done=None,
        dashboard_done=0,
        dashboard_task="P0-T02",
        handoff_task="P0-T02",
        handoff_status="IN_PROGRESS",
    )
    assert _errors(text) == []


def test_desktop_pending_task_rejects_non_desktop_unfinished_verification() -> None:
    text = _runbook(
        first=_provisional_task(desktop_item="Run ordinary unit tests"),
        current_task="P0-T02",
        current_status="IN_PROGRESS",
        last_done=None,
        dashboard_done=0,
        dashboard_task="P0-T02",
        handoff_task="P0-T02",
        handoff_status="IN_PROGRESS",
    )
    assert any("non-Desktop verification" in error for error in _errors(text))


def test_checked_task_may_depend_on_desktop_pending_implementation() -> None:
    second = _task("P0-T02", checked=True, checkpoint="DONE_VERIFIED", dependency="P0-T01")
    text = _runbook(
        first=_provisional_task(),
        second=second,
        current_task="P0-T01",
        current_status="WEB_IMPLEMENTED_PENDING_DESKTOP_VERIFICATION",
        last_done="P0-T02",
        dashboard_done=1,
        dashboard_status="IN_PROGRESS",
        dashboard_task="P0-T01",
        handoff_task="P0-T01",
        handoff_status="WEB_IMPLEMENTED_PENDING_DESKTOP_VERIFICATION",
    )
    assert _errors(text) == []