Harden exploitability check expectations (#84)

src/open_range/validator/exploitability.py

@@ -15,6 +15,16 @@ _META_COMMANDS = {"submit_flag", "submit_evidence", "submit_finding", "auth", "l
 class ExploitabilityCheck:
     """Execute every golden-path step and verify ``expect_in_stdout`` appears."""
 
+    def __init__(self, *, require_expectation: bool = True) -> None:
+        """Create an exploitability check.
+
+        Args:
+            require_expectation: When ``True`` (default), every non-meta golden
+                path step must define ``expect_in_stdout``. Missing expectations
+                are treated as validation failures.
+        """
+        self.require_expectation = require_expectation
+
     async def check(self, snapshot: SnapshotSpec, containers: ContainerSet) -> CheckResult:
         if not snapshot.golden_path:
             return CheckResult(
@@ -43,12 +53,20 @@ class ExploitabilityCheck:
 
             expected = step.expect_in_stdout
             if not expected:
-                logger.warning(
-                    "exploitability: golden path step %d has no expect_in_stdout — "
-                    "output not validated",
-                    step.step,
+                message = (
+                    f"golden path step {step.step} has no expect_in_stdout"
                 )
-                unvalidated_steps.append(step.step)
+                if self.require_expectation:
+                    failed_steps.append({
+                        "step": step.step,
+                        "error": message,
+                    })
+                else:
+                    logger.warning(
+                        "exploitability: %s — output not validated",
+                        message,
+                    )
+                    unvalidated_steps.append(step.step)
             elif expected not in output:
                 failed_steps.append({
                     "step": step.step,
@@ -71,6 +89,7 @@ class ExploitabilityCheck:
                 "unvalidated_steps": unvalidated_steps,
                 "issues": issues,
                 "total_steps": len(snapshot.golden_path),
+                "require_expectation": self.require_expectation,
             },
             error="" if passed else f"{len(failed_steps)} golden-path step(s) failed",
         )

tests/test_validator.py

@@ -336,6 +336,42 @@ async def test_exploitability_skips_meta_commands(mock_containers):
     assert result.details["skipped_steps"] == [2]
 
 
+@pytest.mark.asyncio
+async def test_exploitability_fails_when_expectation_missing_in_strict_mode(mock_containers):
+    from open_range.validator.exploitability import ExploitabilityCheck
+
+    spec = SnapshotSpec(
+        golden_path=[
+            GoldenPathStep(step=1, command="curl http://web/", expect_in_stdout=""),
+        ],
+    )
+    mock_containers.exec_results[("attacker", "curl http://web/")] = "Welcome"
+
+    result = await ExploitabilityCheck().check(spec, mock_containers)
+    assert result.passed is False
+    assert result.details["require_expectation"] is True
+    assert result.details["failed_steps"][0]["error"] == (
+        "golden path step 1 has no expect_in_stdout"
+    )
+
+
+@pytest.mark.asyncio
+async def test_exploitability_allows_missing_expectation_in_lenient_mode(mock_containers):
+    from open_range.validator.exploitability import ExploitabilityCheck
+
+    spec = SnapshotSpec(
+        golden_path=[
+            GoldenPathStep(step=1, command="curl http://web/", expect_in_stdout=""),
+        ],
+    )
+    mock_containers.exec_results[("attacker", "curl http://web/")] = "Welcome"
+
+    result = await ExploitabilityCheck(require_expectation=False).check(spec, mock_containers)
+    assert result.passed is True
+    assert result.details["require_expectation"] is False
+    assert result.details["unvalidated_steps"] == [1]
+
+
 # ---------------------------------------------------------------------------
 # Check 3: Patchability
 # ---------------------------------------------------------------------------