[NOTICKET] fix(eval): don't let a crashed help case pass its rules (flag errored)

Review follow-up on eval/help: if HelpAgent.astream raised, output became "ERROR:..."
and assertions still ran on it — must_not_contain_any trivially passed and language_match
fell back to Indonesian, so an infra failure could inflate compliance. Now such a case is
flagged `errored`, its rules are not scored, it counts as a failure, and it's surfaced
under a separate ERRORED line (not mistaken for a rule violation). Also: type
_build_state -> AnalysisState (mypy strict), and document the known limitations
(temperature variance, detector-graded language, errored handling) in the README.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

eval/help/README.md

@@ -60,3 +60,18 @@ two are reported separately.
 `help_dataset.json` — see the `_about` / `_carried_over` doc keys in the file. Language
 detection reuses `help._detect_reply_language`; `report_ready.missing` uses the codes
 `analysis` / `delta` mapped to the real `is_report_ready` strings in the runner.
+
+## Known limitations
+
+- **Compliance is approximate across runs.** `HelpAgent` runs at `temperature=0.3`, so the
+  reply varies; a borderline case can flip pass/fail between runs. Treat the rate as a
+  signal, not a fixed number — re-run before trusting a single-point drop.
+- **`language_match` grades with the same detector the feature uses** (`_detect_reply_language`
+  over the reply). It verifies the model obeyed the `[Reply language]` directive, assuming the
+  detector is correct — the detector itself is unit-tested separately in
+  `tests/unit/agents/handlers/test_help.py`. It can also misfire on a reply that mixes
+  languages (e.g. an Indonesian reply quoting an English business question).
+- **Errored cases (stream crash) count as failures, not rule violations.** If `astream` raises
+  (Azure down, timeout), the case is flagged `errored` and reported under a separate `ERRORED`
+  line — assertions are NOT run on the error string (a crash must not trivially "pass" a
+  `must_not_contain_any`). A run with errors is not a clean pass; re-run once the cause clears.

eval/help/run_eval.py

@@ -47,7 +47,7 @@ from langchain_core.callbacks import BaseCallbackHandler
 from langchain_core.messages import AIMessage, BaseMessage, HumanMessage
 from langchain_core.outputs import LLMResult
 
-from src.agents.gate import stub_analysis_state
+from src.agents.gate import AnalysisState, stub_analysis_state
 from src.agents.handlers.help import HelpAgent, ReportReadiness, _detect_reply_language
 from src.agents.report.readiness import _MISSING_ANALYSIS, _MISSING_DELTA
 
@@ -143,6 +143,7 @@ class CaseResult:
     all_passed: bool | None  # None when manual_review (not auto-scored)
     latency_ms: float
     tokens: dict[str, int]
+    errored: bool = False  # the astream call raised — infra failure, not a rule verdict
 
 
 def load_cases(path: Path) -> list[dict[str, Any]]:
@@ -151,7 +152,7 @@ def load_cases(path: Path) -> list[dict[str, Any]]:
     return list(data["cases"])
 
 
-def _build_state(spec: dict[str, Any]):
+def _build_state(spec: dict[str, Any]) -> AnalysisState:
     """Build an AnalysisState from a case's `state` block (defaults from the stub)."""
     return stub_analysis_state().model_copy(
         update={
@@ -205,13 +206,23 @@ async def run_case(case: dict[str, Any]) -> CaseResult:
     latency_ms = round((time.perf_counter() - start) * 1000, 1)
 
     manual = bool(case.get("manual_review"))
+    errored = output.startswith("ERROR:")
     asserts: list[AssertResult] = []
-    if not manual:
+    if errored:
+        # Don't run rule checks on an error string — a crash must not "pass" a
+        # must_not_contain_any (the pattern is trivially absent) or a language check.
+        # Count it as a failure, but flag it as errored so it reads as infra, not a
+        # rule violation (overrides manual_review — a crash isn't reviewable).
+        asserts = [AssertResult(type="stream", passed=False, detail=_truncate(output, 100))]
+        all_passed: bool | None = False
+    elif manual:
+        all_passed = None
+    else:
         for spec in case.get("asserts", []):
             check = _ASSERT_CHECKS[spec["type"]]
             passed, detail = check(output, spec)
             asserts.append(AssertResult(type=spec["type"], passed=passed, detail=detail))
-    all_passed = None if manual else all(a.passed for a in asserts)
+        all_passed = all(a.passed for a in asserts)
 
     return CaseResult(
         id=case["id"],
@@ -223,6 +234,7 @@ async def run_case(case: dict[str, Any]) -> CaseResult:
         all_passed=all_passed,
         latency_ms=latency_ms,
         tokens=collector.tokens,
+        errored=errored,
     )
 
 
@@ -246,6 +258,7 @@ def summarize(results: list[CaseResult]) -> dict[str, Any]:
         for g in GROUPS
         if any(r.group == g for r in results)
     }
+    errored = [r for r in results if r.errored]
     return {
         "total": len(results),
         "scored": len(scored),
@@ -257,6 +270,7 @@ def summarize(results: list[CaseResult]) -> dict[str, Any]:
         "by_group": by_group,
         "held_out": _compliance([r for r in scored if not r.carried_over]),
         "carried_over": _compliance([r for r in scored if r.carried_over]),
+        "errored": {"count": len(errored), "ids": [r.id for r in errored]},
     }
 
 
@@ -314,12 +328,21 @@ def format_summary(summary: dict[str, Any], results: list[CaseResult]) -> str:
         f"    carried_over   {co['passed']}/{co['n']}  "
         f"{co['compliance'] * 100:.0f}%   <- regression"
     )
-    failures = [r for r in results if r.all_passed is False]
+    # Rule failures (real disobedience) vs errored (infra/stream crash) — kept apart so
+    # a crashed run isn't misread as the model breaking a rule.
+    failures = [r for r in results if r.all_passed is False and not r.errored]
     lines.append("")
     lines.append(f"  FAILURES ({len(failures)})")
     for r in failures:
         bad = [f"{a.type}({a.detail})" for a in r.asserts if not a.passed]
         lines.append(f"    {r.id:<20} {r.group:<13} {'; '.join(bad)}")
+    err = summary["errored"]
+    if err["count"]:
+        lines.append("")
+        lines.append(
+            f"  ERRORED ({err['count']}) - stream crashed, counted as fail NOT a rule miss"
+            f"  -> {', '.join(err['ids'])}"
+        )
     return "\n".join(lines)
 
 
@@ -339,6 +362,7 @@ def build_report(
         "by_group": summary["by_group"],
         "held_out": summary["held_out"],
         "carried_over": summary["carried_over"],
+        "errored": summary["errored"],
         "cases": [asdict(r) for r in results],
     }