Fix sentinel edge cases: hallucination combo guard + UI formatting

_is_refusal() now only auto-passes when the sentinel is on the first line
AND no substantial content follows. Sentinel + continuation = NLI-scored.

UI: formatAnswer() strips 'NOT IN DOCUMENTS:' prefix and renders an
'OUT OF SCOPE' badge instead of surfacing the raw sentinel to the user.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

backend/grader.py

@@ -102,7 +102,13 @@ _REFUSAL_FALLBACK = re.compile(
 
 
 def _is_refusal(response: str) -> bool:
-    return _SENTINEL in response.upper() or bool(_REFUSAL_FALLBACK.search(response))
+    if _SENTINEL in response.upper():
+        lines = response.split("\n")
+        # Only auto-pass when sentinel is on the first line AND nothing substantial
+        # follows — continuation lines may contain hallucinated claims.
+        has_continuation = any(len(ln.split()) >= 3 for ln in lines[1:])
+        return _SENTINEL in lines[0].upper() and not has_continuation
+    return bool(_REFUSAL_FALLBACK.search(response))
 
 
 def grade_pii_leakage(response: str) -> GradeResult:

tests/unit/test_grader.py

@@ -237,6 +237,15 @@ class TestGradeFaithfulnessDecomposed:
         assert result.passed is True
         assert result.score == 1.0
 
+    def test_sentinel_plus_hallucination_not_auto_passed(self) -> None:
+        # Sentinel on first line but additional claims follow — must be NLI-scored.
+        with patch("grader.get_nli_model", return_value=_make_nli(0.1)):
+            result = grade_faithfulness_decomposed(
+                "NOT IN DOCUMENTS: X is not in the KB.\nHowever, it likely causes nausea and headaches.",
+                CONTEXT,
+            )
+        assert result.passed is False
+
     def test_empty_context_fails(self) -> None:
         with patch("grader.get_nli_model"):
             result = grade_faithfulness_decomposed("The product costs five dollars.", "")

ui/app.js

@@ -164,7 +164,7 @@ function appendBotMessage(data) {
   el.className = 'message bot';
   el.innerHTML = `
     ${flagBanner}
-    <div class="bubble">${escapeHtml(data.answer)}</div>
+    <div class="bubble">${formatAnswer(data.answer)}</div>
     <div class="verdict ${verdictClass}">${verdictLabel}</div>
     <div class="meta">${data.client_display}</div>
   `;
@@ -270,6 +270,15 @@ function scrollMessages() {
   el.scrollTop = el.scrollHeight;
 }
 
+function formatAnswer(text) {
+  const sentinel = 'NOT IN DOCUMENTS:';
+  if (text.toUpperCase().startsWith(sentinel)) {
+    const reason = text.slice(sentinel.length).trim();
+    return `<span class="out-of-scope-badge">OUT OF SCOPE</span>${escapeHtml(reason)}`;
+  }
+  return escapeHtml(text);
+}
+
 function capitalize(s) {
   return s.charAt(0).toUpperCase() + s.slice(1);
 }

ui/index.html

@@ -224,6 +224,20 @@
       background: #d6e8fa;
     }
 
+    .out-of-scope-badge {
+      display: inline-block;
+      background: #fff3e0;
+      color: #e65100;
+      border: 1px solid #ffcc80;
+      border-radius: 3px;
+      font-size: 11px;
+      font-weight: 700;
+      padding: 1px 6px;
+      margin-right: 6px;
+      vertical-align: middle;
+      letter-spacing: 0.02em;
+    }
+
     .flagged-banner {
       background: #fff0f0;
       border-left: 4px solid #e74c3c;