Replace ad-hoc refusal regexes with NOT IN DOCUMENTS sentinel

Prompt now instructs the model to use 'NOT IN DOCUMENTS: ...' for any
out-of-scope response. _is_refusal() checks for this sentinel first —
one deterministic check instead of an open-ended regex list.

Fallback patterns retained for responses that predate the instruction.
Aligns with NOT IN DOCUMENTS pattern from knowledge/problems/keeps-hallucinating.md.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

backend/grader.py

@@ -89,10 +89,12 @@ TOKEN_BUDGET = 512
 RELEVANCY_THRESHOLD = 0.45
 FAITHFULNESS_THRESHOLD = 0.35
 
-_REFUSAL_PATTERNS = re.compile(
+_SENTINEL = "NOT IN DOCUMENTS"
+
+# Fallback patterns for responses that predate the sentinel instruction or
+# where the model ignores the sentinel format.
+_REFUSAL_FALLBACK = re.compile(
     r"(i (don't|do not|cannot|can't|'m not able to) (have|find|provide|answer)|"
-    r"not able to (answer|provide|help)|"
-    r"(falls?|is) outside (of )?(the )?(scope|knowledge base)|"
     r"not enough (information|context)|"
     r"the (context|provided) (does not|doesn't) (contain|include|mention))",
     re.IGNORECASE,
@@ -100,7 +102,7 @@ _REFUSAL_PATTERNS = re.compile(
 
 
 def _is_refusal(response: str) -> bool:
-    return bool(_REFUSAL_PATTERNS.search(response))
+    return _SENTINEL in response.upper() or bool(_REFUSAL_FALLBACK.search(response))
 
 
 def grade_pii_leakage(response: str) -> GradeResult:

backend/pipeline.py

@@ -39,7 +39,11 @@ MIN_RETRIEVAL_SCORE = 0.1
 SYSTEM_PROMPT = """\
 You are a helpful assistant for {client_display} ({domain} domain).
 Answer the user's question using only the information in the provided context.
-Be concise. If the context does not contain enough information to answer, say so clearly rather than speculating.
+Be concise.
+
+If the context does not contain enough information to answer, respond with exactly:
+NOT IN DOCUMENTS: [one sentence explaining what information is missing]
+Do not speculate, infer, or use knowledge outside the provided context.
 
 You MUST use the following terminology. These are the only acceptable terms — do not substitute synonyms:
 {term_list}"""

tests/unit/test_grader.py

@@ -223,16 +223,16 @@ class TestGradeFaithfulnessDecomposed:
         assert result.metadata["claims"][0]["supported"] is True
         assert result.metadata["claims"][1]["supported"] is False
 
-    def test_refusal_auto_passes(self) -> None:
+    def test_refusal_sentinel_auto_passes(self) -> None:
         result = grade_faithfulness_decomposed(
-            "I don't have enough information to answer that.", CONTEXT
+            "NOT IN DOCUMENTS: The context does not contain information about this drug.", CONTEXT
         )
         assert result.passed is True
         assert result.score == 1.0
 
-    def test_refusal_not_able_to_auto_passes(self) -> None:
+    def test_refusal_fallback_auto_passes(self) -> None:
         result = grade_faithfulness_decomposed(
-            "I'm not able to answer that as it falls outside the knowledge base.", CONTEXT
+            "I don't have enough information to answer that.", CONTEXT
         )
         assert result.passed is True
         assert result.score == 1.0