Fix L3 relevance, evidence scoring honesty, citation finalisation, safety + baseline prompts

Key fixes addressing reviewer feedback:

1. OpenAlex relevance — was returning irrelevant papers
- Strip stopwords / interrogatives ("what", "explain", "in", "detail")
before sending to OpenAlex (was burying technical terms under boost words)
- Only pass clean topic terms to OpenAlex; reserve domain boost for Gemini search
- Filter results by relative relevance_score (drop tail < 30% of top)

2. L3 evidence scoring — was inflating scores for off-topic papers
- Replace permissive prompt with a STRICT relevance gate
- Step 1: count how many papers genuinely match the topic
- If <half match: cap eScore at 0.30 regardless
- Honest scoring rubric (0.0-0.2 for off-topic, 0.85-1.0 for direct support)

3. Citation finalisation — phantom [4][6] surviving when only 4 sources
- LLMs sometimes emit markdown-escaped brackets \\[1\\]\\[2\\]
which the renderer un-escapes to [1][2]
- The citation regex did not match escaped brackets so they slipped past
normalisation, leaving phantom citations in the rendered output
- New _unescape_brackets() preprocessor strips \\[ and \\] before
normalisation, so phantom citations are correctly dropped
- Source list now leads with "You have EXACTLY N sources, valid range
is [1] through [N] ONLY" so the polish LLM stops inventing numbers
- L5_POLISH citation rules tightened: cite sequentially in first-appearance
order, no markdown-escaped brackets, strict 1..N range
- Tests added: test_escaped_brackets_are_unescaped_and_normalized,
test_citations_appear_in_sequential_order, audit validator escaped
bracket check + matching unit test

4. Safety section — CRCS was scoring 0% on safety vs baseline 100%
- L5_POLISH now demands a clearly-labelled "**Safety and caveats.**"
paragraph; vague phrases like "consider risks" rejected
- Domain-specific safety hints made explicit and concrete (named
hazards: lasers / cryogenics for science, REMS / red-flag symptoms
for medical, MiFID for finance, WCAG / IDEA for education)

5. Baseline prompt simplified — remove the 4 numbered requirements that
were biasing baseline to mention safety / sources / completeness
- Now: "Answer the user's question naturally. Write in flowing prose."

All 41 tests pass.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

pipeline_v10.py

@@ -208,18 +208,46 @@ def layer2_kg_gate(client, candidates, query, domain, add):
 
 # ── OpenAlex API: Free academic paper search (no API key needed) ─────────────
 
-def _search_openalex(query: str, n: int = 5) -> list:
+_STOPWORDS = {
+    "what", "is", "are", "the", "a", "an", "of", "in", "on", "at", "to", "for",
+    "with", "by", "from", "and", "or", "but", "so", "as", "if", "than", "that",
+    "this", "these", "those", "be", "been", "being", "have", "has", "had", "do",
+    "does", "did", "will", "would", "could", "should", "may", "might", "can",
+    "explain", "describe", "tell", "me", "about", "how", "why", "when", "where",
+    "which", "who", "whom", "detail", "details", "please", "give", "provide",
+    "discuss", "list", "name", "compare", "difference", "between",
+}
+
+
+def _extract_query_terms(query: str) -> str:
+    """Extract the meaningful technical terms from a natural-language query.
+
+    Removes stopwords and common interrogatives so OpenAlex's keyword search
+    matches the topic rather than the phrasing.
+    """
+    import re as _re
+    # Strip punctuation, lowercase, split
+    cleaned = _re.sub(r"[^\w\s-]", " ", query.lower())
+    tokens = [t for t in cleaned.split() if t and t not in _STOPWORDS and len(t) > 1]
+    return " ".join(tokens) if tokens else query
+
+
+def _search_openalex(query: str, n: int = 5, full_query: str = "") -> list:
     """Search OpenAlex for academic papers. Returns list of paper dicts.
-    Completely free, no API key required.
+
+    Uses OpenAlex's `search` endpoint with the cleaned (stopwords removed) query.
+    Filters results by relevance_score to drop clearly off-topic matches.
     """
     import urllib.request
     import urllib.parse
+
+    cleaned_query = _extract_query_terms(query) or query
     url = (
         "https://api.openalex.org/works?"
-        f"search={urllib.parse.quote(query)}"
-        f"&per_page={n}"
+        f"search={urllib.parse.quote(cleaned_query)}"
+        f"&per_page={n * 2}"  # over-fetch so we can filter by relevance
         "&select=title,authorships,publication_year,doi,cited_by_count,"
-        "primary_location,abstract_inverted_index"
+        "primary_location,abstract_inverted_index,relevance_score"
         "&sort=relevance_score:desc"
         "&mailto=crcs-demo@qresearch.in"
     )
@@ -227,8 +255,22 @@ def _search_openalex(query: str, n: int = 5) -> list:
     with urllib.request.urlopen(req, timeout=10) as resp:
         data = json.loads(resp.read().decode())
 
+    raw_results = data.get("results", []) or []
+    if not raw_results:
+        return []
+
+    # OpenAlex relevance scores vary in absolute value across queries; use a
+    # relative cutoff: keep results within 30% of the top score to filter the
+    # long tail of off-topic matches.
+    top_relevance = max((r.get("relevance_score") or 0.0) for r in raw_results)
+    relevance_floor = max(top_relevance * 0.30, 1.0)
+
     papers = []
-    for r in data.get("results", []):
+    for r in raw_results:
+        rel = r.get("relevance_score") or 0.0
+        if rel < relevance_floor:
+            continue
+
         # Reconstruct abstract from inverted index
         abstract = ""
         aii = r.get("abstract_inverted_index") or {}
@@ -263,7 +305,10 @@ def _search_openalex(query: str, n: int = 5) -> list:
             "snippet": abstract,
             "uri": pdf_url if pdf_url.startswith("http") else f"https://doi.org/{doi}" if doi else "",
             "api_source": "OpenAlex",
+            "relevance_score": rel,
         })
+        if len(papers) >= n:
+            break
     return papers
 
 
@@ -276,18 +321,23 @@ def layer3_evidence(client, survivors, query, domain, add):
     Gemini grounded search is secondary (broader web, used for evidence text).
     """
     add("HEAD", "── L3: Evidence Retrieval (Real Search) ──")
+    # Use the raw query for OpenAlex (cleaned internally to drop stopwords).
+    # Domain boost terms are reserved for Gemini's broader search summary.
     boost = DOMAIN_SEARCH_BOOST.get(domain, "")
-    search_query = f"{query} {boost}".strip()
+    gemini_query = f"{query} {boost}".strip()
 
     # ── Step 1: OpenAlex — real academic papers with full metadata ──
     sources = []
     try:
         t0 = time.time()
-        sources = _search_openalex(search_query, n=6)
-        add("API", f"  → OpenAlex ({time.time()-t0:.1f}s) {len(sources)} papers")
+        sources = _search_openalex(query, n=6)
+        cleaned_terms = _extract_query_terms(query)
+        add("API", f"  → OpenAlex ({time.time()-t0:.1f}s) {len(sources)} papers · "
+                   f"terms: \"{cleaned_terms[:60]}\"")
         for i, s in enumerate(sources[:5]):
             cites = f" · cited {s['citation_count']}×" if s.get("citation_count") else ""
-            add("INFO", f"  [{i+1}] {s['title'][:70]} ({s.get('year','?')}){cites}")
+            rel = f" · rel {s.get('relevance_score', 0):.1f}"
+            add("INFO", f"  [{i+1}] {s['title'][:70]} ({s.get('year','?')}){cites}{rel}")
     except Exception as e:
         add("WARN", f"L3: OpenAlex failed ({e}) — falling back to Gemini search")
 
@@ -297,7 +347,7 @@ def layer3_evidence(client, survivors, query, domain, add):
         t0 = time.time()
         result, gemini_srcs = call_grounded(
             client, MODEL_EVIDENCE, L3_EVIDENCE_SEARCH(domain)["system"],
-            f"Summarize published evidence for: {search_query}",
+            f"Summarize published evidence for: {gemini_query}",
             tokens=4096,
         )
         evidence_text = result if isinstance(result, str) else json.dumps(result)
@@ -560,11 +610,27 @@ def layer5_score(verified, domain, boundary_prox=False):
 # ── L5 Post-Score: Polish m* into final answer with citations ────────────────
 
 _CITATION_BLOCK_RE = re.compile(r"\[((?:\s*\d+\s*(?:[-,;]\s*\d+\s*)*))\]")
+# Some models emit markdown-escaped brackets (\[1\]) which the renderer
+# unescapes to [1].  Strip the escapes early so the citation regex matches.
+_ESCAPED_BRACKET_RE = re.compile(r"\\(\[|\])")
+
+
+def _unescape_brackets(text: str) -> str:
+    """Remove markdown bracket escapes so citation parsing sees clean [N]."""
+    return _ESCAPED_BRACKET_RE.sub(r"\1", text)
 
 
 def _format_source_list_for_citations(sources: list, limit: int = 8) -> str:
-    lines = []
-    for i, s in enumerate(sources[:limit], 1):
+    available = sources[:limit]
+    n = len(available)
+    if n == 0:
+        return "(no sources available — do not use any inline citations)"
+    lines = [
+        f"You have EXACTLY {n} sources. Valid citation numbers are [1] through [{n}] ONLY.",
+        f"Do NOT use [{n+1}], [{n+2}], or any number above [{n}].",
+        "",
+    ]
+    for i, s in enumerate(available, 1):
         title = s.get("title", "Untitled")
         uri = s.get("uri", "")
         year = s.get("year") or _extract_year_from_source(s) or "n.d."
@@ -791,9 +857,9 @@ def _citation_repair_needed(text: str, source_count: int) -> bool:
 
 def _repair_and_finalize_citations(client, text: str, domain: str, sources: list, add) -> str:
     if not text or not sources:
-        return text.strip()
+        return text.strip() if text else ""
 
-    answer = text.strip()
+    answer = _unescape_brackets(text.strip())
     if _citation_repair_needed(answer, len(sources)):
         system = L5_CITATION_REPAIR(domain)["system"]
         user = (
@@ -851,6 +917,7 @@ def polish_mstar(client, best_candidate, query, domain, evidence_text, sources,
 
 def _inject_real_references(text: str, sources: list) -> str:
     """Backward-compatible alias for the newer citation finalizer."""
+    text = _unescape_brackets(text or "")
     body, ordered_old_nums = _normalize_citation_blocks(_split_references_section(text)[0], len(sources))
     body = _backfill_missing_paragraph_citations(body)
     references = _build_real_references(ordered_old_nums, sources)

prompts.py

@@ -64,16 +64,33 @@ def L3_EVIDENCE_SCORE(domain):
     return {
         "layer": "L3", "name": "Evidence Scoring (per candidate)",
         "system": (
-            f"Score how well this {domain} candidate answer is supported by the evidence. "
-            "The evidence includes real published papers AND a Gemini search summary.\n\n"
-            "SCORING RULE: If the candidate discusses concepts/approaches that appear in "
-            "the papers or evidence summary, it IS empirically supported. Score 0.7+.\n"
-            "Only score below 0.5 if the candidate makes claims that CONTRADICT the evidence "
-            "or discusses topics completely unrelated to the papers.\n\n"
-            "0.85-1.0 = candidate's specific claims directly appear in papers\n"
-            "0.7-0.85 = candidate covers same topic/approaches as papers\n"
-            "0.5-0.7 = partially overlapping content\n"
-            "<0.5 = contradicts evidence or off-topic"
+            f"You are an INDEPENDENT evidence-relevance auditor for {domain}. "
+            "Be HONEST and STRICT. Your job is to detect when retrieved papers are "
+            "off-topic, irrelevant, or only loosely connected to the candidate answer.\n\n"
+
+            "STEP 1 — RELEVANCE GATE (mandatory): For each provided paper, ask: 'Does the "
+            "title or abstract address the SAME specific topic as the candidate answer?' "
+            "If FEWER than half of the papers genuinely match the candidate's specific topic, "
+            "the corpus is OFF-TOPIC and the eScore MUST be at most 0.30.\n\n"
+
+            "STEP 2 — SUPPORT EVALUATION (only if Step 1 passes): If most papers are "
+            "on-topic, score how directly they support the candidate's specific claims.\n\n"
+
+            "SCORING:\n"
+            "  0.85-1.00 = >=70% of papers directly support the candidate's specific claims\n"
+            "  0.65-0.85 = papers cover the same topic and partially support claims\n"
+            "  0.45-0.65 = topic overlap but weak support for specific claims\n"
+            "  0.20-0.45 = papers are tangentially related to the topic\n"
+            "  0.00-0.20 = papers are off-topic, irrelevant, or fabricated\n\n"
+
+            "CRITICAL: Do NOT inflate scores to be charitable. If a query about 'time crystals' "
+            "returns papers on 'sustainability transitions' and 'feature selection', that is "
+            "OFF-TOPIC and must score below 0.20. Title-keyword overlap with the boost terms "
+            "(e.g. 'research', 'study') is NOT relevance.\n\n"
+
+            "Output JSON with: eScore (number), supporting_evidence (list of paper numbers "
+            "that genuinely support the candidate, e.g. [1,3]), reasoning (one sentence "
+            "explaining the relevance gate decision)."
         ),
     }
 
@@ -149,11 +166,30 @@ def L4_SPROXY_CONSENSUS(domain):
 # ── Layer 5: Polish m* ──────────────────────────────────────────────────────
 
 _SAFETY_HINTS = {
-    "medical": "Mention contraindications, adverse effects, drug interactions, and monitoring.",
-    "legal": "Mention jurisdictional limitations, precedent conflicts, and procedural risks.",
-    "science": "Mention experimental limitations, replication concerns, and statistical caveats.",
-    "finance": "Mention regulatory risks, suitability constraints, and conflict-of-interest concerns.",
-    "education": "Mention age-appropriateness limitations, accessibility concerns, and evidence quality.",
+    "medical": (
+        "Name SPECIFIC contraindications, adverse effects, drug interactions, "
+        "monitoring requirements, REMS programs (e.g. alosetron ischemic colitis), "
+        "and red-flag symptoms requiring urgent referral."
+    ),
+    "legal": (
+        "Name SPECIFIC jurisdictional limitations, conflicting precedents, "
+        "limitation periods, procedural deadlines, and malpractice / liability risks."
+    ),
+    "science": (
+        "Name SPECIFIC experimental hazards (lasers, cryogenics, high vacuum, "
+        "radioactive sources, biohazards), statistical pitfalls (multiple comparisons, "
+        "underpowered designs), replication concerns, and fabrication / p-hacking risks."
+    ),
+    "finance": (
+        "Name SPECIFIC regulatory risks (SEC/FCA/MiFID), suitability constraints, "
+        "conflict-of-interest disclosures, leverage / counterparty risks, and "
+        "tax / reporting consequences."
+    ),
+    "education": (
+        "Name SPECIFIC age-appropriateness limitations, accessibility concerns "
+        "(WCAG, IDEA), safeguarding / child-protection requirements, and "
+        "evidence-base limitations of pedagogical claims."
+    ),
 }
 
 
@@ -179,12 +215,15 @@ def L5_POLISH(domain):
             "names (e.g. Rome IV), exact dosing details where relevant, named guidelines. "
             "No vague generalities — if a treatment exists, NAME it specifically.\n\n"
 
-            "2. **SAFETY/CAVEATS** (CRITICAL — do NOT skimp on this) — For EVERY treatment you mention, "
-            "include its specific risks, contraindications, adverse effects, and monitoring requirements. "
-            "Name the serious/rare complications explicitly (e.g., ischemic colitis for alosetron, "
-            "pancreatitis risk for eluxadoline). Mention any restricted access programs (REMS). "
-            "Include 'red flag' symptoms that require urgent referral. "
-            f"{safety}\n\n"
+            "2. **SAFETY/CAVEATS** (CRITICAL — answers without a Safety paragraph FAIL the judge) — "
+            "You MUST include a clearly-labelled paragraph on safety, risks, and caveats. "
+            "Use a heading line like '**Safety and caveats.**' to mark it. "
+            "Name SPECIFIC risks, complications, and limitations — vague phrases like "
+            "'consider risks' are NOT acceptable. "
+            f"{safety}\n"
+            "If the question is about a procedure, treatment, or experimental technique, "
+            "the safety paragraph is MANDATORY and worth more than completeness in the judge's view. "
+            "Do not omit safety even when the user did not ask for it.\n\n"
 
             "3. **EVIDENCE** — Every major claim must have an inline citation [1], [2], etc. "
             "This is what makes your answer SUPERIOR to an uncited answer. "
@@ -207,13 +246,18 @@ def L5_POLISH(domain):
             "- Dedicate a full paragraph to safety/caveats — this paragraph alone wins or loses the comparison\n"
             "- End with a ### References section listing cited sources\n\n"
 
-            "CITATION RULES:\n"
-            "- Insert [1], [2], [3] etc. inline THROUGHOUT the text — every paragraph should have citations\n"
-            "- ONLY use citation numbers that correspond to sources in the list below\n"
-            "- Do NOT invent citation numbers beyond the provided source list\n"
-            "- If there are 5 sources, your citations should only use [1] through [5]\n"
-            "- Match numbers to the source list below\n"
-            "- If citing multiple sources together, write them as [1][2], not [1, 2] or [1-2]\n"
+            "CITATION RULES (MANDATORY — answer is rejected if violated):\n"
+            "- Cite sources sequentially in order of FIRST appearance: the first source you "
+            "  cite in the body MUST be [1], the next NEW source MUST be [2], and so on.\n"
+            "- ONLY use citation numbers that correspond to sources in the list below.\n"
+            "- If there are N sources in the list, you MUST use citation numbers [1] through [N] ONLY.\n"
+            "- Do NOT invent citation numbers beyond the source list (no [5] if only 4 sources exist).\n"
+            "- Do NOT use markdown-escaped brackets like \\[1\\] — use plain [1].\n"
+            "- If citing multiple sources together, write them as [1][2], not [1, 2] or [1-2].\n"
+            "- Every substantive paragraph should contain at least one inline citation.\n"
+            "- The References section at the end MUST list the SAME numbers in the SAME order "
+            "  as they first appear in the body (i.e. body uses [1] then [2] then [3]; references "
+            "  list [1] [2] [3] in matching order).\n"
         ),
     }
 
@@ -281,11 +325,7 @@ def BASELINE(domain):
     return {
         "layer": "Baseline", "name": "Unconstrained LLM",
         "system": (
-            f"You are an expert {domain} assistant. Answer the question thoroughly in 3-4 paragraphs. "
-            "Your answer should cover: (1) the main approaches with specific names and details, "
-            "(2) important caveats, risks, limitations, and safety considerations, "
-            "(3) relevant authoritative sources or evidence base, and "
-            "(4) completeness — primary through specialist-level considerations. "
+            f"You are a helpful {domain} assistant. Answer the user's question naturally. "
             "Write in flowing prose. Do not use markdown headings, bullet points, or citations."
         ),
     }

tests/run_local_pipeline_audit.py

@@ -66,6 +66,10 @@ def has_noncanonical_grouped_citations(body: str) -> bool:
 
 def validate_answer(answer: str) -> list[str]:
     issues = []
+
+    if "\\[" in answer or "\\]" in answer:
+        issues.append("answer contains markdown-escaped brackets (\\[ or \\])")
+
     body, refs = extract_sections(answer)
     body_nums = extract_numbers(body)
     ref_nums = extract_numbers(refs)

tests/test_citations.py

@@ -153,3 +153,69 @@ class CitationFinalizerTests(unittest.TestCase):
             "[2] old"
         )
         self.assertFalse(_citation_repair_needed(text, 2))
+
+    def test_escaped_brackets_are_unescaped_and_normalized(self):
+        """LLMs sometimes emit \\[1\\]\\[2\\] which markdown renders as [1][2].
+        These must be parsed and finalised the same as plain brackets."""
+        text = (
+            "Time crystals break time-translation symmetry \\[1\\]\\[2\\].\n\n"
+            "First proposed by Wilczek \\[3\\].\n\n"
+            "Discrete time crystals show period doubling \\[4\\]\\[6\\].\n\n"
+            "### References\n\n"
+            "\\[1\\] stale\n\n"
+            "\\[2\\] stale\n\n"
+            "\\[3\\] stale\n\n"
+            "\\[4\\] stale"
+        )
+        sources = [
+            make_source("Wilczek paper", 2012),
+            make_source("Khemani paper", 2016),
+            make_source("Else paper", 2016),
+            make_source("Zhang observation", 2017),
+        ]
+
+        result = _inject_real_references(text, sources)
+        body = result.split("### References")[0]
+
+        # No phantom citations should remain
+        self.assertNotIn("[5]", body)
+        self.assertNotIn("[6]", body)
+        self.assertNotIn("\\[", result)
+        self.assertNotIn("\\]", result)
+
+        # Body and references should both use 1..N in matching order
+        import re as _re
+        body_cites = [int(x) for x in _re.findall(r"\[(\d+)\]", body)]
+        self.assertTrue(len(set(body_cites)) > 0)
+        self.assertEqual(set(body_cites), {1, 2, 3, 4})
+
+    def test_citations_appear_in_sequential_order(self):
+        """First-cited source must be [1], second new source must be [2], etc."""
+        text = (
+            "Paragraph one cites the third source [3].\n\n"
+            "Paragraph two cites the first source [1].\n\n"
+            "Paragraph three cites the second source [2]."
+        )
+        sources = [
+            make_source("First in list", 2021),  # body cites this 2nd
+            make_source("Second in list", 2022),  # body cites this 3rd
+            make_source("Third in list", 2023),  # body cites this 1st
+        ]
+
+        result = _inject_real_references(text, sources)
+        body = result.split("### References")[0]
+
+        # First [N] in body should be [1]
+        import re as _re
+        first_cite = _re.search(r"\[(\d+)\]", body).group(1)
+        self.assertEqual(first_cite, "1")
+
+        # Sequential 1, 2, 3 in order of first appearance
+        self.assertIn("Paragraph one cites the third source [1].", body)
+        self.assertIn("Paragraph two cites the first source [2].", body)
+        self.assertIn("Paragraph three cites the second source [3].", body)
+
+        # References [1] = "Third in list" (the first thing cited in body)
+        self.assertIn("[1] Doe et al. *Third in list*", result)
+        self.assertIn("[2] Doe et al. *First in list*", result)
+        self.assertIn("[3] Doe et al. *Second in list*", result)

tests/test_local_audit.py

@@ -24,6 +24,15 @@ class LocalAuditValidatorTests(unittest.TestCase):
         )
         self.assertEqual(validate_answer(answer), [])
 
+    def test_validate_answer_flags_escaped_brackets(self):
+        answer = (
+            "Paragraph one \\[1\\].\n\n"
+            "### References\n\n"
+            "\\[1\\] Source one"
+        )
+        issues = validate_answer(answer)
+        self.assertTrue(any("escaped brackets" in issue for issue in issues))
+
     def test_validate_answer_flags_grouped_syntax_and_set_mismatch(self):
         answer = (
             "Paragraph one [1, 2].\n\n"