"""Post-generation entailment verification.
Checks whether the generated answer is actually supported by the retrieved
evidence. Uses the same Qwen3-8B model as a verifier (no additional model
needed).
This is a label-free signal: it measures grounding quality, not correctness
against ground truth.
Usage:
from src.verification import build_verification_prompt, parse_verification
prompt = build_verification_prompt(answer, evidence, question)
# ... generate with model ...
entailment_score = parse_verification(model_output)
"""
import re
def build_verification_prompt(
answer: str,
evidence: list[dict],
question_text: str,
category: str,
) -> str:
"""Build a prompt that asks the model to verify its own answer.
The model is asked to classify whether the evidence supports the answer
as: SUPPORTED, PARTIALLY_SUPPORTED, or NOT_SUPPORTED.
"""
evidence_block = "\n".join(
f"[{i + 1}] {e.get('source_name', 'Unknown')} ({e.get('url', '')}): "
f"{e.get('snippet', '')[:500]}"
for i, e in enumerate(evidence[:5])
)
return f"""You are a medical evidence verifier. Your task is to determine whether the following answer is supported by the provided evidence.
QUESTION: {question_text}
PROPOSED ANSWER: {answer}
EVIDENCE:
{evidence_block}
Based ONLY on the evidence above, classify the answer as one of:
- SUPPORTED: The evidence directly supports or entails this answer.
- PARTIALLY_SUPPORTED: The evidence is relevant but does not fully confirm the answer.
- NOT_SUPPORTED: The evidence does not support this answer, or contradicts it.
Classification (respond with exactly one word: SUPPORTED, PARTIALLY_SUPPORTED, or NOT_SUPPORTED):"""
def parse_verification(output: str) -> float:
"""Parse verification model output into an entailment score (0.0-1.0).
Returns:
1.0 for SUPPORTED
0.5 for PARTIALLY_SUPPORTED
0.0 for NOT_SUPPORTED
0.3 for unparseable (conservative default)
"""
text = output.strip().upper()
# Strip thinking tags if present
if "" in text:
end = text.find("")
if end >= 0:
text = text[end + len(""):].strip()
if "NOT_SUPPORTED" in text or "NOT SUPPORTED" in text:
return 0.0
if "PARTIALLY" in text:
return 0.5
if "SUPPORTED" in text:
return 1.0
# Fallback: look for yes/no patterns
if re.search(r"\byes\b", text, re.IGNORECASE):
return 0.8
if re.search(r"\bno\b", text, re.IGNORECASE):
return 0.2
return 0.3 # Unparseable -> conservative default
def build_requery_prompt(
answer: str,
evidence: list[dict],
question_text: str,
contradiction_snippet: str,
) -> str:
"""Build a re-generation prompt when verification fails.
Highlights the contradicting evidence and asks the model to reconsider.
"""
evidence_block = "\n".join(
f"[{i + 1}] {e.get('source_name', 'Unknown')}: {e.get('snippet', '')[:400]}"
for i, e in enumerate(evidence[:5])
)
return f"""You are an expert in rare disease therapeutics and clinical genetics.
QUESTION: {question_text}
EVIDENCE:
{evidence_block}
IMPORTANT: A previous attempt answered "{answer}", but this may not be supported by the evidence.
Please carefully re-read the evidence above and provide the correct answer.
Pay special attention to this evidence passage:
>>> {contradiction_snippet[:300]} <<<
Return ONLY valid JSON:
{{"response": "", "evidence": [{{"source": "", "time_accessed": 0, "justification": ""}}]}}"""