"""Post-generation entailment verification. Checks whether the generated answer is actually supported by the retrieved evidence. Uses the same Qwen3-8B model as a verifier (no additional model needed). This is a label-free signal: it measures grounding quality, not correctness against ground truth. Usage: from src.verification import build_verification_prompt, parse_verification prompt = build_verification_prompt(answer, evidence, question) # ... generate with model ... entailment_score = parse_verification(model_output) """ import re def build_verification_prompt( answer: str, evidence: list[dict], question_text: str, category: str, ) -> str: """Build a prompt that asks the model to verify its own answer. The model is asked to classify whether the evidence supports the answer as: SUPPORTED, PARTIALLY_SUPPORTED, or NOT_SUPPORTED. """ evidence_block = "\n".join( f"[{i + 1}] {e.get('source_name', 'Unknown')} ({e.get('url', '')}): " f"{e.get('snippet', '')[:500]}" for i, e in enumerate(evidence[:5]) ) return f"""You are a medical evidence verifier. Your task is to determine whether the following answer is supported by the provided evidence. QUESTION: {question_text} PROPOSED ANSWER: {answer} EVIDENCE: {evidence_block} Based ONLY on the evidence above, classify the answer as one of: - SUPPORTED: The evidence directly supports or entails this answer. - PARTIALLY_SUPPORTED: The evidence is relevant but does not fully confirm the answer. - NOT_SUPPORTED: The evidence does not support this answer, or contradicts it. Classification (respond with exactly one word: SUPPORTED, PARTIALLY_SUPPORTED, or NOT_SUPPORTED):""" def parse_verification(output: str) -> float: """Parse verification model output into an entailment score (0.0-1.0). Returns: 1.0 for SUPPORTED 0.5 for PARTIALLY_SUPPORTED 0.0 for NOT_SUPPORTED 0.3 for unparseable (conservative default) """ text = output.strip().upper() # Strip thinking tags if present if "" in text: end = text.find("") if end >= 0: text = text[end + len(""):].strip() if "NOT_SUPPORTED" in text or "NOT SUPPORTED" in text: return 0.0 if "PARTIALLY" in text: return 0.5 if "SUPPORTED" in text: return 1.0 # Fallback: look for yes/no patterns if re.search(r"\byes\b", text, re.IGNORECASE): return 0.8 if re.search(r"\bno\b", text, re.IGNORECASE): return 0.2 return 0.3 # Unparseable -> conservative default def build_requery_prompt( answer: str, evidence: list[dict], question_text: str, contradiction_snippet: str, ) -> str: """Build a re-generation prompt when verification fails. Highlights the contradicting evidence and asks the model to reconsider. """ evidence_block = "\n".join( f"[{i + 1}] {e.get('source_name', 'Unknown')}: {e.get('snippet', '')[:400]}" for i, e in enumerate(evidence[:5]) ) return f"""You are an expert in rare disease therapeutics and clinical genetics. QUESTION: {question_text} EVIDENCE: {evidence_block} IMPORTANT: A previous attempt answered "{answer}", but this may not be supported by the evidence. Please carefully re-read the evidence above and provide the correct answer. Pay special attention to this evidence passage: >>> {contradiction_snippet[:300]} <<< Return ONLY valid JSON: {{"response": "", "evidence": [{{"source": "", "time_accessed": 0, "justification": ""}}]}}"""