src/verification.py

"""Post-generation entailment verification.

Checks whether the generated answer is actually supported by the retrieved
evidence. Uses the same Qwen3-8B model as a verifier (no additional model
needed).

This is a label-free signal: it measures grounding quality, not correctness
against ground truth.

Usage:
    from src.verification import build_verification_prompt, parse_verification

    prompt = build_verification_prompt(answer, evidence, question)
    # ... generate with model ...
    entailment_score = parse_verification(model_output)
"""

import re


def build_verification_prompt(
    answer: str,
    evidence: list[dict],
    question_text: str,
    category: str,
) -> str:
    """Build a prompt that asks the model to verify its own answer.

    The model is asked to classify whether the evidence supports the answer
    as: SUPPORTED, PARTIALLY_SUPPORTED, or NOT_SUPPORTED.
    """
    evidence_block = "\n".join(
        f"[{i + 1}] {e.get('source_name', 'Unknown')} ({e.get('url', '')}): "
        f"{e.get('snippet', '')[:500]}"
        for i, e in enumerate(evidence[:5])
    )

    return f"""You are a medical evidence verifier. Your task is to determine whether the following answer is supported by the provided evidence.

QUESTION: {question_text}

PROPOSED ANSWER: {answer}

EVIDENCE:
{evidence_block}

Based ONLY on the evidence above, classify the answer as one of:
- SUPPORTED: The evidence directly supports or entails this answer.
- PARTIALLY_SUPPORTED: The evidence is relevant but does not fully confirm the answer.
- NOT_SUPPORTED: The evidence does not support this answer, or contradicts it.

Classification (respond with exactly one word: SUPPORTED, PARTIALLY_SUPPORTED, or NOT_SUPPORTED):"""


def parse_verification(output: str) -> float:
    """Parse verification model output into an entailment score (0.0-1.0).

    Returns:
        1.0 for SUPPORTED
        0.5 for PARTIALLY_SUPPORTED
        0.0 for NOT_SUPPORTED
        0.3 for unparseable (conservative default)
    """
    text = output.strip().upper()

    # Strip thinking tags if present
    if "<THINK>" in text:
        end = text.find("</THINK>")
        if end >= 0:
            text = text[end + len("</THINK>"):].strip()

    if "NOT_SUPPORTED" in text or "NOT SUPPORTED" in text:
        return 0.0
    if "PARTIALLY" in text:
        return 0.5
    if "SUPPORTED" in text:
        return 1.0

    # Fallback: look for yes/no patterns
    if re.search(r"\byes\b", text, re.IGNORECASE):
        return 0.8
    if re.search(r"\bno\b", text, re.IGNORECASE):
        return 0.2

    return 0.3  # Unparseable -> conservative default


def build_requery_prompt(
    answer: str,
    evidence: list[dict],
    question_text: str,
    contradiction_snippet: str,
) -> str:
    """Build a re-generation prompt when verification fails.

    Highlights the contradicting evidence and asks the model to reconsider.
    """
    evidence_block = "\n".join(
        f"[{i + 1}] {e.get('source_name', 'Unknown')}: {e.get('snippet', '')[:400]}"
        for i, e in enumerate(evidence[:5])
    )

    return f"""You are an expert in rare disease therapeutics and clinical genetics.

QUESTION: {question_text}

EVIDENCE:
{evidence_block}

IMPORTANT: A previous attempt answered "{answer}", but this may not be supported by the evidence.
Please carefully re-read the evidence above and provide the correct answer.

Pay special attention to this evidence passage:
>>> {contradiction_snippet[:300]} <<<

Return ONLY valid JSON:
{{"response": "<your answer>", "evidence": [{{"source": "<url>", "time_accessed": 0, "justification": "<explanation>"}}]}}"""