comparison.py

"""
Method comparison module.

Computes a fair, honest side-by-side comparison of three detection layers
used in this project, all scored on the same 0-100 "misinformation danger"
scale so they can be charted together:

  1. BERT (NER) only    -- a naive baseline using ONLY entity/keyword
                            patterns extracted by the BERT-based NER model.
                            No evidence, no reasoning.
  2. RAG + LLM           -- the real verdict pipeline: evidence retrieved
                            from WHO/CDC/NIH, reasoned over by an LLM.
                            Accurate on truth, but static -- it doesn't say
                            how urgent or dangerous a false claim is.
  3. GNN-Enhanced        -- takes the RAG verdict and adds the trained
                            Graph Attention Network's spread-risk
                            prediction on top. This is the only method
                            that answers "how much should I worry about
                            this, right now" -- not just true/false.

IMPORTANT: this does NOT run three independent LLM calls. Methods 2 and 3
reuse the verify_claim() result and predict_spread() result that the
/api/verify endpoint already computed -- so this comparison is free (no
extra API calls, no extra latency) beyond a couple of arithmetic formulas.
"""

import sys
import os
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "gnn"))
from claim_features import VERDICT_RISK, sensational_score


def _risk_label(score: int) -> str:
    if score >= 60:
        return "High Risk"
    if score >= 30:
        return "Medium Risk"
    return "Low Risk"


def compute_method_comparison(claim_text: str, verify_result: dict, spread_result: dict) -> dict:
    """
    claim_text: the original claim text
    verify_result: the dict returned by verify_claim() (verdict, confidence, entities, ...)
    spread_result: the dict returned by predict_spread() (virality_score, risk_level, ...)
    """
    entities = verify_result.get("entities", []) or []
    verdict = verify_result.get("verdict", "Unverified")
    confidence = verify_result.get("confidence", 0) or 0

    # ---- Method 1: BERT (NER) only -- naive baseline ----
    entity_density = min(len(entities) / 5, 1.0)
    sensational = sensational_score(claim_text)
    risk_bert = round(100 * min(1.0, 0.5 * sensational + 0.5 * entity_density))

    # ---- Method 2: RAG + LLM -- evidence-grounded verdict ----
    verdict_risk = VERDICT_RISK.get(verdict, 0.3)
    risk_rag = round(verdict_risk * confidence)

    # ---- Method 3: GNN-Enhanced -- RAG verdict + spread-risk modeling ----
    virality_score = spread_result.get("virality_score", 0) or 0
    risk_gnn = round(0.4 * risk_rag + 0.6 * virality_score)

    methods = [
        {
            "key": "bert",
            "name": "BERT (NER) Only",
            "score": risk_bert,
            "label": _risk_label(risk_bert),
            "description": "Uses only medical entities and sensational-language patterns extracted by the BERT-based NER model. No evidence, no source-checking -- a naive baseline.",
            "is_winner": False,
        },
        {
            "key": "rag",
            "name": "RAG + LLM",
            "score": risk_rag,
            "label": verdict,
            "description": "Grounds the claim in real evidence retrieved from WHO/CDC/NIH, then an LLM reasons over that evidence. Accurate on truth, but doesn't assess urgency or real-world danger.",
            "is_winner": False,
        },
        {
            "key": "gnn",
            "name": "GNN-Enhanced (Full Pipeline)",
            "score": risk_gnn,
            "label": spread_result.get("risk_level", _risk_label(risk_gnn)),
            "description": "Builds on the RAG-grounded verdict and adds a trained Graph Attention Network's spread-risk prediction. The only method that answers not just 'is this false' but 'how urgently does this need attention'.",
            "is_winner": True,
        },
    ]

    conclusion = (
        f"NER-only pattern matching rates this claim {risk_bert}/100 using surface signals alone, with no "
        f"evidence behind it. The evidence-grounded RAG+LLM verdict is more trustworthy on truth "
        f"(\"{verdict}\", {confidence}% confidence) but stops there. The GNN-enhanced score of {risk_gnn}/100 "
        f"is the most complete: it keeps that evidence-grounded verdict and adds real spread-risk modeling on top, "
        f"so it's the only method that tells you both whether this is false AND how much it deserves urgent attention."
    )

    return {
        "methods": methods,
        "conclusion": conclusion,
        "winner": "gnn",
    }


if __name__ == "__main__":
    # Quick manual test -- run: python comparison.py
    fake_verify_result = {
        "verdict": "False", "confidence": 92,
        "entities": [{"text": "garlic"}, {"text": "COVID-19"}],
    }
    fake_spread_result = {"virality_score": 95, "risk_level": "High Risk"}

    result = compute_method_comparison(
        "Garlic cures COVID-19 instantly, doctors hate this secret!",
        fake_verify_result, fake_spread_result,
    )
    for m in result["methods"]:
        print(f"{m['name']}: {m['score']}/100 ({m['label']}) {'<-- WINNER' if m['is_winner'] else ''}")
    print(f"\nConclusion: {result['conclusion']}")