tests/test_evaluator.py

"""Tests for Evaluator orchestration."""

from datetime import date
import json

from src.data.cascade_extractor import _infer_severity
from src.eval.evaluator import Evaluator, _build_description, judge_fingerprint
from src.models.schemas import (
    CascadeChain,
    CascadeNode,
    EventEvaluation,
    FloodEvent,
    PredictionResult,
)


def _test_event(**overrides):
    base = dict(
        event_id="2025-0001-FRA",
        country="France",
        iso="FRA",
        region="Europe",
        location="Paris",
        start_date=date(2025, 6, 1),
    )
    base.update(overrides)
    return FloodEvent(**base)


def _predicted_chain():
    return CascadeChain(
        event_id="PRED-2025-06-01-FRA",
        trigger_summary="Heavy rain flood in Paris",
        trigger_country="France",
        trigger_iso="FRA",
        trigger_date=date(2025, 6, 1),
        cascade_events=[
            CascadeNode(id="E1", description="Metro flooded",
                        domain="infrastructure/transport",
                        severity="high", mechanism="Drainage overwhelmed"),
        ],
    )


class StubPredictor:
    """Predictor stub that records each predict() call for assertions."""

    def __init__(self, chain, reference_event_ids=None, similarity_scores=None):
        self._chain = chain
        self._refs = reference_event_ids or ["2023-0866-FRA"]
        self._sims = similarity_scores or [0.81]
        self.calls: list[dict] = []

    def predict(self, **kwargs):
        self.calls.append(kwargs)
        return PredictionResult(
            predicted_chain=self._chain,
            confidence_scores={"E1": 0.8},
            reference_event_ids=self._refs,
            similarity_scores=self._sims,
        )


def _base_config(tmp_path, cascade_index: str | None = None) -> dict:
    cfg = {
        "prompts": {"judge_cascade": "prompts/judge_cascade.txt"},
        "knowledge": {"judge": "knowledge/expert_judge.md"},
        "evaluation": {
            "judge_model": None,
            "judge_temperature": 0.0,
            "max_articles_per_event": 6,
            "max_chars_per_article": 4000,
            "output_dir": str(tmp_path / "evaluation"),
        },
    }
    if cascade_index is not None:
        cfg["paths"] = {"cascade_index": cascade_index}
    return cfg


JUDGE_RESPONSE_OK = json.dumps({
    "node_evaluations": [{
        "node_id": "E1",
        "evidence_level": "verified",
        "evidence_quote": "'Metro line closed after flood' (https://lemonde.fr/x)",
        "reasoning": "Article reports the metro closure directly.",
        "timing_alignment": "match",
    }],
    "missed_cascades": [{
        "description": "Power outage",
        "domain": "infrastructure/power",
        "evidence_quote": "'25,000 homes without power' (https://afp.com/y)",
    }],
    "summary": "Captured transport impact, missed power outage.",
})


def test_evaluate_event_returns_structured_result(
    tmp_path, fake_llm, write_article, articles_dir
):
    write_article(event_id="2025-0001-FRA", url="https://lemonde.fr/x",
                  title="Flood hits Paris", date="2025-06-01",
                  text="Metro line 4 was closed after floodwater filled tunnels.",
                  hash_suffix="aa")

    evaluator = Evaluator(
        llm_client=fake_llm([JUDGE_RESPONSE_OK]),
        predictor=StubPredictor(_predicted_chain()),
        articles_dir=articles_dir,
        config=_base_config(tmp_path),
    )

    result = evaluator.evaluate_event(_test_event())

    assert isinstance(result, EventEvaluation)
    assert result.event_id == "2025-0001-FRA"
    assert len(result.node_evaluations) == 1
    assert result.node_evaluations[0].evidence_level == "verified"
    assert len(result.missed_cascades) == 1
    assert result.missed_cascades[0].domain == "infrastructure/power"
    assert result.retrieval_info == [{"event_id": "2023-0866-FRA",
                                       "similarity": 0.81}]
    assert result.predicted_chain.cascade_events[0].id == "E1"
    assert result.news_sources_used == ["https://lemonde.fr/x"]


def test_evaluate_event_writes_cache_file(
    tmp_path, fake_llm, write_article, articles_dir
):
    write_article(event_id="2025-0001-FRA", url="https://x", title="t",
                  date="2025-06-01", text="Metro flooded", hash_suffix="aa")

    evaluator = Evaluator(
        llm_client=fake_llm([JUDGE_RESPONSE_OK]),
        predictor=StubPredictor(_predicted_chain()),
        articles_dir=articles_dir,
        config=_base_config(tmp_path),
    )
    evaluator.evaluate_event(_test_event(), write_cache=True)

    cache_file = tmp_path / "evaluation" / "2025-0001-FRA.json"
    assert cache_file.exists()
    loaded = EventEvaluation.model_validate_json(cache_file.read_text())
    assert loaded.event_id == "2025-0001-FRA"
    # Cache carries the current fingerprint, which is non-empty for a
    # successful judge run.
    assert loaded.judge_fingerprint != ""


def test_evaluate_event_no_articles_skips_judge(
    tmp_path, fake_llm, articles_dir
):
    """When no articles exist, judge is NOT called; a placeholder record is returned."""
    client = fake_llm([])  # empty — would fail if judge is called

    evaluator = Evaluator(
        llm_client=client,
        predictor=StubPredictor(_predicted_chain()),
        articles_dir=articles_dir,  # empty
        config=_base_config(tmp_path),
    )
    result = evaluator.evaluate_event(_test_event())

    assert client.calls == []
    assert result.summary.startswith("No news articles")
    assert result.node_evaluations == []
    assert result.news_sources_used == []
    assert evaluator.last_cache_status == "skip_no_news"


def test_evaluate_event_malformed_judge_json_captures_error(
    tmp_path, fake_llm, write_article, articles_dir
):
    write_article(event_id="2025-0001-FRA", url="u", title="t",
                  date="2025-06-01", text="x", hash_suffix="aa")

    evaluator = Evaluator(
        llm_client=fake_llm(["not actually json {{{"]),
        predictor=StubPredictor(_predicted_chain()),
        articles_dir=articles_dir,
        config=_base_config(tmp_path),
    )
    result = evaluator.evaluate_event(_test_event())

    assert result.node_evaluations == []
    assert "malformed" in result.summary.lower()


def test_judge_fingerprint_changes_with_content():
    a = judge_fingerprint("prompt v1", "knowledge v1")
    b = judge_fingerprint("prompt v2", "knowledge v1")
    c = judge_fingerprint("prompt v1", "knowledge v2")
    assert a != b
    assert a != c
    # Deterministic
    assert a == judge_fingerprint("prompt v1", "knowledge v1")


def test_evaluate_event_invalid_evidence_level_captures_error(
    tmp_path, fake_llm, write_article, articles_dir
):
    """Judge returns valid JSON but with an invalid evidence_level enum value."""
    bad_response = json.dumps({
        "node_evaluations": [{
            "node_id": "E1",
            "evidence_level": "maybe",   # not a valid EvidenceLevel
            "evidence_quote": "q",
            "reasoning": "r",
            "timing_alignment": "match",
        }],
        "missed_cascades": [],
        "summary": "should be swallowed",
    })
    write_article(event_id="2025-0001-FRA", url="u", title="t",
                  date="2025-06-01", text="x", hash_suffix="aa")
    evaluator = Evaluator(
        llm_client=fake_llm([bad_response]),
        predictor=StubPredictor(_predicted_chain()),
        articles_dir=articles_dir,
        config=_base_config(tmp_path),
    )
    result = evaluator.evaluate_event(_test_event())
    assert result.node_evaluations == []
    assert "malformed" in result.summary.lower()


# ---------------------------------------------------------------------------
# v2 alignment coverage: severity derivation, richer description, cache
# invalidation, and retrieval_info enrichment.
# ---------------------------------------------------------------------------


def test_severity_derivation_covers_four_buckets():
    """`_infer_severity` is the v2 shared mapping; confirm all four tiers."""
    assert _infer_severity(_test_event(total_deaths=60)) == "critical"
    assert _infer_severity(_test_event(total_affected=150_000)) == "critical"
    assert _infer_severity(_test_event(total_deaths=11)) == "high"
    assert _infer_severity(_test_event(total_affected=20_000)) == "high"
    assert _infer_severity(_test_event(total_deaths=1)) == "medium"
    assert _infer_severity(_test_event(total_affected=2_000)) == "medium"
    assert _infer_severity(_test_event()) == "low"


def test_predictor_receives_derived_severity_and_rich_description(
    tmp_path, fake_llm, write_article, articles_dir
):
    """Predictor must get v2-derived severity (not hardcoded 'medium') and a
    description that carries the EM-DAT numeric fields."""
    write_article(event_id="2025-0001-FRA", url="u", title="t",
                  date="2025-06-01", text="x", hash_suffix="aa")

    stub = StubPredictor(_predicted_chain())
    evaluator = Evaluator(
        llm_client=fake_llm([JUDGE_RESPONSE_OK]),
        predictor=stub,
        articles_dir=articles_dir,
        config=_base_config(tmp_path),
    )
    event = _test_event(total_deaths=25, total_affected=50_000,
                        total_damage_k_usd=1200.0, origin="Heavy rains")
    evaluator.evaluate_event(event)

    assert len(stub.calls) == 1
    call = stub.calls[0]
    assert call["severity"] == "high"  # 25 deaths + 50k affected → high
    desc = call["description"]
    assert "Deaths: 25" in desc
    assert "Affected: 50000 people" in desc
    assert "Heavy rains" in desc


def test_build_description_tolerates_missing_fields():
    desc = _build_description(_test_event())
    # Location, iso, and date should always land in the string.
    assert "Paris" in desc
    assert "FRA" in desc
    assert "2025-06-01" in desc


def test_cache_hit_skips_predict_and_judge(
    tmp_path, fake_llm, write_article, articles_dir
):
    """When a cached EventEvaluation has a matching fingerprint, neither the
    predictor nor the judge LLM should be invoked."""
    write_article(event_id="2025-0001-FRA", url="u", title="t",
                  date="2025-06-01", text="x", hash_suffix="aa")

    # First pass: populate cache with current fingerprint.
    first_llm = fake_llm([JUDGE_RESPONSE_OK])
    first_stub = StubPredictor(_predicted_chain())
    ev = Evaluator(
        llm_client=first_llm,
        predictor=first_stub,
        articles_dir=articles_dir,
        config=_base_config(tmp_path),
    )
    ev.evaluate_event(_test_event(), write_cache=True)
    assert ev.last_cache_status == "rejudge"
    assert len(first_stub.calls) == 1

    # Second pass: new client/stub. Cache hit should skip everything.
    second_llm = fake_llm([])  # would error if called
    second_stub = StubPredictor(_predicted_chain())
    ev2 = Evaluator(
        llm_client=second_llm,
        predictor=second_stub,
        articles_dir=articles_dir,
        config=_base_config(tmp_path),
    )
    result = ev2.evaluate_event(_test_event(), write_cache=True)

    assert ev2.last_cache_status == "hit"
    assert second_llm.calls == []
    assert second_stub.calls == []
    assert result.event_id == "2025-0001-FRA"
    assert result.judge_fingerprint == ev2.judge_fingerprint_value


def test_fingerprint_mismatch_forces_rejudge(
    tmp_path, fake_llm, write_article, articles_dir
):
    """A cache file with a stale fingerprint must be overwritten by a new
    predict+judge run."""
    write_article(event_id="2025-0001-FRA", url="u", title="t",
                  date="2025-06-01", text="x", hash_suffix="aa")

    # Seed the cache with a bogus fingerprint.
    out_dir = tmp_path / "evaluation"
    out_dir.mkdir()
    stale = EventEvaluation(
        event_id="2025-0001-FRA",
        node_evaluations=[],
        missed_cascades=[],
        summary="stale",
        news_sources_used=[],
        retrieval_info=[],
        predicted_chain=_predicted_chain(),
        evaluated_at=date(2025, 1, 1),
        judge_fingerprint="stale0000fingerprint",
    )
    (out_dir / "2025-0001-FRA.json").write_text(stale.model_dump_json())

    llm = fake_llm([JUDGE_RESPONSE_OK])
    ev = Evaluator(
        llm_client=llm,
        predictor=StubPredictor(_predicted_chain()),
        articles_dir=articles_dir,
        config=_base_config(tmp_path),
    )
    result = ev.evaluate_event(_test_event(), write_cache=True)

    assert ev.last_cache_status == "rejudge"
    assert len(llm.calls) == 1
    assert result.summary != "stale"
    assert result.judge_fingerprint == ev.judge_fingerprint_value


def test_force_rejudge_bypasses_cache(
    tmp_path, fake_llm, write_article, articles_dir
):
    """`force_rejudge=True` must ignore a matching-fingerprint cache."""
    write_article(event_id="2025-0001-FRA", url="u", title="t",
                  date="2025-06-01", text="x", hash_suffix="aa")

    # First, populate cache.
    ev = Evaluator(
        llm_client=fake_llm([JUDGE_RESPONSE_OK]),
        predictor=StubPredictor(_predicted_chain()),
        articles_dir=articles_dir,
        config=_base_config(tmp_path),
    )
    ev.evaluate_event(_test_event(), write_cache=True)

    # Second, force.
    llm2 = fake_llm([JUDGE_RESPONSE_OK])
    stub2 = StubPredictor(_predicted_chain())
    ev2 = Evaluator(
        llm_client=llm2, predictor=stub2,
        articles_dir=articles_dir,
        config=_base_config(tmp_path),
    )
    ev2.evaluate_event(_test_event(), write_cache=True, force_rejudge=True)

    assert ev2.last_cache_status == "rejudge"
    assert len(llm2.calls) == 1
    assert len(stub2.calls) == 1


def test_retrieval_info_enriched_from_chain_index(
    tmp_path, fake_llm, write_article, articles_dir
):
    """retrieval_info should carry country/date/num_cascade_nodes/top_domains
    when a cascade index is available."""
    write_article(event_id="2025-0001-FRA", url="u", title="t",
                  date="2025-06-01", text="x", hash_suffix="aa")

    index_path = tmp_path / "cascade_chains_index.json"
    index_path.write_text(json.dumps([
        {
            "event_id": "2023-0866-FRA",
            "country": "France",
            "date": "2023-11-02",
            "num_cascade_nodes": 24,
            "domains": [
                "infrastructure/transport",
                "infrastructure/power",
                "social/evacuation",
                "economy/agriculture",
            ],
        },
        {
            "event_id": "2024-0100-DEU",
            "country": "Germany",
            "date": "2024-05-30",
            "num_cascade_nodes": 18,
            "domains": ["infrastructure/water"],
        },
    ]))

    ev = Evaluator(
        llm_client=fake_llm([JUDGE_RESPONSE_OK]),
        predictor=StubPredictor(
            _predicted_chain(),
            reference_event_ids=["2023-0866-FRA", "2024-0100-DEU"],
            similarity_scores=[0.81, 0.62],
        ),
        articles_dir=articles_dir,
        config=_base_config(tmp_path, cascade_index=str(index_path)),
    )

    result = ev.evaluate_event(_test_event())
    assert len(result.retrieval_info) == 2
    first = result.retrieval_info[0]
    assert first["event_id"] == "2023-0866-FRA"
    assert first["similarity"] == 0.81
    assert first["country"] == "France"
    assert first["date"] == "2023-11-02"
    assert first["num_cascade_nodes"] == 24
    # top_domains capped at 3.
    assert first["top_domains"] == [
        "infrastructure/transport",
        "infrastructure/power",
        "social/evacuation",
    ]
    assert result.retrieval_info[1]["country"] == "Germany"


def test_retrieval_info_falls_back_when_index_missing(
    tmp_path, fake_llm, write_article, articles_dir
):
    """With no cascade_index configured, retrieval_info stays minimal
    (just event_id + similarity)."""
    write_article(event_id="2025-0001-FRA", url="u", title="t",
                  date="2025-06-01", text="x", hash_suffix="aa")

    ev = Evaluator(
        llm_client=fake_llm([JUDGE_RESPONSE_OK]),
        predictor=StubPredictor(_predicted_chain()),
        articles_dir=articles_dir,
        config=_base_config(tmp_path),
    )
    result = ev.evaluate_event(_test_event())
    assert result.retrieval_info == [
        {"event_id": "2023-0866-FRA", "similarity": 0.81}
    ]


def test_empty_fingerprint_cache_does_not_short_circuit(
    tmp_path, fake_llm, write_article, articles_dir
):
    """A cache file written when there were no articles has judge_fingerprint=''
    and must not count as a fingerprint match — otherwise empty cache entries
    would pin forever."""
    out_dir = tmp_path / "evaluation"
    out_dir.mkdir()
    empty = EventEvaluation(
        event_id="2025-0001-FRA",
        node_evaluations=[],
        missed_cascades=[],
        summary="No news articles available",
        news_sources_used=[],
        retrieval_info=[],
        predicted_chain=_predicted_chain(),
        evaluated_at=date(2025, 1, 1),
        judge_fingerprint="",
    )
    (out_dir / "2025-0001-FRA.json").write_text(empty.model_dump_json())

    # Now add articles — re-run should NOT early-return.
    write_article(event_id="2025-0001-FRA", url="u", title="t",
                  date="2025-06-01", text="x", hash_suffix="aa")

    llm = fake_llm([JUDGE_RESPONSE_OK])
    ev = Evaluator(
        llm_client=llm,
        predictor=StubPredictor(_predicted_chain()),
        articles_dir=articles_dir,
        config=_base_config(tmp_path),
    )
    ev.evaluate_event(_test_event(), write_cache=True)
    assert ev.last_cache_status == "rejudge"
    assert len(llm.calls) == 1