"""Loader contract for app/components/evaluation_panel.py — issue #11 fix. The Streamlit Evaluation tab globs ``data/evaluation/gold/*.json`` and tries to parse each as ``GoldEvaluation``. Issue #11 added co-located ``*.diag.json`` files (NodeMatchCandidate dumps) that share the directory but a different schema. Without filtering, the loader emits one ``st.warning`` per diag file with a noisy 11-error pydantic stack trace. This test pins the filter so the bug doesn't regress. """ from __future__ import annotations import json from datetime import date from pathlib import Path from app.components.evaluation_panel import load_gold_evaluations from src.models.schemas import ( CascadeChain, CascadeNode, GoldEvaluation, ) def _make_gold_evaluation_json(event_id: str) -> str: chain = CascadeChain( event_id=event_id, trigger_summary="trigger", trigger_country="C", trigger_iso="ISO", trigger_date=date(2025, 1, 1), cascade_events=[ CascadeNode( id="E1", description="x", domain="infrastructure/power", severity="high", mechanism="m", ) ], ) ge = GoldEvaluation( event_id=event_id, predicted_chain=chain, gold_event_id=event_id, gold_node_count=1, predicted_node_count=1, matches=[], precision=0.0, recall=0.0, f1=0.0, domain_jaccard=0.0, threshold=0.35, embedder_model="all-MiniLM-L6-v2", algorithm_version="v3", gold_eval_fingerprint="abc123", evaluated_at=date(2025, 1, 1), ) return ge.model_dump_json(indent=2) def _diag_payload(event_id: str) -> str: return json.dumps({ "event_id": event_id, "threshold_used": 0.5, "embedder_model": "all-MiniLM-L6-v2", "algorithm_version": "v3", "candidates": [ {"p_id": "E1", "g_id": "G1", "cosine": 0.42, "severity_match": True, "accepted": False} ], }) def test_loader_skips_diag_json_files(tmp_path: Path): """A directory with both .json and .diag.json should yield only the GoldEvaluation files — diag files are silently skipped, no warning.""" (tmp_path / "2025-0183-ITA.json").write_text( _make_gold_evaluation_json("2025-0183-ITA"), encoding="utf-8" ) (tmp_path / "2025-0183-ITA.diag.json").write_text( _diag_payload("2025-0183-ITA"), encoding="utf-8" ) (tmp_path / "2025-0848-UKR.json").write_text( _make_gold_evaluation_json("2025-0848-UKR"), encoding="utf-8" ) (tmp_path / "2025-0848-UKR.diag.json").write_text( _diag_payload("2025-0848-UKR"), encoding="utf-8" ) evaluations = load_gold_evaluations(tmp_path) assert len(evaluations) == 2 assert {e.event_id for e in evaluations} == {"2025-0183-ITA", "2025-0848-UKR"} def test_loader_returns_empty_when_dir_missing(tmp_path: Path): assert load_gold_evaluations(tmp_path / "nope") == [] def test_loader_handles_diag_only_dir(tmp_path: Path): """Pathological case: only .diag.json files exist. Should return [], not crash, not emit warnings about diag files.""" (tmp_path / "2025-0183-ITA.diag.json").write_text( _diag_payload("2025-0183-ITA"), encoding="utf-8" ) assert load_gold_evaluations(tmp_path) == [] def test_loader_skips_outlier_event_ids(tmp_path: Path): """Issue #14 fix: even with a stale GoldEvaluation file on disk for an event that was later marked as outlier in config.evaluation. outlier_event_ids, the loader must hide it so the Streamlit Evaluation tab matches what scripts/05_evaluate.py aggregates.""" (tmp_path / "2025-0183-ITA.json").write_text( _make_gold_evaluation_json("2025-0183-ITA"), encoding="utf-8" ) (tmp_path / "2025-0632-ROU.json").write_text( _make_gold_evaluation_json("2025-0632-ROU"), encoding="utf-8" ) (tmp_path / "2025-0848-UKR.json").write_text( _make_gold_evaluation_json("2025-0848-UKR"), encoding="utf-8" ) # No outlier set -> all three returned (back-compat baseline). assert {e.event_id for e in load_gold_evaluations(tmp_path)} == { "2025-0183-ITA", "2025-0632-ROU", "2025-0848-UKR", } # ROU marked outlier -> hidden even though the file still sits on disk. filtered = load_gold_evaluations( tmp_path, outlier_event_ids={"2025-0632-ROU"} ) assert {e.event_id for e in filtered} == {"2025-0183-ITA", "2025-0848-UKR"}