tests/test_05_evaluate_cli.py

"""CLI flag parsing for scripts/05_evaluate.py — Step B issue #11.

Loaded as a module via importlib because the script lives outside the
``src`` package and isn't normally importable. Tests focus on argparse
contract — the rest of the script is exercised end-to-end by the gold /
judge eval flows.
"""

from __future__ import annotations

import importlib.util
from pathlib import Path

import pytest

_SCRIPT_PATH = Path(__file__).parent.parent / "scripts" / "05_evaluate.py"


def _load_script_module():
    spec = importlib.util.spec_from_file_location("evaluate_05", _SCRIPT_PATH)
    mod = importlib.util.module_from_spec(spec)
    assert spec.loader is not None
    spec.loader.exec_module(mod)
    return mod


@pytest.fixture(scope="module")
def script():
    return _load_script_module()


def test_default_args(script, monkeypatch):
    monkeypatch.setattr("sys.argv", ["05_evaluate.py"])
    args = script.parse_args()
    assert args.mode == "gold"
    assert args.force is False
    assert args.event_ids is None
    assert args.threshold is None
    assert args.dump_match_debug is False


def test_threshold_flag_parses_float(script, monkeypatch):
    monkeypatch.setattr("sys.argv", ["05_evaluate.py", "--threshold", "0.42"])
    args = script.parse_args()
    assert args.threshold == 0.42


def test_dump_match_debug_flag(script, monkeypatch):
    monkeypatch.setattr(
        "sys.argv", ["05_evaluate.py", "--dump-match-debug"]
    )
    args = script.parse_args()
    assert args.dump_match_debug is True


def test_combined_flags(script, monkeypatch):
    monkeypatch.setattr(
        "sys.argv",
        [
            "05_evaluate.py",
            "--mode", "gold",
            "--force",
            "--threshold", "0.35",
            "--dump-match-debug",
            "--event-id", "2025-0848-UKR",
        ],
    )
    args = script.parse_args()
    assert args.mode == "gold"
    assert args.force is True
    assert args.threshold == 0.35
    assert args.dump_match_debug is True
    assert args.event_ids == ["2025-0848-UKR"]


def test_config_temperature_zero_and_seed_list():
    """Issue A: config locked to temp=0 + 3 seeds."""
    from src.llm.client import load_config
    cfg = load_config()
    assert cfg["llm"]["temperature"] == 0.0
    assert cfg["evaluation"]["seed_list"] == [42, 1337, 2026]


def test_compare_to_flag_parses_string_path(script, monkeypatch):
    """v0.7 issue A: --compare-to records a path string for downstream load."""
    monkeypatch.setattr(
        "sys.argv",
        ["05_evaluate.py",
         "--compare-to", "data/evaluation/aggregate_l3b.json"],
    )
    args = script.parse_args()
    assert args.compare_to == "data/evaluation/aggregate_l3b.json"


def test_default_compare_to_is_none(script, monkeypatch):
    """Default keeps backward-compat: no comparison printed."""
    monkeypatch.setattr("sys.argv", ["05_evaluate.py"])
    args = script.parse_args()
    assert args.compare_to is None


def test_write_aggregate_artefact_schema(script, tmp_path):
    """v0.7 issue A: _write_aggregate_artefact produces a v1-format JSON
    with the 4 public metric names and the required envelope fields.

    Spec §5.3. Avoids the cost of a full eval-pipeline subprocess by
    feeding the helper synthetic CI dicts directly.
    """
    import json

    config = {
        "rag": {"top_k": 5},
        "llm": {"backend": "local", "model": "test"},
        "evaluation": {
            "output_dir": str(tmp_path),
            "cosine_threshold": 0.35,
            "seed_list": [42, 1337, 2026],
        },
    }
    fake_ci = {
        "cat_r": {"mean": 0.5, "ci_low": 0.4, "ci_high": 0.6,
                  "n": 6, "n_resamples": 1000, "confidence": 0.95},
        "cat_sev": {"mean": 0.7, "ci_low": 0.6, "ci_high": 0.8,
                    "n": 6, "n_resamples": 1000, "confidence": 0.95},
        "f1": {"mean": 0.3, "ci_low": 0.2, "ci_high": 0.4,
               "n": 6, "n_resamples": 1000, "confidence": 0.95},
        "cat_f1": {"mean": 0.45, "ci_low": 0.35, "ci_high": 0.55,
                   "n": 6, "n_resamples": 1000, "confidence": 0.95},
    }
    fake_seed_vals = {k: {f"E{i}": [0.5] for i in range(6)} for k in fake_ci}
    ci_metric_attrs = [
        ("Cat Macro Recall", "cat_r"),
        ("Cat Macro Sev",    "cat_sev"),
        ("Cos Macro F1",     "f1"),
        ("Cat Macro F1",     "cat_f1"),
    ]
    out_path = script._write_aggregate_artefact(
        config=config,
        seeds=[42, 1337, 2026],
        ci_per_metric=fake_ci,
        per_event_seed_values_per_metric=fake_seed_vals,
        ci_metric_attrs=ci_metric_attrs,
    )
    assert out_path.exists()
    payload = json.loads(out_path.read_text())
    assert payload["format_version"] == 1
    assert payload["n_events"] == 6
    assert payload["n_seeds"] == 3
    assert payload["bootstrap"] == {
        "n_resamples": 1000, "seed": 42, "confidence": 0.95,
    }
    assert set(payload["metrics"].keys()) == {
        "category_recall", "category_severity_match_rate",
        "cosine_f1", "category_f1",
    }
    assert payload["metrics"]["category_recall"]["mean"] == 0.5
    # config_fingerprint stable across calls with same config
    out2 = script._write_aggregate_artefact(
        config=config, seeds=[42, 1337, 2026],
        ci_per_metric=fake_ci,
        per_event_seed_values_per_metric=fake_seed_vals,
        ci_metric_attrs=ci_metric_attrs,
    )
    assert out2.name == out_path.name  # same fingerprint → same filename