tests/test_pimt_v11_3_powered_eval.py

import json
from pathlib import Path


ROOT = Path(__file__).resolve().parents[1]


def load_jsonl(path: Path):
    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line]


def test_v11_3_eval_set_is_strict_provenance_only():
    rows = load_jsonl(ROOT / "data" / "substitution_eval_set_v11_3.jsonl")
    assert len(rows) >= 80
    assert len({row["pair_id"] for row in rows}) == len(rows)
    assert {row["answer_shape"] for row in rows} <= {"one_to_one", "accord_rebuild", "partial"}
    assert {row["grade"] for row in rows} <= {
        "acceptable", "partial", "functional-only", "odor-only", "not-a-substitute"
    }
    assert all(row["eval_only"] is True for row in rows)
    assert all(row["provenance_flag"] in {
        "authoritative-supplier-page", "authoritative-supplier-catalogue"
    } for row in rows)
    assert all(row["source_url"].startswith(("https://", "local:")) for row in rows)


def test_v11_3_report_keeps_exclusions_out_of_misses():
    report = json.loads((ROOT / "artifacts" / "pimt_v11_3_powered_retriever.json").read_text())
    assert report["eval_set_n"] >= 80
    assert report["primary_rule"].startswith("one_to_one only")
    assert report["verdict"] in {"holds", "collapses", "ambiguous"}
    for conditions in report["by_answer_shape"].values():
        for result in conditions.values():
            assert result["n_eval_rows"] == result["n_evaluable"] + result["n_excluded"]
            assert result["n_evaluable"] == result["n_hit"] + result["n_missed"]