File size: 1,537 Bytes
dc015a1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | import json
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def load_jsonl(path: Path):
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line]
def test_v11_3_eval_set_is_strict_provenance_only():
rows = load_jsonl(ROOT / "data" / "substitution_eval_set_v11_3.jsonl")
assert len(rows) >= 80
assert len({row["pair_id"] for row in rows}) == len(rows)
assert {row["answer_shape"] for row in rows} <= {"one_to_one", "accord_rebuild", "partial"}
assert {row["grade"] for row in rows} <= {
"acceptable", "partial", "functional-only", "odor-only", "not-a-substitute"
}
assert all(row["eval_only"] is True for row in rows)
assert all(row["provenance_flag"] in {
"authoritative-supplier-page", "authoritative-supplier-catalogue"
} for row in rows)
assert all(row["source_url"].startswith(("https://", "local:")) for row in rows)
def test_v11_3_report_keeps_exclusions_out_of_misses():
report = json.loads((ROOT / "artifacts" / "pimt_v11_3_powered_retriever.json").read_text())
assert report["eval_set_n"] >= 80
assert report["primary_rule"].startswith("one_to_one only")
assert report["verdict"] in {"holds", "collapses", "ambiguous"}
for conditions in report["by_answer_shape"].values():
for result in conditions.values():
assert result["n_eval_rows"] == result["n_evaluable"] + result["n_excluded"]
assert result["n_evaluable"] == result["n_hit"] + result["n_missed"]
|