Spaces:
Paused
Paused
File size: 4,668 Bytes
656439d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | """Trust metrics — citation attribution, confidence calibration (ECE), and the
regression gate. Deterministic, no model/network."""
from __future__ import annotations
from auralynq.eval.calibration import answer_correct, calibration_scores
from auralynq.eval.citation_eval import citation_scores
from auralynq.eval.gate import eval_gate
# ── calibration ─────────────────────────────────────────────────────────────
def test_calibration_perfect_is_zero_ece():
# bin [0.9,1.0): confidence 0.9, 9/10 correct → acc 0.9 == conf 0.9 → ECE 0
pairs = [(0.9, True)] * 9 + [(0.9, False)]
s = calibration_scores(pairs)
assert s.accuracy == 0.9
assert s.ece == 0.0
def test_calibration_overconfident_has_max_ece():
# confidence 1.0 everywhere, but only half correct → gap 0.5 in the top bin
pairs = [(1.0, True)] * 5 + [(1.0, False)] * 5
s = calibration_scores(pairs)
assert s.accuracy == 0.5
assert s.avg_confidence == 1.0
assert s.ece == 0.5
assert s.mce == 0.5
assert s.brier == 0.5 # mean((1-1)^2 x5, (1-0)^2 x5) = 0.5
def test_calibration_empty():
s = calibration_scores([])
assert s.n == 0 and s.ece == 0.0
def test_answer_correct():
assert answer_correct("The capital is Paris.", "Paris") is True
assert (
answer_correct("Ericsson filed FRAND patent licensing terms", "FRAND patent licensing")
is True
)
assert answer_correct("The weather is sunny", "Paris") is False
assert answer_correct("", "Paris") is False
# ── citation attribution ────────────────────────────────────────────────────
def test_citation_precision_penalizes_spurious():
good = {
"text": "Ericsson filed fair reasonable FRAND patent licensing terms with standards bodies",
"source": "ericsson.pdf",
}
spurious = {
"text": "The weather in Paris was sunny throughout the summer holidays",
"source": "weather.pdf",
}
answer = "Ericsson filed FRAND patent licensing terms."
both = citation_scores([{"answer": answer, "citations": [good, spurious]}])
only_good = citation_scores([{"answer": answer, "citations": [good]}])
assert only_good.citation_precision == 1.0
# the spurious citation drops precision (it doesn't back the answer)
assert both.citation_precision < only_good.citation_precision
assert both.avg_citations == 2.0
def test_attribution_and_unsupported_rate():
# one supported claim + one unsupported claim
cites = [
{"text": "Ericsson filed fair reasonable FRAND patent licensing terms", "source": "e.pdf"}
]
answer = "Ericsson filed FRAND patent licensing terms. The moon orbits earth every month."
s = citation_scores([{"answer": answer, "citations": cites}])
assert 0.0 < s.attribution_rate < 1.0 # first claim supported, second not
assert round(s.attribution_rate + s.unsupported_claim_rate, 4) == 1.0
def test_citation_empty():
s = citation_scores([])
assert s.n == 0 and s.citation_precision == 0.0
# ── gate ────────────────────────────────────────────────────────────────────
def _report(cit_prec, ece, faith=0.8, recall=0.8, attr=0.8, unsup=0.2):
return {
"agentic": {
"retrieval": {"recall_at_k": recall},
"ragas": {"faithfulness": faith},
"citation": {
"citation_precision": cit_prec,
"attribution_rate": attr,
"unsupported_claim_rate": unsup,
},
"calibration": {"ece": ece},
}
}
def test_gate_passes_when_healthy():
g = eval_gate(_report(cit_prec=0.9, ece=0.05))
assert g["passed"] is True and g["failures"] == []
def test_gate_fails_on_low_citation_precision_and_high_ece():
g = eval_gate(_report(cit_prec=0.3, ece=0.4))
assert g["passed"] is False
failed = {f["metric"] for f in g["failures"]}
assert "citation_precision" in failed
assert "ece" in failed
def test_gate_skips_absent_metrics():
# a report missing citation/calibration must not spuriously fail
g = eval_gate({"agentic": {"retrieval": {"recall_at_k": 0.9}, "ragas": {"faithfulness": 0.9}}})
assert g["passed"] is True
metrics = {c["metric"] for c in g["checks"]}
assert "citation_precision" not in metrics
|