Spaces:
Running
Running
File size: 2,652 Bytes
27716f7 ef2c57c f4b92b8 ef2c57c f4b92b8 ef2c57c 27716f7 e67c48b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | from ragforge.eval_metrics import answer_key_match, citation_metrics, percentile, scalar_value_match, source_metrics
def test_source_metrics_reward_early_relevant_source():
metrics = source_metrics(["wrong.md", "right.md", "other.md"], ["right.md"])
assert metrics["source_recall@5"] == 1.0
assert metrics["source_mrr"] == 0.5
assert metrics["source_hit@1"] == 0.0
assert 0 < metrics["source_precision@5"] < 1
assert 0 <= metrics["source_ap@5"] <= 1
assert 0 <= metrics["source_ndcg@5"] <= 1
def test_source_metrics_deduplicate_repeated_chunks_and_bound_ap():
metrics = source_metrics(["nist.pdf"] * 5, ["nist.pdf"])
assert metrics["source_hit@1"] == 1.0
assert metrics["source_mrr"] == 1.0
assert metrics["source_ap@5"] == 1.0
assert metrics["source_ndcg@5"] == 1.0
assert metrics["source_duplicate_rate@5"] == 0.8
def test_citation_metrics_distinguish_validity_and_coverage():
answer = "The target is five minutes [D1]. A second unsupported statement is also present."
sources = [{"id": "D1", "type": "document", "title": "runbook.md"}]
metrics = citation_metrics(answer, sources)
assert metrics["citation_validity"] == 1.0
assert 0 < metrics["citation_coverage"] < 1.0
def test_citation_metrics_reject_unknown_ids():
metrics = citation_metrics("Claim [D9].", [{"id": "D1"}])
assert metrics["citation_validity"] == 0.0
def test_answer_key_supports_all_and_any():
assert answer_key_match("Govern, Map, Measure, Manage", {"expected_all": ["govern", "map", "measure", "manage"]})
assert answer_key_match("Enterprise is fastest", {"expected_any": ["enterprise", "business"]})
assert not answer_key_match("Team is fastest", {"expected_any": ["enterprise"]})
def test_percentile_interpolates():
assert percentile([100, 200, 300], 0.5) == 200
assert percentile([], 0.95) == 0.0
def test_scalar_value_match_handles_boolean_numeric_and_text_values():
assert scalar_value_match(True, True)
assert scalar_value_match("true", True)
assert scalar_value_match(199, 199)
assert scalar_value_match(199.0, 199)
assert scalar_value_match("Enterprise", "enterprise")
assert not scalar_value_match(False, True)
def test_grouped_citation_ids_are_supported():
from ragforge.eval_metrics import citation_metrics
sources = [{"id": "D1"}, {"id": "D2"}]
metrics = citation_metrics(
"The system uses hybrid retrieval with two supporting sources [D1, D2].",
sources,
)
assert metrics["citation_count"] == 2
assert metrics["citation_validity"] == 1.0
assert metrics["citation_coverage"] == 1.0
|