File size: 2,942 Bytes
39cba11 f5628ad 39cba11 f5628ad | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | from server.eval.precision import compute_precision_at_k, run_batch_precision_eval_multi_k
from server.eval.faithfulness import score_faithfulness
def test_precision_at_k_returns_float_between_0_and_1():
chunks = [
{"source": "npci_upi_report_2024.txt", "content": "UPI volume was 131 billion"},
{"source": "npci_upi_report_2024.txt", "content": "UPI Lite launched"},
{"source": "rbi_annual_report_2024.txt", "content": "GDP growth was 7.6%"},
{"source": "npci_upi_report_2024.txt", "content": "PhonePe 47% market share"},
{"source": "bajaj_finance_q3_2024_transcript.txt", "content": "AUM grew 35%"},
]
ground_truth = {
"relevant_sources": ["npci_upi_report_2024.txt"],
"relevant_chunk_keywords": ["131 billion", "UPI volume"],
}
precision = compute_precision_at_k("UPI volume", chunks, ground_truth, k=5)
assert isinstance(precision, float)
assert 0.0 <= precision <= 1.0
def test_precision_perfect_score():
chunks = [
{"source": "a.txt", "content": "relevant keyword here"},
{"source": "a.txt", "content": "another relevant keyword"},
]
ground_truth = {
"relevant_sources": ["a.txt"],
"relevant_chunk_keywords": ["keyword"],
}
precision = compute_precision_at_k("test", chunks, ground_truth, k=2)
assert precision == 1.0
def test_precision_zero_score():
chunks = [
{"source": "wrong.txt", "content": "nothing relevant"},
{"source": "wrong.txt", "content": "still nothing"},
]
ground_truth = {
"relevant_sources": ["correct.txt"],
"relevant_chunk_keywords": ["specific term"],
}
precision = compute_precision_at_k("test", chunks, ground_truth, k=2)
assert precision == 0.0
def test_multi_k_eval_returns_one_result_per_k(monkeypatch):
def fake_single(eval_pairs_path, k=5):
return {"mean_precision_at_k": round(k / 10, 4), "per_query_results": []}
monkeypatch.setattr("server.eval.precision.run_batch_precision_eval", fake_single)
results = run_batch_precision_eval_multi_k("dummy.json", ks=[1, 3, 5])
assert list(results.keys()) == ["precision@1", "precision@3", "precision@5"]
assert results["precision@1"]["mean_precision_at_k"] == 0.1
assert results["precision@3"]["mean_precision_at_k"] == 0.3
assert results["precision@5"]["mean_precision_at_k"] == 0.5
def test_faithfulness_returns_dict_with_score():
chunks = [{"content": "UPI processed 131 billion transactions in FY2024."}]
answer = "UPI processed 131 billion transactions in FY2024."
result = score_faithfulness(answer, chunks)
assert isinstance(result, dict)
assert "score" in result
assert "reason" in result
assert "raw_response" in result
def test_faithfulness_handles_empty_chunks():
result = score_faithfulness("Some answer", [])
assert isinstance(result, dict)
assert "score" in result
|