Spaces:
Paused
Paused
File size: 3,729 Bytes
8c1b9fe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 | from __future__ import annotations
import pytest
from auralynq.eval.asr_eval import evaluate_asr, word_error_rate
from auralynq.eval.bench import run_bench
from auralynq.eval.provenance import report_provenance
from auralynq.eval.ragas_eval import proxy_scores
from auralynq.eval.report import run_eval
from auralynq.eval.retrieval_metrics import aggregate, mrr, ndcg_at_k, recall_at_k
_PROVENANCE_KEYS = {"git_commit", "generated_at", "hardware", "dataset_version"}
def test_recall_and_mrr():
rel = {"geography"}
ranked = ["other.md", "geography.md", "x.md"]
assert recall_at_k(rel, ranked, 3) == 1.0
assert recall_at_k(rel, ranked, 1) == 0.0
assert mrr(rel, ranked) == pytest.approx(0.5)
def test_ndcg_monotonic_in_position():
rel = {"a"}
top = ndcg_at_k(rel, ["a.md", "b.md"], 2)
bottom = ndcg_at_k(rel, ["b.md", "a.md"], 2)
assert top > bottom
def test_aggregate_shapes():
cases = [({"geography"}, ["geography.md", "x.md"]), ({"pathrag"}, ["pathrag.md"])]
s = aggregate(cases, k=2)
assert s.n == 2 and 0 <= s.recall_at_k <= 1
def test_wer_exact_and_fallback():
assert word_error_rate("the cat sat", "the cat sat") == 0.0
assert word_error_rate("the cat sat", "the dog sat") == pytest.approx(1 / 3, abs=1e-3)
s = evaluate_asr([("paris is the capital", "paris is the capital")])
assert s.wer == 0.0
def test_ragas_proxy_bounds():
samples = [
{
"question": "what is the capital of france",
"answer": "paris is the capital of france",
"contexts": ["paris is the capital of france"],
}
]
sc = proxy_scores(samples)
assert 0 <= sc.faithfulness <= 1
assert sc.faithfulness > 0.5
assert sc.provider == "proxy"
def test_run_eval_smoke_writes_report(corpus_dir, monkeypatch):
# point data_dir corpus at our fixture corpus
from auralynq.config import get_settings
s = get_settings()
monkeypatch.setattr(s, "data_dir", corpus_dir.parent)
(corpus_dir.parent / "corpus").exists() or corpus_dir.rename(corpus_dir.parent / "corpus")
report = run_eval(smoke=True, write_report=True)
assert "retrieval" in report
assert {"naive", "hybrid", "pathrag"} <= set(report["retrieval"])
assert "agentic" in report and "ragas" in report["agentic"]
assert report["drift"]["status"] in ("baseline_created", "ok", "regressed")
assert (s.reports_dir / "eval_report.json").exists()
# every report must be traceable: git commit, timestamp, hardware, dataset
assert set(report["provenance"]) >= _PROVENANCE_KEYS
assert "smoke=True" in report["provenance"]["dataset_version"]
def test_run_bench(corpus_dir, monkeypatch):
from auralynq.config import get_settings
s = get_settings()
report = run_bench(k=5, n_queries=8, write_report=True)
assert set(report["quantization"]) == {"none", "scalar", "binary"}
assert report["quantization"]["none"]["recall_at_k"] == 1.0
assert 0 <= report["quantization"]["binary"]["recall_at_k"] <= 1
# scalar (int8) should retain more recall than binary (1-bit)
assert (
report["quantization"]["scalar"]["recall_at_k"]
>= report["quantization"]["binary"]["recall_at_k"]
)
assert (s.reports_dir / "bench_report.json").exists()
assert set(report["provenance"]) >= _PROVENANCE_KEYS
def test_report_provenance_schema():
p = report_provenance(dataset_version="unit-test-dataset")
assert set(p) >= _PROVENANCE_KEYS
assert p["dataset_version"] == "unit-test-dataset"
assert isinstance(p["hardware"], dict)
# git_commit degrades to "unknown" outside a repo, but must never raise
assert isinstance(p["git_commit"], str) and p["git_commit"]
|