from __future__ import annotations

import pytest
from auralynq.eval.asr_eval import evaluate_asr, word_error_rate
from auralynq.eval.bench import run_bench
from auralynq.eval.provenance import report_provenance
from auralynq.eval.ragas_eval import proxy_scores
from auralynq.eval.report import run_eval
from auralynq.eval.retrieval_metrics import aggregate, mrr, ndcg_at_k, recall_at_k

_PROVENANCE_KEYS = {"git_commit", "generated_at", "hardware", "dataset_version"}


def test_recall_and_mrr():
    rel = {"geography"}
    ranked = ["other.md", "geography.md", "x.md"]
    assert recall_at_k(rel, ranked, 3) == 1.0
    assert recall_at_k(rel, ranked, 1) == 0.0
    assert mrr(rel, ranked) == pytest.approx(0.5)


def test_ndcg_monotonic_in_position():
    rel = {"a"}
    top = ndcg_at_k(rel, ["a.md", "b.md"], 2)
    bottom = ndcg_at_k(rel, ["b.md", "a.md"], 2)
    assert top > bottom


def test_aggregate_shapes():
    cases = [({"geography"}, ["geography.md", "x.md"]), ({"pathrag"}, ["pathrag.md"])]
    s = aggregate(cases, k=2)
    assert s.n == 2 and 0 <= s.recall_at_k <= 1


def test_wer_exact_and_fallback():
    assert word_error_rate("the cat sat", "the cat sat") == 0.0
    assert word_error_rate("the cat sat", "the dog sat") == pytest.approx(1 / 3, abs=1e-3)
    s = evaluate_asr([("paris is the capital", "paris is the capital")])
    assert s.wer == 0.0


def test_ragas_proxy_bounds():
    samples = [
        {
            "question": "what is the capital of france",
            "answer": "paris is the capital of france",
            "contexts": ["paris is the capital of france"],
        }
    ]
    sc = proxy_scores(samples)
    assert 0 <= sc.faithfulness <= 1
    assert sc.faithfulness > 0.5
    assert sc.provider == "proxy"


def test_run_eval_smoke_writes_report(corpus_dir, monkeypatch):
    # point data_dir corpus at our fixture corpus
    from auralynq.config import get_settings

    s = get_settings()
    monkeypatch.setattr(s, "data_dir", corpus_dir.parent)
    (corpus_dir.parent / "corpus").exists() or corpus_dir.rename(corpus_dir.parent / "corpus")
    report = run_eval(smoke=True, write_report=True)
    assert "retrieval" in report
    assert {"naive", "hybrid", "pathrag"} <= set(report["retrieval"])
    assert "agentic" in report and "ragas" in report["agentic"]
    assert report["drift"]["status"] in ("baseline_created", "ok", "regressed")
    assert (s.reports_dir / "eval_report.json").exists()
    # every report must be traceable: git commit, timestamp, hardware, dataset
    assert set(report["provenance"]) >= _PROVENANCE_KEYS
    assert "smoke=True" in report["provenance"]["dataset_version"]


def test_run_bench(corpus_dir, monkeypatch):
    from auralynq.config import get_settings

    s = get_settings()
    report = run_bench(k=5, n_queries=8, write_report=True)
    assert set(report["quantization"]) == {"none", "scalar", "binary"}
    assert report["quantization"]["none"]["recall_at_k"] == 1.0
    assert 0 <= report["quantization"]["binary"]["recall_at_k"] <= 1
    # scalar (int8) should retain more recall than binary (1-bit)
    assert (
        report["quantization"]["scalar"]["recall_at_k"]
        >= report["quantization"]["binary"]["recall_at_k"]
    )
    assert (s.reports_dir / "bench_report.json").exists()
    assert set(report["provenance"]) >= _PROVENANCE_KEYS


def test_report_provenance_schema():
    p = report_provenance(dataset_version="unit-test-dataset")
    assert set(p) >= _PROVENANCE_KEYS
    assert p["dataset_version"] == "unit-test-dataset"
    assert isinstance(p["hardware"], dict)
    # git_commit degrades to "unknown" outside a repo, but must never raise
    assert isinstance(p["git_commit"], str) and p["git_commit"]