tests/test_evaluation_script.py

"""Tests for scripts.evaluation artifact assembly."""

from sage.core.models import EvalCase, EvalCaseProvenance, MetricsReport
import scripts.evaluation as evaluation_script


def _case(
    *,
    query: str,
    query_id: str,
    source_type: str,
    category: str,
    intent: str,
    subset_tags: tuple[str, ...],
    query_slice_tags: tuple[str, ...],
    origin_family: str,
    curation_mode: str,
) -> EvalCase:
    return EvalCase(
        query=query,
        relevant_items={"ASIN1": 3.0},
        query_id=query_id,
        source_type=source_type,
        category=category,
        intent=intent,
        subset_tags=subset_tags,
        query_slice_tags=query_slice_tags,
        provenance=EvalCaseProvenance(
            schema_version="query_provenance_v1",
            origin_family=origin_family,
            curation_mode=curation_mode,
            source_dataset="amazon_esci",
            source_split="test",
            selection_policy="corpus_overlap_min_relevant_items_v1",
            subset_assignment_policy="normalized_query_sha256_v1",
        ),
    )


def test_build_primary_evaluation_artifact_adds_metadata_breakdowns(monkeypatch):
    cases = [
        _case(
            query="wireless keyboard",
            query_id="qb_001",
            source_type="amazon_esci",
            category="keyboards",
            intent="use_case",
            subset_tags=("retrieval_eval",),
            query_slice_tags=(),
            origin_family="amazon_esci_overlap",
            curation_mode="pure_import",
        ),
        _case(
            query="latest earbuds to avoid",
            query_id="qb_002",
            source_type="manual_seed",
            category="audio",
            intent="problem_solving",
            subset_tags=("retrieval_eval", "special_probe"),
            query_slice_tags=(
                "recency_sensitive_query",
                "negative_problem_query",
            ),
            origin_family="manual_seed",
            curation_mode="candidate_bootstrap",
        ),
    ]
    case_results = []
    for case, recommended_ids, metrics in [
        (
            cases[0],
            ["ASIN1", "ASIN3"],
            {
                "ndcg": 1.0,
                "hit": 1.0,
                "mrr": 1.0,
                "precision": 0.5,
                "recall": 1.0,
                "diversity": 0.2,
                "novelty": 2.0,
            },
        ),
        (
            cases[1],
            ["ASIN4", "ASIN1"],
            {
                "ndcg": 0.6309,
                "hit": 1.0,
                "mrr": 0.5,
                "precision": 0.5,
                "recall": 1.0,
                "diversity": 0.6,
                "novelty": 3.0,
            },
        ),
    ]:
        row = case.to_dict()
        row["recommended_product_ids"] = recommended_ids
        row["relevant_item_count"] = 1
        row["relevant_hits"] = [
            {
                "product_id": "ASIN1",
                "rank": recommended_ids.index("ASIN1") + 1,
                "relevance": 3.0,
            }
        ]
        row["first_relevant_rank"] = recommended_ids.index("ASIN1") + 1
        row["metrics"] = metrics
        case_results.append(row)

    monkeypatch.setattr(
        evaluation_script,
        "evaluate_recommendations_with_details",
        lambda **_kwargs: (
            MetricsReport(
                n_cases=2,
                ndcg_at_k=0.8154,
                hit_at_k=1.0,
                mrr=0.75,
                precision_at_k=0.5,
                recall_at_k=1.0,
                diversity=0.4,
                coverage=0.6,
                novelty=2.5,
                k=10,
            ),
            case_results,
        ),
    )

    artifact = evaluation_script.build_primary_evaluation_artifact(
        cases,
        item_embeddings={},
        item_popularity={},
        total_items=5,
    )

    assert artifact["metrics"]["ndcg_at_10"] == 0.8154
    assert artifact["case_metadata_summary"]["total_cases"] == 2
    assert artifact["case_metadata_summary"]["by_origin_family"] == {
        "amazon_esci_overlap": 1,
        "manual_seed": 1,
    }
    assert artifact["case_metadata_summary"]["by_query_slice_tag"] == {
        "recency_sensitive_query": 1,
        "negative_problem_query": 1,
    }
    assert (
        artifact["metric_breakdowns"]["by_curation_mode"]["candidate_bootstrap"][
            "n_cases"
        ]
        == 1
    )
    assert (
        artifact["metric_breakdowns"]["by_query_slice_tag"]["recency_sensitive_query"][
            "n_cases"
        ]
        == 1
    )
    assert (
        artifact["metric_breakdowns"]["by_query_slice_tag"]["recency_sensitive_query"][
            "coverage"
        ]
        == 0.4
    )
    assert (
        artifact["metric_breakdowns"]["by_subset_tag"]["retrieval_eval"]["n_cases"] == 2
    )
    assert "subset_tags" in artifact["breakdown_methodology"]["multi_membership_fields"]