sync_pilot/taxonomy/model_eval_plan.py

"""Sidecar model-evaluation design for culturally specific music tagging."""

from __future__ import annotations

from typing import Any


MODEL_CANDIDATES: list[dict[str, Any]] = [
    {
        "id": "laion/clap-htsat-unfused",
        "role": "current_zero_shot_baseline",
        "canonical_tags_mutated": False,
        "notes": "Keep as candidate evidence until evaluated per taxonomy dimension.",
    },
    {
        "id": "OpenMuQ/MuQ-MuLan-large",
        "role": "zero_shot_music_text_candidate",
        "canonical_tags_mutated": False,
        "notes": "Most relevant drop-in zero-shot candidate; evaluate as sidecar first.",
    },
    {
        "id": "m-a-p/MERT-v1-330M",
        "role": "embedding_candidate",
        "canonical_tags_mutated": False,
        "notes": "Use embeddings for linear probes or nearest-neighbor review labels, not direct tags.",
    },
    {
        "id": "OpenMuQ/MuQ-large-msd-iter",
        "role": "embedding_candidate",
        "canonical_tags_mutated": False,
        "notes": "Alternative music foundation embedding model for supervised probes.",
    },
]

EVALUATION_DIMENSIONS = [
    "genre_family",
    "genre_subtype",
    "instrumentation",
    "vocal_configuration",
    "vocal_technique",
    "mood",
    "arrangement_aesthetic",
    "recording_context",
]

METRICS = [
    "top1_accuracy_for_single_select",
    "top3_recall_for_review_candidates",
    "precision_at_emitted_threshold",
    "false_positive_examples",
    "abstention_rate",
]


def model_eval_plan() -> dict[str, Any]:
    return {
        "canonical_tags_mutated": False,
        "minimum_reviewed_tracks": 30,
        "preferred_reviewed_tracks": 60,
        "candidates": MODEL_CANDIDATES,
        "dimensions": EVALUATION_DIMENSIONS,
        "metrics": METRICS,
        "decision_rule": (
            "Promote a model per dimension only when it improves reviewed-label "
            "precision/recall over CLAP and produces inspectable false-positive examples."
        ),
    }