sync_pilot/groundtruth/schema.py

"""Pydantic schema for per-track ground-truth records.

The ground-truth record is a separate artifact from ``TrackRecord`` (the
inference-pipeline output). They are deliberately decoupled:

- ``TrackRecord`` lives at ``data/outputs/<catalog>/<track_id>.json`` and
  is rebuilt every time the inference pipeline runs.
- ``GroundTruthRecord`` lives at ``data/groundtruth/<catalog>/<track_id>.json``
  and represents the source-of-truth (verified factual baseline) — artist,
  year, region, era — that the inference pipeline does NOT produce.

The two are joined at evaluation time, not at write time. This keeps the
ground-truth populate-once / evaluate-many lifecycle independent of the
inference re-runs.

Confidence semantics — every controlled-vocab field carries a per-field
confidence + an overall_confidence at the record level. The convention is
the same as taxonomy.md §2 (Null semantics):

- ``high``   — explicit source statement, cross-confirmed by ≥2 sources.
- ``medium`` — single authoritative source (Wikipedia, Discogs) explicitly
  stating the value.
- ``low``    — inferred from indirect evidence (artist-known-for-X,
  era-typical-of-Y); the LLM is making a defensible guess.
- ``uncertain`` — sources disagree, OR the field could not be determined.
  Almost always paired with ``value: null``.

``needs_expert_review: true`` is the hard gate from taxonomy.md §2 — the
evaluation harness must EXCLUDE flagged values from gold-comparison metrics
until a human (Aran/Murat) clears the flag.
"""

from __future__ import annotations

from datetime import datetime, timezone
from typing import Any, Literal

from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator

# Per-field + overall confidence enum. Mirrors taxonomy.md "Null semantics"
# vocabulary. ``uncertain`` is distinct from ``low`` because the evaluation
# harness treats them differently: ``low`` still emits a label (with caveat);
# ``uncertain`` means we genuinely don't know.
Confidence = Literal["high", "medium", "low", "uncertain"]
_ARRANGEMENT_AESTHETIC_TERMS = {
    "orchestral-arabesk",
    "bare-bones-halk",
    "electric-anadolu-rock",
    "modern-pop-production",
    "gazino-fantezi",
    "studio-polish",
    "acoustic-singer-songwriter",
    "dizi-OST-orchestral",
}
_RECORDING_CONTEXT_TERMS = {"studio", "live-recording", "lo-fi-cassette", "broadcast"}

# Source-type taxonomy. The list is intentionally finite + LLM-facing —
# extending it requires a schema bump. ``other`` is the escape hatch for
# one-off references; everything else should be one of the named buckets so
# the source-quality scoreboard in the batch summary can group cleanly.
SourceType = Literal[
    "wikipedia_tr",
    "wikipedia_en",
    "discogs",
    "musicbrainz",
    "youtube_description",
    "fizy",
    "spotify",
    "blog",
    "symbtr",  # SymbTr notation corpus — composition-level makam + usul
    "other",
]


class Source(BaseModel):
    """One piece of source material fetched for a track.

    ``excerpt`` is a 200–500 char snippet that supports the extracted fields.
    We do NOT cache the full fetched page — that's an order of magnitude
    more disk and the LLM only sees the excerpt anyway. If we need to
    re-extract later, ``url`` is enough to re-fetch.
    """

    model_config = ConfigDict(extra="forbid")

    url: str
    source_type: SourceType
    fetched_at: datetime
    # 200–500 chars is the empirical sweet spot — short enough that 5–6
    # sources fit in a single LLM context window without trimming, long
    # enough to carry artist/year/album mentions plus surrounding context.
    excerpt: str = Field(min_length=1, max_length=2000)
    # Free-form one-line note from the fetcher — e.g. "matched on song title
    # + artist", "YouTube description first 500 chars", "Discogs master
    # release 1125630". Optional; surfaces in the batch summary for debugging.
    notes: str = ""

    @field_validator("url")
    @classmethod
    def _url_non_empty(cls, v: str) -> str:
        v = v.strip()
        if not v:
            raise ValueError("url must be non-empty")
        return v


class GroundTruthRecord(BaseModel):
    """Source-verified ground-truth metadata for one track.

    Per taxonomy.md §3, confidence and provenance are PER-FIELD, not per-track.
    For v0.1 we collapse the per-field metadata block (`source_url`,
    `source_authority`, `validated_by`, `extraction_method`) to a single
    per-field confidence + a record-level sources list, because the
    research pipeline produces sources that span multiple fields. v0.2
    can split if the evaluation harness needs per-field provenance.

    NULL semantics (taxonomy.md §2):
    - ``null`` value = "not determined / no source consensus". Almost
      always paired with confidence=``uncertain``.
    - Empty list (e.g. ``instrumentation=[]``) = "not analysed", NOT
      "no instruments present".
    """

    model_config = ConfigDict(extra="forbid")

    # Identity
    track_id: str
    research_date: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
    overall_confidence: Confidence

    # ---- Factual identifiers ------------------------------------------------
    artist: str | None
    artist_confidence: Confidence
    year: int | None  # original recording year (taxonomy.md dim 10 convention)
    year_confidence: Confidence
    album: str | None = None
    composer: str | None = None  # bestekâr
    lyricist: str | None = None  # söz yazarı

    # ---- Sync-licensing / supplementary expert data (Ek Veri v2) -------------
    vocal_language: str | None = None        # "Türkçe", "Lazca / Türkçe (karma)", …
    pd_status: str | None = None             # composition public-domain status (sync-critical)
    cover_status: str | None = None          # "Orijinal" / "Cover" / "Türkçe Cover (Orijinal: …)"
    sound_alike: str | None = None           # reference / sound-alike (DRAFT, derived)
    scene_tags: list[str] = Field(default_factory=list)  # usage/scene tags (DRAFT, derived)

    # ---- Taxonomy dimensions (v0.1 draft) -----------------------------------
    # Dimension 1 — genre family (controlled vocab; arabesk / halk / sanat /
    # pop / rock / fantezi / özgün / hip-hop / electronic / other).
    genre_family: str | None = None
    genre_family_confidence: Confidence = "uncertain"

    # Dimension 2 — genre subtype (controlled vocab; constrained by parent).
    # Stored as dot-notation per taxonomy.md Q9 v0.1 choice
    # (e.g. ``halk.oyun-havası.roman-havası``).
    genre_subtype: str | None = None
    genre_subtype_confidence: Confidence = "uncertain"

    # Dimension 3 — yöre / regional style (karadeniz / ege / güneydoğu /
    # iç-anadolu / trakya / akdeniz / doğu-anadolu / roman / urban-istanbul /
    # urban-ankara / urban-izmir / marmara).
    yore: str | None = None
    yore_confidence: Confidence = "uncertain"

    # Dimension 4 — aksak meter / usul (single-select controlled vocab).
    # Populated by expert listening, not by inference outputs.
    aksak_meter: str | None = None
    aksak_meter_confidence: Confidence = "uncertain"

    # Makam (modal structure) — composition-level, sourced from notation
    # (e.g. the SymbTr corpus) when a score match exists; otherwise unset.
    # Free text controlled vocab (hicaz, rast, nihavend, hüzzam, …).
    makam: str | None = None
    makam_confidence: Confidence = "uncertain"

    # Expert felt/measured-tempo note ("his/ölçü") — the perceived tempo read,
    # which corrects librosa's BPM when it double-counts a half-time groove
    # (e.g. "double-time altyapı; hissedilen ~73"). Free text, Turkish.
    tempo_feel: str | None = None

    # Dimension 10 — era (pre-1970 / 1970-1980 / ... / 2020-plus).
    era: str | None = None
    era_confidence: Confidence = "uncertain"

    # Dimension 12a — arrangement aesthetic: genre/arrangement idiom.
    arrangement_aesthetic: str | None = None
    arrangement_aesthetic_confidence: Confidence = "uncertain"

    # Dimension 12b — recording context: capture/release texture.
    recording_context: list[str] = Field(default_factory=list)
    recording_context_confidence: Confidence = "uncertain"

    # Dimension 6 — instrumentation (multi-select; flat tag set from
    # taxonomy.md). Only populated when sources explicitly credit the
    # instruments — not from listening (that's the inference pipeline's job).
    instrumentation: list[str] = Field(default_factory=list)

    # Dimension 7 split — human/expert curated vocal descriptors.
    vocal_configuration: list[str] = Field(default_factory=list)
    vocal_technique: list[str] = Field(default_factory=list)

    # ---- Provenance ---------------------------------------------------------
    sources: list[Source] = Field(default_factory=list)
    # Free-form description of which source combo produced this record.
    # Example values: "wikipedia_tr+discogs+llm", "youtube_description+llm",
    # "wikipedia_tr only (no other sources found)+llm". Used in the source-
    # quality scoreboard so we can see at a glance which combinations work.
    extraction_method: str
    notes: str = ""
    needs_expert_review: bool = False

    @model_validator(mode="before")
    @classmethod
    def _migrate_legacy_production_aesthetic(cls, data: Any) -> Any:
        if not isinstance(data, dict):
            return data
        data = dict(data)
        legacy = data.pop("production_aesthetic", None)
        legacy_conf = data.pop("production_aesthetic_confidence", "uncertain")
        if legacy:
            if (
                not data.get("arrangement_aesthetic")
                and legacy in _ARRANGEMENT_AESTHETIC_TERMS
            ):
                data["arrangement_aesthetic"] = legacy
                data.setdefault("arrangement_aesthetic_confidence", legacy_conf)
            if legacy in _RECORDING_CONTEXT_TERMS:
                contexts = data.get("recording_context") or []
                if isinstance(contexts, str):
                    contexts = [contexts]
                if legacy not in contexts:
                    data["recording_context"] = [*contexts, legacy]
                data.setdefault("recording_context_confidence", legacy_conf)
        return data

    @field_validator("track_id")
    @classmethod
    def _track_id_non_empty(cls, v: str) -> str:
        v = v.strip()
        if not v:
            raise ValueError("track_id must be non-empty")
        return v

    @field_validator("year")
    @classmethod
    def _year_plausible(cls, v: int | None) -> int | None:
        """Recording years for popular Turkish music run roughly 1920–present.
        Reject obvious LLM hallucinations (e.g. ``year: 2050``) so they
        surface as validation errors rather than landing in the dataset."""
        if v is None:
            return v
        if v < 1900 or v > datetime.now(timezone.utc).year + 1:
            raise ValueError(f"year {v} outside plausible range [1900, current+1]")
        return v

    @field_validator("recording_context", mode="before")
    @classmethod
    def _recording_context_list(cls, v: Any) -> list[str]:
        if v is None or v == "":
            return []
        if isinstance(v, str):
            return [v]
        return v

    @field_validator("vocal_configuration", mode="before")
    @classmethod
    def _vocal_configuration_terms(cls, v: Any) -> list[str]:
        if v is None or v == "":
            return []
        values = [v] if isinstance(v, str) else list(v)
        alias_map = {
            "solo-vocal-male": "solo-male",
            "male_vocals": "solo-male",
            "solo-vocal-female": "solo-female",
            "female_vocals": "solo-female",
            "duet": "duet-mf",
        }
        return [alias_map.get(str(item), str(item)) for item in values if item]