"""Pydantic schema for per-track ground-truth records. The ground-truth record is a separate artifact from ``TrackRecord`` (the inference-pipeline output). They are deliberately decoupled: - ``TrackRecord`` lives at ``data/outputs//.json`` and is rebuilt every time the inference pipeline runs. - ``GroundTruthRecord`` lives at ``data/groundtruth//.json`` and represents the source-of-truth (verified factual baseline) — artist, year, region, era — that the inference pipeline does NOT produce. The two are joined at evaluation time, not at write time. This keeps the ground-truth populate-once / evaluate-many lifecycle independent of the inference re-runs. Confidence semantics — every controlled-vocab field carries a per-field confidence + an overall_confidence at the record level. The convention is the same as taxonomy.md §2 (Null semantics): - ``high`` — explicit source statement, cross-confirmed by ≥2 sources. - ``medium`` — single authoritative source (Wikipedia, Discogs) explicitly stating the value. - ``low`` — inferred from indirect evidence (artist-known-for-X, era-typical-of-Y); the LLM is making a defensible guess. - ``uncertain`` — sources disagree, OR the field could not be determined. Almost always paired with ``value: null``. ``needs_expert_review: true`` is the hard gate from taxonomy.md §2 — the evaluation harness must EXCLUDE flagged values from gold-comparison metrics until a human (Aran/Murat) clears the flag. """ from __future__ import annotations from datetime import datetime, timezone from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator # Per-field + overall confidence enum. Mirrors taxonomy.md "Null semantics" # vocabulary. ``uncertain`` is distinct from ``low`` because the evaluation # harness treats them differently: ``low`` still emits a label (with caveat); # ``uncertain`` means we genuinely don't know. Confidence = Literal["high", "medium", "low", "uncertain"] _ARRANGEMENT_AESTHETIC_TERMS = { "orchestral-arabesk", "bare-bones-halk", "electric-anadolu-rock", "modern-pop-production", "gazino-fantezi", "studio-polish", "acoustic-singer-songwriter", "dizi-OST-orchestral", } _RECORDING_CONTEXT_TERMS = {"studio", "live-recording", "lo-fi-cassette", "broadcast"} # Source-type taxonomy. The list is intentionally finite + LLM-facing — # extending it requires a schema bump. ``other`` is the escape hatch for # one-off references; everything else should be one of the named buckets so # the source-quality scoreboard in the batch summary can group cleanly. SourceType = Literal[ "wikipedia_tr", "wikipedia_en", "discogs", "musicbrainz", "youtube_description", "fizy", "spotify", "blog", "symbtr", # SymbTr notation corpus — composition-level makam + usul "other", ] class Source(BaseModel): """One piece of source material fetched for a track. ``excerpt`` is a 200–500 char snippet that supports the extracted fields. We do NOT cache the full fetched page — that's an order of magnitude more disk and the LLM only sees the excerpt anyway. If we need to re-extract later, ``url`` is enough to re-fetch. """ model_config = ConfigDict(extra="forbid") url: str source_type: SourceType fetched_at: datetime # 200–500 chars is the empirical sweet spot — short enough that 5–6 # sources fit in a single LLM context window without trimming, long # enough to carry artist/year/album mentions plus surrounding context. excerpt: str = Field(min_length=1, max_length=2000) # Free-form one-line note from the fetcher — e.g. "matched on song title # + artist", "YouTube description first 500 chars", "Discogs master # release 1125630". Optional; surfaces in the batch summary for debugging. notes: str = "" @field_validator("url") @classmethod def _url_non_empty(cls, v: str) -> str: v = v.strip() if not v: raise ValueError("url must be non-empty") return v class GroundTruthRecord(BaseModel): """Source-verified ground-truth metadata for one track. Per taxonomy.md §3, confidence and provenance are PER-FIELD, not per-track. For v0.1 we collapse the per-field metadata block (`source_url`, `source_authority`, `validated_by`, `extraction_method`) to a single per-field confidence + a record-level sources list, because the research pipeline produces sources that span multiple fields. v0.2 can split if the evaluation harness needs per-field provenance. NULL semantics (taxonomy.md §2): - ``null`` value = "not determined / no source consensus". Almost always paired with confidence=``uncertain``. - Empty list (e.g. ``instrumentation=[]``) = "not analysed", NOT "no instruments present". """ model_config = ConfigDict(extra="forbid") # Identity track_id: str research_date: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) overall_confidence: Confidence # ---- Factual identifiers ------------------------------------------------ artist: str | None artist_confidence: Confidence year: int | None # original recording year (taxonomy.md dim 10 convention) year_confidence: Confidence album: str | None = None composer: str | None = None # bestekâr lyricist: str | None = None # söz yazarı # ---- Sync-licensing / supplementary expert data (Ek Veri v2) ------------- vocal_language: str | None = None # "Türkçe", "Lazca / Türkçe (karma)", … pd_status: str | None = None # composition public-domain status (sync-critical) cover_status: str | None = None # "Orijinal" / "Cover" / "Türkçe Cover (Orijinal: …)" sound_alike: str | None = None # reference / sound-alike (DRAFT, derived) scene_tags: list[str] = Field(default_factory=list) # usage/scene tags (DRAFT, derived) # ---- Taxonomy dimensions (v0.1 draft) ----------------------------------- # Dimension 1 — genre family (controlled vocab; arabesk / halk / sanat / # pop / rock / fantezi / özgün / hip-hop / electronic / other). genre_family: str | None = None genre_family_confidence: Confidence = "uncertain" # Dimension 2 — genre subtype (controlled vocab; constrained by parent). # Stored as dot-notation per taxonomy.md Q9 v0.1 choice # (e.g. ``halk.oyun-havası.roman-havası``). genre_subtype: str | None = None genre_subtype_confidence: Confidence = "uncertain" # Dimension 3 — yöre / regional style (karadeniz / ege / güneydoğu / # iç-anadolu / trakya / akdeniz / doğu-anadolu / roman / urban-istanbul / # urban-ankara / urban-izmir / marmara). yore: str | None = None yore_confidence: Confidence = "uncertain" # Dimension 4 — aksak meter / usul (single-select controlled vocab). # Populated by expert listening, not by inference outputs. aksak_meter: str | None = None aksak_meter_confidence: Confidence = "uncertain" # Makam (modal structure) — composition-level, sourced from notation # (e.g. the SymbTr corpus) when a score match exists; otherwise unset. # Free text controlled vocab (hicaz, rast, nihavend, hüzzam, …). makam: str | None = None makam_confidence: Confidence = "uncertain" # Expert felt/measured-tempo note ("his/ölçü") — the perceived tempo read, # which corrects librosa's BPM when it double-counts a half-time groove # (e.g. "double-time altyapı; hissedilen ~73"). Free text, Turkish. tempo_feel: str | None = None # Dimension 10 — era (pre-1970 / 1970-1980 / ... / 2020-plus). era: str | None = None era_confidence: Confidence = "uncertain" # Dimension 12a — arrangement aesthetic: genre/arrangement idiom. arrangement_aesthetic: str | None = None arrangement_aesthetic_confidence: Confidence = "uncertain" # Dimension 12b — recording context: capture/release texture. recording_context: list[str] = Field(default_factory=list) recording_context_confidence: Confidence = "uncertain" # Dimension 6 — instrumentation (multi-select; flat tag set from # taxonomy.md). Only populated when sources explicitly credit the # instruments — not from listening (that's the inference pipeline's job). instrumentation: list[str] = Field(default_factory=list) # Dimension 7 split — human/expert curated vocal descriptors. vocal_configuration: list[str] = Field(default_factory=list) vocal_technique: list[str] = Field(default_factory=list) # ---- Provenance --------------------------------------------------------- sources: list[Source] = Field(default_factory=list) # Free-form description of which source combo produced this record. # Example values: "wikipedia_tr+discogs+llm", "youtube_description+llm", # "wikipedia_tr only (no other sources found)+llm". Used in the source- # quality scoreboard so we can see at a glance which combinations work. extraction_method: str notes: str = "" needs_expert_review: bool = False @model_validator(mode="before") @classmethod def _migrate_legacy_production_aesthetic(cls, data: Any) -> Any: if not isinstance(data, dict): return data data = dict(data) legacy = data.pop("production_aesthetic", None) legacy_conf = data.pop("production_aesthetic_confidence", "uncertain") if legacy: if ( not data.get("arrangement_aesthetic") and legacy in _ARRANGEMENT_AESTHETIC_TERMS ): data["arrangement_aesthetic"] = legacy data.setdefault("arrangement_aesthetic_confidence", legacy_conf) if legacy in _RECORDING_CONTEXT_TERMS: contexts = data.get("recording_context") or [] if isinstance(contexts, str): contexts = [contexts] if legacy not in contexts: data["recording_context"] = [*contexts, legacy] data.setdefault("recording_context_confidence", legacy_conf) return data @field_validator("track_id") @classmethod def _track_id_non_empty(cls, v: str) -> str: v = v.strip() if not v: raise ValueError("track_id must be non-empty") return v @field_validator("year") @classmethod def _year_plausible(cls, v: int | None) -> int | None: """Recording years for popular Turkish music run roughly 1920–present. Reject obvious LLM hallucinations (e.g. ``year: 2050``) so they surface as validation errors rather than landing in the dataset.""" if v is None: return v if v < 1900 or v > datetime.now(timezone.utc).year + 1: raise ValueError(f"year {v} outside plausible range [1900, current+1]") return v @field_validator("recording_context", mode="before") @classmethod def _recording_context_list(cls, v: Any) -> list[str]: if v is None or v == "": return [] if isinstance(v, str): return [v] return v @field_validator("vocal_configuration", mode="before") @classmethod def _vocal_configuration_terms(cls, v: Any) -> list[str]: if v is None or v == "": return [] values = [v] if isinstance(v, str) else list(v) alias_map = { "solo-vocal-male": "solo-male", "male_vocals": "solo-male", "solo-vocal-female": "solo-female", "female_vocals": "solo-female", "duet": "duet-mf", } return [alias_map.get(str(item), str(item)) for item in values if item]