scripts/phase3_run_baselines.py

"""
Phase 3, Steps 4-7: run EXP-001 (energy/silence), EXP-002 (classical
features + logistic regression), EXP-003 (global vs. global+recent-window
features), plus slice and filler analysis — all on the REAL 300-clip
sample materialized by scripts/phase3_inventory_and_sample.py from the
uploaded pipecat-ai/smart-turn-data-v3.2-train shard.

Labeled everywhere as SMALL REAL-AUDIO VALIDATION — this is 300 clips from
ONE of ten training shards, not a claim about full-dataset performance.
The official test set (pipecat-ai/smart-turn-data-v3.2-test) is never
touched by this script.
"""

from __future__ import annotations

import csv
import json
import sys
import time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))

import numpy as np

from turn_detector.audio_io import load_audio
from turn_detector.baseline_energy import EnergySilenceBaseline, EnergySilenceConfig, tune_threshold
from turn_detector.baseline_classifier import ClassifierBaseline, ClassifierBaselineConfig
from turn_detector.features import FeatureConfig, extract_features
from turn_detector.evaluation import classification_metrics, slice_metrics, measure_inference_latency
from turn_detector.splits import split_random

META_PATH = Path("data/raw/phase3_sample/metadata.csv")
OUT_DIR = Path("data/raw/phase3_sample")


def load_dataset():
    with open(META_PATH) as f:
        rows = list(csv.DictReader(f))
    for r in rows:
        r["endpoint_bool"] = r["endpoint_bool"] == "True"
        r["synthetic"] = r["synthetic"] == "True"
        r["midfiller"] = {"True": True, "False": False, "": None}[r["midfiller"]]
        r["endfiller"] = {"True": True, "False": False, "": None}[r["endfiller"]]
        r["duration_sec"] = float(r["duration_sec"])
    return rows


def main():
    rows = load_dataset()
    print(f"Loaded {len(rows)} real metadata rows.")

    print("Loading real audio into memory (float32 arrays)...")
    t0 = time.time()
    for r in rows:
        r["_audio"] = load_audio(r["audio_path"])
    print(f"  loaded {len(rows)} clips in {time.time()-t0:.1f}s")

    # ---- dev/val split (random, 70/30, seed=42) — NOT the official test set ----
    dev, val = split_random(rows, val_frac=0.3, seed=42)
    print(f"\nDev/val split: {len(dev)} dev / {len(val)} val (random split, seed=42)")
    print(f"Dev label balance: {sum(r['endpoint_bool'] for r in dev)}/{len(dev)} END")
    print(f"Val label balance: {sum(r['endpoint_bool'] for r in val)}/{len(val)} END")

    results = {"label": "SMALL REAL-AUDIO VALIDATION", "n_total": len(rows),
               "n_dev": len(dev), "n_val": len(val), "source_shard": "train-00000-of-00010.parquet"}

    # =========================================================================
    # EXP-001: energy/silence baseline — tune threshold on dev, eval on val
    # =========================================================================
    print("\n" + "=" * 70)
    print("EXP-001: Energy/Silence baseline")
    print("=" * 70)

    cfg = EnergySilenceConfig(silence_rms_threshold=0.02)
    dev_pairs = [(r["_audio"], r["endpoint_bool"]) for r in dev]
    best_thresh, tuning_curve = tune_threshold(dev_pairs, cfg, candidate_thresholds_sec=np.arange(0.05, 1.55, 0.05))
    print(f"Tuned silence_duration_threshold_sec = {best_thresh:.2f} (dev accuracy = {tuning_curve[best_thresh]['accuracy']:.3f})")

    cfg_tuned = EnergySilenceConfig(silence_rms_threshold=0.02, silence_duration_threshold_sec=best_thresh)
    baseline1 = EnergySilenceBaseline(cfg_tuned)
    val_preds_1 = np.array([baseline1.predict(r["_audio"]) for r in val])
    val_true = np.array([r["endpoint_bool"] for r in val])
    metrics_1 = classification_metrics(val_true, val_preds_1)
    print(json.dumps(metrics_1, indent=2))
    results["EXP-001"] = {
        "tuned_threshold_sec": best_thresh,
        "tuning_curve_dev": tuning_curve,
        "val_metrics": metrics_1,
    }

    # =========================================================================
    # EXP-002: classical features + logistic regression
    # =========================================================================
    print("\n" + "=" * 70)
    print("EXP-002: Classical features + Logistic Regression")
    print("=" * 70)

    clf = ClassifierBaseline(ClassifierBaselineConfig())
    dev_audios = [r["_audio"] for r in dev]
    dev_labels = [r["endpoint_bool"] for r in dev]
    val_audios = [r["_audio"] for r in val]

    t0 = time.time()
    clf.fit(dev_audios, dev_labels)
    fit_time = time.time() - t0

    val_preds_2 = clf.predict(val_audios)
    metrics_2 = classification_metrics(val_true, val_preds_2)
    print(json.dumps(metrics_2, indent=2))

    latency_info = measure_inference_latency(lambda a: clf.predict_proba([a]), val_audios, n_warmup=3)
    print("Inference latency (per single clip, this machine):", json.dumps(latency_info, indent=2))

    top_feats = clf.top_features(10)
    print("Top 10 |coefficient| features:", top_feats)

    results["EXP-002"] = {
        "fit_time_sec_dev_set": fit_time,
        "n_features": len(clf.feature_order_),
        "param_count": clf.param_count(),
        "model_size_bytes": clf.model_size_bytes(),
        "val_metrics": metrics_2,
        "inference_latency": latency_info,
        "top_features": [(k, float(v)) for k, v in top_feats],
    }

    # =========================================================================
    # EXP-003: global-only vs global+recent-window features
    # =========================================================================
    print("\n" + "=" * 70)
    print("EXP-003: Global-only vs Global+recent-window features")
    print("=" * 70)

    # A: global-only (disable recent windows in the classifier's feature extraction)
    class GlobalOnlyClassifier(ClassifierBaseline):
        def _featurize(self, audios):
            return [extract_features(a, self.cfg.feature_cfg, include_recent_windows=False) for a in audios]

    clf_global = GlobalOnlyClassifier(ClassifierBaselineConfig())
    clf_global.fit(dev_audios, dev_labels)
    preds_global = clf_global.predict(val_audios)
    metrics_global = classification_metrics(val_true, preds_global)

    # B: global + recent-window (the default ClassifierBaseline, already fit above as `clf`)
    metrics_both = metrics_2  # already computed with recent windows included (default)

    delta_f1 = metrics_both["f1"] - metrics_global["f1"]
    delta_false_end = (
        (metrics_both["false_end_rate"] - metrics_global["false_end_rate"])
        if metrics_both["false_end_rate"] is not None and metrics_global["false_end_rate"] is not None
        else None
    )
    delta_false_continue = (
        (metrics_both["false_continue_rate"] - metrics_global["false_continue_rate"])
        if metrics_both["false_continue_rate"] is not None and metrics_global["false_continue_rate"] is not None
        else None
    )

    print("Global-only metrics:", json.dumps(metrics_global, indent=2))
    print("Global+recent-window metrics:", json.dumps(metrics_both, indent=2))
    print(f"Delta F1 (both - global-only): {delta_f1:+.4f}")
    print(f"Delta false_end_rate: {delta_false_end}")
    print(f"Delta false_continue_rate: {delta_false_continue}")

    results["EXP-003"] = {
        "global_only_val_metrics": metrics_global,
        "global_plus_recent_window_val_metrics": metrics_both,
        "delta_f1": delta_f1,
        "delta_false_end_rate": delta_false_end,
        "delta_false_continue_rate": delta_false_continue,
    }

    # =========================================================================
    # Slice / filler analysis (on EXP-002's val predictions)
    # =========================================================================
    print("\n" + "=" * 70)
    print("Slice analysis (EXP-002 val predictions)")
    print("=" * 70)

    val_language = np.array([r["language"] for r in val])
    val_synthetic = np.array([r["synthetic"] for r in val])

    lang_slices = slice_metrics(val_true, val_preds_2, val_language, min_samples=10)
    synth_slices = slice_metrics(val_true, val_preds_2, val_synthetic, min_samples=10)
    print("By language (min_samples=10):", json.dumps(lang_slices, indent=2))
    print("By synthetic flag (min_samples=10):", json.dumps(synth_slices, indent=2))

    # Filler analysis — NULL != False. Only compare rows where midfiller is
    # actually populated (not None), split True vs False.
    filler_known = [r for r in val if r["midfiller"] is not None]
    filler_unknown_n = len(val) - len(filler_known)
    print(f"\nFiller metadata: {len(filler_known)}/{len(val)} val rows have midfiller populated; "
          f"{filler_unknown_n} are NULL (metadata unavailable, NOT treated as 'no filler').")

    filler_results = None
    if len(filler_known) >= 20:
        fk_audios = [r["_audio"] for r in filler_known]
        fk_true = np.array([r["endpoint_bool"] for r in filler_known])
        fk_preds = clf.predict(fk_audios)
        fk_midfiller = np.array(["has_midfiller" if r["midfiller"] else "no_midfiller" for r in filler_known])
        filler_results = slice_metrics(fk_true, fk_preds, fk_midfiller, min_samples=10)
        print("Filler slice results:", json.dumps(filler_results, indent=2))
    else:
        print("Too few val rows with populated filler metadata for a meaningful slice comparison (exploratory only).")

    results["slice_analysis"] = {
        "by_language": lang_slices,
        "by_synthetic": synth_slices,
        "filler_known_n": len(filler_known),
        "filler_unknown_n": filler_unknown_n,
        "filler_slice_results": filler_results,
    }

    # =========================================================================
    # Save predictions for error analysis + full results JSON
    # =========================================================================
    val_predictions_export = []
    for r, pred1, pred2, true_label in zip(val, val_preds_1, val_preds_2, val_true):
        val_predictions_export.append({
            "id": r["id"], "language": r["language"], "dataset": r["dataset"],
            "synthetic": r["synthetic"], "midfiller": r["midfiller"], "endfiller": r["endfiller"],
            "duration_sec": r["duration_sec"], "endpoint_bool_true": bool(true_label),
            "exp001_pred": bool(pred1), "exp002_pred": bool(pred2),
            "exp001_correct": bool(pred1 == true_label), "exp002_correct": bool(pred2 == true_label),
            "audio_path": r["audio_path"],
        })

    with open(OUT_DIR / "val_predictions.json", "w") as f:
        json.dump(val_predictions_export, f, indent=2)

    with open(OUT_DIR / "phase3_results.json", "w") as f:
        json.dump(results, f, indent=2, default=str)

    print(f"\nSaved val predictions to {OUT_DIR / 'val_predictions.json'}")
    print(f"Saved full results to {OUT_DIR / 'phase3_results.json'}")


if __name__ == "__main__":
    main()